diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 47fa041e9..e16e6c62d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,15 +10,8 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 + with: + fetch-depth: 0 - uses: cachix/install-nix-action@v10 - - run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings - macos_perf_test: - runs-on: macos-latest - steps: - - name: Disable syspolicy assessments - run: | - spctl --status - sudo spctl --master-disable - - uses: actions/checkout@v2 - - uses: cachix/install-nix-action@v10 - - run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings + #- run: nix flake check + - run: nix-build -A checks.$(if [[ `uname` = Linux ]]; then echo x86_64-linux; else echo x86_64-darwin; fi) diff --git a/.version b/.version index 7208c2182..f398a2061 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.4 \ No newline at end of file +3.0 \ No newline at end of file diff --git a/Makefile b/Makefile index 332e6e971..f472ca7e5 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,7 @@ makefiles = \ src/resolve-system-dependencies/local.mk \ scripts/local.mk \ corepkgs/local.mk \ + misc/bash/local.mk \ misc/systemd/local.mk \ misc/launchd/local.mk \ misc/upstart/local.mk \ diff --git a/Makefile.config.in b/Makefile.config.in index b632444e8..5c245b8e9 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -19,6 +19,7 @@ LIBLZMA_LIBS = @LIBLZMA_LIBS@ OPENSSL_LIBS = @OPENSSL_LIBS@ PACKAGE_NAME = @PACKAGE_NAME@ PACKAGE_VERSION = @PACKAGE_VERSION@ +SHELL = @bash@ SODIUM_LIBS = @SODIUM_LIBS@ SQLITE3_LIBS = @SQLITE3_LIBS@ bash = @bash@ diff --git a/README.md b/README.md index a1588284d..03c5deb7b 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ for more details. On Linux and macOS the easiest way to Install Nix is to run the following shell command (as a user other than root): -``` +```console $ curl -L https://nixos.org/nix/install | sh ``` @@ -20,27 +20,8 @@ Information on additional installation methods is available on the [Nix download ## Building And Developing -### Building Nix - -You can build Nix using one of the targets provided by [release.nix](./release.nix): - -``` -$ nix-build ./release.nix -A build.aarch64-linux -$ nix-build ./release.nix -A build.x86_64-darwin -$ nix-build ./release.nix -A build.i686-linux -$ nix-build ./release.nix -A build.x86_64-linux -``` - -### Development Environment - -You can use the provided `shell.nix` to get a working development environment: - -``` -$ nix-shell -$ ./bootstrap.sh -$ ./configure -$ make -``` +See our [Hacking guide](https://hydra.nixos.org/job/nix/master/build.x86_64-linux/latest/download-by-type/doc/manual#chap-hacking) in our manual for instruction on how to +build nix from source with nix-build or how to get a development environment. ## Additional Resources diff --git a/configure.ac b/configure.ac index c3007b4b6..2f29cf864 100644 --- a/configure.ac +++ b/configure.ac @@ -123,6 +123,7 @@ AC_PATH_PROG(flex, flex, false) AC_PATH_PROG(bison, bison, false) AC_PATH_PROG(dot, dot) AC_PATH_PROG(lsof, lsof, lsof) +NEED_PROG(jq, jq) AC_SUBST(coreutils, [$(dirname $(type -p cat))]) diff --git a/default.nix b/default.nix new file mode 100644 index 000000000..71d1a80ad --- /dev/null +++ b/default.nix @@ -0,0 +1,3 @@ +(import (fetchTarball https://github.com/edolstra/flake-compat/archive/master.tar.gz) { + src = ./.; +}).defaultNix diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index 1fa74a143..9c55526a3 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -370,34 +370,6 @@ false. - hashed-mirrors - - A list of web servers used by - builtins.fetchurl to obtain files by - hash. The default is - http://tarballs.nixos.org/. Given a hash type - ht and a base-16 hash - h, Nix will try to download the file - from - hashed-mirror/ht/h. - This allows files to be downloaded even if they have disappeared - from their original URI. For example, given the default mirror - http://tarballs.nixos.org/, when building the derivation - - -builtins.fetchurl { - url = "https://example.org/foo-1.2.3.tar.xz"; - sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"; -} - - - Nix will attempt to download this file from - http://tarballs.nixos.org/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae - first. If it is not available there, if will try the original URI. - - - - http-connections The maximum number of parallel TCP connections diff --git a/doc/manual/expressions/builder-syntax.xml b/doc/manual/expressions/builder-syntax.xml deleted file mode 100644 index e51bade44..000000000 --- a/doc/manual/expressions/builder-syntax.xml +++ /dev/null @@ -1,119 +0,0 @@ -
- -Builder Syntax - -Build script for GNU Hello -(<filename>builder.sh</filename>) - -source $stdenv/setup - -PATH=$perl/bin:$PATH - -tar xvfz $src -cd hello-* -./configure --prefix=$out -make -make install - - - shows the builder referenced -from Hello's Nix expression (stored in -pkgs/applications/misc/hello/ex-1/builder.sh). -The builder can actually be made a lot shorter by using the -generic builder functions provided by -stdenv, but here we write out the build steps to -elucidate what a builder does. It performs the following -steps: - - - - - - When Nix runs a builder, it initially completely clears the - environment (except for the attributes declared in the - derivation). For instance, the PATH variable is - emptyActually, it's initialised to - /path-not-set to prevent Bash from setting it - to a default value.. This is done to prevent - undeclared inputs from being used in the build process. If for - example the PATH contained - /usr/bin, then you might accidentally use - /usr/bin/gcc. - - So the first step is to set up the environment. This is - done by calling the setup script of the - standard environment. The environment variable - stdenv points to the location of the standard - environment being used. (It wasn't specified explicitly as an - attribute in , but - mkDerivation adds it automatically.) - - - - - - Since Hello needs Perl, we have to make sure that Perl is in - the PATH. The perl environment - variable points to the location of the Perl package (since it - was passed in as an attribute to the derivation), so - $perl/bin is the - directory containing the Perl interpreter. - - - - - - Now we have to unpack the sources. The - src attribute was bound to the result of - fetching the Hello source tarball from the network, so the - src environment variable points to the location in - the Nix store to which the tarball was downloaded. After - unpacking, we cd to the resulting source - directory. - - The whole build is performed in a temporary directory - created in /tmp, by the way. This directory is - removed after the builder finishes, so there is no need to clean - up the sources afterwards. Also, the temporary directory is - always newly created, so you don't have to worry about files from - previous builds interfering with the current build. - - - - - - GNU Hello is a typical Autoconf-based package, so we first - have to run its configure script. In Nix - every package is stored in a separate location in the Nix store, - for instance - /nix/store/9a54ba97fb71b65fda531012d0443ce2-hello-2.1.1. - Nix computes this path by cryptographically hashing all attributes - of the derivation. The path is passed to the builder through the - out environment variable. So here we give - configure the parameter - --prefix=$out to cause Hello to be installed in - the expected location. - - - - - - Finally we build Hello (make) and install - it into the location specified by out - (make install). - - - - - -If you are wondering about the absence of error checking on the -result of various commands called in the builder: this is because the -shell script is evaluated with Bash's option, -which causes the script to be aborted if any command fails without an -error check. - -
\ No newline at end of file diff --git a/doc/manual/hacking.xml b/doc/manual/hacking.xml index b671811d3..d25d4b84a 100644 --- a/doc/manual/hacking.xml +++ b/doc/manual/hacking.xml @@ -4,18 +4,37 @@ Hacking -This section provides some notes on how to hack on Nix. To get +This section provides some notes on how to hack on Nix. To get the latest version of Nix from GitHub: -$ git clone git://github.com/NixOS/nix.git +$ git clone https://github.com/NixOS/nix.git $ cd nix -To build it and its dependencies: +To build Nix for the current operating system/architecture use + -$ nix-build release.nix -A build.x86_64-linux +$ nix-build + +or if you have a flakes-enabled nix: + + +$ nix build + + +This will build defaultPackage attribute defined in the flake.nix file. + +To build for other platforms add one of the following suffixes to it: aarch64-linux, +i686-linux, x86_64-darwin, x86_64-linux. + +i.e. + + +nix-build -A defaultPackage.x86_64-linux + + To build all dependencies and start a shell in which all @@ -27,13 +46,27 @@ $ nix-shell To build Nix itself in this shell: [nix-shell]$ ./bootstrap.sh -[nix-shell]$ configurePhase -[nix-shell]$ make +[nix-shell]$ ./configure $configureFlags +[nix-shell]$ make -j $NIX_BUILD_CORES To install it in $(pwd)/inst and test it: [nix-shell]$ make install [nix-shell]$ make installcheck +[nix-shell]$ ./inst/bin/nix --version +nix (Nix) 2.4 + + +If you have a flakes-enabled nix you can replace: + + +$ nix-shell + + +by: + + +$ nix develop diff --git a/flake.lock b/flake.lock new file mode 100644 index 000000000..74326d294 --- /dev/null +++ b/flake.lock @@ -0,0 +1,26 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1591633336, + "narHash": "sha256-oVXv4xAnDJB03LvZGbC72vSVlIbbJr8tpjEW5o/Fdek=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "70717a337f7ae4e486ba71a500367cad697e5f09", + "type": "github" + }, + "original": { + "id": "nixpkgs", + "ref": "nixos-20.03-small", + "type": "indirect" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 6 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000..a707e90e7 --- /dev/null +++ b/flake.nix @@ -0,0 +1,443 @@ +{ + description = "The purely functional package manager"; + + inputs.nixpkgs.url = "nixpkgs/nixos-20.03-small"; + + outputs = { self, nixpkgs }: + + let + + version = builtins.readFile ./.version + versionSuffix; + versionSuffix = + if officialRelease + then "" + else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified)}_${self.shortRev or "dirty"}"; + + officialRelease = false; + + systems = [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]; + + forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system); + + # Memoize nixpkgs for different platforms for efficiency. + nixpkgsFor = forAllSystems (system: + import nixpkgs { + inherit system; + overlays = [ self.overlay ]; + } + ); + + commonDeps = pkgs: with pkgs; rec { + # Use "busybox-sandbox-shell" if present, + # if not (legacy) fallback and hope it's sufficient. + sh = pkgs.busybox-sandbox-shell or (busybox.override { + useMusl = true; + enableStatic = true; + enableMinimal = true; + extraConfig = '' + CONFIG_FEATURE_FANCY_ECHO y + CONFIG_FEATURE_SH_MATH y + CONFIG_FEATURE_SH_MATH_64 y + + CONFIG_ASH y + CONFIG_ASH_OPTIMIZE_FOR_SIZE y + + CONFIG_ASH_ALIAS y + CONFIG_ASH_BASH_COMPAT y + CONFIG_ASH_CMDCMD y + CONFIG_ASH_ECHO y + CONFIG_ASH_GETOPTS y + CONFIG_ASH_INTERNAL_GLOB y + CONFIG_ASH_JOB_CONTROL y + CONFIG_ASH_PRINTF y + CONFIG_ASH_TEST y + ''; + }); + + configureFlags = + lib.optionals stdenv.isLinux [ + "--with-sandbox-shell=${sh}/bin/busybox" + ]; + + buildDeps = + [ bison + flex + libxml2 + libxslt + docbook5 + docbook_xsl_ns + autoconf-archive + autoreconfHook + + curl + bzip2 xz brotli zlib editline + openssl pkgconfig sqlite + libarchive + boost + (if lib.versionAtLeast lib.version "20.03pre" + then nlohmann_json + else nlohmann_json.override { multipleHeaders = true; }) + nlohmann_json + + # Tests + git + mercurial + jq + gmock + ] + ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal] + ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium + ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) + (aws-sdk-cpp.override { + apis = ["s3" "transfer"]; + customMemoryManagement = false; + }); + + propagatedDeps = + [ (boehmgc.override { enableLargeConfig = true; }) + ]; + + perlDeps = + [ perl + perlPackages.DBDSQLite + ]; + }; + + in { + + # A Nixpkgs overlay that overrides the 'nix' and + # 'nix.perl-bindings' packages. + overlay = final: prev: { + + nix = with final; with commonDeps pkgs; (stdenv.mkDerivation { + name = "nix-${version}"; + + src = self; + + VERSION_SUFFIX = versionSuffix; + + outputs = [ "out" "dev" "doc" ]; + + buildInputs = buildDeps; + + propagatedBuildInputs = propagatedDeps; + + preConfigure = + '' + # Copy libboost_context so we don't get all of Boost in our closure. + # https://github.com/NixOS/nixpkgs/issues/45462 + mkdir -p $out/lib + cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib + rm -f $out/lib/*.a + ${lib.optionalString stdenv.isLinux '' + chmod u+w $out/lib/*.so.* + patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* + ''} + ''; + + configureFlags = configureFlags ++ + [ "--sysconfdir=/etc" ]; + + enableParallelBuilding = true; + + makeFlags = "profiledir=$(out)/etc/profile.d"; + + doCheck = true; + + installFlags = "sysconfdir=$(out)/etc"; + + postInstall = '' + mkdir -p $doc/nix-support + echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products + ''; + + doInstallCheck = true; + installCheckFlags = "sysconfdir=$(out)/etc"; + + separateDebugInfo = true; + }) // { + + perl-bindings = with final; stdenv.mkDerivation { + name = "nix-perl-${version}"; + + src = self; + + buildInputs = + [ autoconf-archive + autoreconfHook + nix + curl + bzip2 + xz + pkgconfig + pkgs.perl + boost + ] + ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium; + + configureFlags = '' + --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix} + --with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix} + ''; + + enableParallelBuilding = true; + + postUnpack = "sourceRoot=$sourceRoot/perl"; + }; + + }; + + }; + + hydraJobs = { + + # Binary package for various platforms. + build = nixpkgs.lib.genAttrs systems (system: nixpkgsFor.${system}.nix); + + # Perl bindings for various platforms. + perlBindings = nixpkgs.lib.genAttrs systems (system: nixpkgsFor.${system}.nix.perl-bindings); + + # Binary tarball for various platforms, containing a Nix store + # with the closure of 'nix' package, and the second half of + # the installation script. + binaryTarball = nixpkgs.lib.genAttrs systems (system: + + with nixpkgsFor.${system}; + + let + installerClosureInfo = closureInfo { rootPaths = [ nix cacert ]; }; + in + + runCommand "nix-binary-tarball-${version}" + { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck; + meta.description = "Distribution-independent Nix bootstrap binaries for ${system}"; + } + '' + cp ${installerClosureInfo}/registration $TMPDIR/reginfo + substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + + substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + + if type -p shellcheck; then + # SC1090: Don't worry about not being able to find + # $nix/etc/profile.d/nix.sh + shellcheck --exclude SC1090 $TMPDIR/install + shellcheck $TMPDIR/install-darwin-multi-user.sh + shellcheck $TMPDIR/install-systemd-multi-user.sh + + # SC1091: Don't panic about not being able to source + # /etc/profile + # SC2002: Ignore "useless cat" "error", when loading + # .reginfo, as the cat is a much cleaner + # implementation, even though it is "useless" + # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving + # root's home directory + shellcheck --external-sources \ + --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user + fi + + chmod +x $TMPDIR/install + chmod +x $TMPDIR/install-darwin-multi-user.sh + chmod +x $TMPDIR/install-systemd-multi-user.sh + chmod +x $TMPDIR/install-multi-user + dir=nix-${version}-${system} + fn=$out/$dir.tar.xz + mkdir -p $out/nix-support + echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products + tar cvfJ $fn \ + --owner=0 --group=0 --mode=u+rw,uga+r \ + --absolute-names \ + --hard-dereference \ + --transform "s,$TMPDIR/install,$dir/install," \ + --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \ + --transform "s,$NIX_STORE,$dir/store,S" \ + $TMPDIR/install $TMPDIR/install-darwin-multi-user.sh \ + $TMPDIR/install-systemd-multi-user.sh \ + $TMPDIR/install-multi-user $TMPDIR/reginfo \ + $(cat ${installerClosureInfo}/store-paths) + ''); + + # The first half of the installation script. This is uploaded + # to https://nixos.org/nix/install. It downloads the binary + # tarball for the user's system and calls the second half of the + # installation script. + installerScript = + with nixpkgsFor.x86_64-linux; + runCommand "installer-script" + { buildInputs = [ nix ]; + } + '' + mkdir -p $out/nix-support + + substitute ${./scripts/install.in} $out/install \ + ${pkgs.lib.concatMapStrings + (system: "--replace '@binaryTarball_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) ") + [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ] + } \ + --replace '@nixVersion@' ${version} + + echo "file installer $out/install" >> $out/nix-support/hydra-build-products + ''; + + # Line coverage analysis. + coverage = + with nixpkgsFor.x86_64-linux; + with commonDeps pkgs; + + releaseTools.coverageAnalysis { + name = "nix-coverage-${version}"; + + src = self; + + enableParallelBuilding = true; + + buildInputs = buildDeps ++ propagatedDeps; + + dontInstall = false; + + doInstallCheck = true; + + lcovFilter = [ "*/boost/*" "*-tab.*" ]; + + # We call `dot', and even though we just use it to + # syntax-check generated dot files, it still requires some + # fonts. So provide those. + FONTCONFIG_FILE = texFunctions.fontsConf; + + # To test building without precompiled headers. + makeFlagsArray = [ "PRECOMPILE_HEADERS=0" ]; + }; + + # System tests. + tests.remoteBuilds = import ./tests/remote-builds.nix { + system = "x86_64-linux"; + inherit nixpkgs; + inherit (self) overlay; + }; + + tests.nix-copy-closure = import ./tests/nix-copy-closure.nix { + system = "x86_64-linux"; + inherit nixpkgs; + inherit (self) overlay; + }; + + tests.githubFlakes = (import ./tests/github-flakes.nix rec { + system = "x86_64-linux"; + inherit nixpkgs; + inherit (self) overlay; + }); + + tests.setuid = nixpkgs.lib.genAttrs + ["i686-linux" "x86_64-linux"] + (system: + import ./tests/setuid.nix rec { + inherit nixpkgs system; + inherit (self) overlay; + }); + + # Test whether the binary tarball works in an Ubuntu system. + tests.binaryTarball = + with nixpkgsFor.x86_64-linux; + vmTools.runInLinuxImage (runCommand "nix-binary-tarball-test" + { diskImage = vmTools.diskImages.ubuntu1204x86_64; + } + '' + set -x + useradd -m alice + su - alice -c 'tar xf ${self.hydraJobs.binaryTarball.x86_64-linux}/*.tar.*' + mkdir /dest-nix + mount -o bind /dest-nix /nix # Provide a writable /nix. + chown alice /nix + su - alice -c '_NIX_INSTALLER_TEST=1 ./nix-*/install' + su - alice -c 'nix-store --verify' + su - alice -c 'PAGER= nix-store -qR ${self.hydraJobs.build.x86_64-linux}' + + # Check whether 'nix upgrade-nix' works. + cat > /tmp/paths.nix <queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base32, true); + auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash->to_string(Base32, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -106,7 +106,7 @@ SV * queryPathInfo(char * path, int base32) XPUSHs(&PL_sv_undef); else XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0))); - auto s = info->narHash.to_string(base32 ? Base32 : Base16, true); + auto s = info->narHash->to_string(base32 ? Base32 : Base16, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); @@ -304,7 +304,10 @@ SV * derivationFromPath(char * drvPath) HV * outputs = newHV(); for (auto & i : drv.outputs) - hv_store(outputs, i.first.c_str(), i.first.size(), newSVpv(store()->printStorePath(i.second.path).c_str(), 0), 0); + hv_store( + outputs, i.first.c_str(), i.first.size(), + newSVpv(store()->printStorePath(i.second.path(*store(), drv.name)).c_str(), 0), + 0); hv_stores(hash, "outputs", newRV((SV *) outputs)); AV * inputDrvs = newAV(); diff --git a/release-common.nix b/release-common.nix deleted file mode 100644 index 4316c3c23..000000000 --- a/release-common.nix +++ /dev/null @@ -1,82 +0,0 @@ -{ pkgs }: - -with pkgs; - -rec { - # Use "busybox-sandbox-shell" if present, - # if not (legacy) fallback and hope it's sufficient. - sh = pkgs.busybox-sandbox-shell or (busybox.override { - useMusl = true; - enableStatic = true; - enableMinimal = true; - extraConfig = '' - CONFIG_FEATURE_FANCY_ECHO y - CONFIG_FEATURE_SH_MATH y - CONFIG_FEATURE_SH_MATH_64 y - - CONFIG_ASH y - CONFIG_ASH_OPTIMIZE_FOR_SIZE y - - CONFIG_ASH_ALIAS y - CONFIG_ASH_BASH_COMPAT y - CONFIG_ASH_CMDCMD y - CONFIG_ASH_ECHO y - CONFIG_ASH_GETOPTS y - CONFIG_ASH_INTERNAL_GLOB y - CONFIG_ASH_JOB_CONTROL y - CONFIG_ASH_PRINTF y - CONFIG_ASH_TEST y - ''; - }); - - configureFlags = - lib.optionals stdenv.isLinux [ - "--with-sandbox-shell=${sh}/bin/busybox" - ]; - - buildDeps = - [ bison - flex - libxml2 - libxslt - docbook5 - docbook_xsl_ns - autoconf-archive - autoreconfHook - - curl - bzip2 xz brotli zlib editline - openssl pkgconfig sqlite - libarchive - boost - nlohmann_json - - # Tests - git - mercurial - gmock - ] - ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal] - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) - ((aws-sdk-cpp.override { - apis = ["s3" "transfer"]; - customMemoryManagement = false; - }).overrideDerivation (args: { - /* - patches = args.patches or [] ++ [ (fetchpatch { - url = https://github.com/edolstra/aws-sdk-cpp/commit/3e07e1f1aae41b4c8b340735ff9e8c735f0c063f.patch; - sha256 = "1pij0v449p166f9l29x7ppzk8j7g9k9mp15ilh5qxp29c7fnvxy2"; - }) ]; - */ - })); - - propagatedDeps = - [ (boehmgc.override { enableLargeConfig = true; }) - ]; - - perlDeps = - [ perl - perlPackages.DBDSQLite - ]; -} diff --git a/release.nix b/release.nix deleted file mode 100644 index fbf9e4721..000000000 --- a/release.nix +++ /dev/null @@ -1,303 +0,0 @@ -{ nix ? builtins.fetchGit ./. -, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-20.03-small.tar.gz -, officialRelease ? false -, systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ] -}: - -let - - pkgs = import nixpkgs { system = builtins.currentSystem or "x86_64-linux"; }; - - version = - builtins.readFile ./.version - + (if officialRelease then "" else "pre${toString nix.revCount}_${nix.shortRev}"); - - jobs = rec { - - build = pkgs.lib.genAttrs systems (system: - - let pkgs = import nixpkgs { inherit system; }; in - - with pkgs; - - with import ./release-common.nix { inherit pkgs; }; - - stdenv.mkDerivation { - name = "nix-${version}"; - - src = nix; - - outputs = [ "out" "dev" "doc" ]; - - buildInputs = buildDeps; - - propagatedBuildInputs = propagatedDeps; - - preConfigure = - '' - # Copy libboost_context so we don't get all of Boost in our closure. - # https://github.com/NixOS/nixpkgs/issues/45462 - mkdir -p $out/lib - cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib - rm -f $out/lib/*.a - ${lib.optionalString stdenv.isLinux '' - chmod u+w $out/lib/*.so.* - patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* - ''} - - (cd perl; autoreconf --install --force --verbose) - ''; - - configureFlags = configureFlags ++ - [ "--sysconfdir=/etc" ]; - - enableParallelBuilding = true; - - makeFlags = "profiledir=$(out)/etc/profile.d"; - - installFlags = "sysconfdir=$(out)/etc"; - - postInstall = '' - mkdir -p $doc/nix-support - echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products - ''; - - doCheck = true; - - doInstallCheck = true; - installCheckFlags = "sysconfdir=$(out)/etc"; - - separateDebugInfo = true; - }); - - - perlBindings = pkgs.lib.genAttrs systems (system: - - let pkgs = import nixpkgs { inherit system; }; in with pkgs; - - releaseTools.nixBuild { - name = "nix-perl-${version}"; - - src = nix; - - buildInputs = - [ autoconf-archive - autoreconfHook - jobs.build.${system} - curl - bzip2 - xz - pkgconfig - pkgs.perl - boost - ] - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium; - - configureFlags = '' - --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix} - --with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix} - ''; - - enableParallelBuilding = true; - - postUnpack = "sourceRoot=$sourceRoot/perl"; - }); - - - binaryTarball = pkgs.lib.genAttrs systems (system: - - with import nixpkgs { inherit system; }; - - let - toplevel = builtins.getAttr system jobs.build; - installerClosureInfo = closureInfo { rootPaths = [ toplevel cacert ]; }; - in - - runCommand "nix-binary-tarball-${version}" - { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck; - meta.description = "Distribution-independent Nix bootstrap binaries for ${system}"; - } - '' - cp ${installerClosureInfo}/registration $TMPDIR/reginfo - cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh - substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \ - --subst-var-by nix ${toplevel} \ - --subst-var-by cacert ${cacert} - substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \ - --subst-var-by nix ${toplevel} \ - --subst-var-by cacert ${cacert} - substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \ - --subst-var-by nix ${toplevel} \ - --subst-var-by cacert ${cacert} - substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \ - --subst-var-by nix ${toplevel} \ - --subst-var-by cacert ${cacert} - - if type -p shellcheck; then - # SC1090: Don't worry about not being able to find - # $nix/etc/profile.d/nix.sh - shellcheck --exclude SC1090 $TMPDIR/install - shellcheck $TMPDIR/create-darwin-volume.sh - shellcheck $TMPDIR/install-darwin-multi-user.sh - shellcheck $TMPDIR/install-systemd-multi-user.sh - - # SC1091: Don't panic about not being able to source - # /etc/profile - # SC2002: Ignore "useless cat" "error", when loading - # .reginfo, as the cat is a much cleaner - # implementation, even though it is "useless" - # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving - # root's home directory - shellcheck --external-sources \ - --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user - fi - - chmod +x $TMPDIR/install - chmod +x $TMPDIR/create-darwin-volume.sh - chmod +x $TMPDIR/install-darwin-multi-user.sh - chmod +x $TMPDIR/install-systemd-multi-user.sh - chmod +x $TMPDIR/install-multi-user - dir=nix-${version}-${system} - fn=$out/$dir.tar.xz - mkdir -p $out/nix-support - echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products - tar cvfJ $fn \ - --owner=0 --group=0 --mode=u+rw,uga+r \ - --absolute-names \ - --hard-dereference \ - --transform "s,$TMPDIR/install,$dir/install," \ - --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \ - --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \ - --transform "s,$NIX_STORE,$dir/store,S" \ - $TMPDIR/install \ - $TMPDIR/create-darwin-volume.sh \ - $TMPDIR/install-darwin-multi-user.sh \ - $TMPDIR/install-systemd-multi-user.sh \ - $TMPDIR/install-multi-user \ - $TMPDIR/reginfo \ - $(cat ${installerClosureInfo}/store-paths) - ''); - - - coverage = - with pkgs; - - with import ./release-common.nix { inherit pkgs; }; - - releaseTools.coverageAnalysis { - name = "nix-coverage-${version}"; - - src = nix; - - enableParallelBuilding = true; - - buildInputs = buildDeps ++ propagatedDeps; - - dontInstall = false; - - doInstallCheck = true; - - lcovFilter = [ "*/boost/*" "*-tab.*" ]; - - # We call `dot', and even though we just use it to - # syntax-check generated dot files, it still requires some - # fonts. So provide those. - FONTCONFIG_FILE = texFunctions.fontsConf; - - # To test building without precompiled headers. - makeFlagsArray = [ "PRECOMPILE_HEADERS=0" ]; - }; - - - # System tests. - tests.remoteBuilds = (import ./tests/remote-builds.nix rec { - inherit nixpkgs; - nix = build.x86_64-linux; system = "x86_64-linux"; - }); - - tests.nix-copy-closure = (import ./tests/nix-copy-closure.nix rec { - inherit nixpkgs; - nix = build.x86_64-linux; system = "x86_64-linux"; - }); - - tests.setuid = pkgs.lib.genAttrs - ["i686-linux" "x86_64-linux"] - (system: - import ./tests/setuid.nix rec { - inherit nixpkgs; - nix = build.${system}; inherit system; - }); - - tests.binaryTarball = - with import nixpkgs { system = "x86_64-linux"; }; - vmTools.runInLinuxImage (runCommand "nix-binary-tarball-test" - { diskImage = vmTools.diskImages.ubuntu1204x86_64; - } - '' - set -x - useradd -m alice - su - alice -c 'tar xf ${binaryTarball.x86_64-linux}/*.tar.*' - mkdir /dest-nix - mount -o bind /dest-nix /nix # Provide a writable /nix. - chown alice /nix - su - alice -c '_NIX_INSTALLER_TEST=1 ./nix-*/install' - su - alice -c 'nix-store --verify' - su - alice -c 'PAGER= nix-store -qR ${build.x86_64-linux}' - - # Check whether 'nix upgrade-nix' works. - cat > /tmp/paths.nix <> $out/nix-support/hydra-build-products - ''; - - }; - - -in jobs diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 00c9d540b..e5cc4d7ed 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -37,6 +37,8 @@ readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix- readonly NIX_INSTALLED_NIX="@nix@" readonly NIX_INSTALLED_CACERT="@cacert@" +#readonly NIX_INSTALLED_NIX="/nix/store/j8dbv5w6jl34caywh2ygdy88knx1mdf7-nix-2.3.6" +#readonly NIX_INSTALLED_CACERT="/nix/store/7dxhzymvy330i28ii676fl1pqwcahv2f-nss-cacert-3.49.2" readonly EXTRACTED_NIX_PATH="$(dirname "$0")" readonly ROOT_HOME=$(echo ~root) @@ -69,9 +71,11 @@ uninstall_directions() { subheader "Uninstalling nix:" local step=0 - if poly_service_installed_check; then + if [ -e /run/systemd/system ] && poly_service_installed_check; then step=$((step + 1)) poly_service_uninstall_directions "$step" + else + step=$((step + 1)) fi for profile_target in "${PROFILE_TARGETS[@]}"; do @@ -250,7 +254,9 @@ function finish_success { echo "But fetching the nixpkgs channel failed. (Are you offline?)" echo "To try again later, run \"sudo -i nix-channel --update nixpkgs\"." fi - cat <&2 -elif [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then +elif [ "$(uname -s)" = "Linux" ]; then echo "Note: a multi-user installation is possible. See https://nixos.org/nix/manual/#sect-multi-user-installation" >&2 fi @@ -122,7 +122,7 @@ if [ "$(uname -s)" = "Darwin" ]; then fi if [ "$INSTALL_MODE" = "daemon" ]; then - printf '\e[1;31mSwitching to the Daemon-based Installer\e[0m\n' + printf '\e[1;31mSwitching to the Multi-user Installer\e[0m\n' exec "$self/install-multi-user" exit 0 fi @@ -207,7 +207,7 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then if [ -w "$fn" ]; then if ! grep -q "$p" "$fn"; then echo "modifying $fn..." >&2 - echo "if [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn" + echo -e "\nif [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn" fi added=1 break @@ -218,7 +218,7 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then if [ -w "$fn" ]; then if ! grep -q "$p" "$fn"; then echo "modifying $fn..." >&2 - echo "if [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn" + echo -e "\nif [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn" fi added=1 break diff --git a/shell.nix b/shell.nix index 17aaa05ed..330df0ab6 100644 --- a/shell.nix +++ b/shell.nix @@ -1,25 +1,3 @@ -{ useClang ? false }: - -with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-20.03-small.tar.gz) {}; - -with import ./release-common.nix { inherit pkgs; }; - -(if useClang then clangStdenv else stdenv).mkDerivation { - name = "nix"; - - buildInputs = buildDeps ++ propagatedDeps ++ perlDeps; - - inherit configureFlags; - - enableParallelBuilding = true; - - installFlags = "sysconfdir=$(out)/etc"; - - shellHook = - '' - export prefix=$(pwd)/inst - configureFlags+=" --prefix=$prefix" - PKG_CONFIG_PATH=$prefix/lib/pkgconfig:$PKG_CONFIG_PATH - PATH=$prefix/bin:$PATH - ''; -} +(import (fetchTarball https://github.com/edolstra/flake-compat/archive/master.tar.gz) { + src = ./.; +}).shellNix diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index e07117496..3579d8fff 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -33,7 +33,7 @@ std::string escapeUri(std::string uri) static string currentLoad; -static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot) +static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot) { return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true); } @@ -119,7 +119,7 @@ static int _main(int argc, char * * argv) bool rightType = false; Machine * bestMachine = nullptr; - unsigned long long bestLoad = 0; + uint64_t bestLoad = 0; for (auto & m : machines) { debug("considering building on remote machine '%s'", m.storeUri); @@ -130,8 +130,8 @@ static int _main(int argc, char * * argv) m.mandatoryMet(requiredFeatures)) { rightType = true; AutoCloseFD free; - unsigned long long load = 0; - for (unsigned long long slot = 0; slot < m.maxJobs; ++slot) { + uint64_t load = 0; + for (uint64_t slot = 0; slot < m.maxJobs; ++slot) { auto slotLock = openSlotLock(m, slot); if (lockFile(slotLock.get(), ltWrite, false)) { if (!free) { diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 44baadd53..6b48ead1f 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -4,6 +4,8 @@ #include "util.hh" #include "eval.hh" #include "fetchers.hh" +#include "registry.hh" +#include "flake/flakeref.hh" #include "store-api.hh" namespace nix { @@ -31,6 +33,27 @@ MixEvalArgs::MixEvalArgs() .labels = {"path"}, .handler = {[&](std::string s) { searchPath.push_back(s); }} }); + + addFlag({ + .longName = "impure", + .description = "allow access to mutable paths and repositories", + .handler = {[&]() { + evalSettings.pureEval = false; + }}, + }); + + addFlag({ + .longName = "override-flake", + .description = "override a flake registry value", + .labels = {"original-ref", "resolved-ref"}, + .handler = {[&](std::string _from, std::string _to) { + auto from = parseFlakeRef(_from, absPath(".")); + auto to = parseFlakeRef(_to, absPath(".")); + fetchers::Attrs extraAttrs; + if (to.subdir != "") extraAttrs["dir"] = to.subdir; + fetchers::overrideRegistry(from.input, to.input, extraAttrs); + }} + }); } Bindings * MixEvalArgs::getAutoArgs(EvalState & state) @@ -53,7 +76,7 @@ Path lookupFileArg(EvalState & state, string s) if (isUri(s)) { return state.store->toRealPath( fetchers::downloadTarball( - state.store, resolveUri(s), "source", false).storePath); + state.store, resolveUri(s), "source", false).first.storePath); } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { Path p = s.substr(1, s.size() - 2); return state.findFile(p); diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc new file mode 100644 index 000000000..deb32484f --- /dev/null +++ b/src/libexpr/eval-cache.cc @@ -0,0 +1,616 @@ +#include "eval-cache.hh" +#include "sqlite.hh" +#include "eval.hh" +#include "eval-inline.hh" +#include "store-api.hh" + +namespace nix::eval_cache { + +static const char * schema = R"sql( +create table if not exists Attributes ( + parent integer not null, + name text, + type integer not null, + value text, + context text, + primary key (parent, name) +); +)sql"; + +struct AttrDb +{ + std::atomic_bool failed{false}; + + struct State + { + SQLite db; + SQLiteStmt insertAttribute; + SQLiteStmt insertAttributeWithContext; + SQLiteStmt queryAttribute; + SQLiteStmt queryAttributes; + std::unique_ptr txn; + }; + + std::unique_ptr> _state; + + AttrDb(const Hash & fingerprint) + : _state(std::make_unique>()) + { + auto state(_state->lock()); + + Path cacheDir = getCacheDir() + "/nix/eval-cache-v2"; + createDirs(cacheDir); + + Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite"; + + state->db = SQLite(dbPath); + state->db.isCache(); + state->db.exec(schema); + + state->insertAttribute.create(state->db, + "insert or replace into Attributes(parent, name, type, value) values (?, ?, ?, ?)"); + + state->insertAttributeWithContext.create(state->db, + "insert or replace into Attributes(parent, name, type, value, context) values (?, ?, ?, ?, ?)"); + + state->queryAttribute.create(state->db, + "select rowid, type, value, context from Attributes where parent = ? and name = ?"); + + state->queryAttributes.create(state->db, + "select name from Attributes where parent = ?"); + + state->txn = std::make_unique(state->db); + } + + ~AttrDb() + { + try { + auto state(_state->lock()); + if (!failed) + state->txn->commit(); + state->txn.reset(); + } catch (...) { + ignoreException(); + } + } + + template + AttrId doSQLite(F && fun) + { + if (failed) return 0; + try { + return fun(); + } catch (SQLiteError &) { + ignoreException(); + failed = true; + return 0; + } + } + + AttrId setAttrs( + AttrKey key, + const std::vector & attrs) + { + return doSQLite([&]() + { + auto state(_state->lock()); + + state->insertAttribute.use() + (key.first) + (key.second) + (AttrType::FullAttrs) + (0, false).exec(); + + AttrId rowId = state->db.getLastInsertedRowId(); + assert(rowId); + + for (auto & attr : attrs) + state->insertAttribute.use() + (rowId) + (attr) + (AttrType::Placeholder) + (0, false).exec(); + + return rowId; + }); + } + + AttrId setString( + AttrKey key, + std::string_view s, + const char * * context = nullptr) + { + return doSQLite([&]() + { + auto state(_state->lock()); + + if (context) { + std::string ctx; + for (const char * * p = context; *p; ++p) { + if (p != context) ctx.push_back(' '); + ctx.append(*p); + } + state->insertAttributeWithContext.use() + (key.first) + (key.second) + (AttrType::String) + (s) + (ctx).exec(); + } else { + state->insertAttribute.use() + (key.first) + (key.second) + (AttrType::String) + (s).exec(); + } + + return state->db.getLastInsertedRowId(); + }); + } + + AttrId setBool( + AttrKey key, + bool b) + { + return doSQLite([&]() + { + auto state(_state->lock()); + + state->insertAttribute.use() + (key.first) + (key.second) + (AttrType::Bool) + (b ? 1 : 0).exec(); + + return state->db.getLastInsertedRowId(); + }); + } + + AttrId setPlaceholder(AttrKey key) + { + return doSQLite([&]() + { + auto state(_state->lock()); + + state->insertAttribute.use() + (key.first) + (key.second) + (AttrType::Placeholder) + (0, false).exec(); + + return state->db.getLastInsertedRowId(); + }); + } + + AttrId setMissing(AttrKey key) + { + return doSQLite([&]() + { + auto state(_state->lock()); + + state->insertAttribute.use() + (key.first) + (key.second) + (AttrType::Missing) + (0, false).exec(); + + return state->db.getLastInsertedRowId(); + }); + } + + AttrId setMisc(AttrKey key) + { + return doSQLite([&]() + { + auto state(_state->lock()); + + state->insertAttribute.use() + (key.first) + (key.second) + (AttrType::Misc) + (0, false).exec(); + + return state->db.getLastInsertedRowId(); + }); + } + + AttrId setFailed(AttrKey key) + { + return doSQLite([&]() + { + auto state(_state->lock()); + + state->insertAttribute.use() + (key.first) + (key.second) + (AttrType::Failed) + (0, false).exec(); + + return state->db.getLastInsertedRowId(); + }); + } + + std::optional> getAttr( + AttrKey key, + SymbolTable & symbols) + { + auto state(_state->lock()); + + auto queryAttribute(state->queryAttribute.use()(key.first)(key.second)); + if (!queryAttribute.next()) return {}; + + auto rowId = (AttrType) queryAttribute.getInt(0); + auto type = (AttrType) queryAttribute.getInt(1); + + switch (type) { + case AttrType::Placeholder: + return {{rowId, placeholder_t()}}; + case AttrType::FullAttrs: { + // FIXME: expensive, should separate this out. + std::vector attrs; + auto queryAttributes(state->queryAttributes.use()(rowId)); + while (queryAttributes.next()) + attrs.push_back(symbols.create(queryAttributes.getStr(0))); + return {{rowId, attrs}}; + } + case AttrType::String: { + std::vector> context; + if (!queryAttribute.isNull(3)) + for (auto & s : tokenizeString>(queryAttribute.getStr(3), ";")) + context.push_back(decodeContext(s)); + return {{rowId, string_t{queryAttribute.getStr(2), context}}}; + } + case AttrType::Bool: + return {{rowId, queryAttribute.getInt(2) != 0}}; + case AttrType::Missing: + return {{rowId, missing_t()}}; + case AttrType::Misc: + return {{rowId, misc_t()}}; + case AttrType::Failed: + return {{rowId, failed_t()}}; + default: + throw Error("unexpected type in evaluation cache"); + } + } +}; + +static std::shared_ptr makeAttrDb(const Hash & fingerprint) +{ + try { + return std::make_shared(fingerprint); + } catch (SQLiteError &) { + ignoreException(); + return nullptr; + } +} + +EvalCache::EvalCache( + std::optional> useCache, + EvalState & state, + RootLoader rootLoader) + : db(useCache ? makeAttrDb(*useCache) : nullptr) + , state(state) + , rootLoader(rootLoader) +{ +} + +Value * EvalCache::getRootValue() +{ + if (!value) { + debug("getting root value"); + value = allocRootValue(rootLoader()); + } + return *value; +} + +std::shared_ptr EvalCache::getRoot() +{ + return std::make_shared(ref(shared_from_this()), std::nullopt); +} + +AttrCursor::AttrCursor( + ref root, + Parent parent, + Value * value, + std::optional> && cachedValue) + : root(root), parent(parent), cachedValue(std::move(cachedValue)) +{ + if (value) + _value = allocRootValue(value); +} + +AttrKey AttrCursor::getKey() +{ + if (!parent) + return {0, root->state.sEpsilon}; + if (!parent->first->cachedValue) { + parent->first->cachedValue = root->db->getAttr( + parent->first->getKey(), root->state.symbols); + assert(parent->first->cachedValue); + } + return {parent->first->cachedValue->first, parent->second}; +} + +Value & AttrCursor::getValue() +{ + if (!_value) { + if (parent) { + auto & vParent = parent->first->getValue(); + root->state.forceAttrs(vParent); + auto attr = vParent.attrs->get(parent->second); + if (!attr) + throw Error("attribute '%s' is unexpectedly missing", getAttrPathStr()); + _value = allocRootValue(attr->value); + } else + _value = allocRootValue(root->getRootValue()); + } + return **_value; +} + +std::vector AttrCursor::getAttrPath() const +{ + if (parent) { + auto attrPath = parent->first->getAttrPath(); + attrPath.push_back(parent->second); + return attrPath; + } else + return {}; +} + +std::vector AttrCursor::getAttrPath(Symbol name) const +{ + auto attrPath = getAttrPath(); + attrPath.push_back(name); + return attrPath; +} + +std::string AttrCursor::getAttrPathStr() const +{ + return concatStringsSep(".", getAttrPath()); +} + +std::string AttrCursor::getAttrPathStr(Symbol name) const +{ + return concatStringsSep(".", getAttrPath(name)); +} + +Value & AttrCursor::forceValue() +{ + debug("evaluating uncached attribute %s", getAttrPathStr()); + + auto & v = getValue(); + + try { + root->state.forceValue(v); + } catch (EvalError &) { + debug("setting '%s' to failed", getAttrPathStr()); + if (root->db) + cachedValue = {root->db->setFailed(getKey()), failed_t()}; + throw; + } + + if (root->db && (!cachedValue || std::get_if(&cachedValue->second))) { + if (v.type == tString) + cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context), v.string.s}; + else if (v.type == tPath) + cachedValue = {root->db->setString(getKey(), v.path), v.path}; + else if (v.type == tBool) + cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean}; + else if (v.type == tAttrs) + ; // FIXME: do something? + else + cachedValue = {root->db->setMisc(getKey()), misc_t()}; + } + + return v; +} + +std::shared_ptr AttrCursor::maybeGetAttr(Symbol name) +{ + if (root->db) { + if (!cachedValue) + cachedValue = root->db->getAttr(getKey(), root->state.symbols); + + if (cachedValue) { + if (auto attrs = std::get_if>(&cachedValue->second)) { + for (auto & attr : *attrs) + if (attr == name) + return std::make_shared(root, std::make_pair(shared_from_this(), name)); + return nullptr; + } else if (std::get_if(&cachedValue->second)) { + auto attr = root->db->getAttr({cachedValue->first, name}, root->state.symbols); + if (attr) { + if (std::get_if(&attr->second)) + return nullptr; + else if (std::get_if(&attr->second)) + throw EvalError("cached failure of attribute '%s'", getAttrPathStr(name)); + else + return std::make_shared(root, + std::make_pair(shared_from_this(), name), nullptr, std::move(attr)); + } + // Incomplete attrset, so need to fall thru and + // evaluate to see whether 'name' exists + } else + return nullptr; + //throw TypeError("'%s' is not an attribute set", getAttrPathStr()); + } + } + + auto & v = forceValue(); + + if (v.type != tAttrs) + return nullptr; + //throw TypeError("'%s' is not an attribute set", getAttrPathStr()); + + auto attr = v.attrs->get(name); + + if (!attr) { + if (root->db) { + if (!cachedValue) + cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()}; + root->db->setMissing({cachedValue->first, name}); + } + return nullptr; + } + + std::optional> cachedValue2; + if (root->db) { + if (!cachedValue) + cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()}; + cachedValue2 = {root->db->setPlaceholder({cachedValue->first, name}), placeholder_t()}; + } + + return std::make_shared( + root, std::make_pair(shared_from_this(), name), attr->value, std::move(cachedValue2)); +} + +std::shared_ptr AttrCursor::maybeGetAttr(std::string_view name) +{ + return maybeGetAttr(root->state.symbols.create(name)); +} + +std::shared_ptr AttrCursor::getAttr(Symbol name) +{ + auto p = maybeGetAttr(name); + if (!p) + throw Error("attribute '%s' does not exist", getAttrPathStr(name)); + return p; +} + +std::shared_ptr AttrCursor::getAttr(std::string_view name) +{ + return getAttr(root->state.symbols.create(name)); +} + +std::shared_ptr AttrCursor::findAlongAttrPath(const std::vector & attrPath) +{ + auto res = shared_from_this(); + for (auto & attr : attrPath) { + res = res->maybeGetAttr(attr); + if (!res) return {}; + } + return res; +} + +std::string AttrCursor::getString() +{ + if (root->db) { + if (!cachedValue) + cachedValue = root->db->getAttr(getKey(), root->state.symbols); + if (cachedValue && !std::get_if(&cachedValue->second)) { + if (auto s = std::get_if(&cachedValue->second)) { + debug("using cached string attribute '%s'", getAttrPathStr()); + return s->first; + } else + throw TypeError("'%s' is not a string", getAttrPathStr()); + } + } + + auto & v = forceValue(); + + if (v.type != tString && v.type != tPath) + throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type)); + + return v.type == tString ? v.string.s : v.path; +} + +string_t AttrCursor::getStringWithContext() +{ + if (root->db) { + if (!cachedValue) + cachedValue = root->db->getAttr(getKey(), root->state.symbols); + if (cachedValue && !std::get_if(&cachedValue->second)) { + if (auto s = std::get_if(&cachedValue->second)) { + debug("using cached string attribute '%s'", getAttrPathStr()); + return *s; + } else + throw TypeError("'%s' is not a string", getAttrPathStr()); + } + } + + auto & v = forceValue(); + + if (v.type == tString) + return {v.string.s, v.getContext()}; + else if (v.type == tPath) + return {v.path, {}}; + else + throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type)); +} + +bool AttrCursor::getBool() +{ + if (root->db) { + if (!cachedValue) + cachedValue = root->db->getAttr(getKey(), root->state.symbols); + if (cachedValue && !std::get_if(&cachedValue->second)) { + if (auto b = std::get_if(&cachedValue->second)) { + debug("using cached Boolean attribute '%s'", getAttrPathStr()); + return *b; + } else + throw TypeError("'%s' is not a Boolean", getAttrPathStr()); + } + } + + auto & v = forceValue(); + + if (v.type != tBool) + throw TypeError("'%s' is not a Boolean", getAttrPathStr()); + + return v.boolean; +} + +std::vector AttrCursor::getAttrs() +{ + if (root->db) { + if (!cachedValue) + cachedValue = root->db->getAttr(getKey(), root->state.symbols); + if (cachedValue && !std::get_if(&cachedValue->second)) { + if (auto attrs = std::get_if>(&cachedValue->second)) { + debug("using cached attrset attribute '%s'", getAttrPathStr()); + return *attrs; + } else + throw TypeError("'%s' is not an attribute set", getAttrPathStr()); + } + } + + auto & v = forceValue(); + + if (v.type != tAttrs) + throw TypeError("'%s' is not an attribute set", getAttrPathStr()); + + std::vector attrs; + for (auto & attr : *getValue().attrs) + attrs.push_back(attr.name); + std::sort(attrs.begin(), attrs.end(), [](const Symbol & a, const Symbol & b) { + return (const string &) a < (const string &) b; + }); + + if (root->db) + cachedValue = {root->db->setAttrs(getKey(), attrs), attrs}; + + return attrs; +} + +bool AttrCursor::isDerivation() +{ + auto aType = maybeGetAttr("type"); + return aType && aType->getString() == "derivation"; +} + +StorePath AttrCursor::forceDerivation() +{ + auto aDrvPath = getAttr(root->state.sDrvPath); + auto drvPath = root->state.store->parseStorePath(aDrvPath->getString()); + if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) { + /* The eval cache contains 'drvPath', but the actual path has + been garbage-collected. So force it to be regenerated. */ + aDrvPath->forceValue(); + if (!root->state.store->isValidPath(drvPath)) + throw Error("don't know how to recreate store derivation '%s'!", + root->state.store->printStorePath(drvPath)); + } + return drvPath; +} + +} diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh new file mode 100644 index 000000000..afee85fa9 --- /dev/null +++ b/src/libexpr/eval-cache.hh @@ -0,0 +1,121 @@ +#pragma once + +#include "sync.hh" +#include "hash.hh" +#include "eval.hh" + +#include +#include + +namespace nix::eval_cache { + +class AttrDb; +class AttrCursor; + +class EvalCache : public std::enable_shared_from_this +{ + friend class AttrCursor; + + std::shared_ptr db; + EvalState & state; + typedef std::function RootLoader; + RootLoader rootLoader; + RootValue value; + + Value * getRootValue(); + +public: + + EvalCache( + std::optional> useCache, + EvalState & state, + RootLoader rootLoader); + + std::shared_ptr getRoot(); +}; + +enum AttrType { + Placeholder = 0, + FullAttrs = 1, + String = 2, + Missing = 3, + Misc = 4, + Failed = 5, + Bool = 6, +}; + +struct placeholder_t {}; +struct missing_t {}; +struct misc_t {}; +struct failed_t {}; +typedef uint64_t AttrId; +typedef std::pair AttrKey; +typedef std::pair>> string_t; + +typedef std::variant< + std::vector, + string_t, + placeholder_t, + missing_t, + misc_t, + failed_t, + bool + > AttrValue; + +class AttrCursor : public std::enable_shared_from_this +{ + friend class EvalCache; + + ref root; + typedef std::optional, Symbol>> Parent; + Parent parent; + RootValue _value; + std::optional> cachedValue; + + AttrKey getKey(); + + Value & getValue(); + +public: + + AttrCursor( + ref root, + Parent parent, + Value * value = nullptr, + std::optional> && cachedValue = {}); + + std::vector getAttrPath() const; + + std::vector getAttrPath(Symbol name) const; + + std::string getAttrPathStr() const; + + std::string getAttrPathStr(Symbol name) const; + + std::shared_ptr maybeGetAttr(Symbol name); + + std::shared_ptr maybeGetAttr(std::string_view name); + + std::shared_ptr getAttr(Symbol name); + + std::shared_ptr getAttr(std::string_view name); + + std::shared_ptr findAlongAttrPath(const std::vector & attrPath); + + std::string getString(); + + string_t getStringWithContext(); + + bool getBool(); + + std::vector getAttrs(); + + bool isDerivation(); + + Value & forceValue(); + + /* Force creation of the .drv file in the Nix store. */ + StorePath forceDerivation(); +}; + +} diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 58066fa48..7a2f55504 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -199,6 +199,18 @@ string showType(const Value & v) } +bool Value::isTrivial() const +{ + return + type != tApp + && type != tPrimOpApp + && (type != tThunk + || (dynamic_cast(thunk.expr) + && ((ExprAttrs *) thunk.expr)->dynamicAttrs.empty()) + || dynamic_cast(thunk.expr)); +} + + #if HAVE_BOEHMGC /* Called when the Boehm GC runs out of memory. */ static void * oomHandler(size_t requested) @@ -337,6 +349,9 @@ EvalState::EvalState(const Strings & _searchPath, ref store) , sOutputHashAlgo(symbols.create("outputHashAlgo")) , sOutputHashMode(symbols.create("outputHashMode")) , sRecurseForDerivations(symbols.create("recurseForDerivations")) + , sDescription(symbols.create("description")) + , sSelf(symbols.create("self")) + , sEpsilon(symbols.create("")) , repair(NoRepair) , store(store) , baseEnv(allocEnv(128)) @@ -782,7 +797,7 @@ Value * ExprPath::maybeThunk(EvalState & state, Env & env) } -void EvalState::evalFile(const Path & path_, Value & v) +void EvalState::evalFile(const Path & path_, Value & v, bool mustBeTrivial) { auto path = checkSourcePath(path_); @@ -811,6 +826,11 @@ void EvalState::evalFile(const Path & path_, Value & v) fileParseCache[path2] = e; try { + // Enforce that 'flake.nix' is a direct attrset, not a + // computation. + if (mustBeTrivial && + !(dynamic_cast(e))) + throw Error("file '%s' must be an attribute set", path); eval(e, v); } catch (Error & e) { addErrorTrace(e, "while evaluating the file '%1%':", path2); @@ -1586,6 +1606,18 @@ string EvalState::forceString(Value & v, const Pos & pos) } +/* Decode a context string ‘!!’ into a pair . */ +std::pair decodeContext(std::string_view s) +{ + if (s.at(0) == '!') { + size_t index = s.find("!", 1); + return {std::string(s.substr(index + 1)), std::string(s.substr(1, index - 1))}; + } else + return {s.at(0) == '/' ? std::string(s) : std::string(s.substr(1)), ""}; +} + + void copyContext(const Value & v, PathSet & context) { if (v.string.context) @@ -1594,6 +1626,17 @@ void copyContext(const Value & v, PathSet & context) } +std::vector> Value::getContext() +{ + std::vector> res; + assert(type == tString); + if (string.context) + for (const char * * p = string.context; *p; ++p) + res.push_back(decodeContext(*p)); + return res; +} + + string EvalState::forceString(Value & v, PathSet & context, const Pos & pos) { string s = forceString(v, pos); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 0d52a7f63..8986952e3 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -4,13 +4,13 @@ #include "value.hh" #include "nixexpr.hh" #include "symbol-table.hh" -#include "hash.hh" #include "config.hh" #include #include #include #include +#include namespace nix { @@ -75,7 +75,8 @@ public: sFile, sLine, sColumn, sFunctor, sToString, sRight, sWrong, sStructuredAttrs, sBuilder, sArgs, sOutputHash, sOutputHashAlgo, sOutputHashMode, - sRecurseForDerivations; + sRecurseForDerivations, + sDescription, sSelf, sEpsilon; Symbol sDerivationNix; /* If set, force copying files to the Nix store even if they @@ -90,6 +91,7 @@ public: const ref store; + private: SrcToStore srcToStore; @@ -152,8 +154,9 @@ public: Expr * parseStdin(); /* Evaluate an expression read from the given file to normal - form. */ - void evalFile(const Path & path, Value & v); + form. Optionally enforce that the top-level expression is + trivial (i.e. doesn't require arbitrary computation). */ + void evalFile(const Path & path, Value & v, bool mustBeTrivial = false); void resetFileCache(); @@ -330,7 +333,7 @@ string showType(const Value & v); /* Decode a context string ‘!!’ into a pair . */ -std::pair decodeContext(const string & s); +std::pair decodeContext(std::string_view s); /* If `path' refers to a directory, then append "/default.nix". */ Path resolveExprPath(Path path); diff --git a/src/libexpr/flake/call-flake.nix b/src/libexpr/flake/call-flake.nix new file mode 100644 index 000000000..932ac5e90 --- /dev/null +++ b/src/libexpr/flake/call-flake.nix @@ -0,0 +1,56 @@ +lockFileStr: rootSrc: rootSubdir: + +let + + lockFile = builtins.fromJSON lockFileStr; + + allNodes = + builtins.mapAttrs + (key: node: + let + + sourceInfo = + if key == lockFile.root + then rootSrc + else fetchTree (node.info or {} // removeAttrs node.locked ["dir"]); + + subdir = if key == lockFile.root then rootSubdir else node.locked.dir or ""; + + flake = import (sourceInfo + (if subdir != "" then "/" else "") + subdir + "/flake.nix"); + + inputs = builtins.mapAttrs + (inputName: inputSpec: allNodes.${resolveInput inputSpec}) + (node.inputs or {}); + + # Resolve a input spec into a node name. An input spec is + # either a node name, or a 'follows' path from the root + # node. + resolveInput = inputSpec: + if builtins.isList inputSpec + then getInputByPath lockFile.root inputSpec + else inputSpec; + + # Follow an input path (e.g. ["dwarffs" "nixpkgs"]) from the + # root node, returning the final node. + getInputByPath = nodeName: path: + if path == [] + then nodeName + else + getInputByPath + # Since this could be a 'follows' input, call resolveInput. + (resolveInput lockFile.nodes.${nodeName}.inputs.${builtins.head path}) + (builtins.tail path); + + outputs = flake.outputs (inputs // { self = result; }); + + result = outputs // sourceInfo // { inherit inputs; inherit outputs; inherit sourceInfo; }; + in + if node.flake or true then + assert builtins.isFunction flake.outputs; + result + else + sourceInfo + ) + lockFile.nodes; + +in allNodes.${lockFile.root} diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc new file mode 100644 index 000000000..01f464859 --- /dev/null +++ b/src/libexpr/flake/flake.cc @@ -0,0 +1,609 @@ +#include "flake.hh" +#include "lockfile.hh" +#include "primops.hh" +#include "eval-inline.hh" +#include "store-api.hh" +#include "fetchers.hh" +#include "finally.hh" + +namespace nix { + +using namespace flake; + +namespace flake { + +typedef std::pair FetchedFlake; +typedef std::vector> FlakeCache; + +static std::optional lookupInFlakeCache( + const FlakeCache & flakeCache, + const FlakeRef & flakeRef) +{ + // FIXME: inefficient. + for (auto & i : flakeCache) { + if (flakeRef == i.first) { + debug("mapping '%s' to previously seen input '%s' -> '%s", + flakeRef, i.first, i.second.second); + return i.second; + } + } + + return std::nullopt; +} + +static std::tuple fetchOrSubstituteTree( + EvalState & state, + const FlakeRef & originalRef, + bool allowLookup, + FlakeCache & flakeCache) +{ + auto fetched = lookupInFlakeCache(flakeCache, originalRef); + FlakeRef resolvedRef = originalRef; + + if (!fetched) { + if (originalRef.input.isDirect()) { + fetched.emplace(originalRef.fetchTree(state.store)); + } else { + if (allowLookup) { + resolvedRef = originalRef.resolve(state.store); + auto fetchedResolved = lookupInFlakeCache(flakeCache, originalRef); + if (!fetchedResolved) fetchedResolved.emplace(resolvedRef.fetchTree(state.store)); + flakeCache.push_back({resolvedRef, fetchedResolved.value()}); + fetched.emplace(fetchedResolved.value()); + } + else { + throw Error("'%s' is an indirect flake reference, but registry lookups are not allowed", originalRef); + } + } + flakeCache.push_back({originalRef, fetched.value()}); + } + + auto [tree, lockedRef] = fetched.value(); + + debug("got tree '%s' from '%s'", + state.store->printStorePath(tree.storePath), lockedRef); + + if (state.allowedPaths) + state.allowedPaths->insert(tree.actualPath); + + assert(!originalRef.input.getNarHash() || tree.storePath == originalRef.input.computeStorePath(*state.store)); + + return {std::move(tree), resolvedRef, lockedRef}; +} + +static void expectType(EvalState & state, ValueType type, + Value & value, const Pos & pos) +{ + if (value.type == tThunk && value.isTrivial()) + state.forceValue(value, pos); + if (value.type != type) + throw Error("expected %s but got %s at %s", + showType(type), showType(value.type), pos); +} + +static std::map parseFlakeInputs( + EvalState & state, Value * value, const Pos & pos); + +static FlakeInput parseFlakeInput(EvalState & state, + const std::string & inputName, Value * value, const Pos & pos) +{ + expectType(state, tAttrs, *value, pos); + + FlakeInput input; + + auto sInputs = state.symbols.create("inputs"); + auto sUrl = state.symbols.create("url"); + auto sFlake = state.symbols.create("flake"); + auto sFollows = state.symbols.create("follows"); + + fetchers::Attrs attrs; + std::optional url; + + for (nix::Attr attr : *(value->attrs)) { + try { + if (attr.name == sUrl) { + expectType(state, tString, *attr.value, *attr.pos); + url = attr.value->string.s; + attrs.emplace("url", *url); + } else if (attr.name == sFlake) { + expectType(state, tBool, *attr.value, *attr.pos); + input.isFlake = attr.value->boolean; + } else if (attr.name == sInputs) { + input.overrides = parseFlakeInputs(state, attr.value, *attr.pos); + } else if (attr.name == sFollows) { + expectType(state, tString, *attr.value, *attr.pos); + input.follows = parseInputPath(attr.value->string.s); + } else { + state.forceValue(*attr.value); + if (attr.value->type == tString) + attrs.emplace(attr.name, attr.value->string.s); + else + throw TypeError("flake input attribute '%s' is %s while a string is expected", + attr.name, showType(*attr.value)); + } + } catch (Error & e) { + e.addTrace(*attr.pos, hintfmt("in flake attribute '%s'", attr.name)); + throw; + } + } + + if (attrs.count("type")) + try { + input.ref = FlakeRef::fromAttrs(attrs); + } catch (Error & e) { + e.addTrace(pos, hintfmt("in flake input")); + throw; + } + else { + attrs.erase("url"); + if (!attrs.empty()) + throw Error("unexpected flake input attribute '%s', at %s", attrs.begin()->first, pos); + if (url) + input.ref = parseFlakeRef(*url, {}, true); + } + + if (!input.follows && !input.ref) + input.ref = FlakeRef::fromAttrs({{"type", "indirect"}, {"id", inputName}}); + + return input; +} + +static std::map parseFlakeInputs( + EvalState & state, Value * value, const Pos & pos) +{ + std::map inputs; + + expectType(state, tAttrs, *value, pos); + + for (nix::Attr & inputAttr : *(*value).attrs) { + inputs.emplace(inputAttr.name, + parseFlakeInput(state, + inputAttr.name, + inputAttr.value, + *inputAttr.pos)); + } + + return inputs; +} + +static Flake getFlake( + EvalState & state, + const FlakeRef & originalRef, + bool allowLookup, + FlakeCache & flakeCache) +{ + auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree( + state, originalRef, allowLookup, flakeCache); + + // Guard against symlink attacks. + auto flakeFile = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir + "/flake.nix"); + if (!isInDir(flakeFile, sourceInfo.actualPath)) + throw Error("'flake.nix' file of flake '%s' escapes from '%s'", + lockedRef, state.store->printStorePath(sourceInfo.storePath)); + + Flake flake { + .originalRef = originalRef, + .resolvedRef = resolvedRef, + .lockedRef = lockedRef, + .sourceInfo = std::make_shared(std::move(sourceInfo)) + }; + + if (!pathExists(flakeFile)) + throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir); + + Value vInfo; + state.evalFile(flakeFile, vInfo, true); // FIXME: symlink attack + + expectType(state, tAttrs, vInfo, Pos(foFile, state.symbols.create(flakeFile), 0, 0)); + + auto sEdition = state.symbols.create("edition"); // FIXME: remove soon + + if (vInfo.attrs->get(sEdition)) + warn("flake '%s' has deprecated attribute 'edition'", lockedRef); + + if (auto description = vInfo.attrs->get(state.sDescription)) { + expectType(state, tString, *description->value, *description->pos); + flake.description = description->value->string.s; + } + + auto sInputs = state.symbols.create("inputs"); + + if (auto inputs = vInfo.attrs->get(sInputs)) + flake.inputs = parseFlakeInputs(state, inputs->value, *inputs->pos); + + auto sOutputs = state.symbols.create("outputs"); + + if (auto outputs = vInfo.attrs->get(sOutputs)) { + expectType(state, tLambda, *outputs->value, *outputs->pos); + flake.vOutputs = allocRootValue(outputs->value); + + if ((*flake.vOutputs)->lambda.fun->matchAttrs) { + for (auto & formal : (*flake.vOutputs)->lambda.fun->formals->formals) { + if (formal.name != state.sSelf) + flake.inputs.emplace(formal.name, FlakeInput { + .ref = parseFlakeRef(formal.name) + }); + } + } + + } else + throw Error("flake '%s' lacks attribute 'outputs'", lockedRef); + + for (auto & attr : *vInfo.attrs) { + if (attr.name != sEdition && + attr.name != state.sDescription && + attr.name != sInputs && + attr.name != sOutputs) + throw Error("flake '%s' has an unsupported attribute '%s', at %s", + lockedRef, attr.name, *attr.pos); + } + + return flake; +} + +Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup) +{ + FlakeCache flakeCache; + return getFlake(state, originalRef, allowLookup, flakeCache); +} + +/* Compute an in-memory lock file for the specified top-level flake, + and optionally write it to file, it the flake is writable. */ +LockedFlake lockFlake( + EvalState & state, + const FlakeRef & topRef, + const LockFlags & lockFlags) +{ + settings.requireExperimentalFeature("flakes"); + + FlakeCache flakeCache; + + auto flake = getFlake(state, topRef, lockFlags.useRegistries, flakeCache); + + // FIXME: symlink attack + auto oldLockFile = LockFile::read( + flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock"); + + debug("old lock file: %s", oldLockFile); + + // FIXME: check whether all overrides are used. + std::map overrides; + std::set overridesUsed, updatesUsed; + + for (auto & i : lockFlags.inputOverrides) + overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second }); + + LockFile newLockFile; + + std::vector parents; + + std::function node, + const InputPath & inputPathPrefix, + std::shared_ptr oldNode)> + computeLocks; + + computeLocks = [&]( + const FlakeInputs & flakeInputs, + std::shared_ptr node, + const InputPath & inputPathPrefix, + std::shared_ptr oldNode) + { + debug("computing lock file node '%s'", printInputPath(inputPathPrefix)); + + /* Get the overrides (i.e. attributes of the form + 'inputs.nixops.inputs.nixpkgs.url = ...'). */ + // FIXME: check this + for (auto & [id, input] : flake.inputs) { + for (auto & [idOverride, inputOverride] : input.overrides) { + auto inputPath(inputPathPrefix); + inputPath.push_back(id); + inputPath.push_back(idOverride); + overrides.insert_or_assign(inputPath, inputOverride); + } + } + + /* Go over the flake inputs, resolve/fetch them if + necessary (i.e. if they're new or the flakeref changed + from what's in the lock file). */ + for (auto & [id, input2] : flakeInputs) { + auto inputPath(inputPathPrefix); + inputPath.push_back(id); + auto inputPathS = printInputPath(inputPath); + debug("computing input '%s'", inputPathS); + + /* Do we have an override for this input from one of the + ancestors? */ + auto i = overrides.find(inputPath); + bool hasOverride = i != overrides.end(); + if (hasOverride) overridesUsed.insert(inputPath); + auto & input = hasOverride ? i->second : input2; + + /* Resolve 'follows' later (since it may refer to an input + path we haven't processed yet. */ + if (input.follows) { + InputPath target; + if (hasOverride || input.absolute) + /* 'follows' from an override is relative to the + root of the graph. */ + target = *input.follows; + else { + /* Otherwise, it's relative to the current flake. */ + target = inputPathPrefix; + for (auto & i : *input.follows) target.push_back(i); + } + debug("input '%s' follows '%s'", inputPathS, printInputPath(target)); + node->inputs.insert_or_assign(id, target); + continue; + } + + assert(input.ref); + + /* Do we have an entry in the existing lock file? And we + don't have a --update-input flag for this input? */ + std::shared_ptr oldLock; + + updatesUsed.insert(inputPath); + + if (oldNode && !lockFlags.inputUpdates.count(inputPath)) + if (auto oldLock2 = get(oldNode->inputs, id)) + if (auto oldLock3 = std::get_if<0>(&*oldLock2)) + oldLock = *oldLock3; + + if (oldLock + && oldLock->originalRef == *input.ref + && !hasOverride) + { + debug("keeping existing input '%s'", inputPathS); + + /* Copy the input from the old lock since its flakeref + didn't change and there is no override from a + higher level flake. */ + auto childNode = std::make_shared( + oldLock->lockedRef, oldLock->originalRef, oldLock->isFlake); + + node->inputs.insert_or_assign(id, childNode); + + /* If we have an --update-input flag for an input + of this input, then we must fetch the flake to + to update it. */ + auto lb = lockFlags.inputUpdates.lower_bound(inputPath); + + auto hasChildUpdate = + lb != lockFlags.inputUpdates.end() + && lb->size() > inputPath.size() + && std::equal(inputPath.begin(), inputPath.end(), lb->begin()); + + if (hasChildUpdate) { + auto inputFlake = getFlake( + state, oldLock->lockedRef, false, flakeCache); + computeLocks(inputFlake.inputs, childNode, inputPath, oldLock); + } else { + /* No need to fetch this flake, we can be + lazy. However there may be new overrides on the + inputs of this flake, so we need to check + those. */ + FlakeInputs fakeInputs; + + for (auto & i : oldLock->inputs) { + if (auto lockedNode = std::get_if<0>(&i.second)) { + fakeInputs.emplace(i.first, FlakeInput { + .ref = (*lockedNode)->originalRef, + .isFlake = (*lockedNode)->isFlake, + }); + } else if (auto follows = std::get_if<1>(&i.second)) { + fakeInputs.emplace(i.first, FlakeInput { + .follows = *follows, + .absolute = true + }); + } + } + + computeLocks(fakeInputs, childNode, inputPath, oldLock); + } + + } else { + /* We need to create a new lock file entry. So fetch + this input. */ + debug("creating new input '%s'", inputPathS); + + if (!lockFlags.allowMutable && !input.ref->input.isImmutable()) + throw Error("cannot update flake input '%s' in pure mode", inputPathS); + + if (input.isFlake) { + auto inputFlake = getFlake(state, *input.ref, lockFlags.useRegistries, flakeCache); + + /* Note: in case of an --override-input, we use + the *original* ref (input2.ref) for the + "original" field, rather than the + override. This ensures that the override isn't + nuked the next time we update the lock + file. That is, overrides are sticky unless you + use --no-write-lock-file. */ + auto childNode = std::make_shared( + inputFlake.lockedRef, input2.ref ? *input2.ref : *input.ref); + + node->inputs.insert_or_assign(id, childNode); + + /* Guard against circular flake imports. */ + for (auto & parent : parents) + if (parent == *input.ref) + throw Error("found circular import of flake '%s'", parent); + parents.push_back(*input.ref); + Finally cleanup([&]() { parents.pop_back(); }); + + /* Recursively process the inputs of this + flake. Also, unless we already have this flake + in the top-level lock file, use this flake's + own lock file. */ + computeLocks( + inputFlake.inputs, childNode, inputPath, + oldLock + ? std::dynamic_pointer_cast(oldLock) + : LockFile::read( + inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root); + } + + else { + auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree( + state, *input.ref, lockFlags.useRegistries, flakeCache); + node->inputs.insert_or_assign(id, + std::make_shared(lockedRef, *input.ref, false)); + } + } + } + }; + + computeLocks( + flake.inputs, newLockFile.root, {}, + lockFlags.recreateLockFile ? nullptr : oldLockFile.root); + + for (auto & i : lockFlags.inputOverrides) + if (!overridesUsed.count(i.first)) + warn("the flag '--override-input %s %s' does not match any input", + printInputPath(i.first), i.second); + + for (auto & i : lockFlags.inputUpdates) + if (!updatesUsed.count(i)) + warn("the flag '--update-input %s' does not match any input", printInputPath(i)); + + /* Check 'follows' inputs. */ + newLockFile.check(); + + debug("new lock file: %s", newLockFile); + + /* Check whether we need to / can write the new lock file. */ + if (!(newLockFile == oldLockFile)) { + + auto diff = LockFile::diff(oldLockFile, newLockFile); + + if (lockFlags.writeLockFile) { + if (auto sourcePath = topRef.input.getSourcePath()) { + if (!newLockFile.isImmutable()) { + if (settings.warnDirty) + warn("will not write lock file of flake '%s' because it has a mutable input", topRef); + } else { + if (!lockFlags.updateLockFile) + throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef); + + auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; + + auto path = *sourcePath + "/" + relPath; + + bool lockFileExists = pathExists(path); + + if (lockFileExists) { + auto s = chomp(diff); + if (s.empty()) + warn("updating lock file '%s'", path); + else + warn("updating lock file '%s':\n%s", path, s); + } else + warn("creating lock file '%s'", path); + + newLockFile.write(path); + + topRef.input.markChangedFile( + (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock", + lockFlags.commitLockFile + ? std::optional(fmt("%s: %s\n\nFlake input changes:\n\n%s", + relPath, lockFileExists ? "Update" : "Add", diff)) + : std::nullopt); + + /* Rewriting the lockfile changed the top-level + repo, so we should re-read it. FIXME: we could + also just clear the 'rev' field... */ + auto prevLockedRef = flake.lockedRef; + FlakeCache dummyCache; + flake = getFlake(state, topRef, lockFlags.useRegistries, dummyCache); + + if (lockFlags.commitLockFile && + flake.lockedRef.input.getRev() && + prevLockedRef.input.getRev() != flake.lockedRef.input.getRev()) + warn("committed new revision '%s'", flake.lockedRef.input.getRev()->gitRev()); + + /* Make sure that we picked up the change, + i.e. the tree should usually be dirty + now. Corner case: we could have reverted from a + dirty to a clean tree! */ + if (flake.lockedRef.input == prevLockedRef.input + && !flake.lockedRef.input.isImmutable()) + throw Error("'%s' did not change after I updated its 'flake.lock' file; is 'flake.lock' under version control?", flake.originalRef); + } + } else + throw Error("cannot write modified lock file of flake '%s' (use '--no-write-lock-file' to ignore)", topRef); + } else + warn("not writing modified lock file of flake '%s':\n%s", topRef, chomp(diff)); + } + + return LockedFlake { .flake = std::move(flake), .lockFile = std::move(newLockFile) }; +} + +void callFlake(EvalState & state, + const LockedFlake & lockedFlake, + Value & vRes) +{ + auto vLocks = state.allocValue(); + auto vRootSrc = state.allocValue(); + auto vRootSubdir = state.allocValue(); + auto vTmp1 = state.allocValue(); + auto vTmp2 = state.allocValue(); + + mkString(*vLocks, lockedFlake.lockFile.to_string()); + + emitTreeAttrs(state, *lockedFlake.flake.sourceInfo, lockedFlake.flake.lockedRef.input, *vRootSrc); + + mkString(*vRootSubdir, lockedFlake.flake.lockedRef.subdir); + + static RootValue vCallFlake = nullptr; + + if (!vCallFlake) { + vCallFlake = allocRootValue(state.allocValue()); + state.eval(state.parseExprFromString( + #include "call-flake.nix.gen.hh" + , "/"), **vCallFlake); + } + + state.callFunction(**vCallFlake, *vLocks, *vTmp1, noPos); + state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos); + state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos); +} + +static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + auto flakeRefS = state.forceStringNoCtx(*args[0], pos); + auto flakeRef = parseFlakeRef(flakeRefS, {}, true); + if (evalSettings.pureEval && !flakeRef.input.isImmutable()) + throw Error("cannot call 'getFlake' on mutable flake reference '%s', at %s (use --impure to override)", flakeRefS, pos); + + callFlake(state, + lockFlake(state, flakeRef, + LockFlags { + .updateLockFile = false, + .useRegistries = !evalSettings.pureEval, + .allowMutable = !evalSettings.pureEval, + }), + v); +} + +static RegisterPrimOp r2("__getFlake", 1, prim_getFlake, "flakes"); + +} + +Fingerprint LockedFlake::getFingerprint() const +{ + // FIXME: as an optimization, if the flake contains a lock file + // and we haven't changed it, then it's sufficient to use + // flake.sourceInfo.storePath for the fingerprint. + return hashString(htSHA256, + fmt("%s;%d;%d;%s", + flake.sourceInfo->storePath.to_string(), + flake.lockedRef.input.getRevCount().value_or(0), + flake.lockedRef.input.getLastModified().value_or(0), + lockFile)); +} + +Flake::~Flake() { } + +} diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh new file mode 100644 index 000000000..c2bb2888b --- /dev/null +++ b/src/libexpr/flake/flake.hh @@ -0,0 +1,111 @@ +#pragma once + +#include "types.hh" +#include "flakeref.hh" +#include "lockfile.hh" +#include "value.hh" + +namespace nix { + +class EvalState; + +namespace fetchers { struct Tree; } + +namespace flake { + +struct FlakeInput; + +typedef std::map FlakeInputs; + +struct FlakeInput +{ + std::optional ref; + bool isFlake = true; + std::optional follows; + bool absolute = false; // whether 'follows' is relative to the flake root + FlakeInputs overrides; +}; + +struct Flake +{ + FlakeRef originalRef; + FlakeRef resolvedRef; + FlakeRef lockedRef; + std::optional description; + std::shared_ptr sourceInfo; + FlakeInputs inputs; + RootValue vOutputs; + ~Flake(); +}; + +Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool allowLookup); + +/* Fingerprint of a locked flake; used as a cache key. */ +typedef Hash Fingerprint; + +struct LockedFlake +{ + Flake flake; + LockFile lockFile; + + Fingerprint getFingerprint() const; +}; + +struct LockFlags +{ + /* Whether to ignore the existing lock file, creating a new one + from scratch. */ + bool recreateLockFile = false; + + /* Whether to update the lock file at all. If set to false, if any + change to the lock file is needed (e.g. when an input has been + added to flake.nix), you get a fatal error. */ + bool updateLockFile = true; + + /* Whether to write the lock file to disk. If set to true, if the + any changes to the lock file are needed and the flake is not + writable (i.e. is not a local Git working tree or similar), you + get a fatal error. If set to false, Nix will use the modified + lock file in memory only, without writing it to disk. */ + bool writeLockFile = true; + + /* Whether to use the registries to lookup indirect flake + references like 'nixpkgs'. */ + bool useRegistries = true; + + /* Whether mutable flake references (i.e. those without a Git + revision or similar) without a corresponding lock are + allowed. Mutable flake references with a lock are always + allowed. */ + bool allowMutable = true; + + /* Whether to commit changes to flake.lock. */ + bool commitLockFile = false; + + /* Flake inputs to be overriden. */ + std::map inputOverrides; + + /* Flake inputs to be updated. This means that any existing lock + for those inputs will be ignored. */ + std::set inputUpdates; +}; + +LockedFlake lockFlake( + EvalState & state, + const FlakeRef & flakeRef, + const LockFlags & lockFlags); + +void callFlake( + EvalState & state, + const LockedFlake & lockedFlake, + Value & v); + +} + +void emitTreeAttrs( + EvalState & state, + const fetchers::Tree & tree, + const fetchers::Input & input, + Value & v, bool emptyRevFallback = false); + +} diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc new file mode 100644 index 000000000..6363446f6 --- /dev/null +++ b/src/libexpr/flake/flakeref.cc @@ -0,0 +1,204 @@ +#include "flakeref.hh" +#include "store-api.hh" +#include "url.hh" +#include "fetchers.hh" +#include "registry.hh" + +namespace nix { + +#if 0 +// 'dir' path elements cannot start with a '.'. We also reject +// potentially dangerous characters like ';'. +const static std::string subDirElemRegex = "(?:[a-zA-Z0-9_-]+[a-zA-Z0-9._-]*)"; +const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRegex + ")*"; +#endif + +std::string FlakeRef::to_string() const +{ + auto url = input.toURL(); + if (subdir != "") + url.query.insert_or_assign("dir", subdir); + return url.to_string(); +} + +fetchers::Attrs FlakeRef::toAttrs() const +{ + auto attrs = input.toAttrs(); + if (subdir != "") + attrs.emplace("dir", subdir); + return attrs; +} + +std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef) +{ + str << flakeRef.to_string(); + return str; +} + +bool FlakeRef::operator ==(const FlakeRef & other) const +{ + return input == other.input && subdir == other.subdir; +} + +FlakeRef FlakeRef::resolve(ref store) const +{ + auto [input2, extraAttrs] = lookupInRegistries(store, input); + return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir)); +} + +FlakeRef parseFlakeRef( + const std::string & url, const std::optional & baseDir, bool allowMissing) +{ + auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing); + if (fragment != "") + throw Error("unexpected fragment '%s' in flake reference '%s'", fragment, url); + return flakeRef; +} + +std::optional maybeParseFlakeRef( + const std::string & url, const std::optional & baseDir) +{ + try { + return parseFlakeRef(url, baseDir); + } catch (Error &) { + return {}; + } +} + +std::pair parseFlakeRefWithFragment( + const std::string & url, const std::optional & baseDir, bool allowMissing) +{ + using namespace fetchers; + + static std::string fnRegex = "[0-9a-zA-Z-._~!$&'\"()*+,;=]+"; + + static std::regex pathUrlRegex( + "(/?" + fnRegex + "(?:/" + fnRegex + ")*/?)" + + "(?:\\?(" + queryRegex + "))?" + + "(?:#(" + queryRegex + "))?", + std::regex::ECMAScript); + + static std::regex flakeRegex( + "((" + flakeIdRegexS + ")(?:/(?:" + refAndOrRevRegex + "))?)" + + "(?:#(" + queryRegex + "))?", + std::regex::ECMAScript); + + std::smatch match; + + /* Check if 'url' is a flake ID. This is an abbreviated syntax for + 'flake:?ref=&rev='. */ + + if (std::regex_match(url, match, flakeRegex)) { + auto parsedURL = ParsedURL{ + .url = url, + .base = "flake:" + std::string(match[1]), + .scheme = "flake", + .authority = "", + .path = match[1], + }; + + return std::make_pair( + FlakeRef(Input::fromURL(parsedURL), ""), + percentDecode(std::string(match[6]))); + } + + else if (std::regex_match(url, match, pathUrlRegex)) { + std::string path = match[1]; + std::string fragment = percentDecode(std::string(match[3])); + + if (baseDir) { + /* Check if 'url' is a path (either absolute or relative + to 'baseDir'). If so, search upward to the root of the + repo (i.e. the directory containing .git). */ + + path = absPath(path, baseDir, true); + + if (!S_ISDIR(lstat(path).st_mode)) + throw BadURL("path '%s' is not a flake (because it's not a directory)", path); + + if (!allowMissing && !pathExists(path + "/flake.nix")) + throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path); + + auto flakeRoot = path; + std::string subdir; + + while (flakeRoot != "/") { + if (pathExists(flakeRoot + "/.git")) { + auto base = std::string("git+file://") + flakeRoot; + + auto parsedURL = ParsedURL{ + .url = base, // FIXME + .base = base, + .scheme = "git+file", + .authority = "", + .path = flakeRoot, + .query = decodeQuery(match[2]), + }; + + if (subdir != "") { + if (parsedURL.query.count("dir")) + throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url); + parsedURL.query.insert_or_assign("dir", subdir); + } + + if (pathExists(flakeRoot + "/.git/shallow")) + parsedURL.query.insert_or_assign("shallow", "1"); + + return std::make_pair( + FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")), + fragment); + } + + subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir); + flakeRoot = dirOf(flakeRoot); + } + + } else { + if (!hasPrefix(path, "/")) + throw BadURL("flake reference '%s' is not an absolute path", url); + path = canonPath(path); + } + + fetchers::Attrs attrs; + attrs.insert_or_assign("type", "path"); + attrs.insert_or_assign("path", path); + + return std::make_pair(FlakeRef(Input::fromAttrs(std::move(attrs)), ""), fragment); + } + + else { + auto parsedURL = parseURL(url); + std::string fragment; + std::swap(fragment, parsedURL.fragment); + return std::make_pair( + FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")), + fragment); + } +} + +std::optional> maybeParseFlakeRefWithFragment( + const std::string & url, const std::optional & baseDir) +{ + try { + return parseFlakeRefWithFragment(url, baseDir); + } catch (Error & e) { + return {}; + } +} + +FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs) +{ + auto attrs2(attrs); + attrs2.erase("dir"); + return FlakeRef( + fetchers::Input::fromAttrs(std::move(attrs2)), + fetchers::maybeGetStrAttr(attrs, "dir").value_or("")); +} + +std::pair FlakeRef::fetchTree(ref store) const +{ + auto [tree, lockedInput] = input.fetch(store); + return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)}; +} + +} diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh new file mode 100644 index 000000000..f4eb825a6 --- /dev/null +++ b/src/libexpr/flake/flakeref.hh @@ -0,0 +1,53 @@ +#pragma once + +#include "types.hh" +#include "hash.hh" +#include "fetchers.hh" + +#include + +namespace nix { + +class Store; + +typedef std::string FlakeId; + +struct FlakeRef +{ + fetchers::Input input; + + Path subdir; + + bool operator==(const FlakeRef & other) const; + + FlakeRef(fetchers::Input && input, const Path & subdir) + : input(std::move(input)), subdir(subdir) + { } + + // FIXME: change to operator <<. + std::string to_string() const; + + fetchers::Attrs toAttrs() const; + + FlakeRef resolve(ref store) const; + + static FlakeRef fromAttrs(const fetchers::Attrs & attrs); + + std::pair fetchTree(ref store) const; +}; + +std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef); + +FlakeRef parseFlakeRef( + const std::string & url, const std::optional & baseDir = {}, bool allowMissing = false); + +std::optional maybeParseFlake( + const std::string & url, const std::optional & baseDir = {}); + +std::pair parseFlakeRefWithFragment( + const std::string & url, const std::optional & baseDir = {}, bool allowMissing = false); + +std::optional> maybeParseFlakeRefWithFragment( + const std::string & url, const std::optional & baseDir = {}); + +} diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc new file mode 100644 index 000000000..a74846944 --- /dev/null +++ b/src/libexpr/flake/lockfile.cc @@ -0,0 +1,338 @@ +#include "lockfile.hh" +#include "store-api.hh" + +#include + +namespace nix::flake { + +FlakeRef getFlakeRef( + const nlohmann::json & json, + const char * attr, + const char * info) +{ + auto i = json.find(attr); + if (i != json.end()) { + auto attrs = jsonToAttrs(*i); + // FIXME: remove when we drop support for version 5. + if (info) { + auto j = json.find(info); + if (j != json.end()) { + for (auto k : jsonToAttrs(*j)) + attrs.insert_or_assign(k.first, k.second); + } + } + return FlakeRef::fromAttrs(attrs); + } + + throw Error("attribute '%s' missing in lock file", attr); +} + +LockedNode::LockedNode(const nlohmann::json & json) + : lockedRef(getFlakeRef(json, "locked", "info")) + , originalRef(getFlakeRef(json, "original", nullptr)) + , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) +{ + if (!lockedRef.input.isImmutable()) + throw Error("lockfile contains mutable lock '%s'", attrsToJson(lockedRef.input.toAttrs())); +} + +StorePath LockedNode::computeStorePath(Store & store) const +{ + return lockedRef.input.computeStorePath(store); +} + +std::shared_ptr LockFile::findInput(const InputPath & path) +{ + auto pos = root; + + if (!pos) return {}; + + for (auto & elem : path) { + if (auto i = get(pos->inputs, elem)) { + if (auto node = std::get_if<0>(&*i)) + pos = *node; + else if (auto follows = std::get_if<1>(&*i)) { + pos = findInput(*follows); + if (!pos) return {}; + } + } else + return {}; + } + + return pos; +} + +LockFile::LockFile(const nlohmann::json & json, const Path & path) +{ + auto version = json.value("version", 0); + if (version < 5 || version > 7) + throw Error("lock file '%s' has unsupported version %d", path, version); + + std::unordered_map> nodeMap; + + std::function getInputs; + + getInputs = [&](Node & node, const nlohmann::json & jsonNode) + { + if (jsonNode.find("inputs") == jsonNode.end()) return; + for (auto & i : jsonNode["inputs"].items()) { + if (i.value().is_array()) { + InputPath path; + for (auto & j : i.value()) + path.push_back(j); + node.inputs.insert_or_assign(i.key(), path); + } else { + std::string inputKey = i.value(); + auto k = nodeMap.find(inputKey); + if (k == nodeMap.end()) { + auto jsonNode2 = json["nodes"][inputKey]; + auto input = std::make_shared(jsonNode2); + k = nodeMap.insert_or_assign(inputKey, input).first; + getInputs(*input, jsonNode2); + } + if (auto child = std::dynamic_pointer_cast(k->second)) + node.inputs.insert_or_assign(i.key(), child); + else + // FIXME: replace by follows node + throw Error("lock file contains cycle to root node"); + } + } + }; + + std::string rootKey = json["root"]; + nodeMap.insert_or_assign(rootKey, root); + getInputs(*root, json["nodes"][rootKey]); + + // FIXME: check that there are no cycles in version >= 7. Cycles + // between inputs are only possible using 'follows' indirections. + // Once we drop support for version <= 6, we can simplify the code + // a bit since we don't need to worry about cycles. +} + +nlohmann::json LockFile::toJson() const +{ + nlohmann::json nodes; + std::unordered_map, std::string> nodeKeys; + std::unordered_set keys; + + std::function node)> dumpNode; + + dumpNode = [&](std::string key, std::shared_ptr node) -> std::string + { + auto k = nodeKeys.find(node); + if (k != nodeKeys.end()) + return k->second; + + if (!keys.insert(key).second) { + for (int n = 2; ; ++n) { + auto k = fmt("%s_%d", key, n); + if (keys.insert(k).second) { + key = k; + break; + } + } + } + + nodeKeys.insert_or_assign(node, key); + + auto n = nlohmann::json::object(); + + if (!node->inputs.empty()) { + auto inputs = nlohmann::json::object(); + for (auto & i : node->inputs) { + if (auto child = std::get_if<0>(&i.second)) { + inputs[i.first] = dumpNode(i.first, *child); + } else if (auto follows = std::get_if<1>(&i.second)) { + auto arr = nlohmann::json::array(); + for (auto & x : *follows) + arr.push_back(x); + inputs[i.first] = std::move(arr); + } + } + n["inputs"] = std::move(inputs); + } + + if (auto lockedNode = std::dynamic_pointer_cast(node)) { + n["original"] = fetchers::attrsToJson(lockedNode->originalRef.toAttrs()); + n["locked"] = fetchers::attrsToJson(lockedNode->lockedRef.toAttrs()); + if (!lockedNode->isFlake) n["flake"] = false; + } + + nodes[key] = std::move(n); + + return key; + }; + + nlohmann::json json; + json["version"] = 7; + json["root"] = dumpNode("root", root); + json["nodes"] = std::move(nodes); + + return json; +} + +std::string LockFile::to_string() const +{ + return toJson().dump(2); +} + +LockFile LockFile::read(const Path & path) +{ + if (!pathExists(path)) return LockFile(); + return LockFile(nlohmann::json::parse(readFile(path)), path); +} + +std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile) +{ + stream << lockFile.toJson().dump(2); + return stream; +} + +void LockFile::write(const Path & path) const +{ + createDirs(dirOf(path)); + writeFile(path, fmt("%s\n", *this)); +} + +bool LockFile::isImmutable() const +{ + std::unordered_set> nodes; + + std::function node)> visit; + + visit = [&](std::shared_ptr node) + { + if (!nodes.insert(node).second) return; + for (auto & i : node->inputs) + if (auto child = std::get_if<0>(&i.second)) + visit(*child); + }; + + visit(root); + + for (auto & i : nodes) { + if (i == root) continue; + auto lockedNode = std::dynamic_pointer_cast(i); + if (lockedNode && !lockedNode->lockedRef.input.isImmutable()) return false; + } + + return true; +} + +bool LockFile::operator ==(const LockFile & other) const +{ + // FIXME: slow + return toJson() == other.toJson(); +} + +InputPath parseInputPath(std::string_view s) +{ + InputPath path; + + for (auto & elem : tokenizeString>(s, "/")) { + if (!std::regex_match(elem, flakeIdRegex)) + throw UsageError("invalid flake input path element '%s'", elem); + path.push_back(elem); + } + + return path; +} + +std::map LockFile::getAllInputs() const +{ + std::unordered_set> done; + std::map res; + + std::function node)> recurse; + + recurse = [&](const InputPath & prefix, std::shared_ptr node) + { + if (!done.insert(node).second) return; + + for (auto &[id, input] : node->inputs) { + auto inputPath(prefix); + inputPath.push_back(id); + res.emplace(inputPath, input); + if (auto child = std::get_if<0>(&input)) + recurse(inputPath, *child); + } + }; + + recurse({}, root); + + return res; +} + +std::ostream & operator <<(std::ostream & stream, const Node::Edge & edge) +{ + if (auto node = std::get_if<0>(&edge)) + stream << "'" << (*node)->lockedRef << "'"; + else if (auto follows = std::get_if<1>(&edge)) + stream << fmt("follows '%s'", printInputPath(*follows)); + return stream; +} + +static bool equals(const Node::Edge & e1, const Node::Edge & e2) +{ + if (auto n1 = std::get_if<0>(&e1)) + if (auto n2 = std::get_if<0>(&e2)) + return (*n1)->lockedRef == (*n2)->lockedRef; + if (auto f1 = std::get_if<1>(&e1)) + if (auto f2 = std::get_if<1>(&e2)) + return *f1 == *f2; + return false; +} + +std::string LockFile::diff(const LockFile & oldLocks, const LockFile & newLocks) +{ + auto oldFlat = oldLocks.getAllInputs(); + auto newFlat = newLocks.getAllInputs(); + + auto i = oldFlat.begin(); + auto j = newFlat.begin(); + std::string res; + + while (i != oldFlat.end() || j != newFlat.end()) { + if (j != newFlat.end() && (i == oldFlat.end() || i->first > j->first)) { + res += fmt("* Added '%s': %s\n", printInputPath(j->first), j->second); + ++j; + } else if (i != oldFlat.end() && (j == newFlat.end() || i->first < j->first)) { + res += fmt("* Removed '%s'\n", printInputPath(i->first)); + ++i; + } else { + if (!equals(i->second, j->second)) { + res += fmt("* Updated '%s': %s -> %s\n", + printInputPath(i->first), + i->second, + j->second); + } + ++i; + ++j; + } + } + + return res; +} + +void LockFile::check() +{ + auto inputs = getAllInputs(); + + for (auto & [inputPath, input] : inputs) { + if (auto follows = std::get_if<1>(&input)) { + if (!follows->empty() && !get(inputs, *follows)) + throw Error("input '%s' follows a non-existent input '%s'", + printInputPath(inputPath), + printInputPath(*follows)); + } + } +} + +void check(); + +std::string printInputPath(const InputPath & path) +{ + return concatStringsSep("/", path); +} + +} diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh new file mode 100644 index 000000000..5e7cfda3e --- /dev/null +++ b/src/libexpr/flake/lockfile.hh @@ -0,0 +1,85 @@ +#pragma once + +#include "flakeref.hh" + +#include + +namespace nix { +class Store; +struct StorePath; +} + +namespace nix::flake { + +using namespace fetchers; + +typedef std::vector InputPath; + +struct LockedNode; + +/* A node in the lock file. It has outgoing edges to other nodes (its + inputs). Only the root node has this type; all other nodes have + type LockedNode. */ +struct Node : std::enable_shared_from_this +{ + typedef std::variant, InputPath> Edge; + + std::map inputs; + + virtual ~Node() { } +}; + +/* A non-root node in the lock file. */ +struct LockedNode : Node +{ + FlakeRef lockedRef, originalRef; + bool isFlake = true; + + LockedNode( + const FlakeRef & lockedRef, + const FlakeRef & originalRef, + bool isFlake = true) + : lockedRef(lockedRef), originalRef(originalRef), isFlake(isFlake) + { } + + LockedNode(const nlohmann::json & json); + + StorePath computeStorePath(Store & store) const; +}; + +struct LockFile +{ + std::shared_ptr root = std::make_shared(); + + LockFile() {}; + LockFile(const nlohmann::json & json, const Path & path); + + nlohmann::json toJson() const; + + std::string to_string() const; + + static LockFile read(const Path & path); + + void write(const Path & path) const; + + bool isImmutable() const; + + bool operator ==(const LockFile & other) const; + + std::shared_ptr findInput(const InputPath & path); + + std::map getAllInputs() const; + + static std::string diff(const LockFile & oldLocks, const LockFile & newLocks); + + /* Check that every 'follows' input target exists. */ + void check(); +}; + +std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile); + +InputPath parseInputPath(std::string_view s); + +std::string printInputPath(const InputPath & path); + +} diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 9055f59a1..5d6e39aa0 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -39,7 +39,7 @@ DrvInfo::DrvInfo(EvalState & state, ref store, const std::string & drvPat if (i == drv.outputs.end()) throw Error("derivation '%s' does not have output '%s'", store->printStorePath(drvPath), outputName); - outPath = store->printStorePath(i->second.path); + outPath = store->printStorePath(i->second.path(*store, drv.name)); } diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 9ed39e745..d84b150e0 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -4,7 +4,12 @@ libexpr_NAME = libnixexpr libexpr_DIR := $(d) -libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc +libexpr_SOURCES := \ + $(wildcard $(d)/*.cc) \ + $(wildcard $(d)/primops/*.cc) \ + $(wildcard $(d)/flake/*.cc) \ + $(d)/lexer-tab.cc \ + $(d)/parser-tab.cc libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr @@ -34,4 +39,9 @@ dist-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer $(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644)) +$(foreach i, $(wildcard src/libexpr/flake/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644))) + $(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh + +$(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 878f06c96..24b21f7da 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -719,7 +719,7 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl if (isUri(elem.second)) { try { res = { true, store->toRealPath(fetchers::downloadTarball( - store, resolveUri(elem.second), "source", false).storePath) }; + store, resolveUri(elem.second), "source", false).first.storePath) }; } catch (FileTransferError & e) { logWarning({ .name = "Entry download", diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 4bbfbd3f5..437b2e749 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -30,18 +30,6 @@ namespace nix { *************************************************************/ -/* Decode a context string ‘!!’ into a pair . */ -std::pair decodeContext(const string & s) -{ - if (s.at(0) == '!') { - size_t index = s.find("!", 1); - return std::pair(string(s, index + 1), string(s, 1, index - 1)); - } else - return std::pair(s.at(0) == '/' ? s : string(s, 1), ""); -} - - InvalidPathError::InvalidPathError(const Path & path) : EvalError("path '%s' is not valid", path), path(path) {} @@ -64,7 +52,7 @@ void EvalState::realiseContext(const PathSet & context) DerivationOutputs::iterator i = drv.outputs.find(outputName); if (i == drv.outputs.end()) throw Error("derivation '%s' does not have an output named '%s'", ctxS, outputName); - allowedPaths->insert(store->printStorePath(i->second.path)); + allowedPaths->insert(store->printStorePath(i->second.path(*store, drv.name))); } } } @@ -77,7 +65,7 @@ void EvalState::realiseContext(const PathSet & context) /* For performance, prefetch all substitute info. */ StorePathSet willBuild, willSubstitute, unknown; - unsigned long long downloadSize, narSize; + uint64_t downloadSize, narSize; store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize); store->buildPaths(drvs); @@ -103,8 +91,17 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args Path realPath = state.checkSourcePath(state.toRealPath(path, context)); // FIXME - if (state.store->isStorePath(path) && state.store->isValidPath(state.store->parseStorePath(path)) && isDerivation(path)) { - Derivation drv = state.store->readDerivation(state.store->parseStorePath(path)); + auto isValidDerivationInStore = [&]() -> std::optional { + if (!state.store->isStorePath(path)) + return std::nullopt; + auto storePath = state.store->parseStorePath(path); + if (!(state.store->isValidPath(storePath) && isDerivation(path))) + return std::nullopt; + return storePath; + }; + if (auto optStorePath = isValidDerivationInStore()) { + auto storePath = *optStorePath; + Derivation drv = state.store->readDerivation(storePath); Value & w = *state.allocValue(); state.mkAttrs(w, 3 + drv.outputs.size()); Value * v2 = state.allocAttr(w, state.sDrvPath); @@ -118,7 +115,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args for (const auto & o : drv.outputs) { v2 = state.allocAttr(w, state.symbols.create(o.first)); - mkString(*v2, state.store->printStorePath(o.second.path), {"!" + o.first + "!" + path}); + mkString(*v2, state.store->printStorePath(o.second.path(*state.store, drv.name)), {"!" + o.first + "!" + path}); outputsVal->listElems()[outputs_index] = state.allocValue(); mkString(*(outputsVal->listElems()[outputs_index++]), o.first); } @@ -582,6 +579,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* Build the derivation expression by processing the attributes. */ Derivation drv; + drv.name = drvName; PathSet context; @@ -776,11 +774,12 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName); if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", DerivationOutput { - .path = std::move(outPath), - .hash = FixedOutputHash { - .method = ingestionMethod, - .hash = std::move(h), - }, + .output = DerivationOutputFixed { + .hash = FixedOutputHash { + .method = ingestionMethod, + .hash = std::move(h), + }, + }, }); } @@ -795,8 +794,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (!jsonObject) drv.env[i] = ""; drv.outputs.insert_or_assign(i, DerivationOutput { - .path = StorePath::dummy, - .hash = std::optional {}, + .output = DerivationOutputInputAddressed { + .path = StorePath::dummy, + }, }); } @@ -807,8 +807,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign(i, DerivationOutput { - .path = std::move(outPath), - .hash = std::optional(), + .output = DerivationOutputInputAddressed { + .path = std::move(outPath), + }, }); } } @@ -829,7 +830,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS}); for (auto & i : drv.outputs) { mkString(*state.allocAttr(v, state.symbols.create(i.first)), - state.store->printStorePath(i.second.path), {"!" + i.first + "!" + drvPathS}); + state.store->printStorePath(i.second.path(*state.store, drv.name)), {"!" + i.first + "!" + drvPathS}); } v.attrs->sort(); } @@ -1123,7 +1124,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_, - Value * filterFun, FileIngestionMethod method, const Hash & expectedHash, Value & v) + Value * filterFun, FileIngestionMethod method, const std::optional expectedHash, Value & v) { const auto path = evalSettings.pureEval && expectedHash ? path_ : @@ -1154,7 +1155,7 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con std::optional expectedStorePath; if (expectedHash) - expectedStorePath = state.store->makeFixedOutputPath(method, expectedHash, name); + expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name); Path dstPath; if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { dstPath = state.store->printStorePath(settings.readOnlyMode @@ -1188,7 +1189,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args .errPos = pos }); - addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, Hash(), v); + addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v); } static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -1198,7 +1199,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value string name; Value * filterFun = nullptr; auto method = FileIngestionMethod::Recursive; - Hash expectedHash; + std::optional expectedHash; for (auto & attr : *args[0]->attrs) { const string & n(attr.name); diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc deleted file mode 100644 index 36b0db2bd..000000000 --- a/src/libexpr/primops/fetchGit.cc +++ /dev/null @@ -1,91 +0,0 @@ -#include "primops.hh" -#include "eval-inline.hh" -#include "store-api.hh" -#include "hash.hh" -#include "fetchers.hh" -#include "url.hh" - -namespace nix { - -static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - std::string url; - std::optional ref; - std::optional rev; - std::string name = "source"; - bool fetchSubmodules = false; - PathSet context; - - state.forceValue(*args[0]); - - if (args[0]->type == tAttrs) { - - state.forceAttrs(*args[0], pos); - - for (auto & attr : *args[0]->attrs) { - string n(attr.name); - if (n == "url") - url = state.coerceToString(*attr.pos, *attr.value, context, false, false); - else if (n == "ref") - ref = state.forceStringNoCtx(*attr.value, *attr.pos); - else if (n == "rev") - rev = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA1); - else if (n == "name") - name = state.forceStringNoCtx(*attr.value, *attr.pos); - else if (n == "submodules") - fetchSubmodules = state.forceBool(*attr.value, *attr.pos); - else - throw EvalError({ - .hint = hintfmt("unsupported argument '%s' to 'fetchGit'", attr.name), - .errPos = *attr.pos - }); - } - - if (url.empty()) - throw EvalError({ - .hint = hintfmt("'url' argument required"), - .errPos = pos - }); - - } else - url = state.coerceToString(pos, *args[0], context, false, false); - - // FIXME: git externals probably can be used to bypass the URI - // whitelist. Ah well. - state.checkURI(url); - - if (evalSettings.pureEval && !rev) - throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); - - fetchers::Attrs attrs; - attrs.insert_or_assign("type", "git"); - attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url); - if (ref) attrs.insert_or_assign("ref", *ref); - if (rev) attrs.insert_or_assign("rev", rev->gitRev()); - if (fetchSubmodules) attrs.insert_or_assign("submodules", true); - auto input = fetchers::inputFromAttrs(attrs); - - // FIXME: use name? - auto [tree, input2] = input->fetchTree(state.store); - - state.mkAttrs(v, 8); - auto storePath = state.store->printStorePath(tree.storePath); - mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); - // Backward compatibility: set 'rev' to - // 0000000000000000000000000000000000000000 for a dirty tree. - auto rev2 = input2->getRev().value_or(Hash(htSHA1)); - mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev()); - mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev2.gitShortRev()); - // Backward compatibility: set 'revCount' to 0 for a dirty tree. - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), - tree.info.revCount.value_or(0)); - mkBool(*state.allocAttr(v, state.symbols.create("submodules")), fetchSubmodules); - v.attrs->sort(); - - if (state.allowedPaths) - state.allowedPaths->insert(tree.actualPath); -} - -static RegisterPrimOp r("fetchGit", 1, prim_fetchGit); - -} diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 59166b777..fc2a6a1c2 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -65,23 +65,23 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url); if (ref) attrs.insert_or_assign("ref", *ref); if (rev) attrs.insert_or_assign("rev", rev->gitRev()); - auto input = fetchers::inputFromAttrs(attrs); + auto input = fetchers::Input::fromAttrs(std::move(attrs)); // FIXME: use name - auto [tree, input2] = input->fetchTree(state.store); + auto [tree, input2] = input.fetch(state.store); state.mkAttrs(v, 8); auto storePath = state.store->printStorePath(tree.storePath); mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); - if (input2->getRef()) - mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2->getRef()); + if (input2.getRef()) + mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2.getRef()); // Backward compatibility: set 'rev' to // 0000000000000000000000000000000000000000 for a dirty tree. - auto rev2 = input2->getRev().value_or(Hash(htSHA1)); + auto rev2 = input2.getRev().value_or(Hash(htSHA1)); mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev()); mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(rev2.gitRev(), 0, 12)); - if (tree.info.revCount) - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount); + if (auto revCount = input2.getRevCount()) + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount); v.attrs->sort(); if (state.allowedPaths) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 01d6ad8b0..cddcf0e59 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -3,6 +3,7 @@ #include "store-api.hh" #include "fetchers.hh" #include "filetransfer.hh" +#include "registry.hh" #include #include @@ -12,39 +13,73 @@ namespace nix { void emitTreeAttrs( EvalState & state, const fetchers::Tree & tree, - std::shared_ptr input, - Value & v) + const fetchers::Input & input, + Value & v, + bool emptyRevFallback) { + assert(input.isImmutable()); + state.mkAttrs(v, 8); auto storePath = state.store->printStorePath(tree.storePath); mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); - assert(tree.info.narHash); - mkString(*state.allocAttr(v, state.symbols.create("narHash")), - tree.info.narHash.to_string(SRI, true)); + // FIXME: support arbitrary input attributes. - if (input->getRev()) { - mkString(*state.allocAttr(v, state.symbols.create("rev")), input->getRev()->gitRev()); - mkString(*state.allocAttr(v, state.symbols.create("shortRev")), input->getRev()->gitShortRev()); + auto narHash = input.getNarHash(); + assert(narHash); + mkString(*state.allocAttr(v, state.symbols.create("narHash")), + narHash->to_string(SRI, true)); + + if (auto rev = input.getRev()) { + mkString(*state.allocAttr(v, state.symbols.create("rev")), rev->gitRev()); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev->gitShortRev()); + } else if (emptyRevFallback) { + // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev + auto emptyHash = Hash(htSHA1); + mkString(*state.allocAttr(v, state.symbols.create("rev")), emptyHash.gitRev()); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitRev()); } - if (tree.info.revCount) - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount); + if (input.getType() == "git") + mkBool(*state.allocAttr(v, state.symbols.create("submodules")), maybeGetBoolAttr(input.attrs, "submodules").value_or(false)); - if (tree.info.lastModified) - mkString(*state.allocAttr(v, state.symbols.create("lastModified")), - fmt("%s", std::put_time(std::gmtime(&*tree.info.lastModified), "%Y%m%d%H%M%S"))); + if (auto revCount = input.getRevCount()) + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount); + else if (emptyRevFallback) + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), 0); + + if (auto lastModified = input.getLastModified()) { + mkInt(*state.allocAttr(v, state.symbols.create("lastModified")), *lastModified); + mkString(*state.allocAttr(v, state.symbols.create("lastModifiedDate")), + fmt("%s", std::put_time(std::gmtime(&*lastModified), "%Y%m%d%H%M%S"))); + } v.attrs->sort(); } -static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v) +std::string fixURI(std::string uri, EvalState &state) { - settings.requireExperimentalFeature("flakes"); + state.checkURI(uri); + return uri.find("://") != std::string::npos ? uri : "file://" + uri; +} - std::shared_ptr input; +void addURI(EvalState &state, fetchers::Attrs &attrs, Symbol name, std::string v) +{ + string n(name); + attrs.emplace(name, n == "url" ? fixURI(v, state) : v); +} + +static void fetchTree( + EvalState &state, + const Pos &pos, + Value **args, + Value &v, + const std::optional type, + bool emptyRevFallback = false +) { + fetchers::Input input; PathSet context; state.forceValue(*args[0]); @@ -56,35 +91,65 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V for (auto & attr : *args[0]->attrs) { state.forceValue(*attr.value); - if (attr.value->type == tString) - attrs.emplace(attr.name, attr.value->string.s); + if (attr.value->type == tPath || attr.value->type == tString) + addURI( + state, + attrs, + attr.name, + state.coerceToString(*attr.pos, *attr.value, context, false, false) + ); + else if (attr.value->type == tString) + addURI(state, attrs, attr.name, attr.value->string.s); else if (attr.value->type == tBool) - attrs.emplace(attr.name, attr.value->boolean); + attrs.emplace(attr.name, fetchers::Explicit{attr.value->boolean}); + else if (attr.value->type == tInt) + attrs.emplace(attr.name, attr.value->integer); else - throw TypeError("fetchTree argument '%s' is %s while a string or Boolean is expected", + throw TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected", attr.name, showType(*attr.value)); } + if (type) + attrs.emplace("type", type.value()); + if (!attrs.count("type")) throw Error({ .hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"), .errPos = pos }); - input = fetchers::inputFromAttrs(attrs); - } else - input = fetchers::inputFromURL(state.coerceToString(pos, *args[0], context, false, false)); + input = fetchers::Input::fromAttrs(std::move(attrs)); + } else { + auto url = fixURI(state.coerceToString(pos, *args[0], context, false, false), state); - if (evalSettings.pureEval && !input->isImmutable()) - throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input"); + if (type == "git") { + fetchers::Attrs attrs; + attrs.emplace("type", "git"); + attrs.emplace("url", url); + input = fetchers::Input::fromAttrs(std::move(attrs)); + } else { + input = fetchers::Input::fromURL(url); + } + } - // FIXME: use fetchOrSubstituteTree - auto [tree, input2] = input->fetchTree(state.store); + if (!evalSettings.pureEval && !input.isDirect()) + input = lookupInRegistries(state.store, input).first; + + if (evalSettings.pureEval && !input.isImmutable()) + throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input, at %s", pos); + + auto [tree, input2] = input.fetch(state.store); if (state.allowedPaths) state.allowedPaths->insert(tree.actualPath); - emitTreeAttrs(state, tree, input2, v); + emitTreeAttrs(state, tree, input2, v, emptyRevFallback); +} + +static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + settings.requireExperimentalFeature("flakes"); + fetchTree(state, pos, args, v, std::nullopt); } static RegisterPrimOp r("fetchTree", 1, prim_fetchTree); @@ -136,7 +201,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, auto storePath = unpack - ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath + ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).first.storePath : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath; auto path = state.store->toRealPath(storePath); @@ -147,7 +212,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, : hashFile(htSHA256, path); if (hash != *expectedHash) throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", - *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true)); + *url, expectedHash->to_string(Base32, true), hash->to_string(Base32, true)); } if (state.allowedPaths) @@ -166,7 +231,13 @@ static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args fetch(state, pos, args, v, "fetchTarball", true, "source"); } +static void prim_fetchGit(EvalState &state, const Pos &pos, Value **args, Value &v) +{ + fetchTree(state, pos, args, v, "git", true); +} + static RegisterPrimOp r2("__fetchurl", 1, prim_fetchurl); static RegisterPrimOp r3("fetchTarball", 1, prim_fetchTarball); +static RegisterPrimOp r4("fetchGit", 1, prim_fetchGit); } diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh index 7ba5e1c14..4eb6dac81 100644 --- a/src/libexpr/symbol-table.hh +++ b/src/libexpr/symbol-table.hh @@ -28,6 +28,12 @@ public: return s == s2.s; } + // FIXME: remove + bool operator == (std::string_view s2) const + { + return s->compare(s2) == 0; + } + bool operator != (const Symbol & s2) const { return s != s2.s; @@ -68,9 +74,10 @@ private: Symbols symbols; public: - Symbol create(const string & s) + Symbol create(std::string_view s) { - std::pair res = symbols.insert(s); + // FIXME: avoid allocation if 's' already exists in the symbol table. + std::pair res = symbols.emplace(std::string(s)); return Symbol(&*res.first); } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 71025824e..fe11bb2ed 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -166,6 +166,13 @@ struct Value { return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size; } + + /* Check whether forcing this value requires a trivial amount of + computation. In particular, function applications are + non-trivial. */ + bool isTrivial() const; + + std::vector> getContext(); }; diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index feb0a6085..1e59faa73 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -27,7 +27,7 @@ nlohmann::json attrsToJson(const Attrs & attrs) { nlohmann::json json; for (auto & attr : attrs) { - if (auto v = std::get_if(&attr.second)) { + if (auto v = std::get_if(&attr.second)) { json[attr.first] = *v; } else if (auto v = std::get_if(&attr.second)) { json[attr.first] = *v; @@ -55,16 +55,16 @@ std::string getStrAttr(const Attrs & attrs, const std::string & name) return *s; } -std::optional maybeGetIntAttr(const Attrs & attrs, const std::string & name) +std::optional maybeGetIntAttr(const Attrs & attrs, const std::string & name) { auto i = attrs.find(name); if (i == attrs.end()) return {}; - if (auto v = std::get_if(&i->second)) + if (auto v = std::get_if(&i->second)) return *v; throw Error("input attribute '%s' is not an integer", name); } -int64_t getIntAttr(const Attrs & attrs, const std::string & name) +uint64_t getIntAttr(const Attrs & attrs, const std::string & name) { auto s = maybeGetIntAttr(attrs, name); if (!s) @@ -76,8 +76,8 @@ std::optional maybeGetBoolAttr(const Attrs & attrs, const std::string & na { auto i = attrs.find(name); if (i == attrs.end()) return {}; - if (auto v = std::get_if(&i->second)) - return *v; + if (auto v = std::get_if>(&i->second)) + return v->t; throw Error("input attribute '%s' is not a Boolean", name); } @@ -93,7 +93,7 @@ std::map attrsToQuery(const Attrs & attrs) { std::map query; for (auto & attr : attrs) { - if (auto v = std::get_if(&attr.second)) { + if (auto v = std::get_if(&attr.second)) { query.insert_or_assign(attr.first, fmt("%d", *v)); } else if (auto v = std::get_if(&attr.second)) { query.insert_or_assign(attr.first, *v); diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh index d6e0ae000..4b4630c80 100644 --- a/src/libfetchers/attrs.hh +++ b/src/libfetchers/attrs.hh @@ -13,9 +13,14 @@ namespace nix::fetchers { template struct Explicit { T t; + + bool operator ==(const Explicit & other) const + { + return t == other.t; + } }; -typedef std::variant> Attr; +typedef std::variant> Attr; typedef std::map Attrs; Attrs jsonToAttrs(const nlohmann::json & json); @@ -26,9 +31,9 @@ std::optional maybeGetStrAttr(const Attrs & attrs, const std::strin std::string getStrAttr(const Attrs & attrs, const std::string & name); -std::optional maybeGetIntAttr(const Attrs & attrs, const std::string & name); +std::optional maybeGetIntAttr(const Attrs & attrs, const std::string & name); -int64_t getIntAttr(const Attrs & attrs, const std::string & name); +uint64_t getIntAttr(const Attrs & attrs, const std::string & name); std::optional maybeGetBoolAttr(const Attrs & attrs, const std::string & name); diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 9174c3de4..28db8aa9c 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -5,71 +5,265 @@ namespace nix::fetchers { -std::unique_ptr>> inputSchemes = nullptr; +std::unique_ptr>> inputSchemes = nullptr; -void registerInputScheme(std::unique_ptr && inputScheme) +void registerInputScheme(std::shared_ptr && inputScheme) { - if (!inputSchemes) inputSchemes = std::make_unique>>(); + if (!inputSchemes) inputSchemes = std::make_unique>>(); inputSchemes->push_back(std::move(inputScheme)); } -std::unique_ptr inputFromURL(const ParsedURL & url) +Input Input::fromURL(const std::string & url) +{ + return fromURL(parseURL(url)); +} + +static void fixupInput(Input & input) +{ + // Check common attributes. + input.getType(); + input.getRef(); + if (input.getRev()) + input.immutable = true; + input.getRevCount(); + input.getLastModified(); + if (input.getNarHash()) + input.immutable = true; +} + +Input Input::fromURL(const ParsedURL & url) { for (auto & inputScheme : *inputSchemes) { auto res = inputScheme->inputFromURL(url); - if (res) return res; + if (res) { + res->scheme = inputScheme; + fixupInput(*res); + return std::move(*res); + } } + throw Error("input '%s' is unsupported", url.url); } -std::unique_ptr inputFromURL(const std::string & url) +Input Input::fromAttrs(Attrs && attrs) { - return inputFromURL(parseURL(url)); -} - -std::unique_ptr inputFromAttrs(const Attrs & attrs) -{ - auto attrs2(attrs); - attrs2.erase("narHash"); for (auto & inputScheme : *inputSchemes) { - auto res = inputScheme->inputFromAttrs(attrs2); + auto res = inputScheme->inputFromAttrs(attrs); if (res) { - if (auto narHash = maybeGetStrAttr(attrs, "narHash")) - // FIXME: require SRI hash. - res->narHash = newHashAllowEmpty(*narHash, {}); - return res; + res->scheme = inputScheme; + fixupInput(*res); + return std::move(*res); } } - throw Error("input '%s' is unsupported", attrsToJson(attrs)); + + Input input; + input.attrs = attrs; + fixupInput(input); + return input; +} + +ParsedURL Input::toURL() const +{ + if (!scheme) + throw Error("cannot show unsupported input '%s'", attrsToJson(attrs)); + return scheme->toURL(*this); +} + +std::string Input::to_string() const +{ + return toURL().to_string(); } Attrs Input::toAttrs() const { - auto attrs = toAttrsInternal(); - if (narHash) - attrs.emplace("narHash", narHash->to_string(SRI, true)); - attrs.emplace("type", type()); return attrs; } -std::pair> Input::fetchTree(ref store) const +bool Input::hasAllInfo() const { - auto [tree, input] = fetchTreeInternal(store); + return getNarHash() && scheme && scheme->hasAllInfo(*this); +} + +bool Input::operator ==(const Input & other) const +{ + return attrs == other.attrs; +} + +bool Input::contains(const Input & other) const +{ + if (*this == other) return true; + auto other2(other); + other2.attrs.erase("ref"); + other2.attrs.erase("rev"); + if (*this == other2) return true; + return false; +} + +std::pair Input::fetch(ref store) const +{ + if (!scheme) + throw Error("cannot fetch unsupported input '%s'", attrsToJson(toAttrs())); + + /* The tree may already be in the Nix store, or it could be + substituted (which is often faster than fetching from the + original source). So check that. */ + if (hasAllInfo()) { + try { + auto storePath = computeStorePath(*store); + + store->ensurePath(storePath); + + debug("using substituted/cached input '%s' in '%s'", + to_string(), store->printStorePath(storePath)); + + auto actualPath = store->toRealPath(storePath); + + return {fetchers::Tree(std::move(actualPath), std::move(storePath)), *this}; + } catch (Error & e) { + debug("substitution of input '%s' failed: %s", to_string(), e.what()); + } + } + + auto [tree, input] = scheme->fetch(store, *this); if (tree.actualPath == "") tree.actualPath = store->toRealPath(tree.storePath); - if (!tree.info.narHash) - tree.info.narHash = store->queryPathInfo(tree.storePath)->narHash; + auto narHash = store->queryPathInfo(tree.storePath)->narHash; + input.attrs.insert_or_assign("narHash", narHash->to_string(SRI, true)); - if (input->narHash) - assert(input->narHash == tree.info.narHash); + if (auto prevNarHash = getNarHash()) { + if (narHash != *prevNarHash) + throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", + to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash->to_string(SRI, true)); + } - if (narHash && narHash != input->narHash) - throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", - to_string(), tree.actualPath, narHash->to_string(SRI, true), input->narHash->to_string(SRI, true)); + if (auto prevLastModified = getLastModified()) { + if (input.getLastModified() != prevLastModified) + throw Error("'lastModified' attribute mismatch in input '%s', expected %d", + input.to_string(), *prevLastModified); + } + + if (auto prevRevCount = getRevCount()) { + if (input.getRevCount() != prevRevCount) + throw Error("'revCount' attribute mismatch in input '%s', expected %d", + input.to_string(), *prevRevCount); + } + + input.immutable = true; + + assert(input.hasAllInfo()); return {std::move(tree), input}; } +Input Input::applyOverrides( + std::optional ref, + std::optional rev) const +{ + if (!scheme) return *this; + return scheme->applyOverrides(*this, ref, rev); +} + +void Input::clone(const Path & destDir) const +{ + assert(scheme); + scheme->clone(*this, destDir); +} + +std::optional Input::getSourcePath() const +{ + assert(scheme); + return scheme->getSourcePath(*this); +} + +void Input::markChangedFile( + std::string_view file, + std::optional commitMsg) const +{ + assert(scheme); + return scheme->markChangedFile(*this, file, commitMsg); +} + +StorePath Input::computeStorePath(Store & store) const +{ + auto narHash = getNarHash(); + if (!narHash) + throw Error("cannot compute store path for mutable input '%s'", to_string()); + return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, "source"); +} + +std::string Input::getType() const +{ + return getStrAttr(attrs, "type"); +} + +std::optional Input::getNarHash() const +{ + if (auto s = maybeGetStrAttr(attrs, "narHash")) + // FIXME: require SRI hash. + return newHashAllowEmpty(*s, htSHA256); + return {}; +} + +std::optional Input::getRef() const +{ + if (auto s = maybeGetStrAttr(attrs, "ref")) + return *s; + return {}; +} + +std::optional Input::getRev() const +{ + if (auto s = maybeGetStrAttr(attrs, "rev")) + return Hash(*s, htSHA1); + return {}; +} + +std::optional Input::getRevCount() const +{ + if (auto n = maybeGetIntAttr(attrs, "revCount")) + return *n; + return {}; +} + +std::optional Input::getLastModified() const +{ + if (auto n = maybeGetIntAttr(attrs, "lastModified")) + return *n; + return {}; +} + +ParsedURL InputScheme::toURL(const Input & input) +{ + throw Error("don't know how to convert input '%s' to a URL", attrsToJson(input.attrs)); +} + +Input InputScheme::applyOverrides( + const Input & input, + std::optional ref, + std::optional rev) +{ + if (ref) + throw Error("don't know how to set branch/tag name of input '%s' to '%s'", input.to_string(), *ref); + if (rev) + throw Error("don't know how to set revision of input '%s' to '%s'", input.to_string(), rev->gitRev()); + return input; +} + +std::optional InputScheme::getSourcePath(const Input & input) +{ + return {}; +} + +void InputScheme::markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) +{ + assert(false); +} + +void InputScheme::clone(const Input & input, const Path & destDir) +{ + throw Error("do not know how to clone input '%s'", input.to_string()); +} + } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 59a58ae67..be71b786b 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -3,7 +3,6 @@ #include "types.hh" #include "hash.hh" #include "path.hh" -#include "tree-info.hh" #include "attrs.hh" #include "url.hh" @@ -13,73 +12,101 @@ namespace nix { class Store; } namespace nix::fetchers { -struct Input; - struct Tree { Path actualPath; StorePath storePath; - TreeInfo info; + Tree(Path && actualPath, StorePath && storePath) : actualPath(actualPath), storePath(std::move(storePath)) {} }; -struct Input : std::enable_shared_from_this +struct InputScheme; + +struct Input { - std::optional narHash; // FIXME: implement + friend class InputScheme; - virtual std::string type() const = 0; + std::shared_ptr scheme; // note: can be null + Attrs attrs; + bool immutable = false; + bool direct = true; - virtual ~Input() { } +public: + static Input fromURL(const std::string & url); - virtual bool operator ==(const Input & other) const { return false; } + static Input fromURL(const ParsedURL & url); - /* Check whether this is a "direct" input, that is, not - one that goes through a registry. */ - virtual bool isDirect() const { return true; } + static Input fromAttrs(Attrs && attrs); - /* Check whether this is an "immutable" input, that is, - one that contains a commit hash or content hash. */ - virtual bool isImmutable() const { return (bool) narHash; } + ParsedURL toURL() const; - virtual bool contains(const Input & other) const { return false; } - - virtual std::optional getRef() const { return {}; } - - virtual std::optional getRev() const { return {}; } - - virtual ParsedURL toURL() const = 0; - - std::string to_string() const - { - return toURL().to_string(); - } + std::string to_string() const; Attrs toAttrs() const; - std::pair> fetchTree(ref store) const; + /* Check whether this is a "direct" input, that is, not + one that goes through a registry. */ + bool isDirect() const { return direct; } -private: + /* Check whether this is an "immutable" input, that is, + one that contains a commit hash or content hash. */ + bool isImmutable() const { return immutable; } - virtual std::pair> fetchTreeInternal(ref store) const = 0; + bool hasAllInfo() const; - virtual Attrs toAttrsInternal() const = 0; + bool operator ==(const Input & other) const; + + bool contains(const Input & other) const; + + std::pair fetch(ref store) const; + + Input applyOverrides( + std::optional ref, + std::optional rev) const; + + void clone(const Path & destDir) const; + + std::optional getSourcePath() const; + + void markChangedFile( + std::string_view file, + std::optional commitMsg) const; + + StorePath computeStorePath(Store & store) const; + + // Convience functions for common attributes. + std::string getType() const; + std::optional getNarHash() const; + std::optional getRef() const; + std::optional getRev() const; + std::optional getRevCount() const; + std::optional getLastModified() const; }; struct InputScheme { - virtual ~InputScheme() { } + virtual std::optional inputFromURL(const ParsedURL & url) = 0; - virtual std::unique_ptr inputFromURL(const ParsedURL & url) = 0; + virtual std::optional inputFromAttrs(const Attrs & attrs) = 0; - virtual std::unique_ptr inputFromAttrs(const Attrs & attrs) = 0; + virtual ParsedURL toURL(const Input & input); + + virtual bool hasAllInfo(const Input & input) = 0; + + virtual Input applyOverrides( + const Input & input, + std::optional ref, + std::optional rev); + + virtual void clone(const Input & input, const Path & destDir); + + virtual std::optional getSourcePath(const Input & input); + + virtual void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg); + + virtual std::pair fetch(ref store, const Input & input) = 0; }; -std::unique_ptr inputFromURL(const ParsedURL & url); - -std::unique_ptr inputFromURL(const std::string & url); - -std::unique_ptr inputFromAttrs(const Attrs & attrs); - -void registerInputScheme(std::unique_ptr && fetcher); +void registerInputScheme(std::shared_ptr && fetcher); struct DownloadFileResult { @@ -94,7 +121,7 @@ DownloadFileResult downloadFile( const std::string & name, bool immutable); -Tree downloadTarball( +std::pair downloadTarball( ref store, const std::string & url, const std::string & name, diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 75ce5ee8b..5d38e0c2b 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -22,80 +22,152 @@ static bool isNotDotGitDirectory(const Path & path) return not std::regex_match(path, gitDirRegex); } -struct GitInput : Input +struct GitInputScheme : InputScheme { - ParsedURL url; - std::optional ref; - std::optional rev; - bool shallow = false; - bool submodules = false; - - GitInput(const ParsedURL & url) : url(url) - { } - - std::string type() const override { return "git"; } - - bool operator ==(const Input & other) const override + std::optional inputFromURL(const ParsedURL & url) override { - auto other2 = dynamic_cast(&other); - return - other2 - && url == other2->url - && rev == other2->rev - && ref == other2->ref; - } + if (url.scheme != "git" && + url.scheme != "git+http" && + url.scheme != "git+https" && + url.scheme != "git+ssh" && + url.scheme != "git+file") return {}; - bool isImmutable() const override - { - return (bool) rev || narHash; - } + auto url2(url); + if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4); + url2.query.clear(); - std::optional getRef() const override { return ref; } - - std::optional getRev() const override { return rev; } - - ParsedURL toURL() const override - { - ParsedURL url2(url); - if (url2.scheme != "git") url2.scheme = "git+" + url2.scheme; - if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); - if (ref) url2.query.insert_or_assign("ref", *ref); - if (shallow) url2.query.insert_or_assign("shallow", "1"); - return url2; - } - - Attrs toAttrsInternal() const override - { Attrs attrs; - attrs.emplace("url", url.to_string()); - if (ref) - attrs.emplace("ref", *ref); - if (rev) - attrs.emplace("rev", rev->gitRev()); - if (shallow) - attrs.emplace("shallow", true); - if (submodules) - attrs.emplace("submodules", true); - return attrs; + attrs.emplace("type", "git"); + + for (auto &[name, value] : url.query) { + if (name == "rev" || name == "ref") + attrs.emplace(name, value); + else if (name == "shallow") + attrs.emplace(name, Explicit { value == "1" }); + else + url2.query.emplace(name, value); + } + + attrs.emplace("url", url2.to_string()); + + return inputFromAttrs(attrs); } - std::pair getActualUrl() const + std::optional inputFromAttrs(const Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "git") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash") + throw Error("unsupported Git input attribute '%s'", name); + + parseURL(getStrAttr(attrs, "url")); + maybeGetBoolAttr(attrs, "shallow"); + maybeGetBoolAttr(attrs, "submodules"); + + if (auto ref = maybeGetStrAttr(attrs, "ref")) { + if (std::regex_search(*ref, badGitRefRegex)) + throw BadURL("invalid Git branch/tag name '%s'", *ref); + } + + Input input; + input.attrs = attrs; + return input; + } + + ParsedURL toURL(const Input & input) override + { + auto url = parseURL(getStrAttr(input.attrs, "url")); + if (url.scheme != "git") url.scheme = "git+" + url.scheme; + if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev()); + if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref); + if (maybeGetBoolAttr(input.attrs, "shallow").value_or(false)) + url.query.insert_or_assign("shallow", "1"); + return url; + } + + bool hasAllInfo(const Input & input) override + { + bool maybeDirty = !input.getRef(); + bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); + return + maybeGetIntAttr(input.attrs, "lastModified") + && (shallow || maybeDirty || maybeGetIntAttr(input.attrs, "revCount")); + } + + Input applyOverrides( + const Input & input, + std::optional ref, + std::optional rev) override + { + auto res(input); + if (rev) res.attrs.insert_or_assign("rev", rev->gitRev()); + if (ref) res.attrs.insert_or_assign("ref", *ref); + if (!res.getRef() && res.getRev()) + throw Error("Git input '%s' has a commit hash but no branch/tag name", res.to_string()); + return res; + } + + void clone(const Input & input, const Path & destDir) override + { + auto [isLocal, actualUrl] = getActualUrl(input); + + Strings args = {"clone"}; + + args.push_back(actualUrl); + + if (auto ref = input.getRef()) { + args.push_back("--branch"); + args.push_back(*ref); + } + + if (input.getRev()) throw Error("cloning a specific revision is not implemented"); + + args.push_back(destDir); + + runProgram("git", true, args); + } + + std::optional getSourcePath(const Input & input) override + { + auto url = parseURL(getStrAttr(input.attrs, "url")); + if (url.scheme == "file" && !input.getRef() && !input.getRev()) + return url.path; + return {}; + } + + void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) override + { + auto sourcePath = getSourcePath(input); + assert(sourcePath); + + runProgram("git", true, + { "-C", *sourcePath, "add", "--force", "--intent-to-add", "--", std::string(file) }); + + if (commitMsg) + runProgram("git", true, + { "-C", *sourcePath, "commit", std::string(file), "-m", *commitMsg }); + } + + std::pair getActualUrl(const Input & input) const { // Don't clone file:// URIs (but otherwise treat them the // same as remote URIs, i.e. don't use the working tree or // HEAD). static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing + auto url = parseURL(getStrAttr(input.attrs, "url")); bool isLocal = url.scheme == "file" && !forceHttp; return {isLocal, isLocal ? url.path : url.base}; } - std::pair> fetchTreeInternal(nix::ref store) const override + std::pair fetch(ref store, const Input & _input) override { auto name = "source"; - auto input = std::make_shared(*this); + Input input(_input); - assert(!rev || rev->type == htSHA1); + bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); + bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); std::string cacheType = "git"; if (shallow) cacheType += "-shallow"; @@ -106,39 +178,35 @@ struct GitInput : Input return Attrs({ {"type", cacheType}, {"name", name}, - {"rev", input->rev->gitRev()}, + {"rev", input.getRev()->gitRev()}, }); }; auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) - -> std::pair> + -> std::pair { - assert(input->rev); - assert(!rev || rev == input->rev); + assert(input.getRev()); + assert(!_input.getRev() || _input.getRev() == input.getRev()); + if (!shallow) + input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount")); + input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified")); return { - Tree { - .actualPath = store->toRealPath(storePath), - .storePath = std::move(storePath), - .info = TreeInfo { - .revCount = shallow ? std::nullopt : std::optional(getIntAttr(infoAttrs, "revCount")), - .lastModified = getIntAttr(infoAttrs, "lastModified"), - }, - }, + Tree(store->toRealPath(storePath), std::move(storePath)), input }; }; - if (rev) { + if (input.getRev()) { if (auto res = getCache()->lookup(store, getImmutableAttrs())) return makeResult(res->first, std::move(res->second)); } - auto [isLocal, actualUrl_] = getActualUrl(); + auto [isLocal, actualUrl_] = getActualUrl(input); auto actualUrl = actualUrl_; // work around clang bug // If this is a local directory and no ref or revision is // given, then allow the use of an unclean working tree. - if (!input->ref && !input->rev && isLocal) { + if (!input.getRef() && !input.getRev() && isLocal) { bool clean = false; /* Check whether this repo has any commits. There are @@ -197,35 +265,35 @@ struct GitInput : Input auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter); - auto tree = Tree { - .actualPath = store->printStorePath(storePath), - .storePath = std::move(storePath), - .info = TreeInfo { - // FIXME: maybe we should use the timestamp of the last - // modified dirty file? - .lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0, - } - }; + // FIXME: maybe we should use the timestamp of the last + // modified dirty file? + input.attrs.insert_or_assign( + "lastModified", + haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0); - return {std::move(tree), input}; + return { + Tree(store->printStorePath(storePath), std::move(storePath)), + input + }; } } - if (!input->ref) input->ref = isLocal ? readHead(actualUrl) : "master"; + if (!input.getRef()) input.attrs.insert_or_assign("ref", isLocal ? readHead(actualUrl) : "master"); Attrs mutableAttrs({ {"type", cacheType}, {"name", name}, {"url", actualUrl}, - {"ref", *input->ref}, + {"ref", *input.getRef()}, }); Path repoDir; if (isLocal) { - if (!input->rev) - input->rev = Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input->ref })), htSHA1); + if (!input.getRev()) + input.attrs.insert_or_assign("rev", + Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input.getRef() })), htSHA1).gitRev()); repoDir = actualUrl; @@ -233,8 +301,8 @@ struct GitInput : Input if (auto res = getCache()->lookup(store, mutableAttrs)) { auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1); - if (!rev || rev == rev2) { - input->rev = rev2; + if (!input.getRev() || input.getRev() == rev2) { + input.attrs.insert_or_assign("rev", rev2.gitRev()); return makeResult(res->first, std::move(res->second)); } } @@ -248,18 +316,18 @@ struct GitInput : Input } Path localRefFile = - input->ref->compare(0, 5, "refs/") == 0 - ? cacheDir + "/" + *input->ref - : cacheDir + "/refs/heads/" + *input->ref; + input.getRef()->compare(0, 5, "refs/") == 0 + ? cacheDir + "/" + *input.getRef() + : cacheDir + "/refs/heads/" + *input.getRef(); bool doFetch; time_t now = time(0); /* If a rev was specified, we need to fetch if it's not in the repo. */ - if (input->rev) { + if (input.getRev()) { try { - runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input->rev->gitRev() }); + runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input.getRev()->gitRev() }); doFetch = false; } catch (ExecError & e) { if (WIFEXITED(e.status)) { @@ -282,9 +350,10 @@ struct GitInput : Input // FIXME: git stderr messes up our progress indicator, so // we're using --quiet for now. Should process its stderr. try { - auto fetchRef = input->ref->compare(0, 5, "refs/") == 0 - ? *input->ref - : "refs/heads/" + *input->ref; + auto ref = input.getRef(); + auto fetchRef = ref->compare(0, 5, "refs/") == 0 + ? *ref + : "refs/heads/" + *ref; runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }); } catch (Error & e) { if (!pathExists(localRefFile)) throw; @@ -300,8 +369,8 @@ struct GitInput : Input utimes(localRefFile.c_str(), times); } - if (!input->rev) - input->rev = Hash(chomp(readFile(localRefFile)), htSHA1); + if (!input.getRev()) + input.attrs.insert_or_assign("rev", Hash(chomp(readFile(localRefFile)), htSHA1).gitRev()); } bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true"; @@ -311,7 +380,7 @@ struct GitInput : Input // FIXME: check whether rev is an ancestor of ref. - printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl); + printTalkative("using revision %s of repo '%s'", input.getRev()->gitRev(), actualUrl); /* Now that we know the ref, check again whether we have it in the store. */ @@ -333,7 +402,7 @@ struct GitInput : Input runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", "--update-head-ok", "--", repoDir, "refs/*:refs/*" }); - runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input->rev->gitRev() }); + runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() }); runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", actualUrl }); runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); @@ -342,7 +411,7 @@ struct GitInput : Input // FIXME: should pipe this, or find some better way to extract a // revision. auto source = sinkToSource([&](Sink & sink) { - RunOptions gitOptions("git", { "-C", repoDir, "archive", input->rev->gitRev() }); + RunOptions gitOptions("git", { "-C", repoDir, "archive", input.getRev()->gitRev() }); gitOptions.standardOut = &sink; runProgram2(gitOptions); }); @@ -352,18 +421,18 @@ struct GitInput : Input auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter); - auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() })); + auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input.getRev()->gitRev() })); Attrs infoAttrs({ - {"rev", input->rev->gitRev()}, + {"rev", input.getRev()->gitRev()}, {"lastModified", lastModified}, }); if (!shallow) infoAttrs.insert_or_assign("revCount", - std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() }))); + std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input.getRev()->gitRev() }))); - if (!this->rev) + if (!_input.getRev()) getCache()->add( store, mutableAttrs, @@ -382,60 +451,6 @@ struct GitInput : Input } }; -struct GitInputScheme : InputScheme -{ - std::unique_ptr inputFromURL(const ParsedURL & url) override - { - if (url.scheme != "git" && - url.scheme != "git+http" && - url.scheme != "git+https" && - url.scheme != "git+ssh" && - url.scheme != "git+file") return nullptr; - - auto url2(url); - if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4); - url2.query.clear(); - - Attrs attrs; - attrs.emplace("type", "git"); - - for (auto &[name, value] : url.query) { - if (name == "rev" || name == "ref") - attrs.emplace(name, value); - else - url2.query.emplace(name, value); - } - - attrs.emplace("url", url2.to_string()); - - return inputFromAttrs(attrs); - } - - std::unique_ptr inputFromAttrs(const Attrs & attrs) override - { - if (maybeGetStrAttr(attrs, "type") != "git") return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules") - throw Error("unsupported Git input attribute '%s'", name); - - auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); - if (auto ref = maybeGetStrAttr(attrs, "ref")) { - if (std::regex_search(*ref, badGitRefRegex)) - throw BadURL("invalid Git branch/tag name '%s'", *ref); - input->ref = *ref; - } - if (auto rev = maybeGetStrAttr(attrs, "rev")) - input->rev = Hash(*rev, htSHA1); - - input->shallow = maybeGetBoolAttr(attrs, "shallow").value_or(false); - - input->submodules = maybeGetBoolAttr(attrs, "submodules").value_or(false); - - return input; - } -}; - static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); } diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 0bee1d6b3..8bb7c2c1d 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -8,81 +8,142 @@ namespace nix::fetchers { -std::regex ownerRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); -std::regex repoRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); +// A github or gitlab url +const static std::string urlRegexS = "[a-zA-Z0-9.]*"; // FIXME: check +std::regex urlRegex(urlRegexS, std::regex::ECMAScript); -struct GitHubInput : Input +struct GitArchiveInputScheme : InputScheme { - std::string owner; - std::string repo; - std::optional ref; - std::optional rev; + virtual std::string type() = 0; - std::string type() const override { return "github"; } - - bool operator ==(const Input & other) const override + std::optional inputFromURL(const ParsedURL & url) override { - auto other2 = dynamic_cast(&other); - return - other2 - && owner == other2->owner - && repo == other2->repo - && rev == other2->rev - && ref == other2->ref; + if (url.scheme != type()) return {}; + + auto path = tokenizeString>(url.path, "/"); + + std::optional rev; + std::optional ref; + std::optional host_url; + + if (path.size() == 2) { + } else if (path.size() == 3) { + if (std::regex_match(path[2], revRegex)) + rev = Hash(path[2], htSHA1); + else if (std::regex_match(path[2], refRegex)) + ref = path[2]; + else + throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]); + } else + throw BadURL("URL '%s' is invalid", url.url); + + for (auto &[name, value] : url.query) { + if (name == "rev") { + if (rev) + throw BadURL("URL '%s' contains multiple commit hashes", url.url); + rev = Hash(value, htSHA1); + } + else if (name == "ref") { + if (!std::regex_match(value, refRegex)) + throw BadURL("URL '%s' contains an invalid branch/tag name", url.url); + if (ref) + throw BadURL("URL '%s' contains multiple branch/tag names", url.url); + ref = value; + } + else if (name == "url") { + if (!std::regex_match(value, urlRegex)) + throw BadURL("URL '%s' contains an invalid instance url", url.url); + host_url = value; + } + // FIXME: barf on unsupported attributes + } + + if (ref && rev) + throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url.url, *ref, rev->gitRev()); + + Input input; + input.attrs.insert_or_assign("type", type()); + input.attrs.insert_or_assign("owner", path[0]); + input.attrs.insert_or_assign("repo", path[1]); + if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); + if (ref) input.attrs.insert_or_assign("ref", *ref); + if (host_url) input.attrs.insert_or_assign("url", *host_url); + + return input; } - bool isImmutable() const override + std::optional inputFromAttrs(const Attrs & attrs) override { - return (bool) rev || narHash; + if (maybeGetStrAttr(attrs, "type") != type()) return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified") + throw Error("unsupported input attribute '%s'", name); + + getStrAttr(attrs, "owner"); + getStrAttr(attrs, "repo"); + + Input input; + input.attrs = attrs; + return input; } - std::optional getRef() const override { return ref; } - - std::optional getRev() const override { return rev; } - - ParsedURL toURL() const override + ParsedURL toURL(const Input & input) override { + auto owner = getStrAttr(input.attrs, "owner"); + auto repo = getStrAttr(input.attrs, "repo"); + auto ref = input.getRef(); + auto rev = input.getRev(); auto path = owner + "/" + repo; assert(!(ref && rev)); if (ref) path += "/" + *ref; if (rev) path += "/" + rev->to_string(Base16, false); return ParsedURL { - .scheme = "github", + .scheme = type(), .path = path, }; } - Attrs toAttrsInternal() const override + bool hasAllInfo(const Input & input) override { - Attrs attrs; - attrs.emplace("owner", owner); - attrs.emplace("repo", repo); - if (ref) - attrs.emplace("ref", *ref); - if (rev) - attrs.emplace("rev", rev->gitRev()); - return attrs; + return input.getRev() && maybeGetIntAttr(input.attrs, "lastModified"); } - std::pair> fetchTreeInternal(nix::ref store) const override + Input applyOverrides( + const Input & _input, + std::optional ref, + std::optional rev) override { - auto rev = this->rev; - auto ref = this->ref.value_or("master"); - - if (!rev) { - auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s", - owner, repo, ref); - auto json = nlohmann::json::parse( - readFile( - store->toRealPath( - downloadFile(store, url, "source", false).storePath))); - rev = Hash(std::string { json["sha"] }, htSHA1); - debug("HEAD revision for '%s' is %s", url, rev->gitRev()); + auto input(_input); + if (rev && ref) + throw BadURL("cannot apply both a commit hash (%s) and a branch/tag name ('%s') to input '%s'", + rev->gitRev(), *ref, input.to_string()); + if (rev) { + input.attrs.insert_or_assign("rev", rev->gitRev()); + input.attrs.erase("ref"); } + if (ref) { + input.attrs.insert_or_assign("ref", *ref); + input.attrs.erase("rev"); + } + return input; + } - auto input = std::make_shared(*this); - input->ref = {}; - input->rev = *rev; + virtual Hash getRevFromRef(nix::ref store, const Input & input) const = 0; + + virtual std::string getDownloadUrl(const Input & input) const = 0; + + std::pair fetch(ref store, const Input & _input) override + { + Input input(_input); + + if (!maybeGetStrAttr(input.attrs, "ref")) input.attrs.insert_or_assign("ref", "HEAD"); + + auto rev = input.getRev(); + if (!rev) rev = getRevFromRef(store, input); + + input.attrs.erase("ref"); + input.attrs.insert_or_assign("rev", rev->gitRev()); Attrs immutableAttrs({ {"type", "git-tarball"}, @@ -90,36 +151,25 @@ struct GitHubInput : Input }); if (auto res = getCache()->lookup(store, immutableAttrs)) { + input.attrs.insert_or_assign("lastModified", getIntAttr(res->first, "lastModified")); return { - Tree{ - .actualPath = store->toRealPath(res->second), - .storePath = std::move(res->second), - .info = TreeInfo { - .lastModified = getIntAttr(res->first, "lastModified"), - }, - }, + Tree(store->toRealPath(res->second), std::move(res->second)), input }; } - // FIXME: use regular /archive URLs instead? api.github.com - // might have stricter rate limits. + auto url = getDownloadUrl(input); - auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s", - owner, repo, rev->to_string(Base16, false)); + auto [tree, lastModified] = downloadTarball(store, url, "source", true); - std::string accessToken = settings.githubAccessToken.get(); - if (accessToken != "") - url += "?access_token=" + accessToken; - - auto tree = downloadTarball(store, url, "source", true); + input.attrs.insert_or_assign("lastModified", lastModified); getCache()->add( store, immutableAttrs, { {"rev", rev->gitRev()}, - {"lastModified", *tree.info.lastModified} + {"lastModified", lastModified} }, tree.storePath, true); @@ -128,68 +178,96 @@ struct GitHubInput : Input } }; -struct GitHubInputScheme : InputScheme +struct GitHubInputScheme : GitArchiveInputScheme { - std::unique_ptr inputFromURL(const ParsedURL & url) override + std::string type() override { return "github"; } + + Hash getRevFromRef(nix::ref store, const Input & input) const override { - if (url.scheme != "github") return nullptr; - - auto path = tokenizeString>(url.path, "/"); - auto input = std::make_unique(); - - if (path.size() == 2) { - } else if (path.size() == 3) { - if (std::regex_match(path[2], revRegex)) - input->rev = Hash(path[2], htSHA1); - else if (std::regex_match(path[2], refRegex)) - input->ref = path[2]; - else - throw BadURL("in GitHub URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]); - } else - throw BadURL("GitHub URL '%s' is invalid", url.url); - - for (auto &[name, value] : url.query) { - if (name == "rev") { - if (input->rev) - throw BadURL("GitHub URL '%s' contains multiple commit hashes", url.url); - input->rev = Hash(value, htSHA1); - } - else if (name == "ref") { - if (!std::regex_match(value, refRegex)) - throw BadURL("GitHub URL '%s' contains an invalid branch/tag name", url.url); - if (input->ref) - throw BadURL("GitHub URL '%s' contains multiple branch/tag names", url.url); - input->ref = value; - } - } - - if (input->ref && input->rev) - throw BadURL("GitHub URL '%s' contains both a commit hash and a branch/tag name", url.url); - - input->owner = path[0]; - input->repo = path[1]; - - return input; + auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com"); + auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check + host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); + auto json = nlohmann::json::parse( + readFile( + store->toRealPath( + downloadFile(store, url, "source", false).storePath))); + auto rev = Hash(std::string { json["sha"] }, htSHA1); + debug("HEAD revision for '%s' is %s", url, rev.gitRev()); + return rev; } - std::unique_ptr inputFromAttrs(const Attrs & attrs) override + std::string getDownloadUrl(const Input & input) const override { - if (maybeGetStrAttr(attrs, "type") != "github") return {}; + // FIXME: use regular /archive URLs instead? api.github.com + // might have stricter rate limits. + auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com"); + auto url = fmt("https://api.%s/repos/%s/%s/tarball/%s", // FIXME: check if this is correct for self hosted instances + host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), + input.getRev()->to_string(Base16, false)); - for (auto & [name, value] : attrs) - if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev") - throw Error("unsupported GitHub input attribute '%s'", name); + std::string accessToken = settings.githubAccessToken.get(); + if (accessToken != "") + url += "?access_token=" + accessToken; - auto input = std::make_unique(); - input->owner = getStrAttr(attrs, "owner"); - input->repo = getStrAttr(attrs, "repo"); - input->ref = maybeGetStrAttr(attrs, "ref"); - if (auto rev = maybeGetStrAttr(attrs, "rev")) - input->rev = Hash(*rev, htSHA1); - return input; + return url; + } + + void clone(const Input & input, const Path & destDir) override + { + auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com"); + Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git", + host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) + .applyOverrides(input.getRef().value_or("HEAD"), input.getRev()) + .clone(destDir); + } +}; + +struct GitLabInputScheme : GitArchiveInputScheme +{ + std::string type() override { return "gitlab"; } + + Hash getRevFromRef(nix::ref store, const Input & input) const override + { + auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com"); + auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s", + host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); + auto json = nlohmann::json::parse( + readFile( + store->toRealPath( + downloadFile(store, url, "source", false).storePath))); + auto rev = Hash(std::string(json[0]["id"]), htSHA1); + debug("HEAD revision for '%s' is %s", url, rev.gitRev()); + return rev; + } + + std::string getDownloadUrl(const Input & input) const override + { + // FIXME: This endpoint has a rate limit threshold of 5 requests per minute + auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com"); + auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s", + host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), + input.getRev()->to_string(Base16, false)); + + /* # FIXME: add privat token auth (`curl --header "PRIVATE-TOKEN: "`) + std::string accessToken = settings.githubAccessToken.get(); + if (accessToken != "") + url += "?access_token=" + accessToken;*/ + + return url; + } + + void clone(const Input & input, const Path & destDir) override + { + auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com"); + // FIXME: get username somewhere + Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git", + host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) + .applyOverrides(input.getRef().value_or("HEAD"), input.getRev()) + .clone(destDir); } }; static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); +static auto r2 = OnStartup([] { registerInputScheme(std::make_unique()); }); } diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc new file mode 100644 index 000000000..91dc83740 --- /dev/null +++ b/src/libfetchers/indirect.cc @@ -0,0 +1,104 @@ +#include "fetchers.hh" + +namespace nix::fetchers { + +std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); + +struct IndirectInputScheme : InputScheme +{ + std::optional inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "flake") return {}; + + auto path = tokenizeString>(url.path, "/"); + + std::optional rev; + std::optional ref; + + if (path.size() == 1) { + } else if (path.size() == 2) { + if (std::regex_match(path[1], revRegex)) + rev = Hash(path[1], htSHA1); + else if (std::regex_match(path[1], refRegex)) + ref = path[1]; + else + throw BadURL("in flake URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[1]); + } else if (path.size() == 3) { + if (!std::regex_match(path[1], refRegex)) + throw BadURL("in flake URL '%s', '%s' is not a branch/tag name", url.url, path[1]); + ref = path[1]; + if (!std::regex_match(path[2], revRegex)) + throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]); + rev = Hash(path[2], htSHA1); + } else + throw BadURL("GitHub URL '%s' is invalid", url.url); + + std::string id = path[0]; + if (!std::regex_match(id, flakeRegex)) + throw BadURL("'%s' is not a valid flake ID", id); + + // FIXME: forbid query params? + + Input input; + input.direct = false; + input.attrs.insert_or_assign("type", "indirect"); + input.attrs.insert_or_assign("id", id); + if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); + if (ref) input.attrs.insert_or_assign("ref", *ref); + + return input; + } + + std::optional inputFromAttrs(const Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "indirect") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "id" && name != "ref" && name != "rev" && name != "narHash") + throw Error("unsupported indirect input attribute '%s'", name); + + auto id = getStrAttr(attrs, "id"); + if (!std::regex_match(id, flakeRegex)) + throw BadURL("'%s' is not a valid flake ID", id); + + Input input; + input.direct = false; + input.attrs = attrs; + return input; + } + + ParsedURL toURL(const Input & input) override + { + ParsedURL url; + url.scheme = "flake"; + url.path = getStrAttr(input.attrs, "id"); + if (auto ref = input.getRef()) { url.path += '/'; url.path += *ref; }; + if (auto rev = input.getRev()) { url.path += '/'; url.path += rev->gitRev(); }; + return url; + } + + bool hasAllInfo(const Input & input) override + { + return false; + } + + Input applyOverrides( + const Input & _input, + std::optional ref, + std::optional rev) override + { + auto input(_input); + if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); + if (ref) input.attrs.insert_or_assign("ref", *ref); + return input; + } + + std::pair fetch(ref store, const Input & input) override + { + throw Error("indirect input '%s' cannot be fetched directly", input.to_string()); + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); + +} diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 2e0d4bf4d..c48cb6fd1 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -10,76 +10,124 @@ using namespace std::string_literals; namespace nix::fetchers { -struct MercurialInput : Input +struct MercurialInputScheme : InputScheme { - ParsedURL url; - std::optional ref; - std::optional rev; - - MercurialInput(const ParsedURL & url) : url(url) - { } - - std::string type() const override { return "hg"; } - - bool operator ==(const Input & other) const override + std::optional inputFromURL(const ParsedURL & url) override { - auto other2 = dynamic_cast(&other); - return - other2 - && url == other2->url - && rev == other2->rev - && ref == other2->ref; + if (url.scheme != "hg+http" && + url.scheme != "hg+https" && + url.scheme != "hg+ssh" && + url.scheme != "hg+file") return {}; + + auto url2(url); + url2.scheme = std::string(url2.scheme, 3); + url2.query.clear(); + + Attrs attrs; + attrs.emplace("type", "hg"); + + for (auto &[name, value] : url.query) { + if (name == "rev" || name == "ref") + attrs.emplace(name, value); + else + url2.query.emplace(name, value); + } + + attrs.emplace("url", url2.to_string()); + + return inputFromAttrs(attrs); } - bool isImmutable() const override + std::optional inputFromAttrs(const Attrs & attrs) override { - return (bool) rev || narHash; + if (maybeGetStrAttr(attrs, "type") != "hg") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash") + throw Error("unsupported Mercurial input attribute '%s'", name); + + parseURL(getStrAttr(attrs, "url")); + + if (auto ref = maybeGetStrAttr(attrs, "ref")) { + if (!std::regex_match(*ref, refRegex)) + throw BadURL("invalid Mercurial branch/tag name '%s'", *ref); + } + + Input input; + input.attrs = attrs; + return input; } - std::optional getRef() const override { return ref; } - - std::optional getRev() const override { return rev; } - - ParsedURL toURL() const override + ParsedURL toURL(const Input & input) override { - ParsedURL url2(url); - url2.scheme = "hg+" + url2.scheme; - if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); - if (ref) url2.query.insert_or_assign("ref", *ref); + auto url = parseURL(getStrAttr(input.attrs, "url")); + url.scheme = "hg+" + url.scheme; + if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev()); + if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref); return url; } - Attrs toAttrsInternal() const override + bool hasAllInfo(const Input & input) override { - Attrs attrs; - attrs.emplace("url", url.to_string()); - if (ref) - attrs.emplace("ref", *ref); - if (rev) - attrs.emplace("rev", rev->gitRev()); - return attrs; + // FIXME: ugly, need to distinguish between dirty and clean + // default trees. + return input.getRef() == "default" || maybeGetIntAttr(input.attrs, "revCount"); } - std::pair getActualUrl() const + Input applyOverrides( + const Input & input, + std::optional ref, + std::optional rev) override { + auto res(input); + if (rev) res.attrs.insert_or_assign("rev", rev->gitRev()); + if (ref) res.attrs.insert_or_assign("ref", *ref); + return res; + } + + std::optional getSourcePath(const Input & input) override + { + auto url = parseURL(getStrAttr(input.attrs, "url")); + if (url.scheme == "file" && !input.getRef() && !input.getRev()) + return url.path; + return {}; + } + + void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) override + { + auto sourcePath = getSourcePath(input); + assert(sourcePath); + + // FIXME: shut up if file is already tracked. + runProgram("hg", true, + { "add", *sourcePath + "/" + std::string(file) }); + + if (commitMsg) + runProgram("hg", true, + { "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg }); + } + + std::pair getActualUrl(const Input & input) const + { + auto url = parseURL(getStrAttr(input.attrs, "url")); bool isLocal = url.scheme == "file"; return {isLocal, isLocal ? url.path : url.base}; } - std::pair> fetchTreeInternal(nix::ref store) const override + std::pair fetch(ref store, const Input & _input) override { auto name = "source"; - auto input = std::make_shared(*this); + Input input(_input); - auto [isLocal, actualUrl_] = getActualUrl(); + auto [isLocal, actualUrl_] = getActualUrl(input); auto actualUrl = actualUrl_; // work around clang bug // FIXME: return lastModified. // FIXME: don't clone local repositories. - if (!input->ref && !input->rev && isLocal && pathExists(actualUrl + "/.hg")) { + if (!input.getRef() && !input.getRev() && isLocal && pathExists(actualUrl + "/.hg")) { bool clean = runProgram("hg", true, { "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == ""; @@ -94,7 +142,7 @@ struct MercurialInput : Input if (settings.warnDirty) warn("Mercurial tree '%s' is unclean", actualUrl); - input->ref = chomp(runProgram("hg", true, { "branch", "-R", actualUrl })); + input.attrs.insert_or_assign("ref", chomp(runProgram("hg", true, { "branch", "-R", actualUrl }))); auto files = tokenizeString>( runProgram("hg", true, { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s); @@ -116,60 +164,54 @@ struct MercurialInput : Input auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter); - return {Tree { - .actualPath = store->printStorePath(storePath), - .storePath = std::move(storePath), - }, input}; + return { + Tree(store->printStorePath(storePath), std::move(storePath)), + input + }; } } - if (!input->ref) input->ref = "default"; + if (!input.getRef()) input.attrs.insert_or_assign("ref", "default"); auto getImmutableAttrs = [&]() { return Attrs({ {"type", "hg"}, {"name", name}, - {"rev", input->rev->gitRev()}, + {"rev", input.getRev()->gitRev()}, }); }; auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) - -> std::pair> + -> std::pair { - assert(input->rev); - assert(!rev || rev == input->rev); + assert(input.getRev()); + assert(!_input.getRev() || _input.getRev() == input.getRev()); + input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount")); return { - Tree{ - .actualPath = store->toRealPath(storePath), - .storePath = std::move(storePath), - .info = TreeInfo { - .revCount = getIntAttr(infoAttrs, "revCount"), - }, - }, + Tree(store->toRealPath(storePath), std::move(storePath)), input }; }; - if (input->rev) { + if (input.getRev()) { if (auto res = getCache()->lookup(store, getImmutableAttrs())) return makeResult(res->first, std::move(res->second)); } - assert(input->rev || input->ref); - auto revOrRef = input->rev ? input->rev->gitRev() : *input->ref; + auto revOrRef = input.getRev() ? input.getRev()->gitRev() : *input.getRef(); Attrs mutableAttrs({ {"type", "hg"}, {"name", name}, {"url", actualUrl}, - {"ref", *input->ref}, + {"ref", *input.getRef()}, }); if (auto res = getCache()->lookup(store, mutableAttrs)) { auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1); - if (!rev || rev == rev2) { - input->rev = rev2; + if (!input.getRev() || input.getRev() == rev2) { + input.attrs.insert_or_assign("rev", rev2.gitRev()); return makeResult(res->first, std::move(res->second)); } } @@ -178,10 +220,10 @@ struct MercurialInput : Input /* If this is a commit hash that we already have, we don't have to pull again. */ - if (!(input->rev + if (!(input.getRev() && pathExists(cacheDir) && runProgram( - RunOptions("hg", { "log", "-R", cacheDir, "-r", input->rev->gitRev(), "--template", "1" }) + RunOptions("hg", { "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" }) .killStderr(true)).second == "1")) { Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl)); @@ -210,9 +252,9 @@ struct MercurialInput : Input runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" })); assert(tokens.size() == 3); - input->rev = Hash(tokens[0], htSHA1); + input.attrs.insert_or_assign("rev", Hash(tokens[0], htSHA1).gitRev()); auto revCount = std::stoull(tokens[1]); - input->ref = tokens[2]; + input.attrs.insert_or_assign("ref", tokens[2]); if (auto res = getCache()->lookup(store, getImmutableAttrs())) return makeResult(res->first, std::move(res->second)); @@ -220,18 +262,18 @@ struct MercurialInput : Input Path tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir, true); - runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input->rev->gitRev(), tmpDir }); + runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input.getRev()->gitRev(), tmpDir }); deletePath(tmpDir + "/.hg_archival.txt"); auto storePath = store->addToStore(name, tmpDir); Attrs infoAttrs({ - {"rev", input->rev->gitRev()}, + {"rev", input.getRev()->gitRev()}, {"revCount", (int64_t) revCount}, }); - if (!this->rev) + if (!_input.getRev()) getCache()->add( store, mutableAttrs, @@ -250,54 +292,6 @@ struct MercurialInput : Input } }; -struct MercurialInputScheme : InputScheme -{ - std::unique_ptr inputFromURL(const ParsedURL & url) override - { - if (url.scheme != "hg+http" && - url.scheme != "hg+https" && - url.scheme != "hg+ssh" && - url.scheme != "hg+file") return nullptr; - - auto url2(url); - url2.scheme = std::string(url2.scheme, 3); - url2.query.clear(); - - Attrs attrs; - attrs.emplace("type", "hg"); - - for (auto &[name, value] : url.query) { - if (name == "rev" || name == "ref") - attrs.emplace(name, value); - else - url2.query.emplace(name, value); - } - - attrs.emplace("url", url2.to_string()); - - return inputFromAttrs(attrs); - } - - std::unique_ptr inputFromAttrs(const Attrs & attrs) override - { - if (maybeGetStrAttr(attrs, "type") != "hg") return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev") - throw Error("unsupported Mercurial input attribute '%s'", name); - - auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); - if (auto ref = maybeGetStrAttr(attrs, "ref")) { - if (!std::regex_match(*ref, refRegex)) - throw BadURL("invalid Mercurial branch/tag name '%s'", *ref); - input->ref = *ref; - } - if (auto rev = maybeGetStrAttr(attrs, "rev")) - input->rev = Hash(*rev, htSHA1); - return input; - } -}; - static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); } diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index ba2cc192e..99d4b4e8f 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -3,65 +3,86 @@ namespace nix::fetchers { -struct PathInput : Input +struct PathInputScheme : InputScheme { - Path path; - - /* Allow the user to pass in "fake" tree info attributes. This is - useful for making a pinned tree work the same as the repository - from which is exported - (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */ - std::optional rev; - std::optional revCount; - std::optional lastModified; - - std::string type() const override { return "path"; } - - std::optional getRev() const override { return rev; } - - bool operator ==(const Input & other) const override + std::optional inputFromURL(const ParsedURL & url) override { - auto other2 = dynamic_cast(&other); - return - other2 - && path == other2->path - && rev == other2->rev - && revCount == other2->revCount - && lastModified == other2->lastModified; + if (url.scheme != "path") return {}; + + if (url.authority && *url.authority != "") + throw Error("path URL '%s' should not have an authority ('%s')", url.url, *url.authority); + + Input input; + input.attrs.insert_or_assign("type", "path"); + input.attrs.insert_or_assign("path", url.path); + + for (auto & [name, value] : url.query) + if (name == "rev" || name == "narHash") + input.attrs.insert_or_assign(name, value); + else if (name == "revCount" || name == "lastModified") { + uint64_t n; + if (!string2Int(value, n)) + throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); + input.attrs.insert_or_assign(name, n); + } + else + throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name); + + return input; } - bool isImmutable() const override + std::optional inputFromAttrs(const Attrs & attrs) override { - return (bool) narHash; + if (maybeGetStrAttr(attrs, "type") != "path") return {}; + + getStrAttr(attrs, "path"); + + for (auto & [name, value] : attrs) + /* Allow the user to pass in "fake" tree info + attributes. This is useful for making a pinned tree + work the same as the repository from which is exported + (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */ + if (name == "type" || name == "rev" || name == "revCount" || name == "lastModified" || name == "narHash" || name == "path") + // checked in Input::fromAttrs + ; + else + throw Error("unsupported path input attribute '%s'", name); + + Input input; + input.attrs = attrs; + return input; } - ParsedURL toURL() const override + ParsedURL toURL(const Input & input) override { - auto query = attrsToQuery(toAttrsInternal()); + auto query = attrsToQuery(input.attrs); query.erase("path"); + query.erase("type"); return ParsedURL { .scheme = "path", - .path = path, + .path = getStrAttr(input.attrs, "path"), .query = query, }; } - Attrs toAttrsInternal() const override + bool hasAllInfo(const Input & input) override { - Attrs attrs; - attrs.emplace("path", path); - if (rev) - attrs.emplace("rev", rev->gitRev()); - if (revCount) - attrs.emplace("revCount", *revCount); - if (lastModified) - attrs.emplace("lastModified", *lastModified); - return attrs; + return true; } - std::pair> fetchTreeInternal(nix::ref store) const override + std::optional getSourcePath(const Input & input) override { - auto input = std::make_shared(*this); + return getStrAttr(input.attrs, "path"); + } + + void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) override + { + // nothing to do + } + + std::pair fetch(ref store, const Input & input) override + { + auto path = getStrAttr(input.attrs, "path"); // FIXME: check whether access to 'path' is allowed. @@ -74,72 +95,10 @@ struct PathInput : Input // FIXME: try to substitute storePath. storePath = store->addToStore("source", path); - return - { - Tree { - .actualPath = store->toRealPath(*storePath), - .storePath = std::move(*storePath), - .info = TreeInfo { - .revCount = revCount, - .lastModified = lastModified - } - }, - input - }; - } - -}; - -struct PathInputScheme : InputScheme -{ - std::unique_ptr inputFromURL(const ParsedURL & url) override - { - if (url.scheme != "path") return nullptr; - - auto input = std::make_unique(); - input->path = url.path; - - for (auto & [name, value] : url.query) - if (name == "rev") - input->rev = Hash(value, htSHA1); - else if (name == "revCount") { - uint64_t revCount; - if (!string2Int(value, revCount)) - throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); - input->revCount = revCount; - } - else if (name == "lastModified") { - time_t lastModified; - if (!string2Int(value, lastModified)) - throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); - input->lastModified = lastModified; - } - else - throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name); - - return input; - } - - std::unique_ptr inputFromAttrs(const Attrs & attrs) override - { - if (maybeGetStrAttr(attrs, "type") != "path") return {}; - - auto input = std::make_unique(); - input->path = getStrAttr(attrs, "path"); - - for (auto & [name, value] : attrs) - if (name == "rev") - input->rev = Hash(getStrAttr(attrs, "rev"), htSHA1); - else if (name == "revCount") - input->revCount = getIntAttr(attrs, "revCount"); - else if (name == "lastModified") - input->lastModified = getIntAttr(attrs, "lastModified"); - else if (name == "type" || name == "path") - ; - else - throw Error("unsupported path input attribute '%s'", name); - - return input; + return { + Tree(store->toRealPath(*storePath), std::move(*storePath)), + input + }; } }; diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc new file mode 100644 index 000000000..d4134ce29 --- /dev/null +++ b/src/libfetchers/registry.cc @@ -0,0 +1,212 @@ +#include "registry.hh" +#include "fetchers.hh" +#include "util.hh" +#include "globals.hh" +#include "store-api.hh" + +#include + +namespace nix::fetchers { + +std::shared_ptr Registry::read( + const Path & path, RegistryType type) +{ + auto registry = std::make_shared(type); + + if (!pathExists(path)) + return std::make_shared(type); + + try { + + auto json = nlohmann::json::parse(readFile(path)); + + auto version = json.value("version", 0); + + if (version == 2) { + for (auto & i : json["flakes"]) { + auto toAttrs = jsonToAttrs(i["to"]); + Attrs extraAttrs; + auto j = toAttrs.find("dir"); + if (j != toAttrs.end()) { + extraAttrs.insert(*j); + toAttrs.erase(j); + } + auto exact = i.find("exact"); + registry->entries.push_back( + Entry { + .from = Input::fromAttrs(jsonToAttrs(i["from"])), + .to = Input::fromAttrs(std::move(toAttrs)), + .extraAttrs = extraAttrs, + .exact = exact != i.end() && exact.value() + }); + } + } + + else + throw Error("flake registry '%s' has unsupported version %d", path, version); + + } catch (nlohmann::json::exception & e) { + warn("cannot parse flake registry '%s': %s", path, e.what()); + } catch (Error & e) { + warn("cannot read flake registry '%s': %s", path, e.what()); + } + + return registry; +} + +void Registry::write(const Path & path) +{ + nlohmann::json arr; + for (auto & entry : entries) { + nlohmann::json obj; + obj["from"] = attrsToJson(entry.from.toAttrs()); + obj["to"] = attrsToJson(entry.to.toAttrs()); + if (!entry.extraAttrs.empty()) + obj["to"].update(attrsToJson(entry.extraAttrs)); + if (entry.exact) + obj["exact"] = true; + arr.emplace_back(std::move(obj)); + } + + nlohmann::json json; + json["version"] = 2; + json["flakes"] = std::move(arr); + + createDirs(dirOf(path)); + writeFile(path, json.dump(2)); +} + +void Registry::add( + const Input & from, + const Input & to, + const Attrs & extraAttrs) +{ + entries.emplace_back( + Entry { + .from = from, + .to = to, + .extraAttrs = extraAttrs + }); +} + +void Registry::remove(const Input & input) +{ + // FIXME: use C++20 std::erase. + for (auto i = entries.begin(); i != entries.end(); ) + if (i->from == input) + i = entries.erase(i); + else + ++i; +} + +static Path getSystemRegistryPath() +{ + return settings.nixConfDir + "/registry.json"; +} + +static std::shared_ptr getSystemRegistry() +{ + static auto systemRegistry = + Registry::read(getSystemRegistryPath(), Registry::System); + return systemRegistry; +} + +Path getUserRegistryPath() +{ + return getHome() + "/.config/nix/registry.json"; +} + +std::shared_ptr getUserRegistry() +{ + static auto userRegistry = + Registry::read(getUserRegistryPath(), Registry::User); + return userRegistry; +} + +static std::shared_ptr flagRegistry = + std::make_shared(Registry::Flag); + +std::shared_ptr getFlagRegistry() +{ + return flagRegistry; +} + +void overrideRegistry( + const Input & from, + const Input & to, + const Attrs & extraAttrs) +{ + flagRegistry->add(from, to, extraAttrs); +} + +static std::shared_ptr getGlobalRegistry(ref store) +{ + static auto reg = [&]() { + auto path = settings.flakeRegistry.get(); + + if (!hasPrefix(path, "/")) { + auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath; + if (auto store2 = store.dynamic_pointer_cast()) + store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json", true); + path = store->toRealPath(storePath); + } + + return Registry::read(path, Registry::Global); + }(); + + return reg; +} + +Registries getRegistries(ref store) +{ + Registries registries; + registries.push_back(getFlagRegistry()); + registries.push_back(getUserRegistry()); + registries.push_back(getSystemRegistry()); + registries.push_back(getGlobalRegistry(store)); + return registries; +} + +std::pair lookupInRegistries( + ref store, + const Input & _input) +{ + Attrs extraAttrs; + int n = 0; + Input input(_input); + + restart: + + n++; + if (n > 100) throw Error("cycle detected in flake registry for '%s'", input.to_string()); + + for (auto & registry : getRegistries(store)) { + // FIXME: O(n) + for (auto & entry : registry->entries) { + if (entry.exact) { + if (entry.from == input) { + input = entry.to; + extraAttrs = entry.extraAttrs; + goto restart; + } + } else { + if (entry.from.contains(input)) { + input = entry.to.applyOverrides( + !entry.from.getRef() && input.getRef() ? input.getRef() : std::optional(), + !entry.from.getRev() && input.getRev() ? input.getRev() : std::optional()); + extraAttrs = entry.extraAttrs; + goto restart; + } + } + } + } + + if (!input.isDirect()) + throw Error("cannot find flake '%s' in the flake registries", input.to_string()); + + debug("looked up '%s' -> '%s'", _input.to_string(), input.to_string()); + + return {input, extraAttrs}; +} + +} diff --git a/src/libfetchers/registry.hh b/src/libfetchers/registry.hh new file mode 100644 index 000000000..1077af020 --- /dev/null +++ b/src/libfetchers/registry.hh @@ -0,0 +1,64 @@ +#pragma once + +#include "types.hh" +#include "fetchers.hh" + +namespace nix { class Store; } + +namespace nix::fetchers { + +struct Registry +{ + enum RegistryType { + Flag = 0, + User = 1, + System = 2, + Global = 3, + }; + + RegistryType type; + + struct Entry + { + Input from, to; + Attrs extraAttrs; + bool exact = false; + }; + + std::vector entries; + + Registry(RegistryType type) + : type(type) + { } + + static std::shared_ptr read( + const Path & path, RegistryType type); + + void write(const Path & path); + + void add( + const Input & from, + const Input & to, + const Attrs & extraAttrs); + + void remove(const Input & input); +}; + +typedef std::vector> Registries; + +std::shared_ptr getUserRegistry(); + +Path getUserRegistryPath(); + +Registries getRegistries(ref store); + +void overrideRegistry( + const Input & from, + const Input & to, + const Attrs & extraAttrs); + +std::pair lookupInRegistries( + ref store, + const Input & input); + +} diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index f5356f0af..55158cece 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -105,7 +105,7 @@ DownloadFileResult downloadFile( }; } -Tree downloadTarball( +std::pair downloadTarball( ref store, const std::string & url, const std::string & name, @@ -120,12 +120,9 @@ Tree downloadTarball( auto cached = getCache()->lookupExpired(store, inAttrs); if (cached && !cached->expired) - return Tree { - .actualPath = store->toRealPath(cached->storePath), - .storePath = std::move(cached->storePath), - .info = TreeInfo { - .lastModified = getIntAttr(cached->infoAttrs, "lastModified"), - }, + return { + Tree(store->toRealPath(cached->storePath), std::move(cached->storePath)), + getIntAttr(cached->infoAttrs, "lastModified") }; auto res = downloadFile(store, url, name, immutable); @@ -160,117 +157,72 @@ Tree downloadTarball( *unpackedStorePath, immutable); - return Tree { - .actualPath = store->toRealPath(*unpackedStorePath), - .storePath = std::move(*unpackedStorePath), - .info = TreeInfo { - .lastModified = lastModified, - }, + return { + Tree(store->toRealPath(*unpackedStorePath), std::move(*unpackedStorePath)), + lastModified, }; } -struct TarballInput : Input -{ - ParsedURL url; - std::optional hash; - - TarballInput(const ParsedURL & url) : url(url) - { } - - std::string type() const override { return "tarball"; } - - bool operator ==(const Input & other) const override - { - auto other2 = dynamic_cast(&other); - return - other2 - && to_string() == other2->to_string() - && hash == other2->hash; - } - - bool isImmutable() const override - { - return hash || narHash; - } - - ParsedURL toURL() const override - { - auto url2(url); - // NAR hashes are preferred over file hashes since tar/zip files - // don't have a canonical representation. - if (narHash) - url2.query.insert_or_assign("narHash", narHash->to_string(SRI, true)); - else if (hash) - url2.query.insert_or_assign("hash", hash->to_string(SRI, true)); - return url2; - } - - Attrs toAttrsInternal() const override - { - Attrs attrs; - attrs.emplace("url", url.to_string()); - if (hash) - attrs.emplace("hash", hash->to_string(SRI, true)); - return attrs; - } - - std::pair> fetchTreeInternal(nix::ref store) const override - { - auto tree = downloadTarball(store, url.to_string(), "source", false); - - auto input = std::make_shared(*this); - input->narHash = store->queryPathInfo(tree.storePath)->narHash; - - return {std::move(tree), input}; - } -}; - struct TarballInputScheme : InputScheme { - std::unique_ptr inputFromURL(const ParsedURL & url) override + std::optional inputFromURL(const ParsedURL & url) override { - if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return nullptr; + if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return {}; if (!hasSuffix(url.path, ".zip") && !hasSuffix(url.path, ".tar") && !hasSuffix(url.path, ".tar.gz") && !hasSuffix(url.path, ".tar.xz") && !hasSuffix(url.path, ".tar.bz2")) - return nullptr; - - auto input = std::make_unique(url); - - auto hash = input->url.query.find("hash"); - if (hash != input->url.query.end()) { - // FIXME: require SRI hash. - input->hash = Hash(hash->second); - input->url.query.erase(hash); - } - - auto narHash = input->url.query.find("narHash"); - if (narHash != input->url.query.end()) { - // FIXME: require SRI hash. - input->narHash = Hash(narHash->second); - input->url.query.erase(narHash); - } + return {}; + Input input; + input.attrs.insert_or_assign("type", "tarball"); + input.attrs.insert_or_assign("url", url.to_string()); + auto narHash = url.query.find("narHash"); + if (narHash != url.query.end()) + input.attrs.insert_or_assign("narHash", narHash->second); return input; } - std::unique_ptr inputFromAttrs(const Attrs & attrs) override + std::optional inputFromAttrs(const Attrs & attrs) override { if (maybeGetStrAttr(attrs, "type") != "tarball") return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "hash") + if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash") throw Error("unsupported tarball input attribute '%s'", name); - auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); - if (auto hash = maybeGetStrAttr(attrs, "hash")) - input->hash = newHashAllowEmpty(*hash, {}); - + Input input; + input.attrs = attrs; + //input.immutable = (bool) maybeGetStrAttr(input.attrs, "hash"); return input; } + + ParsedURL toURL(const Input & input) override + { + auto url = parseURL(getStrAttr(input.attrs, "url")); + // NAR hashes are preferred over file hashes since tar/zip files + // don't have a canonical representation. + if (auto narHash = input.getNarHash()) + url.query.insert_or_assign("narHash", narHash->to_string(SRI, true)); + /* + else if (auto hash = maybeGetStrAttr(input.attrs, "hash")) + url.query.insert_or_assign("hash", Hash(*hash).to_string(SRI, true)); + */ + return url; + } + + bool hasAllInfo(const Input & input) override + { + return true; + } + + std::pair fetch(ref store, const Input & input) override + { + auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false).first; + return {std::move(tree), input}; + } }; static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); diff --git a/src/libfetchers/tree-info.cc b/src/libfetchers/tree-info.cc deleted file mode 100644 index b2d8cfc8d..000000000 --- a/src/libfetchers/tree-info.cc +++ /dev/null @@ -1,14 +0,0 @@ -#include "tree-info.hh" -#include "store-api.hh" - -#include - -namespace nix::fetchers { - -StorePath TreeInfo::computeStorePath(Store & store) const -{ - assert(narHash); - return store.makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "source"); -} - -} diff --git a/src/libfetchers/tree-info.hh b/src/libfetchers/tree-info.hh deleted file mode 100644 index 2c7347281..000000000 --- a/src/libfetchers/tree-info.hh +++ /dev/null @@ -1,29 +0,0 @@ -#pragma once - -#include "path.hh" -#include "hash.hh" - -#include - -namespace nix { class Store; } - -namespace nix::fetchers { - -struct TreeInfo -{ - Hash narHash; - std::optional revCount; - std::optional lastModified; - - bool operator ==(const TreeInfo & other) const - { - return - narHash == other.narHash - && revCount == other.revCount - && lastModified == other.lastModified; - } - - StorePath computeStorePath(Store & store) const; -}; - -} diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 051668e53..09f4cd133 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -34,9 +34,19 @@ MixCommonArgs::MixCommonArgs(const string & programName) try { globalConfig.set(name, value); } catch (UsageError & e) { - warn(e.what()); + if (!completions) + warn(e.what()); } }}, + .completer = [](size_t index, std::string_view prefix) { + if (index == 0) { + std::map settings; + globalConfig.getSettings(settings); + for (auto & s : settings) + if (hasPrefix(s.first, prefix)) + completions->insert(s.first); + } + } }); addFlag({ diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc index fa18f991d..0a7291780 100644 --- a/src/libmain/loggers.cc +++ b/src/libmain/loggers.cc @@ -1,12 +1,13 @@ #include "loggers.hh" #include "progress-bar.hh" +#include "util.hh" namespace nix { LogFormat defaultLogFormat = LogFormat::raw; LogFormat parseLogFormat(const std::string & logFormatStr) { - if (logFormatStr == "raw") + if (logFormatStr == "raw" || getEnv("NIX_GET_COMPLETIONS")) return LogFormat::raw; else if (logFormatStr == "raw-with-logs") return LogFormat::rawWithLogs; diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 52718c231..2b1f25ca3 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -36,7 +36,7 @@ void printGCWarning() void printMissing(ref store, const std::vector & paths, Verbosity lvl) { - unsigned long long downloadSize, narSize; + uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); printMissing(store, willBuild, willSubstitute, unknown, downloadSize, narSize, lvl); @@ -45,7 +45,7 @@ void printMissing(ref store, const std::vector & pa void printMissing(ref store, const StorePathSet & willBuild, const StorePathSet & willSubstitute, const StorePathSet & unknown, - unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl) + uint64_t downloadSize, uint64_t narSize, Verbosity lvl) { if (!willBuild.empty()) { if (willBuild.size() == 1) @@ -384,7 +384,7 @@ RunPager::~RunPager() } -string showBytes(unsigned long long bytes) +string showBytes(uint64_t bytes) { return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str(); } diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index f558247c0..ffae5d796 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -47,7 +47,7 @@ void printMissing( void printMissing(ref store, const StorePathSet & willBuild, const StorePathSet & willSubstitute, const StorePathSet & unknown, - unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl = lvlInfo); + uint64_t downloadSize, uint64_t narSize, Verbosity lvl = lvlInfo); string getArg(const string & opt, Strings::iterator & i, const Strings::iterator & end); @@ -110,7 +110,7 @@ extern volatile ::sig_atomic_t blockInt; /* GC helpers. */ -string showBytes(unsigned long long bytes); +string showBytes(uint64_t bytes); struct GCResults; diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index b791c125b..30150eeba 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -178,7 +178,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource auto [fileHash, fileSize] = fileHashSink.finish(); narInfo->fileHash = fileHash; narInfo->fileSize = fileSize; - narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" + narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar" + (compression == "xz" ? ".xz" : compression == "bzip2" ? ".bz2" : compression == "br" ? ".br" : @@ -372,7 +372,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath method for very large paths, but `copyPath' is mainly used for small files. */ StringSink sink; - Hash h; + std::optional h; if (method == FileIngestionMethod::Recursive) { dumpPath(srcPath, sink, filter); h = hashString(hashAlgo, *sink.s); @@ -382,7 +382,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath h = hashString(hashAlgo, s); } - ValidPathInfo info(makeFixedOutputPath(method, h, name)); + ValidPathInfo info(makeFixedOutputPath(method, *h, name)); auto source = StringSource { *sink.s }; addToStore(info, source, repair, CheckSigs); diff --git a/src/libstore/build.cc b/src/libstore/build.cc index ac2e67574..4156cd678 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -297,7 +297,7 @@ public: GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); std::shared_ptr makeBasicDerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal); - GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair); + GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); /* Remove a dead goal. */ void removeGoal(GoalPtr goal); @@ -1047,7 +1047,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation { this->drv = std::make_unique(BasicDerivation(drv)); state = &DerivationGoal::haveDerivation; - name = fmt("building of %s", worker.store.showPaths(drv.outputPaths())); + name = fmt("building of %s", worker.store.showPaths(drv.outputPaths(worker.store))); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); @@ -1182,7 +1182,7 @@ void DerivationGoal::haveDerivation() retrySubstitution = false; for (auto & i : drv->outputs) - worker.store.addTempRoot(i.second.path); + worker.store.addTempRoot(i.second.path(worker.store, drv->name)); /* Check what outputs paths are not already valid. */ auto invalidOutputs = checkPathValidity(false, buildMode == bmRepair); @@ -1206,7 +1206,7 @@ void DerivationGoal::haveDerivation() them. */ if (settings.useSubstitutes && parsedDrv->substitutesAllowed()) for (auto & i : invalidOutputs) - addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair)); + addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair, getDerivationCA(*drv))); if (waitees.empty()) /* to prevent hang (no wake-up event) */ outputsSubstituted(); @@ -1290,12 +1290,12 @@ void DerivationGoal::repairClosure() StorePathSet outputClosure; for (auto & i : drv->outputs) { if (!wantOutput(i.first, wantedOutputs)) continue; - worker.store.computeFSClosure(i.second.path, outputClosure); + worker.store.computeFSClosure(i.second.path(worker.store, drv->name), outputClosure); } /* Filter out our own outputs (which we have already checked). */ for (auto & i : drv->outputs) - outputClosure.erase(i.second.path); + outputClosure.erase(i.second.path(worker.store, drv->name)); /* Get all dependencies of this derivation so that we know which derivation is responsible for which path in the output @@ -1307,7 +1307,7 @@ void DerivationGoal::repairClosure() if (i.isDerivation()) { Derivation drv = worker.store.derivationFromPath(i); for (auto & j : drv.outputs) - outputsToDrv.insert_or_assign(j.second.path, i); + outputsToDrv.insert_or_assign(j.second.path(worker.store, drv.name), i); } /* Check each path (slow!). */ @@ -1379,7 +1379,7 @@ void DerivationGoal::inputsRealised() for (auto & j : i.second) { auto k = inDrv.outputs.find(j); if (k != inDrv.outputs.end()) - worker.store.computeFSClosure(k->second.path, inputPaths); + worker.store.computeFSClosure(k->second.path(worker.store, inDrv.name), inputPaths); else throw Error( "derivation '%s' requires non-existent output '%s' from input derivation '%s'", @@ -1432,7 +1432,7 @@ void DerivationGoal::tryToBuild() goal can start a build, and if not, the main loop will sleep a few seconds and then retry this goal. */ PathSet lockFiles; - for (auto & outPath : drv->outputPaths()) + for (auto & outPath : drv->outputPaths(worker.store)) lockFiles.insert(worker.store.Store::toRealPath(outPath)); if (!outputLocks.lockPaths(lockFiles, "", false)) { @@ -1460,16 +1460,16 @@ void DerivationGoal::tryToBuild() return; } - missingPaths = drv->outputPaths(); + missingPaths = drv->outputPaths(worker.store); if (buildMode != bmCheck) for (auto & i : validPaths) missingPaths.erase(i); /* If any of the outputs already exist but are not valid, delete them. */ for (auto & i : drv->outputs) { - if (worker.store.isValidPath(i.second.path)) continue; - debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path)); - deletePath(worker.store.Store::toRealPath(i.second.path)); + if (worker.store.isValidPath(i.second.path(worker.store, drv->name))) continue; + debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path(worker.store, drv->name))); + deletePath(worker.store.Store::toRealPath(i.second.path(worker.store, drv->name))); } /* Don't do a remote build if the derivation has the attribute @@ -1646,13 +1646,13 @@ void DerivationGoal::buildDone() So instead, check if the disk is (nearly) full now. If so, we don't mark this build as a permanent failure. */ #if HAVE_STATVFS - unsigned long long required = 8ULL * 1024 * 1024; // FIXME: make configurable + uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable struct statvfs st; if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 && - (unsigned long long) st.f_bavail * st.f_bsize < required) + (uint64_t) st.f_bavail * st.f_bsize < required) diskFull = true; if (statvfs(tmpDir.c_str(), &st) == 0 && - (unsigned long long) st.f_bavail * st.f_bsize < required) + (uint64_t) st.f_bavail * st.f_bsize < required) diskFull = true; #endif @@ -1692,7 +1692,7 @@ void DerivationGoal::buildDone() fmt("running post-build-hook '%s'", settings.postBuildHook), Logger::Fields{worker.store.printStorePath(drvPath)}); PushActivity pact(act.id); - auto outputPaths = drv->outputPaths(); + auto outputPaths = drv->outputPaths(worker.store); std::map hookEnvironment = getEnv(); hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath)); @@ -1920,7 +1920,7 @@ StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) if (j.isDerivation()) { Derivation drv = worker.store.derivationFromPath(j); for (auto & k : drv.outputs) - worker.store.computeFSClosure(k.second.path, paths); + worker.store.computeFSClosure(k.second.path(worker.store, drv.name), paths); } } @@ -2015,7 +2015,7 @@ void DerivationGoal::startBuilder() /* Substitute output placeholders with the actual output paths. */ for (auto & output : drv->outputs) - inputRewrites[hashPlaceholder(output.first)] = worker.store.printStorePath(output.second.path); + inputRewrites[hashPlaceholder(output.first)] = worker.store.printStorePath(output.second.path(worker.store, drv->name)); /* Construct the environment passed to the builder. */ initEnv(); @@ -2200,7 +2200,7 @@ void DerivationGoal::startBuilder() (typically the dependencies of /bin/sh). Throw them out. */ for (auto & i : drv->outputs) - dirsInChroot.erase(worker.store.printStorePath(i.second.path)); + dirsInChroot.erase(worker.store.printStorePath(i.second.path(worker.store, drv->name))); #elif __APPLE__ /* We don't really have any parent prep work to do (yet?) @@ -2613,7 +2613,7 @@ void DerivationGoal::writeStructuredAttrs() /* Add an "outputs" object containing the output paths. */ nlohmann::json outputs; for (auto & i : drv->outputs) - outputs[i.first] = rewriteStrings(worker.store.printStorePath(i.second.path), inputRewrites); + outputs[i.first] = rewriteStrings(worker.store.printStorePath(i.second.path(worker.store, drv->name)), inputRewrites); json["outputs"] = outputs; /* Handle exportReferencesGraph. */ @@ -2774,7 +2774,7 @@ struct RestrictedStore : public LocalFSStore goal.addDependency(info.path); } - StorePath addToStoreFromDump(const string & dump, const string & name, + StorePath addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override { auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair); @@ -2817,7 +2817,7 @@ struct RestrictedStore : public LocalFSStore auto drv = derivationFromPath(path.path); for (auto & output : drv.outputs) if (wantOutput(output.first, path.outputs)) - newPaths.insert(output.second.path); + newPaths.insert(output.second.path(*this, drv.name)); } else if (!goal.isAllowed(path.path)) throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); } @@ -2851,7 +2851,7 @@ struct RestrictedStore : public LocalFSStore void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize) override + uint64_t & downloadSize, uint64_t & narSize) override { /* This is slightly impure since it leaks information to the client about what paths will be built/substituted or are @@ -3579,7 +3579,7 @@ StorePathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv if (store.isStorePath(i)) result.insert(store.parseStorePath(i)); else if (drv.outputs.count(i)) - result.insert(drv.outputs.find(i)->second.path); + result.insert(drv.outputs.find(i)->second.path(store, drv.name)); else throw BuildError("derivation contains an illegal reference specifier '%s'", i); } return result; @@ -3617,7 +3617,7 @@ void DerivationGoal::registerOutputs() if (hook) { bool allValid = true; for (auto & i : drv->outputs) - if (!worker.store.isValidPath(i.second.path)) allValid = false; + if (!worker.store.isValidPath(i.second.path(worker.store, drv->name))) allValid = false; if (allValid) return; } @@ -3638,23 +3638,23 @@ void DerivationGoal::registerOutputs() Nix calls. */ StorePathSet referenceablePaths; for (auto & p : inputPaths) referenceablePaths.insert(p); - for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path); + for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path(worker.store, drv->name)); for (auto & p : addedPaths) referenceablePaths.insert(p); /* Check whether the output paths were created, and grep each output path to determine what other paths it references. Also make all output paths read-only. */ for (auto & i : drv->outputs) { - auto path = worker.store.printStorePath(i.second.path); - if (!missingPaths.count(i.second.path)) continue; + auto path = worker.store.printStorePath(i.second.path(worker.store, drv->name)); + if (!missingPaths.count(i.second.path(worker.store, drv->name))) continue; Path actualPath = path; if (needsHashRewrite()) { - auto r = redirectedOutputs.find(i.second.path); + auto r = redirectedOutputs.find(i.second.path(worker.store, drv->name)); if (r != redirectedOutputs.end()) { auto redirected = worker.store.Store::toRealPath(r->second); if (buildMode == bmRepair - && redirectedBadOutputs.count(i.second.path) + && redirectedBadOutputs.count(i.second.path(worker.store, drv->name)) && pathExists(redirected)) replaceValidPath(path, redirected); if (buildMode == bmCheck) @@ -3723,7 +3723,9 @@ void DerivationGoal::registerOutputs() if (fixedOutput) { - if (i.second.hash->method == FileIngestionMethod::Flat) { + FixedOutputHash outputHash = std::get(i.second.output).hash; + + if (outputHash.method == FileIngestionMethod::Flat) { /* The output path should be a regular file without execute permission. */ if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) throw BuildError( @@ -3734,13 +3736,13 @@ void DerivationGoal::registerOutputs() /* Check the hash. In hash mode, move the path produced by the derivation to its content-addressed location. */ - Hash h2 = i.second.hash->method == FileIngestionMethod::Recursive - ? hashPath(*i.second.hash->hash.type, actualPath).first - : hashFile(*i.second.hash->hash.type, actualPath); + Hash h2 = outputHash.method == FileIngestionMethod::Recursive + ? hashPath(outputHash.hash.type, actualPath).first + : hashFile(outputHash.hash.type, actualPath); - auto dest = worker.store.makeFixedOutputPath(i.second.hash->method, h2, i.second.path.name()); + auto dest = worker.store.makeFixedOutputPath(outputHash.method, h2, i.second.path(worker.store, drv->name).name()); - if (i.second.hash->hash != h2) { + if (outputHash.hash != h2) { /* Throw an error after registering the path as valid. */ @@ -3748,7 +3750,7 @@ void DerivationGoal::registerOutputs() delayedException = std::make_exception_ptr( BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", worker.store.printStorePath(dest), - i.second.hash->hash.to_string(SRI, true), + outputHash.hash.to_string(SRI, true), h2.to_string(SRI, true))); Path actualDest = worker.store.Store::toRealPath(dest); @@ -3770,7 +3772,7 @@ void DerivationGoal::registerOutputs() assert(worker.store.parseStorePath(path) == dest); ca = FixedOutputHash { - .method = i.second.hash->method, + .method = outputHash.method, .hash = h2, }; } @@ -3785,8 +3787,10 @@ void DerivationGoal::registerOutputs() time. The hash is stored in the database so that we can verify later on whether nobody has messed with the store. */ debug("scanning for references inside '%1%'", path); - HashResult hash; - auto references = worker.store.parseStorePathSet(scanForReferences(actualPath, worker.store.printStorePathSet(referenceablePaths), hash)); + // HashResult hash; + auto pathSetAndHash = scanForReferences(actualPath, worker.store.printStorePathSet(referenceablePaths)); + auto references = worker.store.parseStorePathSet(pathSetAndHash.first); + HashResult hash = pathSetAndHash.second; if (buildMode == bmCheck) { if (!worker.store.isValidPath(worker.store.parseStorePath(path))) continue; @@ -3894,7 +3898,7 @@ void DerivationGoal::registerOutputs() /* If this is the first round of several, then move the output out of the way. */ if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) { for (auto & i : drv->outputs) { - auto path = worker.store.printStorePath(i.second.path); + auto path = worker.store.printStorePath(i.second.path(worker.store, drv->name)); Path prev = path + checkSuffix; deletePath(prev); Path dst = path + checkSuffix; @@ -3912,7 +3916,7 @@ void DerivationGoal::registerOutputs() if the result was not determistic? */ if (curRound == nrRounds) { for (auto & i : drv->outputs) { - Path prev = worker.store.printStorePath(i.second.path) + checkSuffix; + Path prev = worker.store.printStorePath(i.second.path(worker.store, drv->name)) + checkSuffix; deletePath(prev); } } @@ -4213,9 +4217,9 @@ StorePathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) for (auto & i : drv->outputs) { if (!wantOutput(i.first, wantedOutputs)) continue; bool good = - worker.store.isValidPath(i.second.path) && - (!checkHash || worker.pathContentsGood(i.second.path)); - if (good == returnValid) result.insert(i.second.path); + worker.store.isValidPath(i.second.path(worker.store, drv->name)) && + (!checkHash || worker.pathContentsGood(i.second.path(worker.store, drv->name))); + if (good == returnValid) result.insert(i.second.path(worker.store, drv->name)); } return result; } @@ -4272,6 +4276,10 @@ private: /* The store path that should be realised through a substitute. */ StorePath storePath; + /* The path the substituter refers to the path as. This will be + * different when the stores have different names. */ + std::optional subPath; + /* The remaining substituters. */ std::list> subs; @@ -4305,8 +4313,11 @@ private: typedef void (SubstitutionGoal::*GoalState)(); GoalState state; + /* Content address for recomputing store path */ + std::optional ca; + public: - SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair); + SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); ~SubstitutionGoal(); void timedOut(Error && ex) override { abort(); }; @@ -4336,10 +4347,11 @@ public: }; -SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair) +SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) : Goal(worker) , storePath(storePath) , repair(repair) + , ca(ca) { state = &SubstitutionGoal::init; name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath)); @@ -4414,14 +4426,18 @@ void SubstitutionGoal::tryNext() sub = subs.front(); subs.pop_front(); - if (sub->storeDir != worker.store.storeDir) { + if (ca) { + subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); + if (sub->storeDir == worker.store.storeDir) + assert(subPath == storePath); + } else if (sub->storeDir != worker.store.storeDir) { tryNext(); return; } try { // FIXME: make async - info = sub->queryPathInfo(storePath); + info = sub->queryPathInfo(subPath ? *subPath : storePath); } catch (InvalidPath &) { tryNext(); return; @@ -4440,6 +4456,19 @@ void SubstitutionGoal::tryNext() throw; } + if (info->path != storePath) { + if (info->isContentAddressed(*sub) && info->references.empty()) { + auto info2 = std::make_shared(*info); + info2->path = storePath; + info = info2; + } else { + printError("asked '%s' for '%s' but got '%s'", + sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path)); + tryNext(); + return; + } + } + /* Update the total expected download size. */ auto narInfo = std::dynamic_pointer_cast(info); @@ -4529,7 +4558,7 @@ void SubstitutionGoal::tryToRun() PushActivity pact(act.id); copyStorePath(ref(sub), ref(worker.store.shared_from_this()), - storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); + subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); promise.set_value(); } catch (...) { @@ -4662,11 +4691,11 @@ std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath } -GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair) +GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) { GoalPtr goal = substitutionGoals[path].lock(); // FIXME if (!goal) { - goal = std::make_shared(path, *this, repair); + goal = std::make_shared(path, *this, repair, ca); substitutionGoals.insert_or_assign(path, goal); wakeUp(goal); } @@ -5008,7 +5037,7 @@ bool Worker::pathContentsGood(const StorePath & path) if (!pathExists(store.printStorePath(path))) res = false; else { - HashResult current = hashPath(*info->narHash.type, store.printStorePath(path)); + HashResult current = hashPath(info->narHash->type, store.printStorePath(path)); Hash nullHash(htSHA256); res = info->narHash == nullHash || info->narHash == current.first; } @@ -5034,7 +5063,7 @@ void Worker::markContentsGood(const StorePath & path) static void primeCache(Store & store, const std::vector & paths) { StorePathSet willBuild, willSubstitute, unknown; - unsigned long long downloadSize, narSize; + uint64_t downloadSize, narSize; store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty()) diff --git a/src/libstore/builtins/buildenv.hh b/src/libstore/builtins/buildenv.hh index 0a37459b0..73c0f5f7f 100644 --- a/src/libstore/builtins/buildenv.hh +++ b/src/libstore/builtins/buildenv.hh @@ -9,7 +9,7 @@ struct Package { Path path; bool active; int priority; - Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {} + Package(const Path & path, bool active, int priority) : path{path}, active{active}, priority{priority} {} }; typedef std::vector Packages; diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index e630cf6f1..6585a480d 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -58,23 +58,6 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) } }; - /* We always have one output, and if it's a fixed-output derivation (as - checked below) it must be the only output */ - auto & output = drv.outputs.begin()->second; - - /* Try the hashed mirrors first. */ - if (output.hash && output.hash->method == FileIngestionMethod::Flat) - for (auto hashedMirror : settings.hashedMirrors.get()) - try { - if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; - auto & h = output.hash->hash; - fetch(hashedMirror + printHashType(*h.type) + "/" + h.to_string(Base16, false)); - return; - } catch (Error & e) { - debug(e.what()); - } - - /* Otherwise try the specified URL. */ fetch(mainUrl); } diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 6cb69d0a9..f83b98a98 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -3,7 +3,7 @@ namespace nix { std::string FixedOutputHash::printMethodAlgo() const { - return makeFileIngestionPrefix(method) + printHashType(*hash.type); + return makeFileIngestionPrefix(method) + printHashType(hash.type); } std::string makeFileIngestionPrefix(const FileIngestionMethod m) { @@ -46,7 +46,7 @@ ContentAddress parseContentAddress(std::string_view rawCa) { if (prefix == "text") { auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos); Hash hash = Hash(string(hashTypeAndHash)); - if (*hash.type != htSHA256) { + if (hash.type != htSHA256) { throw Error("parseContentAddress: the text hash should have type SHA256"); } return TextHash { hash }; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index db7139374..b9a750425 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -86,7 +86,7 @@ struct TunnelLogger : public Logger } /* startWork() means that we're starting an operation for which we - want to send out stderr to the client. */ + want to send out stderr to the client. */ void startWork() { auto state(state_.lock()); @@ -173,31 +173,6 @@ struct TunnelSource : BufferedSource } }; -/* If the NAR archive contains a single file at top-level, then save - the contents of the file to `s'. Otherwise barf. */ -struct RetrieveRegularNARSink : ParseSink -{ - bool regular; - string s; - - RetrieveRegularNARSink() : regular(true) { } - - void createDirectory(const Path & path) - { - regular = false; - } - - void receiveContents(unsigned char * data, unsigned int len) - { - s.append((const char *) data, len); - } - - void createSymlink(const Path & path, const string & target) - { - regular = false; - } -}; - struct ClientSettings { bool keepFailed; @@ -314,7 +289,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto hash = store->queryPathInfo(path)->narHash; logger->stopWork(); - to << hash.to_string(Base16, false); + to << hash->to_string(Base16, false); break; } @@ -375,25 +350,28 @@ static void performOp(TunnelLogger * logger, ref store, } case wopAddToStore: { - std::string s, baseName; + HashType hashAlgo; + std::string baseName; FileIngestionMethod method; { - bool fixed; uint8_t recursive; - from >> baseName >> fixed /* obsolete */ >> recursive >> s; + bool fixed; + uint8_t recursive; + std::string hashAlgoRaw; + from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw; if (recursive > (uint8_t) FileIngestionMethod::Recursive) throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive); method = FileIngestionMethod { recursive }; /* Compatibility hack. */ if (!fixed) { - s = "sha256"; + hashAlgoRaw = "sha256"; method = FileIngestionMethod::Recursive; } + hashAlgo = parseHashType(hashAlgoRaw); } - HashType hashAlgo = parseHashType(s); - StringSink savedNAR; - TeeSource savedNARSource(from, savedNAR); - RetrieveRegularNARSink savedRegular; + StringSink saved; + TeeSource savedNARSource(from, saved); + RetrieveRegularNARSink savedRegular { saved }; if (method == FileIngestionMethod::Recursive) { /* Get the entire NAR dump from the client and save it to @@ -407,11 +385,9 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); if (!savedRegular.regular) throw Error("regular file expected"); - auto path = store->addToStoreFromDump( - method == FileIngestionMethod::Recursive ? *savedNAR.s : savedRegular.s, - baseName, - method, - hashAlgo); + // FIXME: try to stream directly from `from`. + StringSource dumpSource { *saved.s }; + auto path = store->addToStoreFromDump(dumpSource, baseName, method, hashAlgo); logger->stopWork(); to << store->printStorePath(path); @@ -475,7 +451,7 @@ static void performOp(TunnelLogger * logger, ref store, case wopBuildDerivation: { auto drvPath = store->parseStorePath(readString(from)); BasicDerivation drv; - readDerivation(from, *store, drv); + readDerivation(from, *store, drv, Derivation::nameFromPath(drvPath)); BuildMode buildMode = (BuildMode) readInt(from); logger->startWork(); if (!trusted) @@ -603,7 +579,7 @@ static void performOp(TunnelLogger * logger, ref store, auto path = store->parseStorePath(readString(from)); logger->startWork(); SubstitutablePathInfos infos; - store->querySubstitutablePathInfos({path}, infos); + store->querySubstitutablePathInfos({{path, std::nullopt}}, infos); logger->stopWork(); auto i = infos.find(path); if (i == infos.end()) @@ -619,10 +595,16 @@ static void performOp(TunnelLogger * logger, ref store, } case wopQuerySubstitutablePathInfos: { - auto paths = readStorePaths(*store, from); - logger->startWork(); SubstitutablePathInfos infos; - store->querySubstitutablePathInfos(paths, infos); + StorePathCAMap pathsMap = {}; + if (GET_PROTOCOL_MINOR(clientVersion) < 22) { + auto paths = readStorePaths(*store, from); + for (auto & path : paths) + pathsMap.emplace(path, std::nullopt); + } else + pathsMap = readStorePathCAMap(*store, from); + logger->startWork(); + store->querySubstitutablePathInfos(pathsMap, infos); logger->stopWork(); to << infos.size(); for (auto & i : infos) { @@ -656,7 +638,7 @@ static void performOp(TunnelLogger * logger, ref store, if (GET_PROTOCOL_MINOR(clientVersion) >= 17) to << 1; to << (info->deriver ? store->printStorePath(*info->deriver) : "") - << info->narHash.to_string(Base16, false); + << info->narHash->to_string(Base16, false); writeStorePaths(*store, to, info->references); to << info->registrationTime << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { @@ -727,24 +709,84 @@ static void performOp(TunnelLogger * logger, ref store, if (!trusted) info.ultimate = false; - std::string saved; - std::unique_ptr source; - if (GET_PROTOCOL_MINOR(clientVersion) >= 21) - source = std::make_unique(from, to); - else { - TeeParseSink tee(from); - parseDump(tee, tee.source); - saved = std::move(*tee.saved.s); - source = std::make_unique(saved); + if (GET_PROTOCOL_MINOR(clientVersion) >= 23) { + + struct FramedSource : Source + { + Source & from; + bool eof = false; + std::vector pending; + size_t pos = 0; + + FramedSource(Source & from) : from(from) + { } + + ~FramedSource() + { + if (!eof) { + while (true) { + auto n = readInt(from); + if (!n) break; + std::vector data(n); + from(data.data(), n); + } + } + } + + size_t read(unsigned char * data, size_t len) override + { + if (eof) throw EndOfFile("reached end of FramedSource"); + + if (pos >= pending.size()) { + size_t len = readInt(from); + if (!len) { + eof = true; + return 0; + } + pending = std::vector(len); + pos = 0; + from(pending.data(), len); + } + + auto n = std::min(len, pending.size() - pos); + memcpy(data, pending.data() + pos, n); + pos += n; + return n; + } + }; + + logger->startWork(); + + { + FramedSource source(from); + store->addToStore(info, source, (RepairFlag) repair, + dontCheckSigs ? NoCheckSigs : CheckSigs); + } + + logger->stopWork(); } - logger->startWork(); + else { + std::unique_ptr source; + if (GET_PROTOCOL_MINOR(clientVersion) >= 21) + source = std::make_unique(from, to); + else { + StringSink saved; + TeeSource tee { from, saved }; + ParseSink ether; + parseDump(ether, tee); + source = std::make_unique(std::move(*saved.s)); + } - // FIXME: race if addToStore doesn't read source? - store->addToStore(info, *source, (RepairFlag) repair, - dontCheckSigs ? NoCheckSigs : CheckSigs); + logger->startWork(); + + // FIXME: race if addToStore doesn't read source? + store->addToStore(info, *source, (RepairFlag) repair, + dontCheckSigs ? NoCheckSigs : CheckSigs); + + logger->stopWork(); + } - logger->stopWork(); break; } @@ -754,7 +796,7 @@ static void performOp(TunnelLogger * logger, ref store, targets.push_back(store->parsePathWithOutputs(s)); logger->startWork(); StorePathSet willBuild, willSubstitute, unknown; - unsigned long long downloadSize, narSize; + uint64_t downloadSize, narSize; store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize); logger->stopWork(); writeStorePaths(*store, to, willBuild); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 67565a704..8815c4632 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -7,12 +7,20 @@ namespace nix { -const StorePath & BasicDerivation::findOutput(const string & id) const +// FIXME Put this somewhere? +template struct overloaded : Ts... { using Ts::operator()...; }; +template overloaded(Ts...) -> overloaded; + +StorePath DerivationOutput::path(const Store & store, std::string_view drvName) const { - auto i = outputs.find(id); - if (i == outputs.end()) - throw Error("derivation has no output '%s'", id); - return i->second.path; + return std::visit(overloaded { + [](DerivationOutputInputAddressed doi) { + return doi.path; + }, + [&](DerivationOutputFixed dof) { + return store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName); + } + }, output); } @@ -107,7 +115,6 @@ static DerivationOutput parseDerivationOutput(const Store & store, std::istrings expect(str, ","); const auto hash = parseString(str); expect(str, ")"); - std::optional fsh; if (hashAlgo != "") { auto method = FileIngestionMethod::Flat; if (string(hashAlgo, 0, 2) == "r:") { @@ -115,22 +122,29 @@ static DerivationOutput parseDerivationOutput(const Store & store, std::istrings hashAlgo = string(hashAlgo, 2); } const HashType hashType = parseHashType(hashAlgo); - fsh = FixedOutputHash { - .method = std::move(method), - .hash = Hash(hash, hashType), - }; - } - return DerivationOutput { - .path = std::move(path), - .hash = std::move(fsh), - }; + return DerivationOutput { + .output = DerivationOutputFixed { + .hash = FixedOutputHash { + .method = std::move(method), + .hash = Hash(hash, hashType), + }, + } + }; + } else + return DerivationOutput { + .output = DerivationOutputInputAddressed { + .path = std::move(path), + } + }; } -Derivation parseDerivation(const Store & store, std::string && s) +Derivation parseDerivation(const Store & store, std::string && s, std::string_view name) { Derivation drv; + drv.name = name; + std::istringstream str(std::move(s)); expect(str, "Derive(["); @@ -235,10 +249,14 @@ string Derivation::unparse(const Store & store, bool maskOutputs, for (auto & i : outputs) { if (first) first = false; else s += ','; s += '('; printUnquotedString(s, i.first); - s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path)); - s += ','; printUnquotedString(s, i.second.hash ? i.second.hash->printMethodAlgo() : ""); - s += ','; printUnquotedString(s, - i.second.hash ? i.second.hash->hash.to_string(Base16, false) : ""); + s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path(store, name))); + if (auto hash = std::get_if(&i.second.output)) { + s += ','; printUnquotedString(s, hash->hash.printMethodAlgo()); + s += ','; printUnquotedString(s, hash->hash.hash.to_string(Base16, false)); + } else { + s += ','; printUnquotedString(s, ""); + s += ','; printUnquotedString(s, ""); + } s += ')'; } @@ -294,7 +312,7 @@ bool BasicDerivation::isFixedOutput() const { return outputs.size() == 1 && outputs.begin()->first == "out" && - outputs.begin()->second.hash; + std::holds_alternative(outputs.begin()->second.output); } @@ -326,10 +344,11 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput /* Return a fixed hash for fixed-output derivations. */ if (drv.isFixedOutput()) { DerivationOutputs::const_iterator i = drv.outputs.begin(); + auto hash = std::get(i->second.output); return hashString(htSHA256, "fixed:out:" - + i->second.hash->printMethodAlgo() + ":" - + i->second.hash->hash.to_string(Base16, false) + ":" - + store.printStorePath(i->second.path)); + + hash.hash.printMethodAlgo() + ":" + + hash.hash.hash.to_string(Base16, false) + ":" + + store.printStorePath(i->second.path(store, drv.name))); } /* For other derivations, replace the inputs paths with recursive @@ -363,11 +382,11 @@ bool wantOutput(const string & output, const std::set & wanted) } -StorePathSet BasicDerivation::outputPaths() const +StorePathSet BasicDerivation::outputPaths(const Store & store) const { StorePathSet paths; for (auto & i : outputs) - paths.insert(i.second.path); + paths.insert(i.second.path(store, name)); return paths; } @@ -377,7 +396,6 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store) auto hashAlgo = readString(in); auto hash = readString(in); - std::optional fsh; if (hashAlgo != "") { auto method = FileIngestionMethod::Flat; if (string(hashAlgo, 0, 2) == "r:") { @@ -385,16 +403,20 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store) hashAlgo = string(hashAlgo, 2); } auto hashType = parseHashType(hashAlgo); - fsh = FixedOutputHash { - .method = std::move(method), - .hash = Hash(hash, hashType), + return DerivationOutput { + .output = DerivationOutputFixed { + .hash = FixedOutputHash { + .method = std::move(method), + .hash = Hash(hash, hashType), + }, + } + }; + } else + return DerivationOutput { + .output = DerivationOutputInputAddressed { + .path = std::move(path), + } }; - } - - return DerivationOutput { - .path = std::move(path), - .hash = std::move(fsh), - }; } StringSet BasicDerivation::outputNames() const @@ -406,8 +428,19 @@ StringSet BasicDerivation::outputNames() const } -Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv) +std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) { + auto nameWithSuffix = drvPath.name(); + constexpr std::string_view extension = ".drv"; + assert(hasSuffix(nameWithSuffix, extension)); + nameWithSuffix.remove_suffix(extension.size()); + return nameWithSuffix; +} + + +Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name) { + drv.name = name; + drv.outputs.clear(); auto nr = readNum(in); for (size_t n = 0; n < nr; n++) { @@ -436,10 +469,10 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr out << drv.outputs.size(); for (auto & i : drv.outputs) { out << i.first - << store.printStorePath(i.second.path); - if (i.second.hash) { - out << i.second.hash->printMethodAlgo() - << i.second.hash->hash.to_string(Base16, false); + << store.printStorePath(i.second.path(store, drv.name)); + if (auto hash = std::get_if(&i.second.output)) { + out << hash->hash.printMethodAlgo() + << hash->hash.hash.to_string(Base16, false); } else { out << "" << ""; } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index d170d7223..112b2a8ca 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -13,10 +13,20 @@ namespace nix { /* Abstract syntax of derivations. */ -struct DerivationOutput +struct DerivationOutputInputAddressed { StorePath path; - std::optional hash; /* hash used for expected hash computation */ +}; + +struct DerivationOutputFixed +{ + FixedOutputHash hash; /* hash used for expected hash computation */ +}; + +struct DerivationOutput +{ + std::variant output; + StorePath path(const Store & store, std::string_view drvName) const; }; typedef std::map DerivationOutputs; @@ -35,24 +45,23 @@ struct BasicDerivation Path builder; Strings args; StringPairs env; + std::string name; BasicDerivation() { } virtual ~BasicDerivation() { }; - /* Return the path corresponding to the output identifier `id' in - the given derivation. */ - const StorePath & findOutput(const std::string & id) const; - bool isBuiltin() const; /* Return true iff this is a fixed-output derivation. */ bool isFixedOutput() const; /* Return the output paths of a derivation. */ - StorePathSet outputPaths() const; + StorePathSet outputPaths(const Store & store) const; /* Return the output names of a derivation. */ StringSet outputNames() const; + + static std::string_view nameFromPath(const StorePath & storePath); }; struct Derivation : BasicDerivation @@ -76,7 +85,7 @@ StorePath writeDerivation(ref store, const Derivation & drv, std::string_view name, RepairFlag repair = NoRepair); /* Read a derivation from a file. */ -Derivation parseDerivation(const Store & store, std::string && s); +Derivation parseDerivation(const Store & store, std::string && s, std::string_view name); // FIXME: remove bool isDerivation(const string & fileName); @@ -93,7 +102,7 @@ bool wantOutput(const string & output, const std::set & wanted); struct Source; struct Sink; -Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv); +Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name); void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv); std::string hashPlaceholder(const std::string & outputName); diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 082d0f1d1..a0fc22264 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -38,9 +38,9 @@ void Store::exportPath(const StorePath & path, Sink & sink) filesystem corruption from spreading to other machines. Don't complain if the stored hash is zero (unknown). */ Hash hash = hashSink.currentHash().first; - if (hash != info->narHash && info->narHash != Hash(*info->narHash.type)) + if (hash != info->narHash && info->narHash != Hash(info->narHash->type)) throw Error("hash of path '%s' has changed from '%s' to '%s'!", - printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true)); + printStorePath(path), info->narHash->to_string(Base32, true), hash.to_string(Base32, true)); teeSink << exportMagic @@ -60,8 +60,10 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) if (n != 1) throw Error("input doesn't look like something created by 'nix-store --export'"); /* Extract the NAR from the source. */ - TeeParseSink tee(source); - parseDump(tee, tee.source); + StringSink saved; + TeeSource tee { source, saved }; + ParseSink ether; + parseDump(ether, tee); uint32_t magic = readInt(source); if (magic != exportMagic) @@ -77,15 +79,15 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) if (deriver != "") info.deriver = parseStorePath(deriver); - info.narHash = hashString(htSHA256, *tee.saved.s); - info.narSize = tee.saved.s->size(); + info.narHash = hashString(htSHA256, *saved.s); + info.narSize = saved.s->size(); // Ignore optional legacy signature. if (readInt(source) == 1) readString(source); // Can't use underlying source, which would have been exhausted - auto source = StringSource { *tee.saved.s }; + auto source = StringSource { *saved.s }; addToStore(info, source, NoRepair, checkSigs); res.push_back(info.path); diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 531b85af8..4149f8155 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -22,6 +22,7 @@ #include #include #include +#include using namespace std::string_literals; @@ -56,7 +57,7 @@ struct curlFileTransfer : public FileTransfer Callback callback; CURL * req = 0; bool active = false; // whether the handle has been added to the multi object - std::string status; + std::string statusMsg; unsigned int attempt = 0; @@ -123,7 +124,7 @@ struct curlFileTransfer : public FileTransfer if (requestHeaders) curl_slist_free_all(requestHeaders); try { if (!done) - fail(FileTransferError(Interrupted, "download of '%s' was interrupted", request.uri)); + fail(FileTransferError(Interrupted, nullptr, "download of '%s' was interrupted", request.uri)); } catch (...) { ignoreException(); } @@ -144,6 +145,7 @@ struct curlFileTransfer : public FileTransfer LambdaSink finalSink; std::shared_ptr decompressionSink; + std::optional errorSink; std::exception_ptr writeException; @@ -153,9 +155,19 @@ struct curlFileTransfer : public FileTransfer size_t realSize = size * nmemb; result.bodySize += realSize; - if (!decompressionSink) + if (!decompressionSink) { decompressionSink = makeDecompressionSink(encoding, finalSink); + if (! successfulStatuses.count(getHTTPStatus())) { + // In this case we want to construct a TeeSink, to keep + // the response around (which we figure won't be big + // like an actual download should be) to improve error + // messages. + errorSink = StringSink { }; + } + } + if (errorSink) + (*errorSink)((unsigned char *) contents, realSize); (*decompressionSink)((unsigned char *) contents, realSize); return realSize; @@ -175,12 +187,13 @@ struct curlFileTransfer : public FileTransfer size_t realSize = size * nmemb; std::string line((char *) contents, realSize); printMsg(lvlVomit, format("got header for '%s': %s") % request.uri % trim(line)); - if (line.compare(0, 5, "HTTP/") == 0) { // new response starts + static std::regex statusLine("HTTP/[^ ]+ +[0-9]+(.*)", std::regex::extended | std::regex::icase); + std::smatch match; + if (std::regex_match(line, match, statusLine)) { result.etag = ""; - auto ss = tokenizeString>(line, " "); - status = ss.size() >= 2 ? ss[1] : ""; result.data = std::make_shared(); result.bodySize = 0; + statusMsg = trim(match[1]); acceptRanges = false; encoding = ""; } else { @@ -194,7 +207,9 @@ struct curlFileTransfer : public FileTransfer the expected ETag on a 200 response, then shut down the connection because we already have the data. */ - if (result.etag == request.expectedETag && status == "200") { + long httpStatus = 0; + curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); + if (result.etag == request.expectedETag && httpStatus == 200) { debug(format("shutting down on 200 HTTP response with expected ETag")); return 0; } @@ -408,16 +423,21 @@ struct curlFileTransfer : public FileTransfer attempt++; + std::shared_ptr response; + if (errorSink) + response = errorSink->s; auto exc = code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted - ? FileTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) + ? FileTransferError(Interrupted, response, "%s of '%s' was interrupted", request.verb(), request.uri) : httpStatus != 0 ? FileTransferError(err, - fmt("unable to %s '%s': HTTP error %d", - request.verb(), request.uri, httpStatus) + response, + fmt("unable to %s '%s': HTTP error %d ('%s')", + request.verb(), request.uri, httpStatus, statusMsg) + (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) ) : FileTransferError(err, + response, fmt("unable to %s '%s': %s (%d)", request.verb(), request.uri, curl_easy_strerror(code), code)); @@ -675,7 +695,7 @@ struct curlFileTransfer : public FileTransfer auto s3Res = s3Helper.getObject(bucketName, key); FileTransferResult res; if (!s3Res.data) - throw FileTransferError(NotFound, fmt("S3 object '%s' does not exist", request.uri)); + throw FileTransferError(NotFound, nullptr, "S3 object '%s' does not exist", request.uri); res.data = s3Res.data; callback(std::move(res)); #else @@ -820,6 +840,21 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink) } } +template +FileTransferError::FileTransferError(FileTransfer::Error error, std::shared_ptr response, const Args & ... args) + : Error(args...), error(error), response(response) +{ + const auto hf = hintfmt(args...); + // FIXME: Due to https://github.com/NixOS/nix/issues/3841 we don't know how + // to print different messages for different verbosity levels. For now + // we add some heuristics for detecting when we want to show the response. + if (response && (response->size() < 1024 || response->find("") != string::npos)) { + err.hint = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), *response); + } else { + err.hint = hf; + } +} + bool isUri(const string & s) { if (s.compare(0, 8, "channel:") == 0) return true; diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 11dca2fe0..25ade0add 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -103,10 +103,12 @@ class FileTransferError : public Error { public: FileTransfer::Error error; + std::shared_ptr response; // intentionally optional + template - FileTransferError(FileTransfer::Error error, const Args & ... args) - : Error(args...), error(error) - { } + FileTransferError(FileTransfer::Error error, std::shared_ptr response, const Args & ... args); + + virtual const char* sname() const override { return "FileTransferError"; } }; bool isUri(const string & s); diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index aaed5c218..e74382ed2 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -500,7 +500,7 @@ struct LocalStore::GCState StorePathSet alive; bool gcKeepOutputs; bool gcKeepDerivations; - unsigned long long bytesInvalidated; + uint64_t bytesInvalidated; bool moveToTrash = true; bool shouldDelete; GCState(const GCOptions & options, GCResults & results) @@ -518,7 +518,7 @@ bool LocalStore::isActiveTempFile(const GCState & state, void LocalStore::deleteGarbage(GCState & state, const Path & path) { - unsigned long long bytesFreed; + uint64_t bytesFreed; deletePath(path, bytesFreed); state.results.bytesFreed += bytesFreed; } @@ -528,7 +528,7 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path) { checkInterrupt(); - unsigned long long size = 0; + uint64_t size = 0; auto storePath = maybeParseStorePath(path); if (storePath && isValidPath(*storePath)) { @@ -687,7 +687,7 @@ void LocalStore::removeUnusedLinks(const GCState & state) AutoCloseDir dir(opendir(linksDir.c_str())); if (!dir) throw SysError("opening directory '%1%'", linksDir); - long long actualSize = 0, unsharedSize = 0; + int64_t actualSize = 0, unsharedSize = 0; struct dirent * dirent; while (errno = 0, dirent = readdir(dir.get())) { @@ -717,10 +717,10 @@ void LocalStore::removeUnusedLinks(const GCState & state) struct stat st; if (stat(linksDir.c_str(), &st) == -1) throw SysError("statting '%1%'", linksDir); - long long overhead = st.st_blocks * 512ULL; + auto overhead = st.st_blocks * 512ULL; - printInfo(format("note: currently hard linking saves %.2f MiB") - % ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))); + printInfo("note: currently hard linking saves %.2f MiB", + ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))); } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 4d5eec7bf..3406a9331 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -335,9 +335,6 @@ public: "setuid/setgid bits or with file capabilities."}; #endif - Setting hashedMirrors{this, {"http://tarballs.nixos.org/"}, "hashed-mirrors", - "A list of servers used by builtins.fetchurl to fetch files by hash."}; - Setting minFree{this, 0, "min-free", "Automatically run the garbage collector when free disk space drops below the specified amount."}; @@ -368,6 +365,9 @@ public: Setting narBufferSize{this, 32 * 1024 * 1024, "nar-buffer-size", "Maximum size of NARs before spilling them to disk."}; + + Setting flakeRegistry{this, "https://github.com/NixOS/flake-registry/raw/master/flake-registry.json", "flake-registry", + "Path or URI of the global flake registry."}; }; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index a8bd8a972..5d7566121 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -113,7 +113,7 @@ struct LegacySSHStore : public Store if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) { auto s = readString(conn->from); - info->narHash = s.empty() ? Hash() : Hash(s); + info->narHash = s.empty() ? std::optional{} : Hash{s}; info->ca = parseContentAddressOpt(readString(conn->from)); info->sigs = readStrings(conn->from); } @@ -138,7 +138,7 @@ struct LegacySSHStore : public Store << cmdAddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(Base16, false); + << info.narHash->to_string(Base16, false); writeStorePaths(*this, conn->to, info.references); conn->to << info.registrationTime diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b3f4b3f7d..f6ce8877e 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -560,19 +560,12 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat DerivationOutputs::const_iterator out = drv.outputs.find("out"); if (out == drv.outputs.end()) throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath)); - - check( - makeFixedOutputPath( - out->second.hash->method, - out->second.hash->hash, - drvName), - out->second.path, "out"); } else { Hash h = hashDerivationModulo(*this, drv, true); for (auto & i : drv.outputs) - check(makeOutputPath(i.first, h, drvName), i.second.path, i.first); + check(makeOutputPath(i.first, h, drvName), i.second.path(*this, drv.name), i.first); } } @@ -586,7 +579,7 @@ uint64_t LocalStore::addValidPath(State & state, state.stmtRegisterValidPath.use() (printStorePath(info.path)) - (info.narHash.to_string(Base16, true)) + (info.narHash->to_string(Base16, true)) (info.registrationTime == 0 ? time(0) : info.registrationTime) (info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver) (info.narSize, info.narSize != 0) @@ -594,7 +587,7 @@ uint64_t LocalStore::addValidPath(State & state, (concatStringsSep(" ", info.sigs), !info.sigs.empty()) (renderContentAddress(info.ca), (bool) info.ca) .exec(); - uint64_t id = sqlite3_last_insert_rowid(state.db); + uint64_t id = state.db.getLastInsertedRowId(); /* If this is a derivation, then store the derivation outputs in the database. This is useful for the garbage collector: it can @@ -614,7 +607,7 @@ uint64_t LocalStore::addValidPath(State & state, state.stmtAddDerivationOutput.use() (id) (i.first) - (printStorePath(i.second.path)) + (printStorePath(i.second.path(*this, drv.name))) .exec(); } } @@ -686,7 +679,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { state.stmtUpdatePathInfo.use() (info.narSize, info.narSize != 0) - (info.narHash.to_string(Base16, true)) + (info.narHash->to_string(Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) (renderContentAddress(info.ca), (bool) info.ca) @@ -846,20 +839,32 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) } -void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos) +void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) { if (!settings.useSubstitutes) return; for (auto & sub : getDefaultSubstituters()) { - if (sub->storeDir != storeDir) continue; for (auto & path : paths) { - if (infos.count(path)) continue; - debug("checking substituter '%s' for path '%s'", sub->getUri(), printStorePath(path)); + auto subPath(path.first); + + // recompute store path so that we can use a different store root + if (path.second) { + subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second); + if (sub->storeDir == storeDir) + assert(subPath == path.first); + if (subPath != path.first) + debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path.first), sub->printStorePath(subPath), sub->getUri()); + } else if (sub->storeDir != storeDir) continue; + + debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath)); try { - auto info = sub->queryPathInfo(path); + auto info = sub->queryPathInfo(subPath); + + if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty())) + continue; + auto narInfo = std::dynamic_pointer_cast( std::shared_ptr(info)); - infos.insert_or_assign(path, SubstitutablePathInfo{ + infos.insert_or_assign(path.first, SubstitutablePathInfo{ info->deriver, info->references, narInfo ? narInfo->fileSize : 0, @@ -900,7 +905,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) StorePathSet paths; for (auto & i : infos) { - assert(i.narHash.type == htSHA256); + assert(i.narHash && i.narHash->type == htSHA256); if (isValidPath_(*state, i.path)) updatePathInfo(*state, i); else @@ -1013,7 +1018,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (hashResult.first != info.narHash) throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s", - printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true)); + printStorePath(info.path), info.narHash->to_string(Base32, true), hashResult.first.to_string(Base32, true)); if (hashResult.second != info.narSize) throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s", @@ -1033,82 +1038,26 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, } -StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) -{ - Hash h = hashString(hashAlgo, dump); - - auto dstPath = makeFixedOutputPath(method, h, name); - - addTempRoot(dstPath); - - if (repair || !isValidPath(dstPath)) { - - /* The first check above is an optimisation to prevent - unnecessary lock acquisition. */ - - auto realPath = Store::toRealPath(dstPath); - - PathLocks outputLock({realPath}); - - if (repair || !isValidPath(dstPath)) { - - deletePath(realPath); - - autoGC(); - - if (method == FileIngestionMethod::Recursive) { - StringSource source(dump); - restorePath(realPath, source); - } else - writeFile(realPath, dump); - - canonicalisePathMetaData(realPath, -1); - - /* Register the SHA-256 hash of the NAR serialisation of - the path in the database. We may just have computed it - above (if called with recursive == true and hashAlgo == - sha256); otherwise, compute it here. */ - HashResult hash; - if (method == FileIngestionMethod::Recursive) { - hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); - hash.second = dump.size(); - } else - hash = hashPath(htSHA256, realPath); - - optimisePath(realPath); // FIXME: combine with hashPath() - - ValidPathInfo info(dstPath); - info.narHash = hash.first; - info.narSize = hash.second; - info.ca = FixedOutputHash { .method = method, .hash = h }; - registerValidPath(info); - } - - outputLock.setDeletion(true); - } - - return dstPath; -} - - StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { Path srcPath(absPath(_srcPath)); + auto source = sinkToSource([&](Sink & sink) { + if (method == FileIngestionMethod::Recursive) + dumpPath(srcPath, sink, filter); + else + readFile(srcPath, sink); + }); + return addToStoreFromDump(*source, name, method, hashAlgo, repair); +} - if (method != FileIngestionMethod::Recursive) - return addToStoreFromDump(readFile(srcPath), name, method, hashAlgo, repair); - /* For computing the NAR hash. */ - auto sha256Sink = std::make_unique(htSHA256); - - /* For computing the store path. In recursive SHA-256 mode, this - is the same as the NAR hash, so no need to do it again. */ - std::unique_ptr hashSink = - hashAlgo == htSHA256 - ? nullptr - : std::make_unique(hashAlgo); +StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) +{ + /* For computing the store path. */ + auto hashSink = std::make_unique(hashAlgo); + TeeSource source { source0, *hashSink }; /* Read the source path into memory, but only if it's up to narBufferSize bytes. If it's larger, write it to a temporary @@ -1116,55 +1065,49 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, destination store path is already valid, we just delete the temporary path. Otherwise, we move it to the destination store path. */ - bool inMemory = true; - std::string nar; + bool inMemory = false; - auto source = sinkToSource([&](Sink & sink) { + std::string dump; - LambdaSink sink2([&](const unsigned char * buf, size_t len) { - (*sha256Sink)(buf, len); - if (hashSink) (*hashSink)(buf, len); - - if (inMemory) { - if (nar.size() + len > settings.narBufferSize) { - inMemory = false; - sink << 1; - sink((const unsigned char *) nar.data(), nar.size()); - nar.clear(); - } else { - nar.append((const char *) buf, len); - } - } - - if (!inMemory) sink(buf, len); - }); - - dumpPath(srcPath, sink2, filter); - }); + /* Fill out buffer, and decide whether we are working strictly in + memory based on whether we break out because the buffer is full + or the original source is empty */ + while (dump.size() < settings.narBufferSize) { + auto oldSize = dump.size(); + constexpr size_t chunkSize = 65536; + auto want = std::min(chunkSize, settings.narBufferSize - oldSize); + dump.resize(oldSize + want); + auto got = 0; + try { + got = source.read((uint8_t *) dump.data() + oldSize, want); + } catch (EndOfFile &) { + inMemory = true; + break; + } + dump.resize(oldSize + got); + } std::unique_ptr delTempDir; Path tempPath; - try { - /* Wait for the source coroutine to give us some dummy - data. This is so that we don't create the temporary - directory if the NAR fits in memory. */ - readInt(*source); + if (!inMemory) { + /* Drain what we pulled so far, and then keep on pulling */ + StringSource dumpSource { dump }; + ChainSource bothSource { dumpSource, source }; auto tempDir = createTempDir(realStoreDir, "add"); delTempDir = std::make_unique(tempDir); tempPath = tempDir + "/x"; - restorePath(tempPath, *source); + if (method == FileIngestionMethod::Recursive) + restorePath(tempPath, bothSource); + else + writeFile(tempPath, bothSource); - } catch (EndOfFile &) { - if (!inMemory) throw; - /* The NAR fits in memory, so we didn't do restorePath(). */ + dump.clear(); } - auto sha256 = sha256Sink->finish(); - - Hash hash = hashSink ? hashSink->finish().first : sha256.first; + auto [hash, size] = hashSink->finish(); auto dstPath = makeFixedOutputPath(method, hash, name); @@ -1186,22 +1129,34 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, autoGC(); if (inMemory) { + StringSource dumpSource { dump }; /* Restore from the NAR in memory. */ - StringSource source(nar); - restorePath(realPath, source); + if (method == FileIngestionMethod::Recursive) + restorePath(realPath, dumpSource); + else + writeFile(realPath, dumpSource); } else { /* Move the temporary path we restored above. */ if (rename(tempPath.c_str(), realPath.c_str())) throw Error("renaming '%s' to '%s'", tempPath, realPath); } + /* For computing the nar hash. In recursive SHA-256 mode, this + is the same as the store hash, so no need to do it again. */ + auto narHash = std::pair { hash, size }; + if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) { + HashSink narSink { htSHA256 }; + dumpPath(realPath, narSink); + narHash = narSink.finish(); + } + canonicalisePathMetaData(realPath, -1); // FIXME: merge into restorePath optimisePath(realPath); ValidPathInfo info(dstPath); - info.narHash = sha256.first; - info.narSize = sha256.second; + info.narHash = narHash.first; + info.narSize = narHash.second; info.ca = FixedOutputHash { .method = method, .hash = hash }; registerValidPath(info); } @@ -1359,9 +1314,9 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) std::unique_ptr hashSink; if (!info->ca || !info->references.count(info->path)) - hashSink = std::make_unique(*info->narHash.type); + hashSink = std::make_unique(info->narHash->type); else - hashSink = std::make_unique(*info->narHash.type, std::string(info->path.hashPart())); + hashSink = std::make_unique(info->narHash->type, std::string(info->path.hashPart())); dumpPath(Store::toRealPath(i), *hashSink); auto current = hashSink->finish(); @@ -1370,7 +1325,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) logError({ .name = "Invalid hash - path modified", .hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'", - printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true)) + printStorePath(i), info->narHash->to_string(Base32, true), current.first.to_string(Base32, true)) }); if (repair) repairPath(i); else errors = true; } else { diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 05f342902..807619d29 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -26,8 +26,8 @@ const int nixSchemaVersion = 10; struct OptimiseStats { unsigned long filesLinked = 0; - unsigned long long bytesFreed = 0; - unsigned long long blocksFreed = 0; + uint64_t bytesFreed = 0; + uint64_t blocksFreed = 0; }; @@ -136,7 +136,7 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; - void querySubstitutablePathInfos(const StorePathSet & paths, + void querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, Source & source, @@ -150,7 +150,7 @@ public: in `dump', which is either a NAR serialisation (if recursive == true) or simply the contents of a regular file (if recursive == false). */ - StorePath addToStoreFromDump(const string & dump, const string & name, + StorePath addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; StorePath addTextToStore(const string & name, const string & s, diff --git a/src/libstore/local.mk b/src/libstore/local.mk index aec4ed493..d266c8efe 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -61,3 +61,6 @@ $(d)/build.cc: clean-files += $(d)/schema.sql.gen.hh $(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644)) + +$(foreach i, $(wildcard src/libstore/builtins/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix/builtins, 0644))) diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index e68edb38c..7f1b62f26 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -108,9 +108,19 @@ void Store::computeFSClosure(const StorePath & startPath, } +std::optional getDerivationCA(const BasicDerivation & drv) +{ + auto out = drv.outputs.find("out"); + if (out != drv.outputs.end()) { + if (auto v = std::get_if(&out->second.output)) + return v->hash; + } + return std::nullopt; +} + void Store::queryMissing(const std::vector & targets, StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_, - unsigned long long & downloadSize_, unsigned long long & narSize_) + uint64_t & downloadSize_, uint64_t & narSize_) { Activity act(*logger, lvlDebug, actUnknown, "querying info about missing paths"); @@ -122,8 +132,8 @@ void Store::queryMissing(const std::vector & targets, { std::unordered_set done; StorePathSet & unknown, & willSubstitute, & willBuild; - unsigned long long & downloadSize; - unsigned long long & narSize; + uint64_t & downloadSize; + uint64_t & narSize; }; struct DrvState @@ -157,7 +167,7 @@ void Store::queryMissing(const std::vector & targets, auto outPath = parseStorePath(outPathS); SubstitutablePathInfos infos; - querySubstitutablePathInfos({outPath}, infos); + querySubstitutablePathInfos({{outPath, getDerivationCA(*drv)}}, infos); if (infos.empty()) { drvState_->lock()->done = true; @@ -198,8 +208,8 @@ void Store::queryMissing(const std::vector & targets, PathSet invalid; for (auto & j : drv->outputs) if (wantOutput(j.first, path.outputs) - && !isValidPath(j.second.path)) - invalid.insert(printStorePath(j.second.path)); + && !isValidPath(j.second.path(*this, drv->name))) + invalid.insert(printStorePath(j.second.path(*this, drv->name))); if (invalid.empty()) return; if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) { @@ -214,7 +224,7 @@ void Store::queryMissing(const std::vector & targets, if (isValidPath(path.path)) return; SubstitutablePathInfos infos; - querySubstitutablePathInfos({path.path}, infos); + querySubstitutablePathInfos({{path.path, std::nullopt}}, infos); if (infos.empty()) { auto state(state_.lock()); diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index d884a131e..59ec164b6 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -79,14 +79,14 @@ struct NarAccessor : public FSAccessor parents.top()->isExecutable = true; } - void preallocateContents(unsigned long long size) override + void preallocateContents(uint64_t size) override { assert(size <= std::numeric_limits::max()); parents.top()->size = (uint64_t) size; parents.top()->start = pos; } - void receiveContents(unsigned char * data, unsigned int len) override + void receiveContents(unsigned char * data, size_t len) override { } void createSymlink(const Path & path, const string & target) override diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 012dea6ea..9ddb9957f 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -230,9 +230,9 @@ public: (std::string(info->path.name())) (narInfo ? narInfo->url : "", narInfo != 0) (narInfo ? narInfo->compression : "", narInfo != 0) - (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string(Base32, true) : "", narInfo && narInfo->fileHash) + (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(Base32, true) : "", narInfo && narInfo->fileHash) (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize) - (info->narHash.to_string(Base32, true)) + (info->narHash->to_string(Base32, true)) (info->narSize) (concatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 04550ed97..ca471463c 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -7,15 +7,14 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & : ValidPathInfo(StorePath(StorePath::dummy)) // FIXME: hack { auto corrupt = [&]() { - throw Error("NAR info file '%1%' is corrupt", whence); + return Error("NAR info file '%1%' is corrupt", whence); }; auto parseHashField = [&](const string & s) { try { return Hash(s); } catch (BadHash &) { - corrupt(); - return Hash(); // never reached + throw corrupt(); } }; @@ -25,12 +24,12 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & while (pos < s.size()) { size_t colon = s.find(':', pos); - if (colon == std::string::npos) corrupt(); + if (colon == std::string::npos) throw corrupt(); std::string name(s, pos, colon - pos); size_t eol = s.find('\n', colon + 2); - if (eol == std::string::npos) corrupt(); + if (eol == std::string::npos) throw corrupt(); std::string value(s, colon + 2, eol - colon - 2); @@ -45,16 +44,16 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "FileHash") fileHash = parseHashField(value); else if (name == "FileSize") { - if (!string2Int(value, fileSize)) corrupt(); + if (!string2Int(value, fileSize)) throw corrupt(); } else if (name == "NarHash") narHash = parseHashField(value); else if (name == "NarSize") { - if (!string2Int(value, narSize)) corrupt(); + if (!string2Int(value, narSize)) throw corrupt(); } else if (name == "References") { auto refs = tokenizeString(value, " "); - if (!references.empty()) corrupt(); + if (!references.empty()) throw corrupt(); for (auto & r : refs) references.insert(StorePath(r)); } @@ -67,7 +66,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "Sig") sigs.insert(value); else if (name == "CA") { - if (ca) corrupt(); + if (ca) throw corrupt(); // FIXME: allow blank ca or require skipping field? ca = parseContentAddressOpt(value); } @@ -77,7 +76,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & if (compression == "") compression = "bzip2"; - if (!havePath || url.empty() || narSize == 0 || !narHash) corrupt(); + if (!havePath || url.empty() || narSize == 0 || !narHash) throw corrupt(); } std::string NarInfo::to_string(const Store & store) const @@ -87,11 +86,11 @@ std::string NarInfo::to_string(const Store & store) const res += "URL: " + url + "\n"; assert(compression != ""); res += "Compression: " + compression + "\n"; - assert(fileHash.type == htSHA256); - res += "FileHash: " + fileHash.to_string(Base32, true) + "\n"; + assert(fileHash && fileHash->type == htSHA256); + res += "FileHash: " + fileHash->to_string(Base32, true) + "\n"; res += "FileSize: " + std::to_string(fileSize) + "\n"; - assert(narHash.type == htSHA256); - res += "NarHash: " + narHash.to_string(Base32, true) + "\n"; + assert(narHash && narHash->type == htSHA256); + res += "NarHash: " + narHash->to_string(Base32, true) + "\n"; res += "NarSize: " + std::to_string(narSize) + "\n"; res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 373c33427..eff19f0ef 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -10,7 +10,7 @@ struct NarInfo : ValidPathInfo { std::string url; std::string compression; - Hash fileHash; + std::optional fileHash; uint64_t fileSize = 0; std::string system; diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index b2b2412a3..e4b4b6213 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -282,7 +282,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats) } } -static string showBytes(unsigned long long bytes) +static string showBytes(uint64_t bytes) { return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str(); } diff --git a/src/libstore/path.hh b/src/libstore/path.hh index e43a8b50c..b03a0f69d 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -64,6 +64,8 @@ typedef std::set StorePathSet; typedef std::vector StorePaths; typedef std::map OutputPathMap; +typedef std::map> StorePathCAMap; + /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 6cfe393a4..6862b42f0 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -12,30 +12,24 @@ namespace nix { -static bool cmpGensByNumber(const Generation & a, const Generation & b) -{ - return a.number < b.number; -} - - /* Parse a generation name of the format `--link'. */ -static int parseName(const string & profileName, const string & name) +static std::optional parseName(const string & profileName, const string & name) { - if (string(name, 0, profileName.size() + 1) != profileName + "-") return -1; + if (string(name, 0, profileName.size() + 1) != profileName + "-") return {}; string s = string(name, profileName.size() + 1); string::size_type p = s.find("-link"); - if (p == string::npos) return -1; - int n; + if (p == string::npos) return {}; + unsigned int n; if (string2Int(string(s, 0, p), n) && n >= 0) return n; else - return -1; + return {}; } -Generations findGenerations(Path profile, int & curGen) +std::pair> findGenerations(Path profile) { Generations gens; @@ -43,30 +37,34 @@ Generations findGenerations(Path profile, int & curGen) auto profileName = std::string(baseNameOf(profile)); for (auto & i : readDirectory(profileDir)) { - int n; - if ((n = parseName(profileName, i.name)) != -1) { - Generation gen; - gen.path = profileDir + "/" + i.name; - gen.number = n; + if (auto n = parseName(profileName, i.name)) { + auto path = profileDir + "/" + i.name; struct stat st; - if (lstat(gen.path.c_str(), &st) != 0) - throw SysError("statting '%1%'", gen.path); - gen.creationTime = st.st_mtime; - gens.push_back(gen); + if (lstat(path.c_str(), &st) != 0) + throw SysError("statting '%1%'", path); + gens.push_back({ + .number = *n, + .path = path, + .creationTime = st.st_mtime + }); } } - gens.sort(cmpGensByNumber); + gens.sort([](const Generation & a, const Generation & b) + { + return a.number < b.number; + }); - curGen = pathExists(profile) + return { + gens, + pathExists(profile) ? parseName(profileName, readLink(profile)) - : -1; - - return gens; + : std::nullopt + }; } -static void makeName(const Path & profile, unsigned int num, +static void makeName(const Path & profile, GenerationNumber num, Path & outLink) { Path prefix = (format("%1%-%2%") % profile % num).str(); @@ -78,10 +76,9 @@ Path createGeneration(ref store, Path profile, Path outPath) { /* The new generation number should be higher than old the previous ones. */ - int dummy; - Generations gens = findGenerations(profile, dummy); + auto [gens, dummy] = findGenerations(profile); - unsigned int num; + GenerationNumber num; if (gens.size() > 0) { Generation last = gens.back(); @@ -121,7 +118,7 @@ static void removeFile(const Path & path) } -void deleteGeneration(const Path & profile, unsigned int gen) +void deleteGeneration(const Path & profile, GenerationNumber gen) { Path generation; makeName(profile, gen, generation); @@ -129,7 +126,7 @@ void deleteGeneration(const Path & profile, unsigned int gen) } -static void deleteGeneration2(const Path & profile, unsigned int gen, bool dryRun) +static void deleteGeneration2(const Path & profile, GenerationNumber gen, bool dryRun) { if (dryRun) printInfo(format("would remove generation %1%") % gen); @@ -140,31 +137,29 @@ static void deleteGeneration2(const Path & profile, unsigned int gen, bool dryRu } -void deleteGenerations(const Path & profile, const std::set & gensToDelete, bool dryRun) +void deleteGenerations(const Path & profile, const std::set & gensToDelete, bool dryRun) { PathLocks lock; lockProfile(lock, profile); - int curGen; - Generations gens = findGenerations(profile, curGen); + auto [gens, curGen] = findGenerations(profile); - if (gensToDelete.find(curGen) != gensToDelete.end()) + if (gensToDelete.count(*curGen)) throw Error("cannot delete current generation of profile %1%'", profile); for (auto & i : gens) { - if (gensToDelete.find(i.number) == gensToDelete.end()) continue; + if (!gensToDelete.count(i.number)) continue; deleteGeneration2(profile, i.number, dryRun); } } -void deleteGenerationsGreaterThan(const Path & profile, int max, bool dryRun) +void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bool dryRun) { PathLocks lock; lockProfile(lock, profile); - int curGen; bool fromCurGen = false; - Generations gens = findGenerations(profile, curGen); + auto [gens, curGen] = findGenerations(profile); for (auto i = gens.rbegin(); i != gens.rend(); ++i) { if (i->number == curGen) { fromCurGen = true; @@ -186,8 +181,7 @@ void deleteOldGenerations(const Path & profile, bool dryRun) PathLocks lock; lockProfile(lock, profile); - int curGen; - Generations gens = findGenerations(profile, curGen); + auto [gens, curGen] = findGenerations(profile); for (auto & i : gens) if (i.number != curGen) @@ -200,8 +194,7 @@ void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun) PathLocks lock; lockProfile(lock, profile); - int curGen; - Generations gens = findGenerations(profile, curGen); + auto [gens, curGen] = findGenerations(profile); bool canDelete = false; for (auto i = gens.rbegin(); i != gens.rend(); ++i) diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh index 78645d8b6..abe507f0e 100644 --- a/src/libstore/profiles.hh +++ b/src/libstore/profiles.hh @@ -9,37 +9,32 @@ namespace nix { +typedef unsigned int GenerationNumber; + struct Generation { - int number; + GenerationNumber number; Path path; time_t creationTime; - Generation() - { - number = -1; - } - operator bool() const - { - return number != -1; - } }; -typedef list Generations; +typedef std::list Generations; /* Returns the list of currently present generations for the specified - profile, sorted by generation number. */ -Generations findGenerations(Path profile, int & curGen); + profile, sorted by generation number. Also returns the number of + the current generation. */ +std::pair> findGenerations(Path profile); class LocalFSStore; Path createGeneration(ref store, Path profile, Path outPath); -void deleteGeneration(const Path & profile, unsigned int gen); +void deleteGeneration(const Path & profile, GenerationNumber gen); -void deleteGenerations(const Path & profile, const std::set & gensToDelete, bool dryRun); +void deleteGenerations(const Path & profile, const std::set & gensToDelete, bool dryRun); -void deleteGenerationsGreaterThan(const Path & profile, const int max, bool dryRun); +void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bool dryRun); void deleteOldGenerations(const Path & profile, bool dryRun); diff --git a/src/libstore/references.cc b/src/libstore/references.cc index a10d536a3..62a3cda61 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -48,13 +48,12 @@ static void search(const unsigned char * s, size_t len, struct RefScanSink : Sink { - HashSink hashSink; StringSet hashes; StringSet seen; string tail; - RefScanSink() : hashSink(htSHA256) { } + RefScanSink() { } void operator () (const unsigned char * data, size_t len); }; @@ -62,8 +61,6 @@ struct RefScanSink : Sink void RefScanSink::operator () (const unsigned char * data, size_t len) { - hashSink(data, len); - /* It's possible that a reference spans the previous and current fragment, so search in the concatenation of the tail of the previous fragment and the start of the current fragment. */ @@ -79,10 +76,12 @@ void RefScanSink::operator () (const unsigned char * data, size_t len) } -PathSet scanForReferences(const string & path, - const PathSet & refs, HashResult & hash) +std::pair scanForReferences(const string & path, + const PathSet & refs) { - RefScanSink sink; + RefScanSink refsSink; + HashSink hashSink { htSHA256 }; + TeeSink sink { refsSink, hashSink }; std::map backMap; /* For efficiency (and a higher hit rate), just search for the @@ -97,7 +96,7 @@ PathSet scanForReferences(const string & path, assert(s.size() == refLength); assert(backMap.find(s) == backMap.end()); // parseHash(htSHA256, s); - sink.hashes.insert(s); + refsSink.hashes.insert(s); backMap[s] = i; } @@ -106,15 +105,15 @@ PathSet scanForReferences(const string & path, /* Map the hashes found back to their store paths. */ PathSet found; - for (auto & i : sink.seen) { + for (auto & i : refsSink.seen) { std::map::iterator j; if ((j = backMap.find(i)) == backMap.end()) abort(); found.insert(j->second); } - hash = sink.hashSink.finish(); + auto hash = hashSink.finish(); - return found; + return std::pair(found, hash); } diff --git a/src/libstore/references.hh b/src/libstore/references.hh index c38bdd720..598a3203a 100644 --- a/src/libstore/references.hh +++ b/src/libstore/references.hh @@ -5,8 +5,7 @@ namespace nix { -PathSet scanForReferences(const Path & path, const PathSet & refs, - HashResult & hash); +std::pair scanForReferences(const Path & path, const PathSet & refs); struct RewritingSink : Sink { diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 9af4364b7..0c65bcd29 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -39,6 +39,24 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths out << store.printStorePath(i); } +StorePathCAMap readStorePathCAMap(const Store & store, Source & from) +{ + StorePathCAMap paths; + auto count = readNum(from); + while (count--) + paths.insert_or_assign(store.parseStorePath(readString(from)), parseContentAddressOpt(readString(from))); + return paths; +} + +void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths) +{ + out << paths.size(); + for (auto & i : paths) { + out << store.printStorePath(i.first); + out << renderContentAddress(i.second); + } +} + std::map readOutputPathMap(const Store & store, Source & from) { std::map pathMap; @@ -332,18 +350,17 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) } -void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos) +void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, SubstitutablePathInfos & infos) { - if (paths.empty()) return; + if (pathsMap.empty()) return; auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { - for (auto & i : paths) { + for (auto & i : pathsMap) { SubstitutablePathInfo info; - conn->to << wopQuerySubstitutablePathInfo << printStorePath(i); + conn->to << wopQuerySubstitutablePathInfo << printStorePath(i.first); conn.processStderr(); unsigned int reply = readInt(conn->from); if (reply == 0) continue; @@ -353,13 +370,19 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, info.references = readStorePaths(*this, conn->from); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); - infos.insert_or_assign(i, std::move(info)); + infos.insert_or_assign(i.first, std::move(info)); } } else { conn->to << wopQuerySubstitutablePathInfos; - writeStorePaths(*this, conn->to, paths); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 22) { + StorePathSet paths; + for (auto & path : pathsMap) + paths.insert(path.first); + writeStorePaths(*this, conn->to, paths); + } else + writeStorePathCAMap(*this, conn->to, pathsMap); conn.processStderr(); size_t count = readNum(conn->from); for (size_t n = 0; n < count; n++) { @@ -498,14 +521,89 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn->to << wopAddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(Base16, false); + << info.narHash->to_string(Base16, false); writeStorePaths(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; - bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21; - if (!tunnel) copyNAR(source, conn->to); - conn.processStderr(0, tunnel ? &source : nullptr); + + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 23) { + + std::exception_ptr ex; + + struct FramedSink : BufferedSink + { + ConnectionHandle & conn; + std::exception_ptr & ex; + + FramedSink(ConnectionHandle & conn, std::exception_ptr & ex) : conn(conn), ex(ex) + { } + + ~FramedSink() + { + try { + conn->to << 0; + conn->to.flush(); + } catch (...) { + ignoreException(); + } + } + + void write(const unsigned char * data, size_t len) override + { + /* Don't send more data if the remote has + encountered an error. */ + if (ex) { + auto ex2 = ex; + ex = nullptr; + std::rethrow_exception(ex2); + } + conn->to << len; + conn->to(data, len); + }; + }; + + /* Handle log messages / exceptions from the remote on a + separate thread. */ + std::thread stderrThread([&]() + { + try { + conn.processStderr(); + } catch (...) { + ex = std::current_exception(); + } + }); + + Finally joinStderrThread([&]() + { + if (stderrThread.joinable()) { + stderrThread.join(); + if (ex) { + try { + std::rethrow_exception(ex); + } catch (...) { + ignoreException(); + } + } + } + }); + + { + FramedSink sink(conn, ex); + copyNAR(source, sink); + sink.flush(); + } + + stderrThread.join(); + if (ex) + std::rethrow_exception(ex); + + } else if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21) { + conn.processStderr(0, &source); + } else { + copyNAR(source, conn->to); + conn.processStderr(0, nullptr); + } } } @@ -707,7 +805,7 @@ void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & s void RemoteStore::queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize) + uint64_t & downloadSize, uint64_t & narSize) { { auto conn(getConnection()); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 3c1b78b6a..72d2a6689 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -56,7 +56,7 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; - void querySubstitutablePathInfos(const StorePathSet & paths, + void querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, Source & nar, @@ -94,7 +94,7 @@ public: void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize) override; + uint64_t & downloadSize, uint64_t & narSize) override; void connect() override; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 1b7dff085..67935f3ba 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -343,13 +343,10 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore std::chrono::duration_cast(now2 - now1) .count(); - auto size = istream->tellg(); - - printInfo("uploaded 's3://%s/%s' (%d bytes) in %d ms", - bucketName, path, size, duration); + printInfo("uploaded 's3://%s/%s' in %d ms", + bucketName, path, duration); stats.putTimeMs += duration; - stats.putBytes += size; stats.put++; } diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index 4d43fe4d2..b2b75d498 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -19,7 +19,6 @@ public: struct Stats { std::atomic put{0}; - std::atomic putBytes{0}; std::atomic putTimeMs{0}; std::atomic get{0}; std::atomic getBytes{0}; diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 76c822c4e..31a1f0cac 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -61,6 +61,11 @@ void SQLite::exec(const std::string & stmt) }); } +uint64_t SQLite::getLastInsertedRowId() +{ + return sqlite3_last_insert_rowid(db); +} + void SQLiteStmt::create(sqlite3 * db, const string & sql) { checkInterrupt(); @@ -95,10 +100,10 @@ SQLiteStmt::Use::~Use() sqlite3_reset(stmt); } -SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool notNull) +SQLiteStmt::Use & SQLiteStmt::Use::operator () (std::string_view value, bool notNull) { if (notNull) { - if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + if (sqlite3_bind_text(stmt, curArg++, value.data(), -1, SQLITE_TRANSIENT) != SQLITE_OK) throwSQLiteError(stmt.db, "binding argument"); } else bind(); diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index dd81ab051..99f0d56ce 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -26,6 +26,8 @@ struct SQLite void isCache(); void exec(const std::string & stmt); + + uint64_t getLastInsertedRowId(); }; /* RAII wrapper to create and destroy SQLite prepared statements. */ @@ -54,7 +56,7 @@ struct SQLiteStmt ~Use(); /* Bind the next parameter. */ - Use & operator () (const std::string & value, bool notNull = true); + Use & operator () (std::string_view value, bool notNull = true); Use & operator () (const unsigned char * data, size_t len, bool notNull = true); Use & operator () (int64_t value, bool notNull = true); Use & bind(); // null diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 5b0b33046..7b0dede49 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -193,6 +193,23 @@ StorePath Store::makeFixedOutputPath( } } +// FIXME Put this somewhere? +template struct overloaded : Ts... { using Ts::operator()...; }; +template overloaded(Ts...) -> overloaded; + +StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, + const StorePathSet & references, bool hasSelfReference) const +{ + // New template + return std::visit(overloaded { + [&](TextHash th) { + return makeTextPath(name, th.hash, references); + }, + [&](FixedOutputHash fsh) { + return makeFixedOutputPath(fsh.method, fsh.hash, name, references, hasSelfReference); + } + }, ca); +} StorePath Store::makeTextPath(std::string_view name, const Hash & hash, const StorePathSet & references) const @@ -222,20 +239,73 @@ StorePath Store::computeStorePathForText(const string & name, const string & s, } +/* +The aim of this function is to compute in one pass the correct ValidPathInfo for +the files that we are trying to add to the store. To accomplish that in one +pass, given the different kind of inputs that we can take (normal nar archives, +nar archives with non SHA-256 hashes, and flat files), we set up a net of sinks +and aliases. Also, since the dataflow is obfuscated by this, we include here a +graphviz diagram: + +digraph graphname { + node [shape=box] + fileSource -> narSink + narSink [style=dashed] + narSink -> unsualHashTee [style = dashed, label = "Recursive && !SHA-256"] + narSink -> narHashSink [style = dashed, label = "else"] + unsualHashTee -> narHashSink + unsualHashTee -> caHashSink + fileSource -> parseSink + parseSink [style=dashed] + parseSink-> fileSink [style = dashed, label = "Flat"] + parseSink -> blank [style = dashed, label = "Recursive"] + fileSink -> caHashSink +} +*/ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, std::optional expectedCAHash) { - /* FIXME: inefficient: we're reading/hashing 'tmpFile' three - times. */ + HashSink narHashSink { htSHA256 }; + HashSink caHashSink { hashAlgo }; - auto [narHash, narSize] = hashPath(htSHA256, srcPath); + /* Note that fileSink and unusualHashTee must be mutually exclusive, since + they both write to caHashSink. Note that that requisite is currently true + because the former is only used in the flat case. */ + RetrieveRegularNARSink fileSink { caHashSink }; + TeeSink unusualHashTee { narHashSink, caHashSink }; - auto hash = method == FileIngestionMethod::Recursive - ? hashAlgo == htSHA256 - ? narHash - : hashPath(hashAlgo, srcPath).first - : hashFile(hashAlgo, srcPath); + auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != htSHA256 + ? static_cast(unusualHashTee) + : narHashSink; + + /* Functionally, this means that fileSource will yield the content of + srcPath. The fact that we use scratchpadSink as a temporary buffer here + is an implementation detail. */ + auto fileSource = sinkToSource([&](Sink & scratchpadSink) { + dumpPath(srcPath, scratchpadSink); + }); + + /* tapped provides the same data as fileSource, but we also write all the + information to narSink. */ + TeeSource tapped { *fileSource, narSink }; + + ParseSink blank; + auto & parseSink = method == FileIngestionMethod::Flat + ? fileSink + : blank; + + /* The information that flows from tapped (besides being replicated in + narSink), is now put in parseSink. */ + parseDump(parseSink, tapped); + + /* We extract the result of the computation from the sink by calling + finish. */ + auto [narHash, narSize] = narHashSink.finish(); + + auto hash = method == FileIngestionMethod::Recursive && hashAlgo == htSHA256 + ? narHash + : caHashSink.finish().first; if (expectedCAHash && expectedCAHash != hash) throw Error("hash mismatch for '%s'", srcPath); @@ -246,8 +316,8 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, info.ca = FixedOutputHash { .method = method, .hash = hash }; if (!isValidPath(info.path)) { - auto source = sinkToSource([&](Sink & sink) { - dumpPath(srcPath, sink); + auto source = sinkToSource([&](Sink & scratchpadSink) { + dumpPath(srcPath, scratchpadSink); }); addToStore(info, *source); } @@ -485,7 +555,7 @@ string Store::makeValidityRegistration(const StorePathSet & paths, auto info = queryPathInfo(i); if (showHash) { - s += info->narHash.to_string(Base16, false) + "\n"; + s += info->narHash->to_string(Base16, false) + "\n"; s += (format("%1%\n") % info->narSize).str(); } @@ -517,7 +587,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store auto info = queryPathInfo(storePath); jsonPath - .attr("narHash", info->narHash.to_string(hashBase, true)) + .attr("narHash", info->narHash->to_string(hashBase, true)) .attr("narSize", info->narSize); { @@ -560,7 +630,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store if (!narInfo->url.empty()) jsonPath.attr("url", narInfo->url); if (narInfo->fileHash) - jsonPath.attr("downloadHash", narInfo->fileHash.to_string(hashBase, true)); + jsonPath.attr("downloadHash", narInfo->fileHash->to_string(hashBase, true)); if (narInfo->fileSize) jsonPath.attr("downloadSize", narInfo->fileSize); if (showClosureSize) @@ -636,6 +706,15 @@ void copyStorePath(ref srcStore, ref dstStore, uint64_t total = 0; + // recompute store path on the chance dstStore does it differently + if (info->ca && info->references.empty()) { + auto info2 = make_ref(*info); + info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), *info->ca); + if (dstStore->storeDir == srcStore->storeDir) + assert(info->path == info2->path); + info = info2; + } + if (!info->narHash) { StringSink sink; srcStore->narFromPath({storePath}, sink); @@ -671,16 +750,20 @@ void copyStorePath(ref srcStore, ref dstStore, } -void copyPaths(ref srcStore, ref dstStore, const StorePathSet & storePaths, +std::map copyPaths(ref srcStore, ref dstStore, const StorePathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) { auto valid = dstStore->queryValidPaths(storePaths, substitute); - PathSet missing; + StorePathSet missing; for (auto & path : storePaths) - if (!valid.count(path)) missing.insert(srcStore->printStorePath(path)); + if (!valid.count(path)) missing.insert(path); - if (missing.empty()) return; + std::map pathsMap; + for (auto & path : storePaths) + pathsMap.insert_or_assign(path, path); + + if (missing.empty()) return pathsMap; Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); @@ -695,30 +778,49 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st ThreadPool pool; - processGraph(pool, - PathSet(missing.begin(), missing.end()), + processGraph(pool, + StorePathSet(missing.begin(), missing.end()), - [&](const Path & storePath) { - if (dstStore->isValidPath(dstStore->parseStorePath(storePath))) { + [&](const StorePath & storePath) { + auto info = srcStore->queryPathInfo(storePath); + auto storePathForDst = storePath; + if (info->ca && info->references.empty()) { + storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca); + if (dstStore->storeDir == srcStore->storeDir) + assert(storePathForDst == storePath); + if (storePathForDst != storePath) + debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); + } + pathsMap.insert_or_assign(storePath, storePathForDst); + + if (dstStore->isValidPath(storePath)) { nrDone++; showProgress(); - return PathSet(); + return StorePathSet(); } - auto info = srcStore->queryPathInfo(srcStore->parseStorePath(storePath)); - bytesExpected += info->narSize; act.setExpected(actCopyPath, bytesExpected); - return srcStore->printStorePathSet(info->references); + return info->references; }, - [&](const Path & storePathS) { + [&](const StorePath & storePath) { checkInterrupt(); - auto storePath = dstStore->parseStorePath(storePathS); + auto info = srcStore->queryPathInfo(storePath); - if (!dstStore->isValidPath(storePath)) { + auto storePathForDst = storePath; + if (info->ca && info->references.empty()) { + storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca); + if (dstStore->storeDir == srcStore->storeDir) + assert(storePathForDst == storePath); + if (storePathForDst != storePath) + debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); + } + pathsMap.insert_or_assign(storePath, storePathForDst); + + if (!dstStore->isValidPath(storePathForDst)) { MaintainCount mc(nrRunning); showProgress(); try { @@ -727,7 +829,7 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st nrFailed++; if (!settings.keepGoing) throw e; - logger->log(lvlError, fmt("could not copy %s: %s", storePathS, e.what())); + logger->log(lvlError, fmt("could not copy %s: %s", dstStore->printStorePath(storePath), e.what())); showProgress(); return; } @@ -736,6 +838,8 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st nrDone++; showProgress(); }); + + return pathsMap; } @@ -801,7 +905,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const store.printStorePath(path)); return "1;" + store.printStorePath(path) + ";" - + narHash.to_string(Base32, true) + ";" + + narHash->to_string(Base32, true) + ";" + std::to_string(narSize) + ";" + concatStringsSep(",", store.printStorePathSet(references)); } @@ -812,10 +916,6 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } -// FIXME Put this somewhere? -template struct overloaded : Ts... { using Ts::operator()...; }; -template overloaded(Ts...) -> overloaded; - bool ValidPathInfo::isContentAddressed(const Store & store) const { if (! ca) return false; @@ -882,7 +982,9 @@ Derivation Store::readDerivation(const StorePath & drvPath) { auto accessor = getFSAccessor(); try { - return parseDerivation(*this, accessor->readFile(printStorePath(drvPath))); + return parseDerivation(*this, + accessor->readFile(printStorePath(drvPath)), + Derivation::nameFromPath(drvPath)); } catch (FormatError & e) { throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg()); } @@ -932,12 +1034,20 @@ ref openStore(const std::string & uri_, throw Error("don't know how to open Nix store '%s'", uri); } +static bool isNonUriPath(const std::string & spec) { + return + // is not a URL + spec.find("://") == std::string::npos + // Has at least one path separator, and so isn't a single word that + // might be special like "auto" + && spec.find("/") != std::string::npos; +} StoreType getStoreType(const std::string & uri, const std::string & stateDir) { if (uri == "daemon") { return tDaemon; - } else if (uri == "local" || hasPrefix(uri, "/")) { + } else if (uri == "local" || isNonUriPath(uri)) { return tLocal; } else if (uri == "" || uri == "auto") { if (access(stateDir.c_str(), R_OK | W_OK) == 0) @@ -961,8 +1071,9 @@ static RegisterStoreImplementation regStore([]( return std::shared_ptr(std::make_shared(params)); case tLocal: { Store::Params params2 = params; - if (hasPrefix(uri, "/")) - params2["root"] = uri; + if (isNonUriPath(uri)) { + params2["root"] = absPath(uri); + } return std::shared_ptr(std::make_shared(params2)); } default: diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index a4be0411e..49da19496 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -85,7 +85,7 @@ struct GCOptions StorePathSet pathsToDelete; /* Stop after at least `maxFreed' bytes have been freed. */ - unsigned long long maxFreed{std::numeric_limits::max()}; + uint64_t maxFreed{std::numeric_limits::max()}; }; @@ -97,7 +97,7 @@ struct GCResults /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the number of bytes that would be or was freed. */ - unsigned long long bytesFreed = 0; + uint64_t bytesFreed = 0; }; @@ -105,8 +105,8 @@ struct SubstitutablePathInfo { std::optional deriver; StorePathSet references; - unsigned long long downloadSize; /* 0 = unknown or inapplicable */ - unsigned long long narSize; /* 0 = unknown */ + uint64_t downloadSize; /* 0 = unknown or inapplicable */ + uint64_t narSize; /* 0 = unknown */ }; typedef std::map SubstitutablePathInfos; @@ -115,7 +115,8 @@ struct ValidPathInfo { StorePath path; std::optional deriver; - Hash narHash; + // TODO document this + std::optional narHash; StorePathSet references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown @@ -343,7 +344,11 @@ public: bool hasSelfReference = false) const; StorePath makeTextPath(std::string_view name, const Hash & hash, - const StorePathSet & references) const; + const StorePathSet & references = {}) const; + + StorePath makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, + const StorePathSet & references = {}, + bool hasSelfReference = false) const; /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store @@ -435,9 +440,10 @@ public: virtual StorePathSet querySubstitutablePaths(const StorePathSet & paths) { return {}; }; /* Query substitute info (i.e. references, derivers and download - sizes) of a set of paths. If a path does not have substitute - info, it's omitted from the resulting ‘infos’ map. */ - virtual void querySubstitutablePathInfos(const StorePathSet & paths, + sizes) of a map of paths to their optional ca values. If a path + does not have substitute info, it's omitted from the resulting + ‘infos’ map. */ + virtual void querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) { return; }; /* Import a path into the store. */ @@ -460,7 +466,7 @@ public: std::optional expectedCAHash = {}); // FIXME: remove? - virtual StorePath addToStoreFromDump(const string & dump, const string & name, + virtual StorePath addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) { throw Error("addToStoreFromDump() is not supported by this store"); @@ -609,7 +615,7 @@ public: that will be substituted. */ virtual void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize); + uint64_t & downloadSize, uint64_t & narSize); /* Sort a set of paths topologically under the references relation. If p refers to q, then p precedes q in this list. */ @@ -739,11 +745,13 @@ void copyStorePath(ref srcStore, ref dstStore, /* Copy store paths from one store to another. The paths may be copied - in parallel. They are copied in a topologically sorted order - (i.e. if A is a reference of B, then A is copied before B), but - the set of store paths is not automatically closed; use - copyClosure() for that. */ -void copyPaths(ref srcStore, ref dstStore, const StorePathSet & storePaths, + in parallel. They are copied in a topologically sorted order (i.e. + if A is a reference of B, then A is copied before B), but the set + of store paths is not automatically closed; use copyClosure() for + that. Returns a map of what each path was copied to the dstStore + as. */ +std::map copyPaths(ref srcStore, ref dstStore, + const StorePathSet & storePaths, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); @@ -842,4 +850,6 @@ std::optional decodeValidPathInfo( /* Split URI into protocol+hierarchy part and its parameter set. */ std::pair splitUriAndParams(const std::string & uri); +std::optional getDerivationCA(const BasicDerivation & drv); + } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 8b538f6da..f76b13fb4 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -6,7 +6,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x116 +#define PROTOCOL_VERSION 0x117 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -70,6 +70,10 @@ template T readStorePaths(const Store & store, Source & from); void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths); +StorePathCAMap readStorePathCAMap(const Store & store, Source & from); + +void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths); + void writeOutputPathMap(const Store & store, Sink & out, const OutputPathMap & paths); } diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 51c88537e..ce7cf9754 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -150,17 +150,17 @@ static void skipGeneric(Source & source) static void parseContents(ParseSink & sink, Source & source, const Path & path) { - unsigned long long size = readLongLong(source); + uint64_t size = readLongLong(source); sink.preallocateContents(size); - unsigned long long left = size; + uint64_t left = size; std::vector buf(65536); while (left) { checkInterrupt(); auto n = buf.size(); - if ((unsigned long long)n > left) n = left; + if ((uint64_t)n > left) n = left; source(buf.data(), n); sink.receiveContents(buf.data(), n); left -= n; @@ -323,7 +323,7 @@ struct RestoreSink : ParseSink throw SysError("fchmod"); } - void preallocateContents(unsigned long long len) + void preallocateContents(uint64_t len) { #if HAVE_POSIX_FALLOCATE if (len) { @@ -338,7 +338,7 @@ struct RestoreSink : ParseSink #endif } - void receiveContents(unsigned char * data, unsigned int len) + void receiveContents(unsigned char * data, size_t len) { writeFull(fd.get(), data, len); } diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 302b1bb18..5665732d2 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -57,18 +57,35 @@ struct ParseSink virtual void createRegularFile(const Path & path) { }; virtual void isExecutable() { }; - virtual void preallocateContents(unsigned long long size) { }; - virtual void receiveContents(unsigned char * data, unsigned int len) { }; + virtual void preallocateContents(uint64_t size) { }; + virtual void receiveContents(unsigned char * data, size_t len) { }; virtual void createSymlink(const Path & path, const string & target) { }; }; -struct TeeParseSink : ParseSink +/* If the NAR archive contains a single file at top-level, then save + the contents of the file to `s'. Otherwise barf. */ +struct RetrieveRegularNARSink : ParseSink { - StringSink saved; - TeeSource source; + bool regular = true; + Sink & sink; - TeeParseSink(Source & source) : source(source, saved) { } + RetrieveRegularNARSink(Sink & sink) : sink(sink) { } + + void createDirectory(const Path & path) + { + regular = false; + } + + void receiveContents(unsigned char * data, size_t len) + { + sink(data, len); + } + + void createSymlink(const Path & path, const string & target) + { + regular = false; + } }; void parseDump(ParseSink & sink, Source & source); diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 1f0e4f803..986c5d1cd 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -1,6 +1,8 @@ #include "args.hh" #include "hash.hh" +#include + namespace nix { void Args::addFlag(Flag && flag_) @@ -13,6 +15,20 @@ void Args::addFlag(Flag && flag_) if (flag->shortName) shortFlags[flag->shortName] = flag; } +bool pathCompletions = false; +std::shared_ptr> completions; + +std::string completionMarker = "___COMPLETE___"; + +std::optional needsCompletion(std::string_view s) +{ + if (!completions) return {}; + auto i = s.find(completionMarker); + if (i != std::string::npos) + return std::string(s.begin(), i); + return {}; +} + void Args::parseCmdline(const Strings & _cmdline) { Strings pendingArgs; @@ -20,6 +36,14 @@ void Args::parseCmdline(const Strings & _cmdline) Strings cmdline(_cmdline); + if (auto s = getEnv("NIX_GET_COMPLETIONS")) { + size_t n = std::stoi(*s); + assert(n > 0 && n <= cmdline.size()); + *std::next(cmdline.begin(), n - 1) += completionMarker; + completions = std::make_shared(); + verbosity = lvlError; + } + for (auto pos = cmdline.begin(); pos != cmdline.end(); ) { auto arg = *pos; @@ -63,7 +87,7 @@ void Args::printHelp(const string & programName, std::ostream & out) for (auto & exp : expectedArgs) { std::cout << renderLabels({exp.label}); // FIXME: handle arity > 1 - if (exp.arity == 0) std::cout << "..."; + if (exp.handler.arity == ArityAny) std::cout << "..."; if (exp.optional) std::cout << "?"; } std::cout << "\n"; @@ -99,18 +123,32 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) auto process = [&](const std::string & name, const Flag & flag) -> bool { ++pos; std::vector args; + bool anyCompleted = false; for (size_t n = 0 ; n < flag.handler.arity; ++n) { if (pos == end) { if (flag.handler.arity == ArityAny) break; throw UsageError("flag '%s' requires %d argument(s)", name, flag.handler.arity); } + if (flag.completer) + if (auto prefix = needsCompletion(*pos)) { + anyCompleted = true; + flag.completer(n, *prefix); + } args.push_back(*pos++); } - flag.handler.fun(std::move(args)); + if (!anyCompleted) + flag.handler.fun(std::move(args)); return true; }; if (string(*pos, 0, 2) == "--") { + if (auto prefix = needsCompletion(*pos)) { + for (auto & [name, flag] : longFlags) { + if (!hiddenCategories.count(flag->category) + && hasPrefix(name, std::string(*prefix, 2))) + completions->insert("--" + name); + } + } auto i = longFlags.find(string(*pos, 2)); if (i == longFlags.end()) return false; return process("--" + i->first, *i->second); @@ -123,6 +161,14 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) return process(std::string("-") + c, *i->second); } + if (auto prefix = needsCompletion(*pos)) { + if (prefix == "-") { + completions->insert("--"); + for (auto & [flag, _] : shortFlags) + completions->insert(std::string("-") + flag); + } + } + return false; } @@ -138,12 +184,17 @@ bool Args::processArgs(const Strings & args, bool finish) bool res = false; - if ((exp.arity == 0 && finish) || - (exp.arity > 0 && args.size() == exp.arity)) + if ((exp.handler.arity == ArityAny && finish) || + (exp.handler.arity != ArityAny && args.size() == exp.handler.arity)) { std::vector ss; - for (auto & s : args) ss.push_back(s); - exp.handler(std::move(ss)); + for (const auto &[n, s] : enumerate(args)) { + ss.push_back(s); + if (exp.completer) + if (auto prefix = needsCompletion(s)) + exp.completer(n, *prefix); + } + exp.handler.fun(ss); expectedArgs.pop_front(); res = true; } @@ -154,6 +205,13 @@ bool Args::processArgs(const Strings & args, bool finish) return res; } +static void hashTypeCompleter(size_t index, std::string_view prefix) +{ + for (auto & type : hashTypes) + if (hasPrefix(type, prefix)) + completions->insert(type); +} + Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht) { return Flag { @@ -162,7 +220,8 @@ Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht) .labels = {"hash-algo"}, .handler = {[ht](std::string s) { *ht = parseHashType(s); - }} + }}, + .completer = hashTypeCompleter }; } @@ -174,10 +233,42 @@ Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional< .labels = {"hash-algo"}, .handler = {[oht](std::string s) { *oht = std::optional { parseHashType(s) }; - }} + }}, + .completer = hashTypeCompleter }; } +static void completePath(std::string_view prefix, bool onlyDirs) +{ + pathCompletions = true; + glob_t globbuf; + int flags = GLOB_NOESCAPE | GLOB_TILDE; + #ifdef GLOB_ONLYDIR + if (onlyDirs) + flags |= GLOB_ONLYDIR; + #endif + if (glob((std::string(prefix) + "*").c_str(), flags, nullptr, &globbuf) == 0) { + for (size_t i = 0; i < globbuf.gl_pathc; ++i) { + if (onlyDirs) { + auto st = lstat(globbuf.gl_pathv[i]); + if (!S_ISDIR(st.st_mode)) continue; + } + completions->insert(globbuf.gl_pathv[i]); + } + globfree(&globbuf); + } +} + +void completePath(size_t, std::string_view prefix) +{ + completePath(prefix, false); +} + +void completeDir(size_t, std::string_view prefix) +{ + completePath(prefix, true); +} + Strings argvToStrings(int argc, char * * argv) { Strings args; @@ -225,18 +316,26 @@ void Command::printHelp(const string & programName, std::ostream & out) MultiCommand::MultiCommand(const Commands & commands) : commands(commands) { - expectedArgs.push_back(ExpectedArg{"command", 1, true, [=](std::vector ss) { - assert(!command); - auto cmd = ss[0]; - if (auto alias = get(deprecatedAliases, cmd)) { - warn("'%s' is a deprecated alias for '%s'", cmd, *alias); - cmd = *alias; - } - auto i = commands.find(cmd); - if (i == commands.end()) - throw UsageError("'%s' is not a recognised command", cmd); - command = {cmd, i->second()}; - }}); + expectArgs({ + .label = "command", + .optional = true, + .handler = {[=](std::string s) { + assert(!command); + if (auto alias = get(deprecatedAliases, s)) { + warn("'%s' is a deprecated alias for '%s'", s, *alias); + s = *alias; + } + if (auto prefix = needsCompletion(s)) { + for (auto & [name, command] : commands) + if (hasPrefix(name, *prefix)) + completions->insert(name); + } + auto i = commands.find(s); + if (i == commands.end()) + throw UsageError("'%s' is not a recognised command", s); + command = {s, i->second()}; + }} + }); categories[Command::catDefault] = "Available commands"; } diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 433d26bba..97a517344 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -8,8 +8,6 @@ namespace nix { -MakeError(UsageError, Error); - enum HashType : char; class Args @@ -28,61 +26,67 @@ protected: static const size_t ArityAny = std::numeric_limits::max(); + struct Handler + { + std::function)> fun; + size_t arity; + + Handler() {} + + Handler(std::function)> && fun) + : fun(std::move(fun)) + , arity(ArityAny) + { } + + Handler(std::function && handler) + : fun([handler{std::move(handler)}](std::vector) { handler(); }) + , arity(0) + { } + + Handler(std::function && handler) + : fun([handler{std::move(handler)}](std::vector ss) { + handler(std::move(ss[0])); + }) + , arity(1) + { } + + Handler(std::function && handler) + : fun([handler{std::move(handler)}](std::vector ss) { + handler(std::move(ss[0]), std::move(ss[1])); + }) + , arity(2) + { } + + Handler(std::vector * dest) + : fun([=](std::vector ss) { *dest = ss; }) + , arity(ArityAny) + { } + + template + Handler(T * dest) + : fun([=](std::vector ss) { *dest = ss[0]; }) + , arity(1) + { } + + template + Handler(T * dest, const T & val) + : fun([=](std::vector ss) { *dest = val; }) + , arity(0) + { } + }; + /* Flags. */ struct Flag { typedef std::shared_ptr ptr; - struct Handler - { - std::function)> fun; - size_t arity; - - Handler() {} - - Handler(std::function)> && fun) - : fun(std::move(fun)) - , arity(ArityAny) - { } - - Handler(std::function && handler) - : fun([handler{std::move(handler)}](std::vector) { handler(); }) - , arity(0) - { } - - Handler(std::function && handler) - : fun([handler{std::move(handler)}](std::vector ss) { - handler(std::move(ss[0])); - }) - , arity(1) - { } - - Handler(std::function && handler) - : fun([handler{std::move(handler)}](std::vector ss) { - handler(std::move(ss[0]), std::move(ss[1])); - }) - , arity(2) - { } - - template - Handler(T * dest) - : fun([=](std::vector ss) { *dest = ss[0]; }) - , arity(1) - { } - - template - Handler(T * dest, const T & val) - : fun([=](std::vector ss) { *dest = val; }) - , arity(0) - { } - }; - std::string longName; char shortName = 0; std::string description; std::string category; Strings labels; Handler handler; + std::function completer; static Flag mkHashTypeFlag(std::string && longName, HashType * ht); static Flag mkHashTypeOptFlag(std::string && longName, std::optional * oht); @@ -99,9 +103,9 @@ protected: struct ExpectedArg { std::string label; - size_t arity; // 0 = any - bool optional; - std::function)> handler; + bool optional = false; + Handler handler; + std::function completer; }; std::list expectedArgs; @@ -175,20 +179,28 @@ public: }); } + void expectArgs(ExpectedArg && arg) + { + expectedArgs.emplace_back(std::move(arg)); + } + /* Expect a string argument. */ void expectArg(const std::string & label, string * dest, bool optional = false) { - expectedArgs.push_back(ExpectedArg{label, 1, optional, [=](std::vector ss) { - *dest = ss[0]; - }}); + expectArgs({ + .label = label, + .optional = true, + .handler = {dest} + }); } /* Expect 0 or more arguments. */ void expectArgs(const std::string & label, std::vector * dest) { - expectedArgs.push_back(ExpectedArg{label, 0, false, [=](std::vector ss) { - *dest = std::move(ss); - }}); + expectArgs({ + .label = label, + .handler = {dest} + }); } friend class MultiCommand; @@ -259,4 +271,13 @@ typedef std::vector> Table2; void printTable(std::ostream & out, const Table2 & table); +extern std::shared_ptr> completions; +extern bool pathCompletions; + +std::optional needsCompletion(std::string_view s); + +void completePath(size_t, std::string_view prefix); + +void completeDir(size_t, std::string_view prefix); + } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index f4b3f11bb..0daaf3be2 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -191,6 +191,7 @@ public: } MakeError(Error, BaseError); +MakeError(UsageError, Error); class SysError : public Error { diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index f0d7754d1..2b0390da4 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -15,16 +15,22 @@ namespace nix { +static size_t regularHashSize(HashType type) { + switch (type) { + case htMD5: return md5HashSize; + case htSHA1: return sha1HashSize; + case htSHA256: return sha256HashSize; + case htSHA512: return sha512HashSize; + } + abort(); +} + +std::set hashTypes = { "md5", "sha1", "sha256", "sha512" }; + void Hash::init() { - assert(type); - switch (*type) { - case htMD5: hashSize = md5HashSize; break; - case htSHA1: hashSize = sha1HashSize; break; - case htSHA256: hashSize = sha256HashSize; break; - case htSHA512: hashSize = sha512HashSize; break; - } + hashSize = regularHashSize(type); assert(hashSize <= maxHashSize); memset(hash, 0, maxHashSize); } @@ -105,17 +111,11 @@ string printHash16or32(const Hash & hash) } -HashType assertInitHashType(const Hash & h) -{ - assert(h.type); - return *h.type; -} - std::string Hash::to_string(Base base, bool includeType) const { std::string s; if (base == SRI || includeType) { - s += printHashType(assertInitHashType(*this)); + s += printHashType(type); s += base == SRI ? '-' : ':'; } switch (base) { @@ -136,60 +136,66 @@ std::string Hash::to_string(Base base, bool includeType) const Hash::Hash(std::string_view s, HashType type) : Hash(s, std::optional { type }) { } Hash::Hash(std::string_view s) : Hash(s, std::optional{}) { } -Hash::Hash(std::string_view s, std::optional type) - : type(type) +Hash::Hash(std::string_view original, std::optional optType) { + auto rest = original; + size_t pos = 0; bool isSRI = false; - auto sep = s.find(':'); - if (sep == string::npos) { - sep = s.find('-'); - if (sep != string::npos) { - isSRI = true; - } else if (! type) - throw BadHash("hash '%s' does not include a type", s); + // Parse the has type before the separater, if there was one. + std::optional optParsedType; + { + auto sep = rest.find(':'); + if (sep == std::string_view::npos) { + sep = rest.find('-'); + if (sep != std::string_view::npos) + isSRI = true; + } + if (sep != std::string_view::npos) { + auto hashRaw = rest.substr(0, sep); + optParsedType = parseHashType(hashRaw); + rest = rest.substr(sep + 1); + } } - if (sep != string::npos) { - string hts = string(s, 0, sep); - this->type = parseHashType(hts); - if (!this->type) - throw BadHash("unknown hash type '%s'", hts); - if (type && type != this->type) - throw BadHash("hash '%s' should have type '%s'", s, printHashType(*type)); - pos = sep + 1; + // Either the string or user must provide the type, if they both do they + // must agree. + if (!optParsedType && !optType) { + throw BadHash("hash '%s' does not include a type, nor is the type otherwise known from context.", rest); + } else { + this->type = optParsedType ? *optParsedType : *optType; + if (optParsedType && optType && *optParsedType != *optType) + throw BadHash("hash '%s' should have type '%s'", original, printHashType(*optType)); } init(); - size_t size = s.size() - pos; - - if (!isSRI && size == base16Len()) { + if (!isSRI && rest.size() == base16Len()) { auto parseHexDigit = [&](char c) { if (c >= '0' && c <= '9') return c - '0'; if (c >= 'A' && c <= 'F') return c - 'A' + 10; if (c >= 'a' && c <= 'f') return c - 'a' + 10; - throw BadHash("invalid base-16 hash '%s'", s); + throw BadHash("invalid base-16 hash '%s'", original); }; for (unsigned int i = 0; i < hashSize; i++) { hash[i] = - parseHexDigit(s[pos + i * 2]) << 4 - | parseHexDigit(s[pos + i * 2 + 1]); + parseHexDigit(rest[pos + i * 2]) << 4 + | parseHexDigit(rest[pos + i * 2 + 1]); } } - else if (!isSRI && size == base32Len()) { + else if (!isSRI && rest.size() == base32Len()) { - for (unsigned int n = 0; n < size; ++n) { - char c = s[pos + size - n - 1]; + for (unsigned int n = 0; n < rest.size(); ++n) { + char c = rest[rest.size() - n - 1]; unsigned char digit; for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */ if (base32Chars[digit] == c) break; if (digit >= 32) - throw BadHash("invalid base-32 hash '%s'", s); + throw BadHash("invalid base-32 hash '%s'", original); unsigned int b = n * 5; unsigned int i = b / 8; unsigned int j = b % 8; @@ -199,21 +205,21 @@ Hash::Hash(std::string_view s, std::optional type) hash[i + 1] |= digit >> (8 - j); } else { if (digit >> (8 - j)) - throw BadHash("invalid base-32 hash '%s'", s); + throw BadHash("invalid base-32 hash '%s'", original); } } } - else if (isSRI || size == base64Len()) { - auto d = base64Decode(s.substr(pos)); + else if (isSRI || rest.size() == base64Len()) { + auto d = base64Decode(rest); if (d.size() != hashSize) - throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", s); + throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", original); assert(hashSize); memcpy(hash, d.data(), hashSize); } else - throw BadHash("hash '%s' has wrong length for hash type '%s'", s, printHashType(*type)); + throw BadHash("hash '%s' has wrong length for hash type '%s'", rest, printHashType(this->type)); } Hash newHashAllowEmpty(std::string hashStr, std::optional ht) @@ -266,7 +272,7 @@ static void finish(HashType ht, Ctx & ctx, unsigned char * hash) } -Hash hashString(HashType ht, const string & s) +Hash hashString(HashType ht, std::string_view s) { Ctx ctx; Hash hash(ht); @@ -333,7 +339,7 @@ HashResult hashPath( Hash compressHash(const Hash & hash, unsigned int newSize) { - Hash h; + Hash h(hash.type); h.hashSize = newSize; for (unsigned int i = 0; i < hash.hashSize; ++i) h.hash[i % newSize] ^= hash.hash[i]; @@ -341,7 +347,7 @@ Hash compressHash(const Hash & hash, unsigned int newSize) } -std::optional parseHashTypeOpt(const string & s) +std::optional parseHashTypeOpt(std::string_view s) { if (s == "md5") return htMD5; else if (s == "sha1") return htSHA1; @@ -350,7 +356,7 @@ std::optional parseHashTypeOpt(const string & s) else return std::optional {}; } -HashType parseHashType(const string & s) +HashType parseHashType(std::string_view s) { auto opt_h = parseHashTypeOpt(s); if (opt_h) diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 23259dced..abcd58f24 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -18,6 +18,8 @@ const int sha1HashSize = 20; const int sha256HashSize = 32; const int sha512HashSize = 64; +extern std::set hashTypes; + extern const string base32Chars; enum Base : int { Base64, Base32, Base16, SRI }; @@ -25,14 +27,11 @@ enum Base : int { Base64, Base32, Base16, SRI }; struct Hash { - static const unsigned int maxHashSize = 64; - unsigned int hashSize = 0; - unsigned char hash[maxHashSize] = {}; + constexpr static size_t maxHashSize = 64; + size_t hashSize = 0; + uint8_t hash[maxHashSize] = {}; - std::optional type = {}; - - /* Create an unset hash object. */ - Hash() { }; + HashType type; /* Create a zero-filled hash object. */ Hash(HashType type) : type(type) { init(); }; @@ -105,14 +104,14 @@ Hash newHashAllowEmpty(std::string hashStr, std::optional ht); string printHash16or32(const Hash & hash); /* Compute the hash of the given string. */ -Hash hashString(HashType ht, const string & s); +Hash hashString(HashType ht, std::string_view s); /* Compute the hash of the given file. */ Hash hashFile(HashType ht, const Path & path); /* Compute the hash of the given path. The hash is defined as (essentially) hashString(ht, dumpPath(path)). */ -typedef std::pair HashResult; +typedef std::pair HashResult; HashResult hashPath(HashType ht, const Path & path, PathFilter & filter = defaultPathFilter); @@ -121,9 +120,10 @@ HashResult hashPath(HashType ht, const Path & path, Hash compressHash(const Hash & hash, unsigned int newSize); /* Parse a string representing a hash type. */ -HashType parseHashType(const string & s); +HashType parseHashType(std::string_view s); + /* Will return nothing on parse error */ -std::optional parseHashTypeOpt(const string & s); +std::optional parseHashTypeOpt(std::string_view s); /* And the reverse. */ string printHashType(HashType ht); @@ -141,7 +141,7 @@ class HashSink : public BufferedSink, public AbstractHashSink private: HashType ht; Ctx * ctx; - unsigned long long bytes; + uint64_t bytes; public: HashSink(HashType ht); diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index c8b71188f..00c945113 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -322,5 +322,18 @@ void StringSink::operator () (const unsigned char * data, size_t len) s->append((const char *) data, len); } +size_t ChainSource::read(unsigned char * data, size_t len) +{ + if (useSecond) { + return source2.read(data, len); + } else { + try { + return source1.read(data, len); + } catch (EndOfFile &) { + useSecond = true; + return this->read(data, len); + } + } +} } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 8386a4991..c29c6b29b 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -189,7 +189,7 @@ struct TeeSource : Source size_t read(unsigned char * data, size_t len) { size_t n = orig.read(data, len); - sink(data, len); + sink(data, n); return n; } }; @@ -256,6 +256,19 @@ struct LambdaSource : Source } }; +/* Chain two sources together so after the first is exhausted, the second is + used */ +struct ChainSource : Source +{ + Source & source1, & source2; + bool useSecond = false; + ChainSource(Source & s1, Source & s2) + : source1(s1), source2(s2) + { } + + size_t read(unsigned char * data, size_t len) override; +}; + /* Convert a function that feeds data into a Sink into a Source. The Source executes the function as a coroutine. */ @@ -299,14 +312,14 @@ T readNum(Source & source) source(buf, sizeof(buf)); uint64_t n = - ((unsigned long long) buf[0]) | - ((unsigned long long) buf[1] << 8) | - ((unsigned long long) buf[2] << 16) | - ((unsigned long long) buf[3] << 24) | - ((unsigned long long) buf[4] << 32) | - ((unsigned long long) buf[5] << 40) | - ((unsigned long long) buf[6] << 48) | - ((unsigned long long) buf[7] << 56); + ((uint64_t) buf[0]) | + ((uint64_t) buf[1] << 8) | + ((uint64_t) buf[2] << 16) | + ((uint64_t) buf[3] << 24) | + ((uint64_t) buf[4] << 32) | + ((uint64_t) buf[5] << 40) | + ((uint64_t) buf[6] << 48) | + ((uint64_t) buf[7] << 56); if (n > std::numeric_limits::max()) throw SerialisationError("serialised integer %d is too large for type '%s'", n, typeid(T).name()); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 1268b146a..97d278581 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -79,7 +80,7 @@ void replaceEnv(std::map newEnv) } -Path absPath(Path path, std::optional dir) +Path absPath(Path path, std::optional dir, bool resolveSymlinks) { if (path[0] != '/') { if (!dir) { @@ -100,7 +101,7 @@ Path absPath(Path path, std::optional dir) } path = *dir + "/" + path; } - return canonPath(path); + return canonPath(path, resolveSymlinks); } @@ -345,7 +346,6 @@ void writeFile(const Path & path, Source & source, mode_t mode) } } - string readLine(int fd) { string s; @@ -374,7 +374,7 @@ void writeLine(int fd, string s) } -static void _deletePath(int parentfd, const Path & path, unsigned long long & bytesFreed) +static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed) { checkInterrupt(); @@ -414,7 +414,7 @@ static void _deletePath(int parentfd, const Path & path, unsigned long long & by } } -static void _deletePath(const Path & path, unsigned long long & bytesFreed) +static void _deletePath(const Path & path, uint64_t & bytesFreed) { Path dir = dirOf(path); if (dir == "") @@ -435,12 +435,12 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed) void deletePath(const Path & path) { - unsigned long long dummy; + uint64_t dummy; deletePath(path, dummy); } -void deletePath(const Path & path, unsigned long long & bytesFreed) +void deletePath(const Path & path, uint64_t & bytesFreed) { //Activity act(*logger, lvlDebug, format("recursively deleting path '%1%'") % path); bytesFreed = 0; @@ -581,20 +581,31 @@ Paths createDirs(const Path & path) } -void createSymlink(const Path & target, const Path & link) +void createSymlink(const Path & target, const Path & link, + std::optional mtime) { if (symlink(target.c_str(), link.c_str())) throw SysError("creating symlink from '%1%' to '%2%'", link, target); + if (mtime) { + struct timeval times[2]; + times[0].tv_sec = *mtime; + times[0].tv_usec = 0; + times[1].tv_sec = *mtime; + times[1].tv_usec = 0; + if (lutimes(link.c_str(), times)) + throw SysError("setting time of symlink '%s'", link); + } } -void replaceSymlink(const Path & target, const Path & link) +void replaceSymlink(const Path & target, const Path & link, + std::optional mtime) { for (unsigned int n = 0; true; n++) { Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); try { - createSymlink(target, tmp); + createSymlink(target, tmp, mtime); } catch (SysError & e) { if (e.errNo == EEXIST) continue; throw; @@ -1006,12 +1017,14 @@ std::vector stringsToCharPtrs(const Strings & ss) return res; } - +// Output = "standard out" output stream string runProgram(Path program, bool searchPath, const Strings & args, const std::optional & input) { RunOptions opts(program, args); opts.searchPath = searchPath; + // This allows you to refer to a program with a pathname relative to the + // PATH variable. opts.input = input; auto res = runProgram(opts); @@ -1022,6 +1035,7 @@ string runProgram(Path program, bool searchPath, const Strings & args, return res.second; } +// Output = error code + "standard out" output stream std::pair runProgram(const RunOptions & options_) { RunOptions options(options_); @@ -1094,6 +1108,8 @@ void runProgram2(const RunOptions & options) if (options.searchPath) execvp(options.program.c_str(), stringsToCharPtrs(args_).data()); + // This allows you to refer to a program with a pathname relative + // to the PATH variable. else execv(options.program.c_str(), stringsToCharPtrs(args_).data()); @@ -1565,7 +1581,7 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) struct sockaddr_un addr; addr.sun_family = AF_UNIX; - if (path.size() >= sizeof(addr.sun_path)) + if (path.size() + 1 >= sizeof(addr.sun_path)) throw Error("socket path '%1%' is too long", path); strcpy(addr.sun_path, path.c_str()); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 3641daaec..6850b5a7a 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -49,7 +49,9 @@ void clearEnv(); /* Return an absolutized path, resolving paths relative to the specified directory, or the current directory otherwise. The path is also canonicalised. */ -Path absPath(Path path, std::optional dir = {}); +Path absPath(Path path, + std::optional dir = {}, + bool resolveSymlinks = false); /* Canonicalise a path by removing all `.' or `..' components and double or trailing slashes. Optionally resolves all symlink @@ -123,7 +125,7 @@ void writeLine(int fd, string s); second variant returns the number of bytes and blocks freed. */ void deletePath(const Path & path); -void deletePath(const Path & path, unsigned long long & bytesFreed); +void deletePath(const Path & path, uint64_t & bytesFreed); std::string getUserName(); @@ -147,10 +149,12 @@ Path getDataDir(); Paths createDirs(const Path & path); /* Create a symlink. */ -void createSymlink(const Path & target, const Path & link); +void createSymlink(const Path & target, const Path & link, + std::optional mtime = {}); /* Atomically create or replace a symlink. */ -void replaceSymlink(const Path & target, const Path & link); +void replaceSymlink(const Path & target, const Path & link, + std::optional mtime = {}); /* Wrappers arount read()/write() that read/write exactly the diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index f77de56ea..94412042f 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -174,7 +174,7 @@ static void _main(int argc, char * * argv) else if (*arg == "--run-env") // obsolete runEnv = true; - else if (*arg == "--command" || *arg == "--run") { + else if (runEnv && (*arg == "--command" || *arg == "--run")) { if (*arg == "--run") interactive = false; envCommand = getArg(*arg, arg, end) + "\nexit"; @@ -192,7 +192,7 @@ static void _main(int argc, char * * argv) else if (*arg == "--pure") pure = true; else if (*arg == "--impure") pure = false; - else if (*arg == "--packages" || *arg == "-p") + else if (runEnv && (*arg == "--packages" || *arg == "-p")) packages = true; else if (inShebang && *arg == "-i") { @@ -325,7 +325,7 @@ static void _main(int argc, char * * argv) auto buildPaths = [&](const std::vector & paths) { /* Note: we do this even when !printMissing to efficiently fetch binary cache data. */ - unsigned long long downloadSize, narSize; + uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index aa5ada3a6..bcf1d8518 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -67,10 +67,8 @@ static int _main(int argc, char * * argv) deleteOlderThan = getArg(*arg, arg, end); } else if (*arg == "--dry-run") dryRun = true; - else if (*arg == "--max-freed") { - long long maxFreed = getIntArg(*arg, arg, end, true); - options.maxFreed = maxFreed >= 0 ? maxFreed : 0; - } + else if (*arg == "--max-freed") + options.maxFreed = std::max(getIntArg(*arg, arg, end, true), (int64_t) 0); else return false; return true; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index c992b7d74..ddd036070 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -381,7 +381,8 @@ static void queryInstSources(EvalState & state, if (path.isDerivation()) { elem.setDrvPath(state.store->printStorePath(path)); - elem.setOutPath(state.store->printStorePath(state.store->derivationFromPath(path).findOutput("out"))); + auto outputs = state.store->queryDerivationOutputMap(path); + elem.setOutPath(state.store->printStorePath(outputs.at("out"))); if (name.size() >= drvExtension.size() && string(name, name.size() - drvExtension.size()) == drvExtension) name = string(name, 0, name.size() - drvExtension.size()); @@ -1208,18 +1209,17 @@ static void opSwitchProfile(Globals & globals, Strings opFlags, Strings opArgs) } -static const int prevGen = -2; +static constexpr GenerationNumber prevGen = std::numeric_limits::max(); -static void switchGeneration(Globals & globals, int dstGen) +static void switchGeneration(Globals & globals, GenerationNumber dstGen) { PathLocks lock; lockProfile(lock, globals.profile); - int curGen; - Generations gens = findGenerations(globals.profile, curGen); + auto [gens, curGen] = findGenerations(globals.profile); - Generation dst; + std::optional dst; for (auto & i : gens) if ((dstGen == prevGen && i.number < curGen) || (dstGen >= 0 && i.number == dstGen)) @@ -1227,18 +1227,16 @@ static void switchGeneration(Globals & globals, int dstGen) if (!dst) { if (dstGen == prevGen) - throw Error("no generation older than the current (%1%) exists", - curGen); + throw Error("no generation older than the current (%1%) exists", curGen.value_or(0)); else throw Error("generation %1% does not exist", dstGen); } - printInfo(format("switching from generation %1% to %2%") - % curGen % dst.number); + printInfo("switching from generation %1% to %2%", curGen.value_or(0), dst->number); if (globals.dryRun) return; - switchLink(globals.profile, dst.path); + switchLink(globals.profile, dst->path); } @@ -1249,7 +1247,7 @@ static void opSwitchGeneration(Globals & globals, Strings opFlags, Strings opArg if (opArgs.size() != 1) throw UsageError("exactly one argument expected"); - int dstGen; + GenerationNumber dstGen; if (!string2Int(opArgs.front(), dstGen)) throw UsageError("expected a generation number"); @@ -1278,8 +1276,7 @@ static void opListGenerations(Globals & globals, Strings opFlags, Strings opArgs PathLocks lock; lockProfile(lock, globals.profile); - int curGen; - Generations gens = findGenerations(globals.profile, curGen); + auto [gens, curGen] = findGenerations(globals.profile); RunPager pager; @@ -1308,14 +1305,14 @@ static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opAr if(opArgs.front().size() < 2) throw Error("invalid number of generations ‘%1%’", opArgs.front()); string str_max = string(opArgs.front(), 1, opArgs.front().size()); - int max; + GenerationNumber max; if (!string2Int(str_max, max) || max == 0) throw Error("invalid number of generations to keep ‘%1%’", opArgs.front()); deleteGenerationsGreaterThan(globals.profile, max, globals.dryRun); } else { - std::set gens; + std::set gens; for (auto & i : opArgs) { - unsigned int n; + GenerationNumber n; if (!string2Int(i, n)) throw UsageError("invalid generation number '%1%'", i); gens.insert(n); diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 961e7fb6d..65d8ec6b6 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -154,7 +154,7 @@ static int _main(int argc, char * * argv) /* If an expected hash is given, the file may already exist in the store. */ std::optional expectedHash; - Hash hash; + Hash hash(ht); std::optional storePath; if (args.size() == 2) { expectedHash = Hash(args[1], ht); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 23bb48d88..7b26970ef 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -77,7 +77,7 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true) if (i == drv.outputs.end()) throw Error("derivation '%s' does not have an output named '%s'", store2->printStorePath(path.path), j); - auto outPath = store2->printStorePath(i->second.path); + auto outPath = store2->printStorePath(i->second.path(*store, drv.name)); if (store2) { if (gcRoot == "") printGCWarning(); @@ -130,7 +130,7 @@ static void opRealise(Strings opFlags, Strings opArgs) for (auto & i : opArgs) paths.push_back(store->followLinksToStorePathWithOutputs(i)); - unsigned long long downloadSize, narSize; + uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); @@ -219,7 +219,7 @@ static StorePathSet maybeUseOutputs(const StorePath & storePath, bool useOutput, auto drv = store->derivationFromPath(storePath); StorePathSet outputs; for (auto & i : drv.outputs) - outputs.insert(i.second.path); + outputs.insert(i.second.path(*store, drv.name)); return outputs; } else return {storePath}; @@ -313,7 +313,7 @@ static void opQuery(Strings opFlags, Strings opArgs) if (forceRealise) realisePath({i2}); Derivation drv = store->derivationFromPath(i2); for (auto & j : drv.outputs) - cout << fmt("%1%\n", store->printStorePath(j.second.path)); + cout << fmt("%1%\n", store->printStorePath(j.second.path(*store, drv.name))); } break; } @@ -372,8 +372,8 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & j : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) { auto info = store->queryPathInfo(j); if (query == qHash) { - assert(info->narHash.type == htSHA256); - cout << fmt("%s\n", info->narHash.to_string(Base32, true)); + assert(info->narHash && info->narHash->type == htSHA256); + cout << fmt("%s\n", info->narHash->to_string(Base32, true)); } else if (query == qSize) cout << fmt("%d\n", info->narSize); } @@ -572,10 +572,8 @@ static void opGC(Strings opFlags, Strings opArgs) if (*i == "--print-roots") printRoots = true; else if (*i == "--print-live") options.action = GCOptions::gcReturnLive; else if (*i == "--print-dead") options.action = GCOptions::gcReturnDead; - else if (*i == "--max-freed") { - long long maxFreed = getIntArg(*i, i, opFlags.end(), true); - options.maxFreed = maxFreed >= 0 ? maxFreed : 0; - } + else if (*i == "--max-freed") + options.maxFreed = std::max(getIntArg(*i, i, opFlags.end(), true), (int64_t) 0); else throw UsageError("bad sub-operation '%1%' in GC", *i); if (!opArgs.empty()) throw UsageError("no arguments expected"); @@ -725,7 +723,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) auto path = store->followLinksToStorePath(i); printMsg(lvlTalkative, "checking path '%s'...", store->printStorePath(path)); auto info = store->queryPathInfo(path); - HashSink sink(*info->narHash.type); + HashSink sink(info->narHash->type); store->narFromPath(path, sink); auto current = sink.finish(); if (current.first != info->narHash) { @@ -734,7 +732,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) .hint = hintfmt( "path '%s' was modified! expected hash '%s', got '%s'", store->printStorePath(path), - info->narHash.to_string(Base32, true), + info->narHash->to_string(Base32, true), current.first.to_string(Base32, true)) }); status = 1; @@ -831,7 +829,7 @@ static void opServe(Strings opFlags, Strings opArgs) for (auto & path : paths) if (!path.isDerivation()) paths2.push_back({path}); - unsigned long long downloadSize, narSize; + uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; store->queryMissing(paths2, willBuild, willSubstitute, unknown, downloadSize, narSize); @@ -864,7 +862,9 @@ static void opServe(Strings opFlags, Strings opArgs) out << info->narSize // downloadSize << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 4) - out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << renderContentAddress(info->ca) << info->sigs; + out << (info->narHash ? info->narHash->to_string(Base32, true) : "") + << renderContentAddress(info->ca) + << info->sigs; } catch (InvalidPath &) { } } @@ -914,9 +914,9 @@ static void opServe(Strings opFlags, Strings opArgs) if (!writeAllowed) throw Error("building paths is not allowed"); - auto drvPath = store->parseStorePath(readString(in)); // informational only + auto drvPath = store->parseStorePath(readString(in)); BasicDerivation drv; - readDerivation(in, *store, drv); + readDerivation(in, *store, drv, Derivation::nameFromPath(drvPath)); getBuildSettings(); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index f9d6de16e..e183cb8b5 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -9,6 +9,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand { Path path; std::optional namePart; + FileIngestionMethod ingestionMethod = FileIngestionMethod::Recursive; CmdAddToStore() { @@ -21,6 +22,13 @@ struct CmdAddToStore : MixDryRun, StoreCommand .labels = {"name"}, .handler = {&namePart}, }); + + addFlag({ + .longName = "flat", + .shortName = 0, + .description = "add flat file to the Nix store", + .handler = {&ingestionMethod, FileIngestionMethod::Flat}, + }); } std::string description() override @@ -45,12 +53,19 @@ struct CmdAddToStore : MixDryRun, StoreCommand auto narHash = hashString(htSHA256, *sink.s); - ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, *namePart)); + Hash hash = narHash; + if (ingestionMethod == FileIngestionMethod::Flat) { + HashSink hsink(htSHA256); + readFile(path, hsink); + hash = hsink.finish().first; + } + + ValidPathInfo info(store->makeFixedOutputPath(ingestionMethod, hash, *namePart)); info.narHash = narHash; info.narSize = sink.s->size(); info.ca = std::optional { FixedOutputHash { - .method = FileIngestionMethod::Recursive, - .hash = info.narHash, + .method = ingestionMethod, + .hash = hash, } }; if (!dryRun) { diff --git a/src/nix/app.cc b/src/nix/app.cc new file mode 100644 index 000000000..3935297cf --- /dev/null +++ b/src/nix/app.cc @@ -0,0 +1,46 @@ +#include "installables.hh" +#include "store-api.hh" +#include "eval-inline.hh" +#include "eval-cache.hh" +#include "names.hh" + +namespace nix { + +App Installable::toApp(EvalState & state) +{ + auto [cursor, attrPath] = getCursor(state, true); + + auto type = cursor->getAttr("type")->getString(); + + if (type == "app") { + auto [program, context] = cursor->getAttr("program")->getStringWithContext(); + + if (!state.store->isInStore(program)) + throw Error("app program '%s' is not in the Nix store", program); + + std::vector context2; + for (auto & [path, name] : context) + context2.push_back({state.store->parseStorePath(path), {name}}); + + return App { + .context = std::move(context2), + .program = program, + }; + } + + else if (type == "derivation") { + auto drvPath = cursor->forceDerivation(); + auto outPath = cursor->getAttr(state.sOutPath)->getString(); + auto outputName = cursor->getAttr(state.sOutputName)->getString(); + auto name = cursor->getAttr(state.sName)->getString(); + return App { + .context = { { drvPath, {outputName} } }, + .program = outPath + "/bin/" + DrvName(name).name, + }; + } + + else + throw Error("attribute '%s' has unsupported type '%s'", attrPath, type); +} + +} diff --git a/src/nix/build.cc b/src/nix/build.cc index 850e09ce8..613cc15eb 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -1,3 +1,4 @@ +#include "eval.hh" #include "command.hh" #include "common-args.hh" #include "shared.hh" @@ -8,6 +9,7 @@ using namespace nix; struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile { Path outLink = "result"; + BuildMode buildMode = bmNormal; CmdBuild() { @@ -17,6 +19,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile .description = "path of the symlink to the build result", .labels = {"path"}, .handler = {&outLink}, + .completer = completePath }); addFlag({ @@ -24,6 +27,12 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile .description = "do not create a symlink to the build result", .handler = {&outLink, Path("")}, }); + + addFlag({ + .longName = "rebuild", + .description = "rebuild an already built package and compare the result to the existing store paths", + .handler = {&buildMode, bmCheck}, + }); } std::string description() override @@ -44,14 +53,14 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile }, Example{ "To make a profile point at GNU Hello:", - "nix build --profile /tmp/profile nixpkgs.hello" + "nix build --profile /tmp/profile nixpkgs#hello" }, }; } void run(ref store) override { - auto buildables = build(store, dryRun ? DryRun : Build, installables); + auto buildables = build(store, dryRun ? Realise::Nothing : Realise::Outputs, installables, buildMode); if (dryRun) return; diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc new file mode 100644 index 000000000..eb3339f5d --- /dev/null +++ b/src/nix/bundle.cc @@ -0,0 +1,129 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "fs-accessor.hh" + +using namespace nix; + +struct CmdBundle : InstallableCommand +{ + std::string bundler = "github:matthewbauer/nix-bundle"; + std::optional outLink; + + CmdBundle() + { + addFlag({ + .longName = "bundler", + .description = "use custom bundler", + .labels = {"flake-url"}, + .handler = {&bundler}, + .completer = {[&](size_t, std::string_view prefix) { + completeFlakeRef(getStore(), prefix); + }} + }); + + addFlag({ + .longName = "out-link", + .shortName = 'o', + .description = "path of the symlink to the build result", + .labels = {"path"}, + .handler = {&outLink}, + .completer = completePath + }); + } + + std::string description() override + { + return "bundle an application so that it works outside of the Nix store"; + } + + Examples examples() override + { + return { + Example{ + "To bundle Hello:", + "nix bundle hello" + }, + }; + } + + Category category() override { return catSecondary; } + + Strings getDefaultFlakeAttrPaths() override + { + Strings res{"defaultApp." + settings.thisSystem.get()}; + for (auto & s : SourceExprCommand::getDefaultFlakeAttrPaths()) + res.push_back(s); + return res; + } + + Strings getDefaultFlakeAttrPathPrefixes() override + { + Strings res{"apps." + settings.thisSystem.get() + ".", "packages"}; + for (auto & s : SourceExprCommand::getDefaultFlakeAttrPathPrefixes()) + res.push_back(s); + return res; + } + + void run(ref store) override + { + auto evalState = getEvalState(); + + auto app = installable->toApp(*evalState); + store->buildPaths(app.context); + + auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath(".")); + const flake::LockFlags lockFlags{ .writeLockFile = false }; + auto bundler = InstallableFlake( + evalState, std::move(bundlerFlakeRef), + Strings{bundlerName == "" ? "defaultBundler" : bundlerName}, + Strings({"bundlers."}), lockFlags); + + Value * arg = evalState->allocValue(); + evalState->mkAttrs(*arg, 2); + + PathSet context; + for (auto & i : app.context) + context.insert("=" + store->printStorePath(i.path)); + mkString(*evalState->allocAttr(*arg, evalState->symbols.create("program")), app.program, context); + + mkString(*evalState->allocAttr(*arg, evalState->symbols.create("system")), settings.thisSystem.get()); + + arg->attrs->sort(); + + auto vRes = evalState->allocValue(); + evalState->callFunction(*bundler.toValue(*evalState).first, *arg, *vRes, noPos); + + if (!evalState->isDerivation(*vRes)) + throw Error("the bundler '%s' does not produce a derivation", bundler.what()); + + auto attr1 = vRes->attrs->find(evalState->sDrvPath); + if (!attr1) + throw Error("the bundler '%s' does not produce a derivation", bundler.what()); + + PathSet context2; + StorePath drvPath = store->parseStorePath(evalState->coerceToPath(*attr1->pos, *attr1->value, context2)); + + auto attr2 = vRes->attrs->find(evalState->sOutPath); + if (!attr2) + throw Error("the bundler '%s' does not produce a derivation", bundler.what()); + + StorePath outPath = store->parseStorePath(evalState->coerceToPath(*attr2->pos, *attr2->value, context2)); + + store->buildPaths({{drvPath}}); + + auto outPathS = store->printStorePath(outPath); + + auto info = store->queryPathInfo(outPath); + if (!info->references.empty()) + throw Error("'%s' has references; a bundler must not leave any references", outPathS); + + if (!outLink) + outLink = baseNameOf(app.program); + + store.dynamic_pointer_cast()->addPermRoot(outPath, absPath(*outLink), true); + } +}; + +static auto r2 = registerCommand("bundle"); diff --git a/src/nix/cat.cc b/src/nix/cat.cc index c82819af8..97306107c 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -25,7 +25,11 @@ struct CmdCatStore : StoreCommand, MixCat { CmdCatStore() { - expectArg("path", &path); + expectArgs({ + .label = "path", + .handler = {&path}, + .completer = completePath + }); } std::string description() override @@ -47,7 +51,11 @@ struct CmdCatNar : StoreCommand, MixCat CmdCatNar() { - expectArg("nar", &narPath); + expectArgs({ + .label = "nar", + .handler = {&narPath}, + .completer = completePath + }); expectArg("path", &path); } diff --git a/src/nix/command.cc b/src/nix/command.cc index 3651a9e9c..af36dda89 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -63,7 +63,7 @@ void StorePathsCommand::run(ref store) } else { - for (auto & p : toStorePaths(store, realiseMode, installables)) + for (auto & p : toStorePaths(store, realiseMode, operateOn, installables)) storePaths.push_back(p); if (recursive) { @@ -80,7 +80,7 @@ void StorePathsCommand::run(ref store) void StorePathCommand::run(ref store) { - auto storePaths = toStorePaths(store, NoBuild, installables); + auto storePaths = toStorePaths(store, Realise::Nothing, operateOn, installables); if (storePaths.size() != 1) throw UsageError("this command requires exactly one store path"); @@ -108,6 +108,7 @@ MixProfile::MixProfile() .description = "profile to update", .labels = {"path"}, .handler = {&profile}, + .completer = completePath }); } diff --git a/src/nix/command.hh b/src/nix/command.hh index 959d5f19d..bc46a2028 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -4,12 +4,19 @@ #include "args.hh" #include "common-eval-args.hh" #include "path.hh" -#include "eval.hh" +#include "flake/lockfile.hh" +#include "store-api.hh" + +#include namespace nix { extern std::string programPath; +class EvalState; +struct Pos; +class Store; + static constexpr Command::Category catSecondary = 100; static constexpr Command::Category catUtility = 101; static constexpr Command::Category catNixInstallation = 102; @@ -27,28 +34,64 @@ private: std::shared_ptr _store; }; -struct SourceExprCommand : virtual StoreCommand, MixEvalArgs +struct EvalCommand : virtual StoreCommand, MixEvalArgs { - Path file; + ref getEvalState(); + + std::shared_ptr evalState; +}; + +struct MixFlakeOptions : virtual Args, EvalCommand +{ + flake::LockFlags lockFlags; + + MixFlakeOptions(); + + virtual std::optional getFlakeRefForCompletion() + { return {}; } +}; + +/* How to handle derivations in commands that operate on store paths. */ +enum class OperateOn { + /* Operate on the output path. */ + Output, + /* Operate on the .drv path. */ + Derivation +}; + +struct SourceExprCommand : virtual Args, MixFlakeOptions +{ + std::optional file; + std::optional expr; + + // FIXME: move this; not all commands (e.g. 'nix run') use it. + OperateOn operateOn = OperateOn::Output; SourceExprCommand(); - /* Return a value representing the Nix expression from which we - are installing. This is either the file specified by ‘--file’, - or an attribute set constructed from $NIX_PATH, e.g. ‘{ nixpkgs - = import ...; bla = import ...; }’. */ - Value * getSourceExpr(EvalState & state); + std::vector> parseInstallables( + ref store, std::vector ss); - ref getEvalState(); + std::shared_ptr parseInstallable( + ref store, const std::string & installable); -private: + virtual Strings getDefaultFlakeAttrPaths(); - std::shared_ptr evalState; + virtual Strings getDefaultFlakeAttrPathPrefixes(); - RootValue vSourceExpr; + void completeInstallable(std::string_view prefix); }; -enum RealiseMode { Build, NoBuild, DryRun }; +enum class Realise { + /* Build the derivation. Postcondition: the + derivation outputs exist. */ + Outputs, + /* Don't build the derivation. Postcondition: the store derivation + exists. */ + Derivation, + /* Evaluate in dry-run mode. Postcondition: nothing. */ + Nothing +}; /* A command that operates on a list of "installables", which can be store paths, attribute paths, Nix expressions, etc. */ @@ -56,15 +99,14 @@ struct InstallablesCommand : virtual Args, SourceExprCommand { std::vector> installables; - InstallablesCommand() - { - expectArgs("installables", &_installables); - } + InstallablesCommand(); void prepare() override; virtual bool useDefaultInstallables() { return true; } + std::optional getFlakeRefForCompletion() override; + private: std::vector _installables; @@ -75,16 +117,18 @@ struct InstallableCommand : virtual Args, SourceExprCommand { std::shared_ptr installable; - InstallableCommand() - { - expectArg("installable", &_installable); - } + InstallableCommand(); void prepare() override; + std::optional getFlakeRefForCompletion() override + { + return parseFlakeRef(_installable, absPath(".")); + } + private: - std::string _installable; + std::string _installable{"."}; }; /* A command that operates on zero or more store paths. */ @@ -97,7 +141,7 @@ private: protected: - RealiseMode realiseMode = NoBuild; + Realise realiseMode = Realise::Derivation; public: @@ -141,17 +185,15 @@ static RegisterCommand registerCommand(const std::string & name) return RegisterCommand(name, [](){ return make_ref(); }); } -std::shared_ptr parseInstallable( - SourceExprCommand & cmd, ref store, const std::string & installable, - bool useDefaultInstallables); +Buildables build(ref store, Realise mode, + std::vector> installables, BuildMode bMode = bmNormal); -Buildables build(ref store, RealiseMode mode, +std::set toStorePaths(ref store, + Realise mode, OperateOn operateOn, std::vector> installables); -std::set toStorePaths(ref store, RealiseMode mode, - std::vector> installables); - -StorePath toStorePath(ref store, RealiseMode mode, +StorePath toStorePath(ref store, + Realise mode, OperateOn operateOn, std::shared_ptr installable); std::set toDerivations(ref store, @@ -194,4 +236,19 @@ struct MixEnvironment : virtual Args { void setEnviron(); }; +void completeFlakeRef(ref store, std::string_view prefix); + +void completeFlakeRefWithFragment( + ref evalState, + flake::LockFlags lockFlags, + Strings attrPathPrefixes, + const Strings & defaultFlakeAttrPaths, + std::string_view prefix); + +void printClosureDiff( + ref store, + const StorePath & beforePath, + const StorePath & afterPath, + std::string_view indent); + } diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 64099f476..811c8ab34 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -45,6 +45,8 @@ struct CmdCopy : StorePathsCommand .description = "whether to try substitutes on the destination store (only supported by SSH)", .handler = {&substitute, Substitute}, }); + + realiseMode = Realise::Outputs; } std::string description() override @@ -70,11 +72,11 @@ struct CmdCopy : StorePathsCommand #ifdef ENABLE_S3 Example{ "To copy Hello to an S3 binary cache:", - "nix copy --to s3://my-bucket?region=eu-west-1 nixpkgs.hello" + "nix copy --to s3://my-bucket?region=eu-west-1 nixpkgs#hello" }, Example{ "To copy Hello to an S3-compatible binary cache:", - "nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com nixpkgs.hello" + "nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com nixpkgs#hello" }, #endif }; @@ -87,11 +89,16 @@ struct CmdCopy : StorePathsCommand return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri); } - void run(ref srcStore, StorePaths storePaths) override + void run(ref store) override { if (srcUri.empty() && dstUri.empty()) throw UsageError("you must pass '--from' and/or '--to'"); + StorePathsCommand::run(store); + } + + void run(ref srcStore, StorePaths storePaths) override + { ref dstStore = dstUri.empty() ? openStore() : openStore(dstUri); copyPaths(srcStore, dstStore, StorePathSet(storePaths.begin(), storePaths.end()), diff --git a/src/nix/develop.cc b/src/nix/develop.cc index eb93f56fc..a0c119e43 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -68,22 +68,22 @@ BuildEnvironment readEnvironment(const Path & path) std::smatch match; - if (std::regex_search(pos, file.cend(), match, declareRegex)) { + if (std::regex_search(pos, file.cend(), match, declareRegex, std::regex_constants::match_continuous)) { pos = match[0].second; exported.insert(match[1]); } - else if (std::regex_search(pos, file.cend(), match, varRegex)) { + else if (std::regex_search(pos, file.cend(), match, varRegex, std::regex_constants::match_continuous)) { pos = match[0].second; res.env.insert({match[1], Var { .exported = exported.count(match[1]) > 0, .value = match[2] }}); } - else if (std::regex_search(pos, file.cend(), match, assocArrayRegex)) { + else if (std::regex_search(pos, file.cend(), match, assocArrayRegex, std::regex_constants::match_continuous)) { pos = match[0].second; res.env.insert({match[1], Var { .associative = true, .value = match[2] }}); } - else if (std::regex_search(pos, file.cend(), match, functionRegex)) { + else if (std::regex_search(pos, file.cend(), match, functionRegex, std::regex_constants::match_continuous)) { res.bashFunctions = std::string(pos, file.cend()); break; } @@ -130,12 +130,16 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) drvName += "-env"; for (auto & output : drv.outputs) drv.env.erase(output.first); + drv.outputs = {{"out", DerivationOutput { .output = DerivationOutputInputAddressed { .path = StorePath::dummy }}}}; drv.env["out"] = ""; + drv.env["_outputs_saved"] = drv.env["outputs"]; drv.env["outputs"] = "out"; drv.inputSrcs.insert(std::move(getEnvShPath)); Hash h = hashDerivationModulo(*store, drv, true); auto shellOutPath = store->makeOutputPath("out", h, drvName); - drv.outputs.insert_or_assign("out", DerivationOutput { .path = shellOutPath }); + drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputInputAddressed { + .path = shellOutPath + } }); drv.env["out"] = store->printStorePath(shellOutPath); auto shellDrvPath2 = writeDerivation(store, drv, drvName); @@ -201,6 +205,11 @@ struct Common : InstallableCommand, MixProfile out << "eval \"$shellHook\"\n"; } + Strings getDefaultFlakeAttrPaths() override + { + return {"devShell." + settings.thisSystem.get(), "defaultPackage." + settings.thisSystem.get()}; + } + StorePath getShellOutPath(ref store) { auto path = installable->getStorePath(); @@ -259,11 +268,15 @@ struct CmdDevelop : Common, MixEnvironment return { Example{ "To get the build environment of GNU hello:", - "nix develop nixpkgs.hello" + "nix develop nixpkgs#hello" + }, + Example{ + "To get the build environment of the default package of flake in the current directory:", + "nix develop" }, Example{ "To store the build environment in a profile:", - "nix develop --profile /tmp/my-shell nixpkgs.hello" + "nix develop --profile /tmp/my-shell nixpkgs#hello" }, Example{ "To use a build environment previously recorded in a profile:", @@ -294,12 +307,28 @@ struct CmdDevelop : Common, MixEnvironment stopProgressBar(); - auto shell = getEnv("SHELL").value_or("bash"); - setEnviron(); // prevent garbage collection until shell exits setenv("NIX_GCROOT", gcroot.data(), 1); + Path shell = "bash"; + + try { + auto state = getEvalState(); + + auto bashInstallable = std::make_shared( + state, + std::move(installable->nixpkgsFlakeRef()), + Strings{"bashInteractive"}, + Strings{"legacyPackages." + settings.thisSystem.get() + "."}, + lockFlags); + + shell = state->store->printStorePath( + toStorePath(state->store, Realise::Outputs, OperateOn::Output, bashInstallable)) + "/bin/bash"; + } catch (Error &) { + ignoreException(); + } + auto args = Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath}; restoreAffinity(); @@ -323,7 +352,7 @@ struct CmdPrintDevEnv : Common return { Example{ "To apply the build environment of GNU hello to the current shell:", - ". <(nix print-dev-env nixpkgs.hello)" + ". <(nix print-dev-env nixpkgs#hello)" }, }; } diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc new file mode 100644 index 000000000..4199dae0f --- /dev/null +++ b/src/nix/diff-closures.cc @@ -0,0 +1,146 @@ +#include "command.hh" +#include "shared.hh" +#include "store-api.hh" +#include "common-args.hh" +#include "names.hh" + +#include + +namespace nix { + +struct Info +{ + std::string outputName; +}; + +// name -> version -> store paths +typedef std::map>> GroupedPaths; + +GroupedPaths getClosureInfo(ref store, const StorePath & toplevel) +{ + StorePathSet closure; + store->computeFSClosure({toplevel}, closure); + + GroupedPaths groupedPaths; + + for (auto & path : closure) { + /* Strip the output name. Unfortunately this is ambiguous (we + can't distinguish between output names like "bin" and + version suffixes like "unstable"). */ + static std::regex regex("(.*)-([a-z]+|lib32|lib64)"); + std::smatch match; + std::string name(path.name()); + std::string outputName; + if (std::regex_match(name, match, regex)) { + name = match[1]; + outputName = match[2]; + } + + DrvName drvName(name); + groupedPaths[drvName.name][drvName.version].emplace(path, Info { .outputName = outputName }); + } + + return groupedPaths; +} + +std::string showVersions(const std::set & versions) +{ + if (versions.empty()) return "∅"; + std::set versions2; + for (auto & version : versions) + versions2.insert(version.empty() ? "ε" : version); + return concatStringsSep(", ", versions2); +} + +void printClosureDiff( + ref store, + const StorePath & beforePath, + const StorePath & afterPath, + std::string_view indent) +{ + auto beforeClosure = getClosureInfo(store, beforePath); + auto afterClosure = getClosureInfo(store, afterPath); + + std::set allNames; + for (auto & [name, _] : beforeClosure) allNames.insert(name); + for (auto & [name, _] : afterClosure) allNames.insert(name); + + for (auto & name : allNames) { + auto & beforeVersions = beforeClosure[name]; + auto & afterVersions = afterClosure[name]; + + auto totalSize = [&](const std::map> & versions) + { + uint64_t sum = 0; + for (auto & [_, paths] : versions) + for (auto & [path, _] : paths) + sum += store->queryPathInfo(path)->narSize; + return sum; + }; + + auto beforeSize = totalSize(beforeVersions); + auto afterSize = totalSize(afterVersions); + auto sizeDelta = (int64_t) afterSize - (int64_t) beforeSize; + auto showDelta = abs(sizeDelta) >= 8 * 1024; + + std::set removed, unchanged; + for (auto & [version, _] : beforeVersions) + if (!afterVersions.count(version)) removed.insert(version); else unchanged.insert(version); + + std::set added; + for (auto & [version, _] : afterVersions) + if (!beforeVersions.count(version)) added.insert(version); + + if (showDelta || !removed.empty() || !added.empty()) { + std::vector items; + if (!removed.empty() || !added.empty()) + items.push_back(fmt("%s → %s", showVersions(removed), showVersions(added))); + if (showDelta) + items.push_back(fmt("%s%+.1f KiB" ANSI_NORMAL, sizeDelta > 0 ? ANSI_RED : ANSI_GREEN, sizeDelta / 1024.0)); + std::cout << fmt("%s%s: %s\n", indent, name, concatStringsSep(", ", items)); + } + } +} + +} + +using namespace nix; + +struct CmdDiffClosures : SourceExprCommand +{ + std::string _before, _after; + + CmdDiffClosures() + { + expectArg("before", &_before); + expectArg("after", &_after); + } + + std::string description() override + { + return "show what packages and versions were added and removed between two closures"; + } + + Category category() override { return catSecondary; } + + Examples examples() override + { + return { + { + "To show what got added and removed between two versions of the NixOS system profile:", + "nix diff-closures /nix/var/nix/profiles/system-655-link /nix/var/nix/profiles/system-658-link", + }, + }; + } + + void run(ref store) override + { + auto before = parseInstallable(store, _before); + auto beforePath = toStorePath(store, Realise::Outputs, operateOn, before); + auto after = parseInstallable(store, _after); + auto afterPath = toStorePath(store, Realise::Outputs, operateOn, after); + printClosureDiff(store, beforePath, afterPath, ""); + } +}; + +static auto r1 = registerCommand("diff-closures"); diff --git a/src/nix/edit.cc b/src/nix/edit.cc index 067d3a973..378a3739c 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -20,7 +20,7 @@ struct CmdEdit : InstallableCommand return { Example{ "To open the Nix expression of the GNU Hello package:", - "nix edit nixpkgs.hello" + "nix edit nixpkgs#hello" }, }; } @@ -45,6 +45,7 @@ struct CmdEdit : InstallableCommand auto args = editorFor(pos); + restoreSignals(); execvp(args.front().c_str(), stringsToCharPtrs(args).data()); std::string command; diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 26e98ac2a..a8ca446be 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -12,10 +12,18 @@ using namespace nix; struct CmdEval : MixJSON, InstallableCommand { bool raw = false; + std::optional apply; CmdEval() { mkFlag(0, "raw", "print strings unquoted", &raw); + + addFlag({ + .longName = "apply", + .description = "apply a function to each argument", + .labels = {"expr"}, + .handler = {&apply}, + }); } std::string description() override @@ -26,21 +34,25 @@ struct CmdEval : MixJSON, InstallableCommand Examples examples() override { return { - Example{ + { "To evaluate a Nix expression given on the command line:", - "nix eval '(1 + 2)'" + "nix eval --expr '1 + 2'" }, - Example{ + { "To evaluate a Nix expression from a file or URI:", - "nix eval -f channel:nixos-17.09 hello.name" + "nix eval -f ./my-nixpkgs hello.name" }, - Example{ + { "To get the current version of Nixpkgs:", - "nix eval --raw nixpkgs.lib.version" + "nix eval --raw nixpkgs#lib.version" }, - Example{ + { "To print the store path of the Hello package:", - "nix eval --raw nixpkgs.hello" + "nix eval --raw nixpkgs#hello" + }, + { + "To get a list of checks in the 'nix' flake:", + "nix eval nix#checks.x86_64-linux --apply builtins.attrNames" }, }; } @@ -57,6 +69,14 @@ struct CmdEval : MixJSON, InstallableCommand auto v = installable->toValue(*state).first; PathSet context; + if (apply) { + auto vApply = state->allocValue(); + state->eval(state->parseExprFromString(*apply, absPath(".")), *vApply); + auto vRes = state->allocValue(); + state->callFunction(*vApply, *v, *vRes, noPos); + v = vRes; + } + if (raw) { stopProgressBar(); std::cout << state->coerceToString(noPos, *v, context); diff --git a/src/nix/flake.cc b/src/nix/flake.cc new file mode 100644 index 000000000..80d8654bc --- /dev/null +++ b/src/nix/flake.cc @@ -0,0 +1,980 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "eval.hh" +#include "eval-inline.hh" +#include "flake/flake.hh" +#include "get-drvs.hh" +#include "store-api.hh" +#include "derivations.hh" +#include "attr-path.hh" +#include "fetchers.hh" +#include "registry.hh" +#include "json.hh" +#include "eval-cache.hh" + +#include +#include +#include + +using namespace nix; +using namespace nix::flake; + +class FlakeCommand : virtual Args, public MixFlakeOptions +{ + std::string flakeUrl = "."; + +public: + + FlakeCommand() + { + expectArgs({ + .label = "flake-url", + .optional = true, + .handler = {&flakeUrl}, + .completer = {[&](size_t, std::string_view prefix) { + completeFlakeRef(getStore(), prefix); + }} + }); + } + + FlakeRef getFlakeRef() + { + return parseFlakeRef(flakeUrl, absPath(".")); //FIXME + } + + Flake getFlake() + { + auto evalState = getEvalState(); + return flake::getFlake(*evalState, getFlakeRef(), lockFlags.useRegistries); + } + + LockedFlake lockFlake() + { + return flake::lockFlake(*getEvalState(), getFlakeRef(), lockFlags); + } + + std::optional getFlakeRefForCompletion() override + { + return getFlakeRef(); + } +}; + +static void printFlakeInfo(const Store & store, const Flake & flake) +{ + logger->stdout("Resolved URL: %s", flake.resolvedRef.to_string()); + logger->stdout("Locked URL: %s", flake.lockedRef.to_string()); + if (flake.description) + logger->stdout("Description: %s", *flake.description); + logger->stdout("Path: %s", store.printStorePath(flake.sourceInfo->storePath)); + if (auto rev = flake.lockedRef.input.getRev()) + logger->stdout("Revision: %s", rev->to_string(Base16, false)); + if (auto revCount = flake.lockedRef.input.getRevCount()) + logger->stdout("Revisions: %s", *revCount); + if (auto lastModified = flake.lockedRef.input.getLastModified()) + logger->stdout("Last modified: %s", + std::put_time(std::localtime(&*lastModified), "%F %T")); +} + +static nlohmann::json flakeToJson(const Store & store, const Flake & flake) +{ + nlohmann::json j; + if (flake.description) + j["description"] = *flake.description; + j["originalUrl"] = flake.originalRef.to_string(); + j["original"] = attrsToJson(flake.originalRef.toAttrs()); + j["resolvedUrl"] = flake.resolvedRef.to_string(); + j["resolved"] = attrsToJson(flake.resolvedRef.toAttrs()); + j["url"] = flake.lockedRef.to_string(); // FIXME: rename to lockedUrl + j["locked"] = attrsToJson(flake.lockedRef.toAttrs()); + if (auto rev = flake.lockedRef.input.getRev()) + j["revision"] = rev->to_string(Base16, false); + if (auto revCount = flake.lockedRef.input.getRevCount()) + j["revCount"] = *revCount; + if (auto lastModified = flake.lockedRef.input.getLastModified()) + j["lastModified"] = *lastModified; + j["path"] = store.printStorePath(flake.sourceInfo->storePath); + return j; +} + +struct CmdFlakeUpdate : FlakeCommand +{ + std::string description() override + { + return "update flake lock file"; + } + + void run(nix::ref store) override + { + /* Use --refresh by default for 'nix flake update'. */ + settings.tarballTtl = 0; + + lockFlake(); + } +}; + +static void enumerateOutputs(EvalState & state, Value & vFlake, + std::function callback) +{ + state.forceAttrs(vFlake); + + auto aOutputs = vFlake.attrs->get(state.symbols.create("outputs")); + assert(aOutputs); + + state.forceAttrs(*aOutputs->value); + + for (auto & attr : *aOutputs->value->attrs) + callback(attr.name, *attr.value, *attr.pos); +} + +struct CmdFlakeInfo : FlakeCommand, MixJSON +{ + std::string description() override + { + return "list info about a given flake"; + } + + void run(nix::ref store) override + { + auto flake = getFlake(); + + if (json) { + auto json = flakeToJson(*store, flake); + logger->stdout("%s", json.dump()); + } else + printFlakeInfo(*store, flake); + } +}; + +struct CmdFlakeListInputs : FlakeCommand, MixJSON +{ + std::string description() override + { + return "list flake inputs"; + } + + void run(nix::ref store) override + { + auto flake = lockFlake(); + + if (json) + logger->stdout("%s", flake.lockFile.toJson()); + else { + logger->stdout("%s", flake.flake.lockedRef); + + std::unordered_set> visited; + + std::function recurse; + + recurse = [&](const Node & node, const std::string & prefix) + { + for (const auto & [i, input] : enumerate(node.inputs)) { + bool last = i + 1 == node.inputs.size(); + + if (auto lockedNode = std::get_if<0>(&input.second)) { + logger->stdout("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s", + prefix + (last ? treeLast : treeConn), input.first, + *lockedNode ? (*lockedNode)->lockedRef : flake.flake.lockedRef); + + bool firstVisit = visited.insert(*lockedNode).second; + + if (firstVisit) recurse(**lockedNode, prefix + (last ? treeNull : treeLine)); + } else if (auto follows = std::get_if<1>(&input.second)) { + logger->stdout("%s" ANSI_BOLD "%s" ANSI_NORMAL " follows input '%s'", + prefix + (last ? treeLast : treeConn), input.first, + printInputPath(*follows)); + } + } + }; + + visited.insert(flake.lockFile.root); + recurse(*flake.lockFile.root, ""); + } + } +}; + +struct CmdFlakeCheck : FlakeCommand +{ + bool build = true; + + CmdFlakeCheck() + { + addFlag({ + .longName = "no-build", + .description = "do not build checks", + .handler = {&build, false} + }); + } + + std::string description() override + { + return "check whether the flake evaluates and run its tests"; + } + + void run(nix::ref store) override + { + settings.readOnlyMode = !build; + + auto state = getEvalState(); + auto flake = lockFlake(); + + // FIXME: rewrite to use EvalCache. + + auto checkSystemName = [&](const std::string & system, const Pos & pos) { + // FIXME: what's the format of "system"? + if (system.find('-') == std::string::npos) + throw Error("'%s' is not a valid system type, at %s", system, pos); + }; + + auto checkDerivation = [&](const std::string & attrPath, Value & v, const Pos & pos) { + try { + auto drvInfo = getDerivation(*state, v, false); + if (!drvInfo) + throw Error("flake attribute '%s' is not a derivation", attrPath); + // FIXME: check meta attributes + return store->parseStorePath(drvInfo->queryDrvPath()); + } catch (Error & e) { + e.addTrace(pos, hintfmt("while checking the derivation '%s'", attrPath)); + throw; + } + }; + + std::vector drvPaths; + + auto checkApp = [&](const std::string & attrPath, Value & v, const Pos & pos) { + try { + #if 0 + // FIXME + auto app = App(*state, v); + for (auto & i : app.context) { + auto [drvPathS, outputName] = decodeContext(i); + store->parseStorePath(drvPathS); + } + #endif + } catch (Error & e) { + e.addTrace(pos, hintfmt("while checking the app definition '%s'", attrPath)); + throw; + } + }; + + auto checkOverlay = [&](const std::string & attrPath, Value & v, const Pos & pos) { + try { + state->forceValue(v, pos); + if (v.type != tLambda || v.lambda.fun->matchAttrs || std::string(v.lambda.fun->arg) != "final") + throw Error("overlay does not take an argument named 'final'"); + auto body = dynamic_cast(v.lambda.fun->body); + if (!body || body->matchAttrs || std::string(body->arg) != "prev") + throw Error("overlay does not take an argument named 'prev'"); + // FIXME: if we have a 'nixpkgs' input, use it to + // evaluate the overlay. + } catch (Error & e) { + e.addTrace(pos, hintfmt("while checking the overlay '%s'", attrPath)); + throw; + } + }; + + auto checkModule = [&](const std::string & attrPath, Value & v, const Pos & pos) { + try { + state->forceValue(v, pos); + if (v.type == tLambda) { + if (!v.lambda.fun->matchAttrs || !v.lambda.fun->formals->ellipsis) + throw Error("module must match an open attribute set ('{ config, ... }')"); + } else if (v.type == tAttrs) { + for (auto & attr : *v.attrs) + try { + state->forceValue(*attr.value, *attr.pos); + } catch (Error & e) { + e.addTrace(*attr.pos, hintfmt("while evaluating the option '%s'", attr.name)); + throw; + } + } else + throw Error("module must be a function or an attribute set"); + // FIXME: if we have a 'nixpkgs' input, use it to + // check the module. + } catch (Error & e) { + e.addTrace(pos, hintfmt("while checking the NixOS module '%s'", attrPath)); + throw; + } + }; + + std::function checkHydraJobs; + + checkHydraJobs = [&](const std::string & attrPath, Value & v, const Pos & pos) { + try { + state->forceAttrs(v, pos); + + if (state->isDerivation(v)) + throw Error("jobset should not be a derivation at top-level"); + + for (auto & attr : *v.attrs) { + state->forceAttrs(*attr.value, *attr.pos); + if (!state->isDerivation(*attr.value)) + checkHydraJobs(attrPath + "." + (std::string) attr.name, + *attr.value, *attr.pos); + } + + } catch (Error & e) { + e.addTrace(pos, hintfmt("while checking the Hydra jobset '%s'", attrPath)); + throw; + } + }; + + auto checkNixOSConfiguration = [&](const std::string & attrPath, Value & v, const Pos & pos) { + try { + Activity act(*logger, lvlChatty, actUnknown, + fmt("checking NixOS configuration '%s'", attrPath)); + Bindings & bindings(*state->allocBindings(0)); + auto vToplevel = findAlongAttrPath(*state, "config.system.build.toplevel", bindings, v).first; + state->forceAttrs(*vToplevel, pos); + if (!state->isDerivation(*vToplevel)) + throw Error("attribute 'config.system.build.toplevel' is not a derivation"); + } catch (Error & e) { + e.addTrace(pos, hintfmt("while checking the NixOS configuration '%s'", attrPath)); + throw; + } + }; + + auto checkTemplate = [&](const std::string & attrPath, Value & v, const Pos & pos) { + try { + Activity act(*logger, lvlChatty, actUnknown, + fmt("checking template '%s'", attrPath)); + + state->forceAttrs(v, pos); + + if (auto attr = v.attrs->get(state->symbols.create("path"))) { + if (attr->name == state->symbols.create("path")) { + PathSet context; + auto path = state->coerceToPath(*attr->pos, *attr->value, context); + if (!store->isInStore(path)) + throw Error("template '%s' has a bad 'path' attribute"); + // TODO: recursively check the flake in 'path'. + } + } else + throw Error("template '%s' lacks attribute 'path'", attrPath); + + if (auto attr = v.attrs->get(state->symbols.create("description"))) + state->forceStringNoCtx(*attr->value, *attr->pos); + else + throw Error("template '%s' lacks attribute 'description'", attrPath); + + for (auto & attr : *v.attrs) { + std::string name(attr.name); + if (name != "path" && name != "description") + throw Error("template '%s' has unsupported attribute '%s'", attrPath, name); + } + } catch (Error & e) { + e.addTrace(pos, hintfmt("while checking the template '%s'", attrPath)); + throw; + } + }; + + auto checkBundler = [&](const std::string & attrPath, Value & v, const Pos & pos) { + try { + state->forceValue(v, pos); + if (v.type != tLambda) + throw Error("bundler must be a function"); + if (!v.lambda.fun->formals || + v.lambda.fun->formals->argNames.find(state->symbols.create("program")) == v.lambda.fun->formals->argNames.end() || + v.lambda.fun->formals->argNames.find(state->symbols.create("system")) == v.lambda.fun->formals->argNames.end()) + throw Error("bundler must take formal arguments 'program' and 'system'"); + } catch (Error & e) { + e.addTrace(pos, hintfmt("while checking the template '%s'", attrPath)); + throw; + } + }; + + { + Activity act(*logger, lvlInfo, actUnknown, "evaluating flake"); + + auto vFlake = state->allocValue(); + flake::callFlake(*state, flake, *vFlake); + + enumerateOutputs(*state, + *vFlake, + [&](const std::string & name, Value & vOutput, const Pos & pos) { + Activity act(*logger, lvlChatty, actUnknown, + fmt("checking flake output '%s'", name)); + + try { + state->forceValue(vOutput, pos); + + if (name == "checks") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + state->forceAttrs(*attr.value, *attr.pos); + for (auto & attr2 : *attr.value->attrs) { + auto drvPath = checkDerivation( + fmt("%s.%s.%s", name, attr.name, attr2.name), + *attr2.value, *attr2.pos); + if ((std::string) attr.name == settings.thisSystem.get()) + drvPaths.push_back({drvPath}); + } + } + } + + else if (name == "packages") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + state->forceAttrs(*attr.value, *attr.pos); + for (auto & attr2 : *attr.value->attrs) + checkDerivation( + fmt("%s.%s.%s", name, attr.name, attr2.name), + *attr2.value, *attr2.pos); + } + } + + else if (name == "apps") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + state->forceAttrs(*attr.value, *attr.pos); + for (auto & attr2 : *attr.value->attrs) + checkApp( + fmt("%s.%s.%s", name, attr.name, attr2.name), + *attr2.value, *attr2.pos); + } + } + + else if (name == "defaultPackage" || name == "devShell") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + checkDerivation( + fmt("%s.%s", name, attr.name), + *attr.value, *attr.pos); + } + } + + else if (name == "defaultApp") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + checkApp( + fmt("%s.%s", name, attr.name), + *attr.value, *attr.pos); + } + } + + else if (name == "legacyPackages") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + // FIXME: do getDerivations? + } + } + + else if (name == "overlay") + checkOverlay(name, vOutput, pos); + + else if (name == "overlays") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) + checkOverlay(fmt("%s.%s", name, attr.name), + *attr.value, *attr.pos); + } + + else if (name == "nixosModule") + checkModule(name, vOutput, pos); + + else if (name == "nixosModules") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) + checkModule(fmt("%s.%s", name, attr.name), + *attr.value, *attr.pos); + } + + else if (name == "nixosConfigurations") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) + checkNixOSConfiguration(fmt("%s.%s", name, attr.name), + *attr.value, *attr.pos); + } + + else if (name == "hydraJobs") + checkHydraJobs(name, vOutput, pos); + + else if (name == "defaultTemplate") + checkTemplate(name, vOutput, pos); + + else if (name == "templates") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) + checkTemplate(fmt("%s.%s", name, attr.name), + *attr.value, *attr.pos); + } + + else if (name == "defaultBundler") + checkBundler(name, vOutput, pos); + + else if (name == "bundlers") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) + checkBundler(fmt("%s.%s", name, attr.name), + *attr.value, *attr.pos); + } + + else + warn("unknown flake output '%s'", name); + + } catch (Error & e) { + e.addTrace(pos, hintfmt("while checking flake output '%s'", name)); + throw; + } + }); + } + + if (build && !drvPaths.empty()) { + Activity act(*logger, lvlInfo, actUnknown, "running flake checks"); + store->buildPaths(drvPaths); + } + } +}; + +struct CmdFlakeInitCommon : virtual Args, EvalCommand +{ + std::string templateUrl = "templates"; + Path destDir; + + const Strings attrsPathPrefixes{"templates."}; + const LockFlags lockFlags{ .writeLockFile = false }; + + CmdFlakeInitCommon() + { + addFlag({ + .longName = "template", + .shortName = 't', + .description = "the template to use", + .labels = {"template"}, + .handler = {&templateUrl}, + .completer = {[&](size_t, std::string_view prefix) { + completeFlakeRefWithFragment( + getEvalState(), + lockFlags, + attrsPathPrefixes, + {"defaultTemplate"}, + prefix); + }} + }); + } + + void run(nix::ref store) override + { + auto flakeDir = absPath(destDir); + + auto evalState = getEvalState(); + + auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment(templateUrl, absPath(".")); + + auto installable = InstallableFlake( + evalState, std::move(templateFlakeRef), + Strings{templateName == "" ? "defaultTemplate" : templateName}, + Strings(attrsPathPrefixes), lockFlags); + + auto [cursor, attrPath] = installable.getCursor(*evalState, true); + + auto templateDir = cursor->getAttr("path")->getString(); + + assert(store->isInStore(templateDir)); + + std::vector files; + + std::function copyDir; + copyDir = [&](const Path & from, const Path & to) + { + createDirs(to); + + for (auto & entry : readDirectory(from)) { + auto from2 = from + "/" + entry.name; + auto to2 = to + "/" + entry.name; + auto st = lstat(from2); + if (S_ISDIR(st.st_mode)) + copyDir(from2, to2); + else if (S_ISREG(st.st_mode)) { + auto contents = readFile(from2); + if (pathExists(to2)) { + auto contents2 = readFile(to2); + if (contents != contents2) + throw Error("refusing to overwrite existing file '%s'", to2); + } else + writeFile(to2, contents); + } + else if (S_ISLNK(st.st_mode)) { + auto target = readLink(from2); + if (pathExists(to2)) { + if (readLink(to2) != target) + throw Error("refusing to overwrite existing symlink '%s'", to2); + } else + createSymlink(target, to2); + } + else + throw Error("file '%s' has unsupported type", from2); + files.push_back(to2); + } + }; + + copyDir(templateDir, flakeDir); + + if (pathExists(flakeDir + "/.git")) { + Strings args = { "-C", flakeDir, "add", "--intent-to-add", "--force", "--" }; + for (auto & s : files) args.push_back(s); + runProgram("git", true, args); + } + } +}; + +struct CmdFlakeInit : CmdFlakeInitCommon +{ + std::string description() override + { + return "create a flake in the current directory from a template"; + } + + Examples examples() override + { + return { + Example{ + "To create a flake using the default template:", + "nix flake init" + }, + Example{ + "To see available templates:", + "nix flake show templates" + }, + Example{ + "To create a flake from a specific template:", + "nix flake init -t templates#nixos-container" + }, + }; + } + + CmdFlakeInit() + { + destDir = "."; + } +}; + +struct CmdFlakeNew : CmdFlakeInitCommon +{ + std::string description() override + { + return "create a flake in the specified directory from a template"; + } + + CmdFlakeNew() + { + expectArgs({ + .label = "dest-dir", + .handler = {&destDir}, + .completer = completePath + }); + } +}; + +struct CmdFlakeClone : FlakeCommand +{ + Path destDir; + + std::string description() override + { + return "clone flake repository"; + } + + CmdFlakeClone() + { + addFlag({ + .longName = "dest", + .shortName = 'f', + .description = "destination path", + .labels = {"path"}, + .handler = {&destDir} + }); + } + + void run(nix::ref store) override + { + if (destDir.empty()) + throw Error("missing flag '--dest'"); + + getFlakeRef().resolve(store).input.clone(destDir); + } +}; + +struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun +{ + std::string dstUri; + + CmdFlakeArchive() + { + addFlag({ + .longName = "to", + .description = "URI of the destination Nix store", + .labels = {"store-uri"}, + .handler = {&dstUri} + }); + } + + std::string description() override + { + return "copy a flake and all its inputs to a store"; + } + + Examples examples() override + { + return { + Example{ + "To copy the dwarffs flake and its dependencies to a binary cache:", + "nix flake archive --to file:///tmp/my-cache dwarffs" + }, + Example{ + "To fetch the dwarffs flake and its dependencies to the local Nix store:", + "nix flake archive dwarffs" + }, + Example{ + "To print the store paths of the flake sources of NixOps without fetching them:", + "nix flake archive --json --dry-run nixops" + }, + }; + } + + void run(nix::ref store) override + { + auto flake = lockFlake(); + + auto jsonRoot = json ? std::optional(std::cout) : std::nullopt; + + StorePathSet sources; + + sources.insert(flake.flake.sourceInfo->storePath); + if (jsonRoot) + jsonRoot->attr("path", store->printStorePath(flake.flake.sourceInfo->storePath)); + + // FIXME: use graph output, handle cycles. + std::function & jsonObj)> traverse; + traverse = [&](const Node & node, std::optional & jsonObj) + { + auto jsonObj2 = jsonObj ? jsonObj->object("inputs") : std::optional(); + for (auto & [inputName, input] : node.inputs) { + if (auto inputNode = std::get_if<0>(&input)) { + auto jsonObj3 = jsonObj2 ? jsonObj2->object(inputName) : std::optional(); + auto storePath = + dryRun + ? (*inputNode)->lockedRef.input.computeStorePath(*store) + : (*inputNode)->lockedRef.input.fetch(store).first.storePath; + if (jsonObj3) + jsonObj3->attr("path", store->printStorePath(storePath)); + sources.insert(std::move(storePath)); + traverse(**inputNode, jsonObj3); + } + } + }; + + traverse(*flake.lockFile.root, jsonRoot); + + if (!dryRun && !dstUri.empty()) { + ref dstStore = dstUri.empty() ? openStore() : openStore(dstUri); + copyPaths(store, dstStore, sources); + } + } +}; + +struct CmdFlakeShow : FlakeCommand +{ + bool showLegacy = false; + bool useEvalCache = true; + + CmdFlakeShow() + { + addFlag({ + .longName = "legacy", + .description = "show the contents of the 'legacyPackages' output", + .handler = {&showLegacy, true} + }); + + addFlag({ + .longName = "no-eval-cache", + .description = "do not use the flake evaluation cache", + .handler = {[&]() { useEvalCache = false; }} + }); + } + + std::string description() override + { + return "show the outputs provided by a flake"; + } + + void run(nix::ref store) override + { + auto state = getEvalState(); + auto flake = std::make_shared(lockFlake()); + + std::function & attrPath, const std::string & headerPrefix, const std::string & nextPrefix)> visit; + + visit = [&](eval_cache::AttrCursor & visitor, const std::vector & attrPath, const std::string & headerPrefix, const std::string & nextPrefix) + { + Activity act(*logger, lvlInfo, actUnknown, + fmt("evaluating '%s'", concatStringsSep(".", attrPath))); + try { + auto recurse = [&]() + { + logger->stdout("%s", headerPrefix); + auto attrs = visitor.getAttrs(); + for (const auto & [i, attr] : enumerate(attrs)) { + bool last = i + 1 == attrs.size(); + auto visitor2 = visitor.getAttr(attr); + auto attrPath2(attrPath); + attrPath2.push_back(attr); + visit(*visitor2, attrPath2, + fmt(ANSI_GREEN "%s%s" ANSI_NORMAL ANSI_BOLD "%s" ANSI_NORMAL, nextPrefix, last ? treeLast : treeConn, attr), + nextPrefix + (last ? treeNull : treeLine)); + } + }; + + auto showDerivation = [&]() + { + auto name = visitor.getAttr(state->sName)->getString(); + + /* + std::string description; + + if (auto aMeta = visitor.maybeGetAttr("meta")) { + if (auto aDescription = aMeta->maybeGetAttr("description")) + description = aDescription->getString(); + } + */ + + logger->stdout("%s: %s '%s'", + headerPrefix, + attrPath.size() == 2 && attrPath[0] == "devShell" ? "development environment" : + attrPath.size() == 3 && attrPath[0] == "checks" ? "derivation" : + attrPath.size() >= 1 && attrPath[0] == "hydraJobs" ? "derivation" : + "package", + name); + }; + + if (attrPath.size() == 0 + || (attrPath.size() == 1 && ( + attrPath[0] == "defaultPackage" + || attrPath[0] == "devShell" + || attrPath[0] == "nixosConfigurations" + || attrPath[0] == "nixosModules" + || attrPath[0] == "defaultApp" + || attrPath[0] == "templates")) + || ((attrPath.size() == 1 || attrPath.size() == 2) + && (attrPath[0] == "checks" + || attrPath[0] == "packages" + || attrPath[0] == "apps")) + ) + { + recurse(); + } + + else if ( + (attrPath.size() == 2 && (attrPath[0] == "defaultPackage" || attrPath[0] == "devShell")) + || (attrPath.size() == 3 && (attrPath[0] == "checks" || attrPath[0] == "packages")) + ) + { + if (visitor.isDerivation()) + showDerivation(); + else + throw Error("expected a derivation"); + } + + else if (attrPath.size() > 0 && attrPath[0] == "hydraJobs") { + if (visitor.isDerivation()) + showDerivation(); + else + recurse(); + } + + else if (attrPath.size() > 0 && attrPath[0] == "legacyPackages") { + if (attrPath.size() == 1) + recurse(); + else if (!showLegacy) + logger->stdout("%s: " ANSI_YELLOW "omitted" ANSI_NORMAL " (use '--legacy' to show)", headerPrefix); + else { + if (visitor.isDerivation()) + showDerivation(); + else if (attrPath.size() <= 2) + // FIXME: handle recurseIntoAttrs + recurse(); + } + } + + else if ( + (attrPath.size() == 2 && attrPath[0] == "defaultApp") || + (attrPath.size() == 3 && attrPath[0] == "apps")) + { + auto aType = visitor.maybeGetAttr("type"); + if (!aType || aType->getString() != "app") + throw EvalError("not an app definition"); + logger->stdout("%s: app", headerPrefix); + } + + else if ( + (attrPath.size() == 1 && attrPath[0] == "defaultTemplate") || + (attrPath.size() == 2 && attrPath[0] == "templates")) + { + auto description = visitor.getAttr("description")->getString(); + logger->stdout("%s: template: " ANSI_BOLD "%s" ANSI_NORMAL, headerPrefix, description); + } + + else { + logger->stdout("%s: %s", + headerPrefix, + attrPath.size() == 1 && attrPath[0] == "overlay" ? "Nixpkgs overlay" : + attrPath.size() == 2 && attrPath[0] == "nixosConfigurations" ? "NixOS configuration" : + attrPath.size() == 2 && attrPath[0] == "nixosModules" ? "NixOS module" : + ANSI_YELLOW "unknown" ANSI_NORMAL); + } + } catch (EvalError & e) { + if (!(attrPath.size() > 0 && attrPath[0] == "legacyPackages")) + throw; + } + }; + + auto cache = openEvalCache(*state, flake, useEvalCache); + + visit(*cache->getRoot(), {}, fmt(ANSI_BOLD "%s" ANSI_NORMAL, flake->flake.lockedRef), ""); + } +}; + +struct CmdFlake : virtual MultiCommand, virtual Command +{ + CmdFlake() + : MultiCommand({ + {"update", []() { return make_ref(); }}, + {"info", []() { return make_ref(); }}, + {"list-inputs", []() { return make_ref(); }}, + {"check", []() { return make_ref(); }}, + {"init", []() { return make_ref(); }}, + {"new", []() { return make_ref(); }}, + {"clone", []() { return make_ref(); }}, + {"archive", []() { return make_ref(); }}, + {"show", []() { return make_ref(); }}, + }) + { + } + + std::string description() override + { + return "manage Nix flakes"; + } + + void run() override + { + if (!command) + throw UsageError("'nix flake' requires a sub-command."); + settings.requireExperimentalFeature("flakes"); + command->second->prepare(); + command->second->run(); + } + + void printHelp(const string & programName, std::ostream & out) override + { + MultiCommand::printHelp(programName, out); + } +}; + +static auto r1 = registerCommand("flake"); diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh index a25ec43a9..2e0e83561 100644 --- a/src/nix/get-env.sh +++ b/src/nix/get-env.sh @@ -1,9 +1,18 @@ set -e if [ -e .attrs.sh ]; then source .attrs.sh; fi + +outputs=$_outputs_saved +for __output in $_outputs_saved; do + declare "$__output"="$out" +done +unset _outputs_saved __output + export IN_NIX_SHELL=impure export dontAddDisableDepTrack=1 + if [[ -n $stdenv ]]; then source $stdenv/setup fi + export > $out set >> $out diff --git a/src/nix/hash.cc b/src/nix/hash.cc index b97c6d21f..b94751e45 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -31,7 +31,11 @@ struct CmdHash : Command .labels({"modulus"}) .dest(&modulus); #endif - expectArgs("paths", &paths); + expectArgs({ + .label = "paths", + .handler = {&paths}, + .completer = completePath + }); } std::string description() override diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 1b04b10d5..babb0e5fe 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -1,3 +1,4 @@ +#include "installables.hh" #include "command.hh" #include "attr-path.hh" #include "common-eval-args.hh" @@ -7,11 +8,108 @@ #include "get-drvs.hh" #include "store-api.hh" #include "shared.hh" +#include "flake/flake.hh" +#include "eval-cache.hh" +#include "url.hh" +#include "registry.hh" #include +#include namespace nix { +void completeFlakeInputPath( + ref evalState, + const FlakeRef & flakeRef, + std::string_view prefix) +{ + auto flake = flake::getFlake(*evalState, flakeRef, true); + for (auto & input : flake.inputs) + if (hasPrefix(input.first, prefix)) + completions->insert(input.first); +} + +MixFlakeOptions::MixFlakeOptions() +{ + addFlag({ + .longName = "recreate-lock-file", + .description = "recreate lock file from scratch", + .handler = {&lockFlags.recreateLockFile, true} + }); + + addFlag({ + .longName = "no-update-lock-file", + .description = "do not allow any updates to the lock file", + .handler = {&lockFlags.updateLockFile, false} + }); + + addFlag({ + .longName = "no-write-lock-file", + .description = "do not write the newly generated lock file", + .handler = {&lockFlags.writeLockFile, false} + }); + + addFlag({ + .longName = "no-registries", + .description = "don't use flake registries", + .handler = {&lockFlags.useRegistries, false} + }); + + addFlag({ + .longName = "commit-lock-file", + .description = "commit changes to the lock file", + .handler = {&lockFlags.commitLockFile, true} + }); + + addFlag({ + .longName = "update-input", + .description = "update a specific flake input", + .labels = {"input-path"}, + .handler = {[&](std::string s) { + lockFlags.inputUpdates.insert(flake::parseInputPath(s)); + }}, + .completer = {[&](size_t, std::string_view prefix) { + if (auto flakeRef = getFlakeRefForCompletion()) + completeFlakeInputPath(getEvalState(), *flakeRef, prefix); + }} + }); + + addFlag({ + .longName = "override-input", + .description = "override a specific flake input (e.g. 'dwarffs/nixpkgs')", + .labels = {"input-path", "flake-url"}, + .handler = {[&](std::string inputPath, std::string flakeRef) { + lockFlags.inputOverrides.insert_or_assign( + flake::parseInputPath(inputPath), + parseFlakeRef(flakeRef, absPath("."))); + }} + }); + + addFlag({ + .longName = "inputs-from", + .description = "use the inputs of the specified flake as registry entries", + .labels = {"flake-url"}, + .handler = {[&](std::string flakeRef) { + auto evalState = getEvalState(); + auto flake = flake::lockFlake( + *evalState, + parseFlakeRef(flakeRef, absPath(".")), + { .writeLockFile = false }); + for (auto & [inputName, input] : flake.lockFile.root->inputs) { + auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes + if (auto input3 = std::dynamic_pointer_cast(input2)) { + overrideRegistry( + fetchers::Input::fromAttrs({{"type","indirect"}, {"id", inputName}}), + input3->lockedRef.input, + {}); + } + } + }}, + .completer = {[&](size_t, std::string_view prefix) { + completeFlakeRef(getEvalState()->store, prefix); + }} + }); +} SourceExprCommand::SourceExprCommand() { @@ -20,67 +118,152 @@ SourceExprCommand::SourceExprCommand() .shortName = 'f', .description = "evaluate FILE rather than the default", .labels = {"file"}, - .handler = {&file} + .handler = {&file}, + .completer = completePath + }); + + addFlag({ + .longName ="expr", + .description = "evaluate attributes from EXPR", + .labels = {"expr"}, + .handler = {&expr} + }); + + addFlag({ + .longName ="derivation", + .description = "operate on the store derivation rather than its outputs", + .handler = {&operateOn, OperateOn::Derivation}, }); } -Value * SourceExprCommand::getSourceExpr(EvalState & state) +Strings SourceExprCommand::getDefaultFlakeAttrPaths() { - if (vSourceExpr) return *vSourceExpr; - - auto sToplevel = state.symbols.create("_toplevel"); - - vSourceExpr = allocRootValue(state.allocValue()); - - if (file != "") - state.evalFile(lookupFileArg(state, file), **vSourceExpr); - - else { - - /* Construct the installation source from $NIX_PATH. */ - - auto searchPath = state.getSearchPath(); - - state.mkAttrs(**vSourceExpr, 1024); - - mkBool(*state.allocAttr(**vSourceExpr, sToplevel), true); - - std::unordered_set seen; - - auto addEntry = [&](const std::string & name) { - if (name == "") return; - if (!seen.insert(name).second) return; - Value * v1 = state.allocValue(); - mkPrimOpApp(*v1, state.getBuiltin("findFile"), state.getBuiltin("nixPath")); - Value * v2 = state.allocValue(); - mkApp(*v2, *v1, mkString(*state.allocValue(), name)); - mkApp(*state.allocAttr(**vSourceExpr, state.symbols.create(name)), - state.getBuiltin("import"), *v2); - }; - - for (auto & i : searchPath) - /* Hack to handle channels. */ - if (i.first.empty() && pathExists(i.second + "/manifest.nix")) { - for (auto & j : readDirectory(i.second)) - if (j.name != "manifest.nix" - && pathExists(fmt("%s/%s/default.nix", i.second, j.name))) - addEntry(j.name); - } else - addEntry(i.first); - - (*vSourceExpr)->attrs->sort(); - } - - return *vSourceExpr; + return {"defaultPackage." + settings.thisSystem.get()}; } -ref SourceExprCommand::getEvalState() +Strings SourceExprCommand::getDefaultFlakeAttrPathPrefixes() +{ + return { + // As a convenience, look for the attribute in + // 'outputs.packages'. + "packages." + settings.thisSystem.get() + ".", + // As a temporary hack until Nixpkgs is properly converted + // to provide a clean 'packages' set, look in 'legacyPackages'. + "legacyPackages." + settings.thisSystem.get() + "." + }; +} + +void SourceExprCommand::completeInstallable(std::string_view prefix) +{ + if (file) return; // FIXME + + completeFlakeRefWithFragment( + getEvalState(), + lockFlags, + getDefaultFlakeAttrPathPrefixes(), + getDefaultFlakeAttrPaths(), + prefix); +} + +void completeFlakeRefWithFragment( + ref evalState, + flake::LockFlags lockFlags, + Strings attrPathPrefixes, + const Strings & defaultFlakeAttrPaths, + std::string_view prefix) +{ + /* Look for flake output attributes that match the + prefix. */ + try { + auto hash = prefix.find('#'); + if (hash != std::string::npos) { + auto fragment = prefix.substr(hash + 1); + auto flakeRefS = std::string(prefix.substr(0, hash)); + // FIXME: do tilde expansion. + auto flakeRef = parseFlakeRef(flakeRefS, absPath(".")); + + auto evalCache = openEvalCache(*evalState, + std::make_shared(lockFlake(*evalState, flakeRef, lockFlags)), + true); + + auto root = evalCache->getRoot(); + + /* Complete 'fragment' relative to all the + attrpath prefixes as well as the root of the + flake. */ + attrPathPrefixes.push_back(""); + + for (auto & attrPathPrefixS : attrPathPrefixes) { + auto attrPathPrefix = parseAttrPath(*evalState, attrPathPrefixS); + auto attrPathS = attrPathPrefixS + std::string(fragment); + auto attrPath = parseAttrPath(*evalState, attrPathS); + + std::string lastAttr; + if (!attrPath.empty() && !hasSuffix(attrPathS, ".")) { + lastAttr = attrPath.back(); + attrPath.pop_back(); + } + + auto attr = root->findAlongAttrPath(attrPath); + if (!attr) continue; + + for (auto & attr2 : attr->getAttrs()) { + if (hasPrefix(attr2, lastAttr)) { + auto attrPath2 = attr->getAttrPath(attr2); + /* Strip the attrpath prefix. */ + attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size()); + completions->insert(flakeRefS + "#" + concatStringsSep(".", attrPath2)); + } + } + } + + /* And add an empty completion for the default + attrpaths. */ + if (fragment.empty()) { + for (auto & attrPath : defaultFlakeAttrPaths) { + auto attr = root->findAlongAttrPath(parseAttrPath(*evalState, attrPath)); + if (!attr) continue; + completions->insert(flakeRefS + "#"); + } + } + } + } catch (Error & e) { + warn(e.msg()); + } + + completeFlakeRef(evalState->store, prefix); +} + +ref EvalCommand::getEvalState() { if (!evalState) evalState = std::make_shared(searchPath, getStore()); return ref(evalState); } +void completeFlakeRef(ref store, std::string_view prefix) +{ + if (prefix == "") + completions->insert("."); + + completeDir(0, prefix); + + /* Look for registry entries that match the prefix. */ + for (auto & registry : fetchers::getRegistries(store)) { + for (auto & entry : registry->entries) { + auto from = entry.from.to_string(); + if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) { + std::string from2(from, 6); + if (hasPrefix(from2, prefix)) + completions->insert(from2); + } else { + if (hasPrefix(from, prefix)) + completions->insert(from); + } + } + } +} + Buildable Installable::toBuildable() { auto buildables = toBuildables(); @@ -89,6 +272,24 @@ Buildable Installable::toBuildable() return std::move(buildables[0]); } +std::vector, std::string>> +Installable::getCursors(EvalState & state, bool useEvalCache) +{ + auto evalCache = + std::make_shared(std::nullopt, state, + [&]() { return toValue(state).first; }); + return {{evalCache->getRoot(), ""}}; +} + +std::pair, std::string> +Installable::getCursor(EvalState & state, bool useEvalCache) +{ + auto cursors = getCursors(state, useEvalCache); + if (cursors.empty()) + throw Error("cannot find flake attribute '%s'", what()); + return cursors[0]; +} + struct InstallableStorePath : Installable { ref store; @@ -101,15 +302,25 @@ struct InstallableStorePath : Installable Buildables toBuildables() override { - std::map outputs; - outputs.insert_or_assign("out", storePath); - Buildable b{ - .drvPath = storePath.isDerivation() ? storePath : std::optional(), - .outputs = std::move(outputs) - }; - Buildables bs; - bs.push_back(std::move(b)); - return bs; + if (storePath.isDerivation()) { + std::map outputs; + auto drv = store->readDerivation(storePath); + for (auto & [name, output] : drv.outputs) + outputs.emplace(name, output.path(*store, drv.name)); + return { + Buildable { + .drvPath = storePath, + .outputs = std::move(outputs) + } + }; + } else { + return { + Buildable { + .drvPath = {}, + .outputs = {{"out", storePath}} + } + }; + } } std::optional getStorePath() override @@ -118,146 +329,325 @@ struct InstallableStorePath : Installable } }; -struct InstallableValue : Installable +Buildables InstallableValue::toBuildables() { - SourceExprCommand & cmd; + Buildables res; - InstallableValue(SourceExprCommand & cmd) : cmd(cmd) { } + StorePathSet drvPaths; - Buildables toBuildables() override - { - auto state = cmd.getEvalState(); + for (auto & drv : toDerivations()) { + Buildable b{.drvPath = drv.drvPath}; + drvPaths.insert(drv.drvPath); - auto v = toValue(*state).first; + auto outputName = drv.outputName; + if (outputName == "") + throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(*b.drvPath)); - Bindings & autoArgs = *cmd.getAutoArgs(*state); + b.outputs.emplace(outputName, drv.outPath); - DrvInfos drvs; - getDerivations(*state, *v, "", autoArgs, drvs, false); - - Buildables res; - - StorePathSet drvPaths; - - for (auto & drv : drvs) { - Buildable b{.drvPath = state->store->parseStorePath(drv.queryDrvPath())}; - drvPaths.insert(*b.drvPath); - - auto outputName = drv.queryOutputName(); - if (outputName == "") - throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(*b.drvPath)); - - b.outputs.emplace(outputName, state->store->parseStorePath(drv.queryOutPath())); - - res.push_back(std::move(b)); - } - - // Hack to recognize .all: if all drvs have the same drvPath, - // merge the buildables. - if (drvPaths.size() == 1) { - Buildable b{.drvPath = *drvPaths.begin()}; - for (auto & b2 : res) - for (auto & output : b2.outputs) - b.outputs.insert_or_assign(output.first, output.second); - Buildables bs; - bs.push_back(std::move(b)); - return bs; - } else - return res; + res.push_back(std::move(b)); } -}; -struct InstallableExpr : InstallableValue -{ - std::string text; - - InstallableExpr(SourceExprCommand & cmd, const std::string & text) - : InstallableValue(cmd), text(text) { } - - std::string what() override { return text; } - - std::pair toValue(EvalState & state) override - { - auto v = state.allocValue(); - state.eval(state.parseExprFromString(text, absPath(".")), *v); - return {v, noPos}; - } -}; + // Hack to recognize .all: if all drvs have the same drvPath, + // merge the buildables. + if (drvPaths.size() == 1) { + Buildable b{.drvPath = *drvPaths.begin()}; + for (auto & b2 : res) + for (auto & output : b2.outputs) + b.outputs.insert_or_assign(output.first, output.second); + Buildables bs; + bs.push_back(std::move(b)); + return bs; + } else + return res; +} struct InstallableAttrPath : InstallableValue { + SourceExprCommand & cmd; + RootValue v; std::string attrPath; - InstallableAttrPath(SourceExprCommand & cmd, const std::string & attrPath) - : InstallableValue(cmd), attrPath(attrPath) + InstallableAttrPath(ref state, SourceExprCommand & cmd, Value * v, const std::string & attrPath) + : InstallableValue(state), cmd(cmd), v(allocRootValue(v)), attrPath(attrPath) { } std::string what() override { return attrPath; } std::pair toValue(EvalState & state) override { - auto source = cmd.getSourceExpr(state); - - Bindings & autoArgs = *cmd.getAutoArgs(state); - - auto v = findAlongAttrPath(state, attrPath, autoArgs, *source).first; - state.forceValue(*v); - - return {v, noPos}; + auto [vRes, pos] = findAlongAttrPath(state, attrPath, *cmd.getAutoArgs(state), **v); + state.forceValue(*vRes); + return {vRes, pos}; } + + virtual std::vector toDerivations() override; }; -// FIXME: extend -std::string attrRegex = R"([A-Za-z_][A-Za-z0-9-_+]*)"; -static std::regex attrPathRegex(fmt(R"(%1%(\.%1%)*)", attrRegex)); +std::vector InstallableAttrPath::toDerivations() +{ + auto v = toValue(*state).first; -static std::vector> parseInstallables( - SourceExprCommand & cmd, ref store, std::vector ss, bool useDefaultInstallables) + Bindings & autoArgs = *cmd.getAutoArgs(*state); + + DrvInfos drvInfos; + getDerivations(*state, *v, "", autoArgs, drvInfos, false); + + std::vector res; + for (auto & drvInfo : drvInfos) { + res.push_back({ + state->store->parseStorePath(drvInfo.queryDrvPath()), + state->store->parseStorePath(drvInfo.queryOutPath()), + drvInfo.queryOutputName() + }); + } + + return res; +} + +std::vector InstallableFlake::getActualAttrPaths() +{ + std::vector res; + + for (auto & prefix : prefixes) + res.push_back(prefix + *attrPaths.begin()); + + for (auto & s : attrPaths) + res.push_back(s); + + return res; +} + +Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake) +{ + auto vFlake = state.allocValue(); + + callFlake(state, lockedFlake, *vFlake); + + auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); + assert(aOutputs); + + state.forceValue(*aOutputs->value); + + return aOutputs->value; +} + +ref openEvalCache( + EvalState & state, + std::shared_ptr lockedFlake, + bool useEvalCache) +{ + auto fingerprint = lockedFlake->getFingerprint(); + return make_ref( + useEvalCache && evalSettings.pureEval + ? std::optional { std::cref(fingerprint) } + : std::nullopt, + state, + [&state, lockedFlake]() + { + /* For testing whether the evaluation cache is + complete. */ + if (getEnv("NIX_ALLOW_EVAL").value_or("1") == "0") + throw Error("not everything is cached, but evaluation is not allowed"); + + auto vFlake = state.allocValue(); + flake::callFlake(state, *lockedFlake, *vFlake); + + state.forceAttrs(*vFlake); + + auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); + assert(aOutputs); + + return aOutputs->value; + }); +} + +static std::string showAttrPaths(const std::vector & paths) +{ + std::string s; + for (const auto & [n, i] : enumerate(paths)) { + if (n > 0) s += n + 1 == paths.size() ? " or " : ", "; + s += '\''; s += i; s += '\''; + } + return s; +} + +std::tuple InstallableFlake::toDerivation() +{ + + auto lockedFlake = getLockedFlake(); + + auto cache = openEvalCache(*state, lockedFlake, true); + auto root = cache->getRoot(); + + for (auto & attrPath : getActualAttrPaths()) { + auto attr = root->findAlongAttrPath(parseAttrPath(*state, attrPath)); + if (!attr) continue; + + if (!attr->isDerivation()) + throw Error("flake output attribute '%s' is not a derivation", attrPath); + + auto drvPath = attr->forceDerivation(); + + auto drvInfo = DerivationInfo{ + std::move(drvPath), + state->store->parseStorePath(attr->getAttr(state->sOutPath)->getString()), + attr->getAttr(state->sOutputName)->getString() + }; + + return {attrPath, lockedFlake->flake.lockedRef, std::move(drvInfo)}; + } + + throw Error("flake '%s' does not provide attribute %s", + flakeRef, showAttrPaths(getActualAttrPaths())); +} + +std::vector InstallableFlake::toDerivations() +{ + std::vector res; + res.push_back(std::get<2>(toDerivation())); + return res; +} + +std::pair InstallableFlake::toValue(EvalState & state) +{ + auto lockedFlake = getLockedFlake(); + + auto vOutputs = getFlakeOutputs(state, *lockedFlake); + + auto emptyArgs = state.allocBindings(0); + + for (auto & attrPath : getActualAttrPaths()) { + try { + auto [v, pos] = findAlongAttrPath(state, attrPath, *emptyArgs, *vOutputs); + state.forceValue(*v); + return {v, pos}; + } catch (AttrPathNotFound & e) { + } + } + + throw Error("flake '%s' does not provide attribute %s", + flakeRef, showAttrPaths(getActualAttrPaths())); +} + +std::vector, std::string>> +InstallableFlake::getCursors(EvalState & state, bool useEvalCache) +{ + auto evalCache = openEvalCache(state, + std::make_shared(lockFlake(state, flakeRef, lockFlags)), + useEvalCache); + + auto root = evalCache->getRoot(); + + std::vector, std::string>> res; + + for (auto & attrPath : getActualAttrPaths()) { + auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath)); + if (attr) res.push_back({attr, attrPath}); + } + + return res; +} + +std::shared_ptr InstallableFlake::getLockedFlake() const +{ + if (!_lockedFlake) + _lockedFlake = std::make_shared(lockFlake(*state, flakeRef, lockFlags)); + return _lockedFlake; +} + +FlakeRef InstallableFlake::nixpkgsFlakeRef() const +{ + auto lockedFlake = getLockedFlake(); + + if (auto nixpkgsInput = lockedFlake->lockFile.findInput({"nixpkgs"})) { + if (auto lockedNode = std::dynamic_pointer_cast(nixpkgsInput)) { + debug("using nixpkgs flake '%s'", lockedNode->lockedRef); + return std::move(lockedNode->lockedRef); + } + } + + return Installable::nixpkgsFlakeRef(); +} + +std::vector> SourceExprCommand::parseInstallables( + ref store, std::vector ss) { std::vector> result; - if (ss.empty() && useDefaultInstallables) { - if (cmd.file == "") - cmd.file = "."; - ss = {""}; - } + if (file || expr) { + if (file && expr) + throw UsageError("'--file' and '--expr' are exclusive"); - for (auto & s : ss) { + // FIXME: backward compatibility hack + if (file) evalSettings.pureEval = false; - if (s.compare(0, 1, "(") == 0) - result.push_back(std::make_shared(cmd, s)); + auto state = getEvalState(); + auto vFile = state->allocValue(); - else if (s.find("/") != std::string::npos) { - try { - result.push_back(std::make_shared( - store, - store->toStorePath(store->followLinksToStore(s)).first)); - } catch (BadStorePath &) { } + if (file) + state->evalFile(lookupFileArg(*state, *file), *vFile); + else { + auto e = state->parseExprFromString(*expr, absPath(".")); + state->eval(e, *vFile); } - else if (s == "" || std::regex_match(s, attrPathRegex)) - result.push_back(std::make_shared(cmd, s)); + for (auto & s : ss) + result.push_back(std::make_shared(state, *this, vFile, s == "." ? "" : s)); - else - throw UsageError("don't know what to do with argument '%s'", s); + } else { + + for (auto & s : ss) { + std::exception_ptr ex; + + try { + auto [flakeRef, fragment] = parseFlakeRefWithFragment(s, absPath(".")); + result.push_back(std::make_shared( + getEvalState(), std::move(flakeRef), + fragment == "" ? getDefaultFlakeAttrPaths() : Strings{fragment}, + getDefaultFlakeAttrPathPrefixes(), lockFlags)); + continue; + } catch (...) { + ex = std::current_exception(); + } + + if (s.find('/') != std::string::npos) { + try { + result.push_back(std::make_shared(store, store->followLinksToStorePath(s))); + continue; + } catch (BadStorePath &) { + } catch (...) { + if (!ex) + ex = std::current_exception(); + } + } + + std::rethrow_exception(ex); + + /* + throw Error( + pathExists(s) + ? "path '%s' is not a flake or a store path" + : "don't know how to handle argument '%s'", s); + */ + } } return result; } -std::shared_ptr parseInstallable( - SourceExprCommand & cmd, ref store, const std::string & installable, - bool useDefaultInstallables) +std::shared_ptr SourceExprCommand::parseInstallable( + ref store, const std::string & installable) { - auto installables = parseInstallables(cmd, store, {installable}, false); + auto installables = parseInstallables(store, {installable}); assert(installables.size() == 1); return installables.front(); } -Buildables build(ref store, RealiseMode mode, - std::vector> installables) +Buildables build(ref store, Realise mode, + std::vector> installables, BuildMode bMode) { - if (mode != Build) + if (mode == Realise::Nothing) settings.readOnlyMode = true; Buildables buildables; @@ -278,33 +668,45 @@ Buildables build(ref store, RealiseMode mode, } } - if (mode == DryRun) + if (mode == Realise::Nothing) printMissing(store, pathsToBuild, lvlError); - else if (mode == Build) - store->buildPaths(pathsToBuild); + else if (mode == Realise::Outputs) + store->buildPaths(pathsToBuild, bMode); return buildables; } -StorePathSet toStorePaths(ref store, RealiseMode mode, +StorePathSet toStorePaths(ref store, + Realise mode, OperateOn operateOn, std::vector> installables) { StorePathSet outPaths; - for (auto & b : build(store, mode, installables)) - for (auto & output : b.outputs) - outPaths.insert(output.second); + if (operateOn == OperateOn::Output) { + for (auto & b : build(store, mode, installables)) + for (auto & output : b.outputs) + outPaths.insert(output.second); + } else { + if (mode == Realise::Nothing) + settings.readOnlyMode = true; + + for (auto & i : installables) + for (auto & b : i->toBuildables()) + if (b.drvPath) + outPaths.insert(*b.drvPath); + } return outPaths; } -StorePath toStorePath(ref store, RealiseMode mode, +StorePath toStorePath(ref store, + Realise mode, OperateOn operateOn, std::shared_ptr installable) { - auto paths = toStorePaths(store, mode, {installable}); + auto paths = toStorePaths(store, mode, operateOn, {installable}); if (paths.size() != 1) - throw Error("argument '%s' should evaluate to one store path", installable->what()); + throw Error("argument '%s' should evaluate to one store path", installable->what()); return *paths.begin(); } @@ -333,14 +735,51 @@ StorePathSet toDerivations(ref store, return drvPaths; } +InstallablesCommand::InstallablesCommand() +{ + expectArgs({ + .label = "installables", + .handler = {&_installables}, + .completer = {[&](size_t, std::string_view prefix) { + completeInstallable(prefix); + }} + }); +} + void InstallablesCommand::prepare() { - installables = parseInstallables(*this, getStore(), _installables, useDefaultInstallables()); + if (_installables.empty() && useDefaultInstallables()) + // FIXME: commands like "nix install" should not have a + // default, probably. + _installables.push_back("."); + installables = parseInstallables(getStore(), _installables); +} + +std::optional InstallablesCommand::getFlakeRefForCompletion() +{ + if (_installables.empty()) { + if (useDefaultInstallables()) + return parseFlakeRef(".", absPath(".")); + return {}; + } + return parseFlakeRef(_installables.front(), absPath(".")); +} + +InstallableCommand::InstallableCommand() +{ + expectArgs({ + .label = "installable", + .optional = true, + .handler = {&_installable}, + .completer = {[&](size_t, std::string_view prefix) { + completeInstallable(prefix); + }} + }); } void InstallableCommand::prepare() { - installable = parseInstallable(*this, getStore(), _installable, false); + installable = parseInstallable(getStore(), _installable); } } diff --git a/src/nix/installables.hh b/src/nix/installables.hh index 503984220..eb34365d4 100644 --- a/src/nix/installables.hh +++ b/src/nix/installables.hh @@ -3,11 +3,17 @@ #include "util.hh" #include "path.hh" #include "eval.hh" +#include "flake/flake.hh" #include namespace nix { +struct DrvInfo; +struct SourceExprCommand; + +namespace eval_cache { class EvalCache; class AttrCursor; } + struct Buildable { std::optional drvPath; @@ -16,19 +22,25 @@ struct Buildable typedef std::vector Buildables; +struct App +{ + std::vector context; + Path program; + // FIXME: add args, sandbox settings, metadata, ... +}; + struct Installable { virtual ~Installable() { } virtual std::string what() = 0; - virtual Buildables toBuildables() - { - throw Error("argument '%s' cannot be built", what()); - } + virtual Buildables toBuildables() = 0; Buildable toBuildable(); + App toApp(EvalState & state); + virtual std::pair toValue(EvalState & state) { throw Error("argument '%s' cannot be evaluated", what()); @@ -40,6 +52,74 @@ struct Installable { return {}; } + + virtual std::vector, std::string>> + getCursors(EvalState & state, bool useEvalCache); + + std::pair, std::string> + getCursor(EvalState & state, bool useEvalCache); + + virtual FlakeRef nixpkgsFlakeRef() const + { + return std::move(FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}})); + } }; +struct InstallableValue : Installable +{ + ref state; + + InstallableValue(ref state) : state(state) {} + + struct DerivationInfo + { + StorePath drvPath; + StorePath outPath; + std::string outputName; + }; + + virtual std::vector toDerivations() = 0; + + Buildables toBuildables() override; +}; + +struct InstallableFlake : InstallableValue +{ + FlakeRef flakeRef; + Strings attrPaths; + Strings prefixes; + const flake::LockFlags & lockFlags; + mutable std::shared_ptr _lockedFlake; + + InstallableFlake(ref state, FlakeRef && flakeRef, + Strings && attrPaths, Strings && prefixes, const flake::LockFlags & lockFlags) + : InstallableValue(state), flakeRef(flakeRef), attrPaths(attrPaths), + prefixes(prefixes), lockFlags(lockFlags) + { } + + std::string what() override { return flakeRef.to_string() + "#" + *attrPaths.begin(); } + + std::vector getActualAttrPaths(); + + Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake); + + std::tuple toDerivation(); + + std::vector toDerivations() override; + + std::pair toValue(EvalState & state) override; + + std::vector, std::string>> + getCursors(EvalState & state, bool useEvalCache) override; + + std::shared_ptr getLockedFlake() const; + + FlakeRef nixpkgsFlakeRef() const override; +}; + +ref openEvalCache( + EvalState & state, + std::shared_ptr lockedFlake, + bool useEvalCache); + } diff --git a/src/nix/log.cc b/src/nix/log.cc index 3fe22f6c2..7e10d373a 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -18,7 +18,7 @@ struct CmdLog : InstallableCommand return { Example{ "To get the build log of GNU Hello:", - "nix log nixpkgs.hello" + "nix log nixpkgs#hello" }, Example{ "To get the build log of a specific path:", @@ -26,7 +26,7 @@ struct CmdLog : InstallableCommand }, Example{ "To get a build log from a specific binary cache:", - "nix log --store https://cache.nixos.org nixpkgs.hello" + "nix log --store https://cache.nixos.org nixpkgs#hello" }, }; } diff --git a/src/nix/ls.cc b/src/nix/ls.cc index d2157f2d4..76c8bc9a3 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -85,7 +85,11 @@ struct CmdLsStore : StoreCommand, MixLs { CmdLsStore() { - expectArg("path", &path); + expectArgs({ + .label = "path", + .handler = {&path}, + .completer = completePath + }); } Examples examples() override @@ -117,7 +121,11 @@ struct CmdLsNar : Command, MixLs CmdLsNar() { - expectArg("nar", &narPath); + expectArgs({ + .label = "nar", + .handler = {&narPath}, + .completer = completePath + }); expectArg("path", &path); } diff --git a/src/nix/main.cc b/src/nix/main.cc index 203901168..e62657e95 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -7,7 +7,6 @@ #include "legacy.hh" #include "shared.hh" #include "store-api.hh" -#include "progress-bar.hh" #include "filetransfer.hh" #include "finally.hh" #include "loggers.hh" @@ -69,7 +68,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs addFlag({ .longName = "help", .description = "show usage information", - .handler = {[&]() { showHelpAndExit(); }}, + .handler = {[&]() { if (!completions) showHelpAndExit(); }}, }); addFlag({ @@ -97,7 +96,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs addFlag({ .longName = "version", .description = "show version information", - .handler = {[&]() { printVersion(programName); }}, + .handler = {[&]() { if (!completions) printVersion(programName); }}, }); addFlag({ @@ -165,6 +164,7 @@ void mainWrapped(int argc, char * * argv) verbosity = lvlWarn; settings.verboseBuild = false; + evalSettings.pureEval = true; setLogFormat("bar"); @@ -172,7 +172,22 @@ void mainWrapped(int argc, char * * argv) NixArgs args; - args.parseCmdline(argvToStrings(argc, argv)); + Finally printCompletions([&]() + { + if (completions) { + std::cout << (pathCompletions ? "filenames\n" : "no-filenames\n"); + for (auto & s : *completions) + std::cout << s << "\n"; + } + }); + + try { + args.parseCmdline(argvToStrings(argc, argv)); + } catch (UsageError &) { + if (!completions) throw; + } + + if (completions) return; initPlugins(); diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index fb36fc410..2fe2e2fb2 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -10,7 +10,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON { CmdMakeContentAddressable() { - realiseMode = Build; + realiseMode = Realise::Outputs; } std::string description() override @@ -23,7 +23,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON return { Example{ "To create a content-addressable representation of GNU Hello (but not its dependencies):", - "nix make-content-addressable nixpkgs.hello" + "nix make-content-addressable nixpkgs#hello" }, Example{ "To compute a content-addressable representation of the current NixOS system closure:", @@ -84,7 +84,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON info.narSize = sink.s->size(); info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, - .hash = info.narHash, + .hash = *info.narHash, }; if (!json) diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index b89a44f83..0c12efaf0 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -40,7 +40,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON }, Example{ "To show a package's closure size and all its dependencies with human readable sizes:", - "nix path-info -rsSh nixpkgs.rust" + "nix path-info -rsSh nixpkgs#rust" }, Example{ "To check the existence of a path in a binary cache:", @@ -61,7 +61,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON }; } - void printSize(unsigned long long value) + void printSize(uint64_t value) { if (!humanReadable) { std::cout << fmt("\t%11d", value); diff --git a/src/nix/profile.cc b/src/nix/profile.cc new file mode 100644 index 000000000..7dcc0b6d4 --- /dev/null +++ b/src/nix/profile.cc @@ -0,0 +1,469 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "derivations.hh" +#include "archive.hh" +#include "builtins/buildenv.hh" +#include "flake/flakeref.hh" +#include "../nix-env/user-env.hh" +#include "profiles.hh" + +#include +#include + +using namespace nix; + +struct ProfileElementSource +{ + FlakeRef originalRef; + // FIXME: record original attrpath. + FlakeRef resolvedRef; + std::string attrPath; + // FIXME: output names +}; + +struct ProfileElement +{ + StorePathSet storePaths; + std::optional source; + bool active = true; + // FIXME: priority +}; + +struct ProfileManifest +{ + std::vector elements; + + ProfileManifest() { } + + ProfileManifest(EvalState & state, const Path & profile) + { + auto manifestPath = profile + "/manifest.json"; + + if (pathExists(manifestPath)) { + auto json = nlohmann::json::parse(readFile(manifestPath)); + + auto version = json.value("version", 0); + if (version != 1) + throw Error("profile manifest '%s' has unsupported version %d", manifestPath, version); + + for (auto & e : json["elements"]) { + ProfileElement element; + for (auto & p : e["storePaths"]) + element.storePaths.insert(state.store->parseStorePath((std::string) p)); + element.active = e["active"]; + if (e.value("uri", "") != "") { + element.source = ProfileElementSource{ + parseFlakeRef(e["originalUri"]), + parseFlakeRef(e["uri"]), + e["attrPath"] + }; + } + elements.emplace_back(std::move(element)); + } + } + + else if (pathExists(profile + "/manifest.nix")) { + // FIXME: needed because of pure mode; ugly. + if (state.allowedPaths) { + state.allowedPaths->insert(state.store->followLinksToStore(profile)); + state.allowedPaths->insert(state.store->followLinksToStore(profile + "/manifest.nix")); + } + + auto drvInfos = queryInstalled(state, state.store->followLinksToStore(profile)); + + for (auto & drvInfo : drvInfos) { + ProfileElement element; + element.storePaths = {state.store->parseStorePath(drvInfo.queryOutPath())}; + elements.emplace_back(std::move(element)); + } + } + } + + std::string toJSON(Store & store) const + { + auto array = nlohmann::json::array(); + for (auto & element : elements) { + auto paths = nlohmann::json::array(); + for (auto & path : element.storePaths) + paths.push_back(store.printStorePath(path)); + nlohmann::json obj; + obj["storePaths"] = paths; + obj["active"] = element.active; + if (element.source) { + obj["originalUri"] = element.source->originalRef.to_string(); + obj["uri"] = element.source->resolvedRef.to_string(); + obj["attrPath"] = element.source->attrPath; + } + array.push_back(obj); + } + nlohmann::json json; + json["version"] = 1; + json["elements"] = array; + return json.dump(); + } + + StorePath build(ref store) + { + auto tempDir = createTempDir(); + + StorePathSet references; + + Packages pkgs; + for (auto & element : elements) { + for (auto & path : element.storePaths) { + if (element.active) + pkgs.emplace_back(store->printStorePath(path), true, 5); + references.insert(path); + } + } + + buildProfile(tempDir, std::move(pkgs)); + + writeFile(tempDir + "/manifest.json", toJSON(*store)); + + /* Add the symlink tree to the store. */ + StringSink sink; + dumpPath(tempDir, sink); + + auto narHash = hashString(htSHA256, *sink.s); + + ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "profile", references)); + info.references = std::move(references); + info.narHash = narHash; + info.narSize = sink.s->size(); + info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, .hash = *info.narHash }; + + auto source = StringSource { *sink.s }; + store->addToStore(info, source); + + return std::move(info.path); + } +}; + +struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile +{ + std::string description() override + { + return "install a package into a profile"; + } + + Examples examples() override + { + return { + Example{ + "To install a package from Nixpkgs:", + "nix profile install nixpkgs#hello" + }, + Example{ + "To install a package from a specific branch of Nixpkgs:", + "nix profile install nixpkgs/release-19.09#hello" + }, + Example{ + "To install a package from a specific revision of Nixpkgs:", + "nix profile install nixpkgs/1028bb33859f8dfad7f98e1c8d185f3d1aaa7340#hello" + }, + }; + } + + void run(ref store) override + { + ProfileManifest manifest(*getEvalState(), *profile); + + std::vector pathsToBuild; + + for (auto & installable : installables) { + if (auto installable2 = std::dynamic_pointer_cast(installable)) { + auto [attrPath, resolvedRef, drv] = installable2->toDerivation(); + + ProfileElement element; + element.storePaths = {drv.outPath}; // FIXME + element.source = ProfileElementSource{ + installable2->flakeRef, + resolvedRef, + attrPath, + }; + + pathsToBuild.push_back({drv.drvPath, StringSet{"out"}}); // FIXME + + manifest.elements.emplace_back(std::move(element)); + } else + throw Error("'nix profile install' does not support argument '%s'", installable->what()); + } + + store->buildPaths(pathsToBuild); + + updateProfile(manifest.build(store)); + } +}; + +class MixProfileElementMatchers : virtual Args +{ + std::vector _matchers; + +public: + + MixProfileElementMatchers() + { + expectArgs("elements", &_matchers); + } + + typedef std::variant Matcher; + + std::vector getMatchers(ref store) + { + std::vector res; + + for (auto & s : _matchers) { + size_t n; + if (string2Int(s, n)) + res.push_back(n); + else if (store->isStorePath(s)) + res.push_back(s); + else + res.push_back(std::regex(s, std::regex::extended | std::regex::icase)); + } + + return res; + } + + bool matches(const Store & store, const ProfileElement & element, size_t pos, const std::vector & matchers) + { + for (auto & matcher : matchers) { + if (auto n = std::get_if(&matcher)) { + if (*n == pos) return true; + } else if (auto path = std::get_if(&matcher)) { + if (element.storePaths.count(store.parseStorePath(*path))) return true; + } else if (auto regex = std::get_if(&matcher)) { + if (element.source + && std::regex_match(element.source->attrPath, *regex)) + return true; + } + } + + return false; + } +}; + +struct CmdProfileRemove : virtual EvalCommand, MixDefaultProfile, MixProfileElementMatchers +{ + std::string description() override + { + return "remove packages from a profile"; + } + + Examples examples() override + { + return { + Example{ + "To remove a package by attribute path:", + "nix profile remove packages.x86_64-linux.hello" + }, + Example{ + "To remove all packages:", + "nix profile remove '.*'" + }, + Example{ + "To remove a package by store path:", + "nix profile remove /nix/store/rr3y0c6zyk7kjjl8y19s4lsrhn4aiq1z-hello-2.10" + }, + Example{ + "To remove a package by position:", + "nix profile remove 3" + }, + }; + } + + void run(ref store) override + { + ProfileManifest oldManifest(*getEvalState(), *profile); + + auto matchers = getMatchers(store); + + ProfileManifest newManifest; + + for (size_t i = 0; i < oldManifest.elements.size(); ++i) { + auto & element(oldManifest.elements[i]); + if (!matches(*store, element, i, matchers)) + newManifest.elements.push_back(std::move(element)); + } + + // FIXME: warn about unused matchers? + + printInfo("removed %d packages, kept %d packages", + oldManifest.elements.size() - newManifest.elements.size(), + newManifest.elements.size()); + + updateProfile(newManifest.build(store)); + } +}; + +struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProfileElementMatchers +{ + std::string description() override + { + return "upgrade packages using their most recent flake"; + } + + Examples examples() override + { + return { + Example{ + "To upgrade all packages that were installed using a mutable flake reference:", + "nix profile upgrade '.*'" + }, + Example{ + "To upgrade a specific package:", + "nix profile upgrade packages.x86_64-linux.hello" + }, + }; + } + + void run(ref store) override + { + ProfileManifest manifest(*getEvalState(), *profile); + + auto matchers = getMatchers(store); + + // FIXME: code duplication + std::vector pathsToBuild; + + for (size_t i = 0; i < manifest.elements.size(); ++i) { + auto & element(manifest.elements[i]); + if (element.source + && !element.source->originalRef.input.isImmutable() + && matches(*store, element, i, matchers)) + { + Activity act(*logger, lvlChatty, actUnknown, + fmt("checking '%s' for updates", element.source->attrPath)); + + InstallableFlake installable(getEvalState(), FlakeRef(element.source->originalRef), {element.source->attrPath}, {}, lockFlags); + + auto [attrPath, resolvedRef, drv] = installable.toDerivation(); + + if (element.source->resolvedRef == resolvedRef) continue; + + printInfo("upgrading '%s' from flake '%s' to '%s'", + element.source->attrPath, element.source->resolvedRef, resolvedRef); + + element.storePaths = {drv.outPath}; // FIXME + element.source = ProfileElementSource{ + installable.flakeRef, + resolvedRef, + attrPath, + }; + + pathsToBuild.push_back({drv.drvPath, StringSet{"out"}}); // FIXME + } + } + + store->buildPaths(pathsToBuild); + + updateProfile(manifest.build(store)); + } +}; + +struct CmdProfileInfo : virtual EvalCommand, virtual StoreCommand, MixDefaultProfile +{ + std::string description() override + { + return "list installed packages"; + } + + Examples examples() override + { + return { + Example{ + "To show what packages are installed in the default profile:", + "nix profile info" + }, + }; + } + + void run(ref store) override + { + ProfileManifest manifest(*getEvalState(), *profile); + + for (size_t i = 0; i < manifest.elements.size(); ++i) { + auto & element(manifest.elements[i]); + logger->stdout("%d %s %s %s", i, + element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath : "-", + element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath : "-", + concatStringsSep(" ", store->printStorePathSet(element.storePaths))); + } + } +}; + +struct CmdProfileDiffClosures : virtual StoreCommand, MixDefaultProfile +{ + std::string description() override + { + return "show the closure difference between each generation of a profile"; + } + + Examples examples() override + { + return { + Example{ + "To show what changed between each generation of the NixOS system profile:", + "nix profile diff-closure --profile /nix/var/nix/profiles/system" + }, + }; + } + + void run(ref store) override + { + auto [gens, curGen] = findGenerations(*profile); + + std::optional prevGen; + bool first = true; + + for (auto & gen : gens) { + if (prevGen) { + if (!first) std::cout << "\n"; + first = false; + std::cout << fmt("Generation %d -> %d:\n", prevGen->number, gen.number); + printClosureDiff(store, + store->followLinksToStorePath(prevGen->path), + store->followLinksToStorePath(gen.path), + " "); + } + + prevGen = gen; + } + } +}; + +struct CmdProfile : virtual MultiCommand, virtual Command +{ + CmdProfile() + : MultiCommand({ + {"install", []() { return make_ref(); }}, + {"remove", []() { return make_ref(); }}, + {"upgrade", []() { return make_ref(); }}, + {"info", []() { return make_ref(); }}, + {"diff-closures", []() { return make_ref(); }}, + }) + { } + + std::string description() override + { + return "manage Nix profiles"; + } + + void run() override + { + if (!command) + throw UsageError("'nix profile' requires a sub-command."); + command->second->prepare(); + command->second->run(); + } + + void printHelp(const string & programName, std::ostream & out) override + { + MultiCommand::printHelp(programName, out); + } +}; + +static auto r1 = registerCommand("profile"); diff --git a/src/nix/registry.cc b/src/nix/registry.cc new file mode 100644 index 000000000..ebee4545c --- /dev/null +++ b/src/nix/registry.cc @@ -0,0 +1,151 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "eval.hh" +#include "flake/flake.hh" +#include "store-api.hh" +#include "fetchers.hh" +#include "registry.hh" + +using namespace nix; +using namespace nix::flake; + +struct CmdRegistryList : StoreCommand +{ + std::string description() override + { + return "list available Nix flakes"; + } + + void run(nix::ref store) override + { + using namespace fetchers; + + auto registries = getRegistries(store); + + for (auto & registry : registries) { + for (auto & entry : registry->entries) { + // FIXME: format nicely + logger->stdout("%s %s %s", + registry->type == Registry::Flag ? "flags " : + registry->type == Registry::User ? "user " : + registry->type == Registry::System ? "system" : + "global", + entry.from.to_string(), + entry.to.to_string()); + } + } + } +}; + +struct CmdRegistryAdd : MixEvalArgs, Command +{ + std::string fromUrl, toUrl; + + std::string description() override + { + return "add/replace flake in user flake registry"; + } + + CmdRegistryAdd() + { + expectArg("from-url", &fromUrl); + expectArg("to-url", &toUrl); + } + + void run() override + { + auto fromRef = parseFlakeRef(fromUrl); + auto toRef = parseFlakeRef(toUrl); + fetchers::Attrs extraAttrs; + if (toRef.subdir != "") extraAttrs["dir"] = toRef.subdir; + auto userRegistry = fetchers::getUserRegistry(); + userRegistry->remove(fromRef.input); + userRegistry->add(fromRef.input, toRef.input, extraAttrs); + userRegistry->write(fetchers::getUserRegistryPath()); + } +}; + +struct CmdRegistryRemove : virtual Args, MixEvalArgs, Command +{ + std::string url; + + std::string description() override + { + return "remove flake from user flake registry"; + } + + CmdRegistryRemove() + { + expectArg("url", &url); + } + + void run() override + { + auto userRegistry = fetchers::getUserRegistry(); + userRegistry->remove(parseFlakeRef(url).input); + userRegistry->write(fetchers::getUserRegistryPath()); + } +}; + +struct CmdRegistryPin : virtual Args, EvalCommand +{ + std::string url; + + std::string description() override + { + return "pin a flake to its current version in user flake registry"; + } + + CmdRegistryPin() + { + expectArg("url", &url); + } + + void run(nix::ref store) override + { + auto ref = parseFlakeRef(url); + auto userRegistry = fetchers::getUserRegistry(); + userRegistry->remove(ref.input); + auto [tree, resolved] = ref.resolve(store).input.fetch(store); + fetchers::Attrs extraAttrs; + if (ref.subdir != "") extraAttrs["dir"] = ref.subdir; + userRegistry->add(ref.input, resolved, extraAttrs); + userRegistry->write(fetchers::getUserRegistryPath()); + } +}; + +struct CmdRegistry : virtual MultiCommand, virtual Command +{ + CmdRegistry() + : MultiCommand({ + {"list", []() { return make_ref(); }}, + {"add", []() { return make_ref(); }}, + {"remove", []() { return make_ref(); }}, + {"pin", []() { return make_ref(); }}, + }) + { + } + + std::string description() override + { + return "manage the flake registry"; + } + + Category category() override { return catSecondary; } + + void run() override + { + if (!command) + throw UsageError("'nix registry' requires a sub-command."); + command->second->prepare(); + command->second->run(); + } + + void printHelp(const string & programName, std::ostream & out) override + { + MultiCommand::printHelp(programName, out); + } +}; + +static auto r1 = registerCommand("registry"); diff --git a/src/nix/repl.cc b/src/nix/repl.cc index ab8ae41e8..33ca1dbf7 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -490,7 +490,7 @@ bool NixRepl::processLine(string line) auto drv = state->store->readDerivation(drvPath); std::cout << std::endl << "this derivation produced the following outputs:" << std::endl; for (auto & i : drv.outputs) - std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(i.second.path)); + std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(i.second.path(*state->store, drv.name))); } } else if (command == ":i") { runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPathRaw}); @@ -764,7 +764,11 @@ struct CmdRepl : StoreCommand, MixEvalArgs CmdRepl() { - expectArgs("files", &files); + expectArgs({ + .label = "files", + .handler = {&files}, + .completer = completePath + }); } std::string description() override @@ -784,6 +788,7 @@ struct CmdRepl : StoreCommand, MixEvalArgs void run(ref store) override { + evalSettings.pureEval = false; auto repl = std::make_unique(searchPath, openStore()); repl->autoArgs = getAutoArgs(*repl->state); repl->mainLoop(files); diff --git a/src/nix/run.cc b/src/nix/run.cc index 321ee1d11..cbaba9d90 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -84,31 +84,30 @@ struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment { return { Example{ - "To start a shell providing GNU Hello from NixOS 17.03:", - "nix shell -f channel:nixos-17.03 hello" + "To start a shell providing GNU Hello from NixOS 20.03:", + "nix shell nixpkgs/nixos-20.03#hello" }, Example{ "To start a shell providing youtube-dl from your 'nixpkgs' channel:", - "nix shell nixpkgs.youtube-dl" + "nix shell nixpkgs#youtube-dl" }, Example{ "To run GNU Hello:", - "nix shell nixpkgs.hello -c hello --greeting 'Hi everybody!'" + "nix shell nixpkgs#hello -c hello --greeting 'Hi everybody!'" }, Example{ "To run GNU Hello in a chroot store:", - "nix shell --store ~/my-nix nixpkgs.hello -c hello" + "nix shell --store ~/my-nix nixpkgs#hello -c hello" }, }; } void run(ref store) override { - auto outPaths = toStorePaths(store, Build, installables); + auto outPaths = toStorePaths(store, Realise::Outputs, OperateOn::Output, installables); auto accessor = store->getFSAccessor(); - std::unordered_set done; std::queue todo; for (auto & path : outPaths) todo.push(path); @@ -143,6 +142,67 @@ struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment static auto r1 = registerCommand("shell"); +struct CmdRun : InstallableCommand, RunCommon +{ + std::vector args; + + CmdRun() + { + expectArgs({ + .label = "args", + .handler = {&args}, + .completer = completePath + }); + } + + std::string description() override + { + return "run a Nix application"; + } + + Examples examples() override + { + return { + Example{ + "To run Blender:", + "nix run blender-bin" + }, + }; + } + + Strings getDefaultFlakeAttrPaths() override + { + Strings res{"defaultApp." + settings.thisSystem.get()}; + for (auto & s : SourceExprCommand::getDefaultFlakeAttrPaths()) + res.push_back(s); + return res; + } + + Strings getDefaultFlakeAttrPathPrefixes() override + { + Strings res{"apps." + settings.thisSystem.get() + ".", "packages"}; + for (auto & s : SourceExprCommand::getDefaultFlakeAttrPathPrefixes()) + res.push_back(s); + return res; + } + + void run(ref store) override + { + auto state = getEvalState(); + + auto app = installable->toApp(*state); + + state->store->buildPaths(app.context); + + Strings allArgs{app.program}; + for (auto & i : args) allArgs.push_back(i); + + runProgram(store, app.program, allArgs); + } +}; + +static auto r2 = registerCommand("run"); + void chrootHelper(int argc, char * * argv) { int p = 1; diff --git a/src/nix/search.cc b/src/nix/search.cc index 93c3f3f83..65a1e1818 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -6,8 +6,9 @@ #include "get-drvs.hh" #include "common-args.hh" #include "json.hh" -#include "json-to-value.hh" #include "shared.hh" +#include "eval-cache.hh" +#include "attr-path.hh" #include #include @@ -25,33 +26,17 @@ std::string hilite(const std::string & s, const std::smatch & m, std::string pos m.empty() ? s : std::string(m.prefix()) - + ANSI_RED + std::string(m.str()) + postfix + + ANSI_GREEN + std::string(m.str()) + postfix + std::string(m.suffix()); } -struct CmdSearch : SourceExprCommand, MixJSON +struct CmdSearch : InstallableCommand, MixJSON { std::vector res; - bool writeCache = true; - bool useCache = true; - CmdSearch() { expectArgs("regex", &res); - - addFlag({ - .longName = "update-cache", - .shortName = 'u', - .description = "update the package search cache", - .handler = {[&]() { writeCache = true; useCache = false; }} - }); - - addFlag({ - .longName = "no-cache", - .description = "do not use or update the package search cache", - .handler = {[&]() { writeCache = false; useCache = false; }} - }); } std::string description() override @@ -63,24 +48,32 @@ struct CmdSearch : SourceExprCommand, MixJSON { return { Example{ - "To show all available packages:", + "To show all packages in the flake in the current directory:", "nix search" }, Example{ - "To show any packages containing 'blender' in its name or description:", - "nix search blender" + "To show packages in the 'nixpkgs' flake containing 'blender' in its name or description:", + "nix search nixpkgs blender" }, Example{ "To search for Firefox or Chromium:", - "nix search 'firefox|chromium'" + "nix search nixpkgs 'firefox|chromium'" }, Example{ - "To search for git and frontend or gui:", - "nix search git 'frontend|gui'" + "To search for packages containing 'git' and either 'frontend' or 'gui':", + "nix search nixpkgs git 'frontend|gui'" } }; } + Strings getDefaultFlakeAttrPaths() override + { + return { + "packages." + settings.thisSystem.get() + ".", + "legacyPackages." + settings.thisSystem.get() + "." + }; + } + void run(ref store) override { settings.readOnlyMode = true; @@ -88,189 +81,107 @@ struct CmdSearch : SourceExprCommand, MixJSON // Empty search string should match all packages // Use "^" here instead of ".*" due to differences in resulting highlighting // (see #1893 -- libc++ claims empty search string is not in POSIX grammar) - if (res.empty()) { + if (res.empty()) res.push_back("^"); - } std::vector regexes; regexes.reserve(res.size()); - for (auto &re : res) { + for (auto & re : res) regexes.push_back(std::regex(re, std::regex::extended | std::regex::icase)); - } auto state = getEvalState(); auto jsonOut = json ? std::make_unique(std::cout) : nullptr; - auto sToplevel = state->symbols.create("_toplevel"); - auto sRecurse = state->symbols.create("recurseForDerivations"); + uint64_t results = 0; - bool fromCache = false; - - std::map results; - - std::function doExpr; - - doExpr = [&](Value * v, std::string attrPath, bool toplevel, JSONObject * cache) { - debug("at attribute '%s'", attrPath); + std::function & attrPath)> visit; + visit = [&](eval_cache::AttrCursor & cursor, const std::vector & attrPath) + { + Activity act(*logger, lvlInfo, actUnknown, + fmt("evaluating '%s'", concatStringsSep(".", attrPath))); try { - uint found = 0; + auto recurse = [&]() + { + for (const auto & attr : cursor.getAttrs()) { + auto cursor2 = cursor.getAttr(attr); + auto attrPath2(attrPath); + attrPath2.push_back(attr); + visit(*cursor2, attrPath2); + } + }; - state->forceValue(*v); + if (cursor.isDerivation()) { + size_t found = 0; - if (v->type == tLambda && toplevel) { - Value * v2 = state->allocValue(); - state->autoCallFunction(*state->allocBindings(1), *v, *v2); - v = v2; - state->forceValue(*v); - } + DrvName name(cursor.getAttr("name")->getString()); - if (state->isDerivation(*v)) { + auto aMeta = cursor.maybeGetAttr("meta"); + auto aDescription = aMeta ? aMeta->maybeGetAttr("description") : nullptr; + auto description = aDescription ? aDescription->getString() : ""; + std::replace(description.begin(), description.end(), '\n', ' '); + auto attrPath2 = concatStringsSep(".", attrPath); - DrvInfo drv(*state, attrPath, v->attrs); - std::string description; std::smatch attrPathMatch; std::smatch descriptionMatch; std::smatch nameMatch; - std::string name; - DrvName parsed(drv.queryName()); - - for (auto ®ex : regexes) { - std::regex_search(attrPath, attrPathMatch, regex); - - name = parsed.name; - std::regex_search(name, nameMatch, regex); - - description = drv.queryMetaString("description"); - std::replace(description.begin(), description.end(), '\n', ' '); + for (auto & regex : regexes) { + std::regex_search(attrPath2, attrPathMatch, regex); + std::regex_search(name.name, nameMatch, regex); std::regex_search(description, descriptionMatch, regex); - if (!attrPathMatch.empty() || !nameMatch.empty() || !descriptionMatch.empty()) - { found++; - } } if (found == res.size()) { + results++; if (json) { - - auto jsonElem = jsonOut->object(attrPath); - - jsonElem.attr("pkgName", parsed.name); - jsonElem.attr("version", parsed.version); + auto jsonElem = jsonOut->object(attrPath2); + jsonElem.attr("pname", name.name); + jsonElem.attr("version", name.version); jsonElem.attr("description", description); - } else { - auto name = hilite(parsed.name, nameMatch, "\e[0;2m") - + std::string(parsed.fullName, parsed.name.length()); - results[attrPath] = fmt( - "* %s (%s)\n %s\n", - wrap("\e[0;1m", hilite(attrPath, attrPathMatch, "\e[0;1m")), - wrap("\e[0;2m", hilite(name, nameMatch, "\e[0;2m")), - hilite(description, descriptionMatch, ANSI_NORMAL)); - } - } - - if (cache) { - cache->attr("type", "derivation"); - cache->attr("name", drv.queryName()); - cache->attr("system", drv.querySystem()); - if (description != "") { - auto meta(cache->object("meta")); - meta.attr("description", description); + auto name2 = hilite(name.name, nameMatch, "\e[0;2m"); + if (results > 1) logger->stdout(""); + logger->stdout( + "* %s%s", + wrap("\e[0;1m", hilite(attrPath2, attrPathMatch, "\e[0;1m")), + name.version != "" ? " (" + name.version + ")" : ""); + if (description != "") + logger->stdout( + " %s", hilite(description, descriptionMatch, ANSI_NORMAL)); } } } - else if (v->type == tAttrs) { + else if ( + attrPath.size() == 0 + || (attrPath[0] == "legacyPackages" && attrPath.size() <= 2) + || (attrPath[0] == "packages" && attrPath.size() <= 2)) + recurse(); - if (!toplevel) { - auto attrs = v->attrs; - Bindings::iterator j = attrs->find(sRecurse); - if (j == attrs->end() || !state->forceBool(*j->value, *j->pos)) { - debug("skip attribute '%s'", attrPath); - return; - } - } - - bool toplevel2 = false; - if (!fromCache) { - Bindings::iterator j = v->attrs->find(sToplevel); - toplevel2 = j != v->attrs->end() && state->forceBool(*j->value, *j->pos); - } - - for (auto & i : *v->attrs) { - auto cache2 = - cache ? std::make_unique(cache->object(i.name)) : nullptr; - doExpr(i.value, - attrPath == "" ? (std::string) i.name : attrPath + "." + (std::string) i.name, - toplevel2 || fromCache, cache2 ? cache2.get() : nullptr); - } + else if (attrPath[0] == "legacyPackages" && attrPath.size() > 2) { + auto attr = cursor.maybeGetAttr(state->sRecurseForDerivations); + if (attr && attr->getBool()) + recurse(); } - } catch (AssertionError & e) { - } catch (Error & e) { - if (!toplevel) { - e.addTrace(std::nullopt, "While evaluating the attribute '%s'", attrPath); + } catch (EvalError & e) { + if (!(attrPath.size() > 0 && attrPath[0] == "legacyPackages")) throw; - } } }; - Path jsonCacheFileName = getCacheDir() + "/nix/package-search.json"; + for (auto & [cursor, prefix] : installable->getCursors(*state, true)) + visit(*cursor, parseAttrPath(*state, prefix)); - if (useCache && pathExists(jsonCacheFileName)) { - - warn("using cached results; pass '-u' to update the cache"); - - Value vRoot; - parseJSON(*state, readFile(jsonCacheFileName), vRoot); - - fromCache = true; - - doExpr(&vRoot, "", true, nullptr); - } - - else { - createDirs(dirOf(jsonCacheFileName)); - - Path tmpFile = fmt("%s.tmp.%d", jsonCacheFileName, getpid()); - - std::ofstream jsonCacheFile; - - try { - // iostream considered harmful - jsonCacheFile.exceptions(std::ofstream::failbit); - jsonCacheFile.open(tmpFile); - - auto cache = writeCache ? std::make_unique(jsonCacheFile, false) : nullptr; - - doExpr(getSourceExpr(*state), "", true, cache.get()); - - } catch (std::exception &) { - /* Fun fact: catching std::ios::failure does not work - due to C++11 ABI shenanigans. - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=66145 */ - if (!jsonCacheFile) - throw Error("error writing to %s", tmpFile); - throw; - } - - if (writeCache && rename(tmpFile.c_str(), jsonCacheFileName.c_str()) == -1) - throw SysError("cannot rename '%s' to '%s'", tmpFile, jsonCacheFileName); - } - - if (!json && results.size() == 0) + if (!json && !results) throw Error("no results for the given search term(s)!"); - - RunPager pager; - for (auto el : results) std::cout << el.second << "\n"; - } }; diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 5d77cfdca..9fd26e2d7 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -33,7 +33,7 @@ struct CmdShowDerivation : InstallablesCommand return { Example{ "To show the store derivation that results from evaluating the Hello package:", - "nix show-derivation nixpkgs.hello" + "nix show-derivation nixpkgs#hello" }, Example{ "To show the full derivation graph (if available) that produced your NixOS system:", @@ -69,10 +69,10 @@ struct CmdShowDerivation : InstallablesCommand auto outputsObj(drvObj.object("outputs")); for (auto & output : drv.outputs) { auto outputObj(outputsObj.object(output.first)); - outputObj.attr("path", store->printStorePath(output.second.path)); - if (output.second.hash) { - outputObj.attr("hashAlgo", output.second.hash->printMethodAlgo()); - outputObj.attr("hash", output.second.hash->hash.to_string(Base16, false)); + outputObj.attr("path", store->printStorePath(output.second.path(*store, drv.name))); + if (auto hash = std::get_if(&output.second.output)) { + outputObj.attr("hashAlgo", hash->hash.printMethodAlgo()); + outputObj.attr("hash", hash->hash.hash.to_string(Base16, false)); } } } diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 6c9b9a792..7821a5432 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -105,7 +105,8 @@ struct CmdSignPaths : StorePathsCommand .shortName = 'k', .description = "file containing the secret signing key", .labels = {"file"}, - .handler = {&secretKeyFile} + .handler = {&secretKeyFile}, + .completer = completePath }); } diff --git a/src/nix/verify.cc b/src/nix/verify.cc index ce90b0f6d..fc7a9765c 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -91,15 +91,15 @@ struct CmdVerify : StorePathsCommand std::unique_ptr hashSink; if (!info->ca) - hashSink = std::make_unique(*info->narHash.type); + hashSink = std::make_unique(info->narHash->type); else - hashSink = std::make_unique(*info->narHash.type, std::string(info->path.hashPart())); + hashSink = std::make_unique(info->narHash->type, std::string(info->path.hashPart())); store->narFromPath(info->path, *hashSink); auto hash = hashSink->finish(); - if (hash.first != info->narHash) { + if (hash.first != *info->narHash) { corrupted++; act2.result(resCorruptedPath, store->printStorePath(info->path)); logError({ @@ -107,7 +107,7 @@ struct CmdVerify : StorePathsCommand .hint = hintfmt( "path '%s' was modified! expected hash '%s', got '%s'", store->printStorePath(info->path), - info->narHash.to_string(Base32, true), + info->narHash->to_string(Base32, true), hash.first.to_string(Base32, true)) }); } diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 49da01c0a..7e630b745 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -55,15 +55,15 @@ struct CmdWhyDepends : SourceExprCommand return { Example{ "To show one path through the dependency graph leading from Hello to Glibc:", - "nix why-depends nixpkgs.hello nixpkgs.glibc" + "nix why-depends nixpkgs#hello nixpkgs#glibc" }, Example{ "To show all files and paths in the dependency graph leading from Thunderbird to libX11:", - "nix why-depends --all nixpkgs.thunderbird nixpkgs.xorg.libX11" + "nix why-depends --all nixpkgs#thunderbird nixpkgs#xorg.libX11" }, Example{ "To show why Glibc depends on itself:", - "nix why-depends nixpkgs.glibc nixpkgs.glibc" + "nix why-depends nixpkgs#glibc nixpkgs#glibc" }, }; } @@ -72,17 +72,19 @@ struct CmdWhyDepends : SourceExprCommand void run(ref store) override { - auto package = parseInstallable(*this, store, _package, false); - auto packagePath = toStorePath(store, Build, package); - auto dependency = parseInstallable(*this, store, _dependency, false); - auto dependencyPath = toStorePath(store, NoBuild, dependency); + auto package = parseInstallable(store, _package); + auto packagePath = toStorePath(store, Realise::Outputs, operateOn, package); + auto dependency = parseInstallable(store, _dependency); + auto dependencyPath = toStorePath(store, Realise::Derivation, operateOn, dependency); auto dependencyPathHash = dependencyPath.hashPart(); StorePathSet closure; store->computeFSClosure({packagePath}, closure, false, false); if (!closure.count(dependencyPath)) { - printError("'%s' does not depend on '%s'", package->what(), dependency->what()); + printError("'%s' does not depend on '%s'", + store->printStorePath(packagePath), + store->printStorePath(dependencyPath)); return; } diff --git a/tests/check.sh b/tests/check.sh index 5f25d04cb..5f4997e28 100644 --- a/tests/check.sh +++ b/tests/check.sh @@ -61,30 +61,30 @@ nix-build check.nix -A nondeterministic --no-out-link --repeat 1 2> $TEST_ROOT/l [ "$status" = "1" ] grep 'differs from previous round' $TEST_ROOT/log -path=$(nix-build check.nix -A fetchurl --no-out-link --hashed-mirrors '') +path=$(nix-build check.nix -A fetchurl --no-out-link) chmod +w $path echo foo > $path chmod -w $path -nix-build check.nix -A fetchurl --no-out-link --check --hashed-mirrors '' +nix-build check.nix -A fetchurl --no-out-link --check # Note: "check" doesn't repair anything, it just compares to the hash stored in the database. [[ $(cat $path) = foo ]] -nix-build check.nix -A fetchurl --no-out-link --repair --hashed-mirrors '' +nix-build check.nix -A fetchurl --no-out-link --repair [[ $(cat $path) != foo ]] -nix-build check.nix -A hashmismatch --no-out-link --hashed-mirrors '' || status=$? +nix-build check.nix -A hashmismatch --no-out-link || status=$? [ "$status" = "102" ] echo -n > ./dummy -nix-build check.nix -A hashmismatch --no-out-link --hashed-mirrors '' +nix-build check.nix -A hashmismatch --no-out-link echo 'Hello World' > ./dummy -nix-build check.nix -A hashmismatch --no-out-link --check --hashed-mirrors '' || status=$? +nix-build check.nix -A hashmismatch --no-out-link --check || status=$? [ "$status" = "102" ] # Multiple failures with --keep-going nix-build check.nix -A nondeterministic --no-out-link -nix-build check.nix -A nondeterministic -A hashmismatch --no-out-link --check --keep-going --hashed-mirrors '' || status=$? +nix-build check.nix -A nondeterministic -A hashmismatch --no-out-link --check --keep-going || status=$? [ "$status" = "110" ] diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh index d9c9874f5..cedd796f7 100644 --- a/tests/fetchGit.sh +++ b/tests/fetchGit.sh @@ -31,44 +31,46 @@ rev2=$(git -C $repo rev-parse HEAD) # Fetch a worktree unset _NIX_FORCE_HTTP -path0=$(nix eval --raw "(builtins.fetchGit file://$TEST_ROOT/worktree).outPath") +path0=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$TEST_ROOT/worktree).outPath") +path0_=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = file://$TEST_ROOT/worktree; }).outPath") +[[ $path0 = $path0_ ]] export _NIX_FORCE_HTTP=1 [[ $(tail -n 1 $path0/hello) = "hello" ]] # Fetch the default branch. -path=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath") +path=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath") [[ $(cat $path/hello) = world ]] # In pure eval mode, fetchGit without a revision should fail. -[[ $(nix eval --raw "(builtins.readFile (fetchGit file://$repo + \"/hello\"))") = world ]] -(! nix eval --pure-eval --raw "(builtins.readFile (fetchGit file://$repo + \"/hello\"))") +[[ $(nix eval --impure --raw --expr "builtins.readFile (fetchGit file://$repo + \"/hello\")") = world ]] +(! nix eval --raw --expr "builtins.readFile (fetchGit file://$repo + \"/hello\")") # Fetch using an explicit revision hash. -path2=$(nix eval --raw "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath") +path2=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath") [[ $path = $path2 ]] # In pure eval mode, fetchGit with a revision should succeed. -[[ $(nix eval --pure-eval --raw "(builtins.readFile (fetchGit { url = file://$repo; rev = \"$rev2\"; } + \"/hello\"))") = world ]] +[[ $(nix eval --raw --expr "builtins.readFile (fetchGit { url = file://$repo; rev = \"$rev2\"; } + \"/hello\")") = world ]] # Fetch again. This should be cached. mv $repo ${repo}-tmp -path2=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath") +path2=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath") [[ $path = $path2 ]] -[[ $(nix eval "(builtins.fetchGit file://$repo).revCount") = 2 ]] -[[ $(nix eval --raw "(builtins.fetchGit file://$repo).rev") = $rev2 ]] +[[ $(nix eval --impure --expr "(builtins.fetchGit file://$repo).revCount") = 2 ]] +[[ $(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).rev") = $rev2 ]] # Fetching with a explicit hash should succeed. -path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath") +path2=$(nix eval --refresh --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath") [[ $path = $path2 ]] -path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchGit { url = file://$repo; rev = \"$rev1\"; }).outPath") +path2=$(nix eval --refresh --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev1\"; }).outPath") [[ $(cat $path2/hello) = utrecht ]] mv ${repo}-tmp $repo # Using a clean working tree should produce the same result. -path2=$(nix eval --raw "(builtins.fetchGit $repo).outPath") +path2=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath") [[ $path = $path2 ]] # Using an unclean tree should yield the tracked but uncommitted changes. @@ -80,59 +82,65 @@ git -C $repo add dir1/foo git -C $repo rm hello unset _NIX_FORCE_HTTP -path2=$(nix eval --raw "(builtins.fetchGit $repo).outPath") +path2=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath") [ ! -e $path2/hello ] [ ! -e $path2/bar ] [ ! -e $path2/dir2/bar ] [ ! -e $path2/.git ] [[ $(cat $path2/dir1/foo) = foo ]] -[[ $(nix eval --raw "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]] +[[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]] # ... unless we're using an explicit ref or rev. -path3=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"master\"; }).outPath") +path3=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"master\"; }).outPath") [[ $path = $path3 ]] -path3=$(nix eval --raw "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; }).outPath") +path3=$(nix eval --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; }).outPath") [[ $path = $path3 ]] # Committing should not affect the store path. git -C $repo commit -m 'Bla3' -a -path4=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchGit file://$repo).outPath") +path4=$(nix eval --impure --refresh --raw --expr "(builtins.fetchGit file://$repo).outPath") [[ $path2 = $path4 ]] +nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; narHash = \"sha256-B5yIPHhEm0eysJKEsO7nqxprh9vcblFxpJG11gXJus1=\"; }).outPath" || status=$? +[[ "$status" = "102" ]] + +path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; narHash = \"sha256-Hr8g6AqANb3xqX28eu1XnjK/3ab8Gv6TJSnkb1LezG9=\"; }).outPath") +[[ $path = $path5 ]] + # tarball-ttl should be ignored if we specify a rev echo delft > $repo/hello git -C $repo add hello git -C $repo commit -m 'Bla4' rev3=$(git -C $repo rev-parse HEAD) -nix eval --tarball-ttl 3600 "(builtins.fetchGit { url = $repo; rev = \"$rev3\"; })" >/dev/null +nix eval --tarball-ttl 3600 --expr "builtins.fetchGit { url = $repo; rev = \"$rev3\"; }" >/dev/null # Update 'path' to reflect latest master -path=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath") +path=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath") # Check behavior when non-master branch is used git -C $repo checkout $rev2 -b dev echo dev > $repo/hello # File URI uses dirty tree unless specified otherwise -path2=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath") +path2=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath") [ $(cat $path2/hello) = dev ] # Using local path with branch other than 'master' should work when clean or dirty -path3=$(nix eval --raw "(builtins.fetchGit $repo).outPath") +path3=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath") # (check dirty-tree handling was used) -[[ $(nix eval --raw "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]] +[[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]] # Committing shouldn't change store path, or switch to using 'master' git -C $repo commit -m 'Bla5' -a -path4=$(nix eval --raw "(builtins.fetchGit $repo).outPath") +path4=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath") [[ $(cat $path4/hello) = dev ]] [[ $path3 = $path4 ]] # Confirm same as 'dev' branch -path5=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath") +path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath") [[ $path3 = $path5 ]] @@ -141,19 +149,18 @@ rm -rf $TEST_HOME/.cache/nix # Try again, but without 'git' on PATH. This should fail. NIX=$(command -v nix) -# This should fail -(! PATH= $NIX eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath" ) +(! PATH= $NIX eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath" ) # Try again, with 'git' available. This should work. -path5=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath") +path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath") [[ $path3 = $path5 ]] # Fetching a shallow repo shouldn't work by default, because we can't # return a revCount. git clone --depth 1 file://$repo $TEST_ROOT/shallow -(! nix eval --raw "(builtins.fetchGit { url = $TEST_ROOT/shallow; ref = \"dev\"; }).outPath") +(! nix eval --impure --raw --expr "(builtins.fetchGit { url = $TEST_ROOT/shallow; ref = \"dev\"; }).outPath") # But you can request a shallow clone, which won't return a revCount. -path6=$(nix eval --raw "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow\"; ref = \"dev\"; shallow = true; }).outPath") +path6=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow\"; ref = \"dev\"; shallow = true; }).outPath") [[ $path3 = $path6 ]] -[[ $(nix eval "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow\"; ref = \"dev\"; shallow = true; }).revCount or 123") == 123 ]] +[[ $(nix eval --impure --expr "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow\"; ref = \"dev\"; shallow = true; }).revCount or 123") == 123 ]] diff --git a/tests/fetchGitRefs.sh b/tests/fetchGitRefs.sh index 23934698e..52926040b 100644 --- a/tests/fetchGitRefs.sh +++ b/tests/fetchGitRefs.sh @@ -19,7 +19,7 @@ echo utrecht > "$repo"/hello git -C "$repo" add hello git -C "$repo" commit -m 'Bla1' -path=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"master\"; }).outPath") +path=$(nix eval --raw --impure --expr "(builtins.fetchGit { url = $repo; ref = \"master\"; }).outPath") # Test various combinations of ref names # (taken from the git project) @@ -42,7 +42,7 @@ valid_ref() { { set +x; printf >&2 '\n>>>>>>>>>> valid_ref %s\b <<<<<<<<<<\n' $(printf %s "$1" | sed -n -e l); set -x; } git check-ref-format --branch "$1" >/dev/null git -C "$repo" branch "$1" master >/dev/null - path1=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = ''$1''; }).outPath") + path1=$(nix eval --raw --impure --expr "(builtins.fetchGit { url = $repo; ref = ''$1''; }).outPath") [[ $path1 = $path ]] git -C "$repo" branch -D "$1" >/dev/null } @@ -56,7 +56,7 @@ invalid_ref() { else (! git check-ref-format --branch "$1" >/dev/null 2>&1) fi - nix --debug eval --raw "(builtins.fetchGit { url = $repo; ref = ''$1''; }).outPath" 2>&1 | grep 'invalid Git branch/tag name' >/dev/null + nix --debug eval --raw --impure --expr "(builtins.fetchGit { url = $repo; ref = ''$1''; }).outPath" 2>&1 | grep 'invalid Git branch/tag name' >/dev/null } diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh index 4c2c13f1a..5f104355f 100644 --- a/tests/fetchGitSubmodules.sh +++ b/tests/fetchGitSubmodules.sh @@ -38,18 +38,18 @@ git -C $rootRepo commit -m "Add submodule" rev=$(git -C $rootRepo rev-parse HEAD) -r1=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") -r2=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = false; }).outPath") -r3=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") +r1=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") +r2=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = false; }).outPath") +r3=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") [[ $r1 == $r2 ]] [[ $r2 != $r3 ]] -r4=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; }).outPath") -r5=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = false; }).outPath") -r6=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath") -r7=$(nix eval --raw "(builtins.fetchGit { url = $rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath") -r8=$(nix eval --raw "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).outPath") +r4=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; }).outPath") +r5=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = false; }).outPath") +r6=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath") +r7=$(nix eval --raw --expr "(builtins.fetchGit { url = $rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath") +r8=$(nix eval --raw --expr "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).outPath") [[ $r1 == $r4 ]] [[ $r4 == $r5 ]] @@ -57,19 +57,19 @@ r8=$(nix eval --raw "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submo [[ $r6 == $r7 ]] [[ $r7 == $r8 ]] -have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; }).submodules") +have_submodules=$(nix eval --expr "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; }).submodules") [[ $have_submodules == false ]] -have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = false; }).submodules") +have_submodules=$(nix eval --expr "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = false; }).submodules") [[ $have_submodules == false ]] -have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).submodules") +have_submodules=$(nix eval --expr "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).submodules") [[ $have_submodules == true ]] -pathWithoutSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") -pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") -pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") -pathWithSubmodulesAgainWithRef=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath") +pathWithoutSubmodules=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") +pathWithSubmodules=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") +pathWithSubmodulesAgain=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") +pathWithSubmodulesAgainWithRef=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath") # The resulting store path cannot be the same. [[ $pathWithoutSubmodules != $pathWithSubmodules ]] @@ -91,7 +91,7 @@ test "$(find "$pathWithSubmodules" -name .git)" = "" # Git repos without submodules can be fetched with submodules = true. subRev=$(git -C $subRepo rev-parse HEAD) -noSubmoduleRepoBaseline=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; }).outPath") -noSubmoduleRepo=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; submodules = true; }).outPath") +noSubmoduleRepoBaseline=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; }).outPath") +noSubmoduleRepo=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; submodules = true; }).outPath") [[ $noSubmoduleRepoBaseline == $noSubmoduleRepo ]] diff --git a/tests/fetchMercurial.sh b/tests/fetchMercurial.sh index 4088dbd39..af8ef8d5b 100644 --- a/tests/fetchMercurial.sh +++ b/tests/fetchMercurial.sh @@ -9,7 +9,7 @@ clearStore repo=$TEST_ROOT/hg -rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix/hg +rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix hg init $repo echo '[ui]' >> $repo/.hg/hgrc @@ -26,43 +26,43 @@ hg commit --cwd $repo -m 'Bla2' rev2=$(hg log --cwd $repo -r tip --template '{node}') # Fetch the default branch. -path=$(nix eval --raw "(builtins.fetchMercurial file://$repo).outPath") +path=$(nix eval --impure --raw --expr "(builtins.fetchMercurial file://$repo).outPath") [[ $(cat $path/hello) = world ]] # In pure eval mode, fetchGit without a revision should fail. -[[ $(nix eval --raw "(builtins.readFile (fetchMercurial file://$repo + \"/hello\"))") = world ]] -(! nix eval --pure-eval --raw "(builtins.readFile (fetchMercurial file://$repo + \"/hello\"))") +[[ $(nix eval --impure --raw --expr "(builtins.readFile (fetchMercurial file://$repo + \"/hello\"))") = world ]] +(! nix eval --raw --expr "builtins.readFile (fetchMercurial file://$repo + \"/hello\")") # Fetch using an explicit revision hash. -path2=$(nix eval --raw "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev2\"; }).outPath") +path2=$(nix eval --impure --raw --expr "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev2\"; }).outPath") [[ $path = $path2 ]] # In pure eval mode, fetchGit with a revision should succeed. -[[ $(nix eval --pure-eval --raw "(builtins.readFile (fetchMercurial { url = file://$repo; rev = \"$rev2\"; } + \"/hello\"))") = world ]] +[[ $(nix eval --raw --expr "builtins.readFile (fetchMercurial { url = file://$repo; rev = \"$rev2\"; } + \"/hello\")") = world ]] # Fetch again. This should be cached. mv $repo ${repo}-tmp -path2=$(nix eval --raw "(builtins.fetchMercurial file://$repo).outPath") +path2=$(nix eval --impure --raw --expr "(builtins.fetchMercurial file://$repo).outPath") [[ $path = $path2 ]] -[[ $(nix eval --raw "(builtins.fetchMercurial file://$repo).branch") = default ]] -[[ $(nix eval "(builtins.fetchMercurial file://$repo).revCount") = 1 ]] -[[ $(nix eval --raw "(builtins.fetchMercurial file://$repo).rev") = $rev2 ]] +[[ $(nix eval --impure --raw --expr "(builtins.fetchMercurial file://$repo).branch") = default ]] +[[ $(nix eval --impure --expr "(builtins.fetchMercurial file://$repo).revCount") = 1 ]] +[[ $(nix eval --impure --raw --expr "(builtins.fetchMercurial file://$repo).rev") = $rev2 ]] # But with TTL 0, it should fail. -(! nix eval --tarball-ttl 0 "(builtins.fetchMercurial file://$repo)") +(! nix eval --impure --refresh --expr "builtins.fetchMercurial file://$repo") # Fetching with a explicit hash should succeed. -path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev2\"; }).outPath") +path2=$(nix eval --refresh --raw --expr "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev2\"; }).outPath") [[ $path = $path2 ]] -path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev1\"; }).outPath") +path2=$(nix eval --refresh --raw --expr "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev1\"; }).outPath") [[ $(cat $path2/hello) = utrecht ]] mv ${repo}-tmp $repo # Using a clean working tree should produce the same result. -path2=$(nix eval --raw "(builtins.fetchMercurial $repo).outPath") +path2=$(nix eval --impure --raw --expr "(builtins.fetchMercurial $repo).outPath") [[ $path = $path2 ]] # Using an unclean tree should yield the tracked but uncommitted changes. @@ -73,21 +73,21 @@ echo bar > $repo/dir2/bar hg add --cwd $repo dir1/foo hg rm --cwd $repo hello -path2=$(nix eval --raw "(builtins.fetchMercurial $repo).outPath") +path2=$(nix eval --impure --raw --expr "(builtins.fetchMercurial $repo).outPath") [ ! -e $path2/hello ] [ ! -e $path2/bar ] [ ! -e $path2/dir2/bar ] [ ! -e $path2/.hg ] [[ $(cat $path2/dir1/foo) = foo ]] -[[ $(nix eval --raw "(builtins.fetchMercurial $repo).rev") = 0000000000000000000000000000000000000000 ]] +[[ $(nix eval --impure --raw --expr "(builtins.fetchMercurial $repo).rev") = 0000000000000000000000000000000000000000 ]] -# ... unless we're using an explicit rev. -path3=$(nix eval --raw "(builtins.fetchMercurial { url = $repo; rev = \"default\"; }).outPath") +# ... unless we're using an explicit ref. +path3=$(nix eval --impure --raw --expr "(builtins.fetchMercurial { url = $repo; rev = \"default\"; }).outPath") [[ $path = $path3 ]] # Committing should not affect the store path. hg commit --cwd $repo -m 'Bla3' -path4=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchMercurial file://$repo).outPath") +path4=$(nix eval --impure --refresh --raw --expr "(builtins.fetchMercurial file://$repo).outPath") [[ $path2 = $path4 ]] diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh index 2535651b0..0f2044342 100644 --- a/tests/fetchurl.sh +++ b/tests/fetchurl.sh @@ -5,7 +5,7 @@ clearStore # Test fetching a flat file. hash=$(nix-hash --flat --type sha256 ./fetchurl.sh) -outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link --hashed-mirrors '') +outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link) cmp $outPath fetchurl.sh @@ -14,7 +14,7 @@ clearStore hash=$(nix hash-file --type sha512 --base64 ./fetchurl.sh) -outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link --hashed-mirrors '') +outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link) cmp $outPath fetchurl.sh @@ -25,26 +25,24 @@ hash=$(nix hash-file ./fetchurl.sh) [[ $hash =~ ^sha256- ]] -outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr hash $hash --no-out-link --hashed-mirrors '') +outPath=$(nix-build '' --argstr url file://$(pwd)/fetchurl.sh --argstr hash $hash --no-out-link) cmp $outPath fetchurl.sh -# Test the hashed mirror feature. +# Test that we can substitute from a different store dir. clearStore -hash=$(nix hash-file --type sha512 --base64 ./fetchurl.sh) -hash32=$(nix hash-file --type sha512 --base16 ./fetchurl.sh) +other_store=file://$TEST_ROOT/other_store?store=/fnord/store -mirror=$TEST_ROOT/hashed-mirror -rm -rf $mirror -mkdir -p $mirror/sha512 -ln -s $(pwd)/fetchurl.sh $mirror/sha512/$hash32 +hash=$(nix hash-file --type sha256 --base16 ./fetchurl.sh) -outPath=$(nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha512 $hash --no-out-link --hashed-mirrors "file://$mirror") +storePath=$(nix --store $other_store add-to-store --flat ./fetchurl.sh) + +outPath=$(nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store) # Test hashed mirrors with an SRI hash. -nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha512 $hash) \ - --argstr name bla --no-out-link --hashed-mirrors "file://$mirror" +nix-build '' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha256 $hash) \ + --no-out-link --substituters $other_store # Test unpacking a NAR. rm -rf $TEST_ROOT/archive diff --git a/tests/filter-source.sh b/tests/filter-source.sh index 1f8dceee5..ba34d2eac 100644 --- a/tests/filter-source.sh +++ b/tests/filter-source.sh @@ -10,10 +10,16 @@ touch $TEST_ROOT/filterin/bak touch $TEST_ROOT/filterin/bla.c.bak ln -s xyzzy $TEST_ROOT/filterin/link -nix-build ./filter-source.nix -o $TEST_ROOT/filterout +checkFilter() { + test ! -e $1/foo/bar + test -e $1/xyzzy + test -e $1/bak + test ! -e $1/bla.c.bak + test ! -L $1/link +} -test ! -e $TEST_ROOT/filterout/foo/bar -test -e $TEST_ROOT/filterout/xyzzy -test -e $TEST_ROOT/filterout/bak -test ! -e $TEST_ROOT/filterout/bla.c.bak -test ! -L $TEST_ROOT/filterout/link +nix-build ./filter-source.nix -o $TEST_ROOT/filterout1 +checkFilter $TEST_ROOT/filterout1 + +nix-build ./path.nix -o $TEST_ROOT/filterout2 +checkFilter $TEST_ROOT/filterout2 diff --git a/tests/flakes.sh b/tests/flakes.sh new file mode 100644 index 000000000..5aec563ac --- /dev/null +++ b/tests/flakes.sh @@ -0,0 +1,718 @@ +source common.sh + +if [[ -z $(type -p git) ]]; then + echo "Git not installed; skipping flake tests" + exit 99 +fi + +if [[ -z $(type -p hg) ]]; then + echo "Mercurial not installed; skipping flake tests" + exit 99 +fi + +clearStore +rm -rf $TEST_HOME/.cache $TEST_HOME/.config + +registry=$TEST_ROOT/registry.json + +flake1Dir=$TEST_ROOT/flake1 +flake2Dir=$TEST_ROOT/flake2 +flake3Dir=$TEST_ROOT/flake3 +flake5Dir=$TEST_ROOT/flake5 +flake6Dir=$TEST_ROOT/flake6 +flake7Dir=$TEST_ROOT/flake7 +templatesDir=$TEST_ROOT/templates +nonFlakeDir=$TEST_ROOT/nonFlake +flakeA=$TEST_ROOT/flakeA +flakeB=$TEST_ROOT/flakeB + +for repo in $flake1Dir $flake2Dir $flake3Dir $flake7Dir $templatesDir $nonFlakeDir $flakeA $flakeB; do + rm -rf $repo $repo.tmp + mkdir $repo + git -C $repo init + git -C $repo config user.email "foobar@example.com" + git -C $repo config user.name "Foobar" +done + +cat > $flake1Dir/flake.nix < $flake2Dir/flake.nix < $flake3Dir/flake.nix < $nonFlakeDir/README.md < $registry <> $flake1Dir/flake.nix +git -C $flake1Dir commit -a -m 'Foo' +hash2=$(nix flake info flake1 --json --refresh | jq -r .revision) +[[ $hash1 != $hash2 ]] + +# Test 'nix build' on a flake. +nix build -o $TEST_ROOT/result flake1#foo +[[ -e $TEST_ROOT/result/hello ]] + +# Test defaultPackage. +nix build -o $TEST_ROOT/result flake1 +[[ -e $TEST_ROOT/result/hello ]] + +nix build -o $TEST_ROOT/result $flake1Dir +nix build -o $TEST_ROOT/result git+file://$flake1Dir + +# Check that store symlinks inside a flake are not interpreted as flakes. +nix build -o $flake1Dir/result git+file://$flake1Dir +nix path-info $flake1Dir/result + +# 'getFlake' on a mutable flakeref should fail in pure mode, but succeed in impure mode. +(! nix build -o $TEST_ROOT/result --expr "(builtins.getFlake \"$flake1Dir\").defaultPackage.$system") +nix build -o $TEST_ROOT/result --expr "(builtins.getFlake \"$flake1Dir\").defaultPackage.$system" --impure + +# 'getFlake' on an immutable flakeref should succeed even in pure mode. +nix build -o $TEST_ROOT/result --expr "(builtins.getFlake \"git+file://$flake1Dir?rev=$hash2\").defaultPackage.$system" + +# Building a flake with an unlocked dependency should fail in pure mode. +(! nix build -o $TEST_ROOT/result flake2#bar --no-registries) +(! nix eval --expr "builtins.getFlake \"$flake2Dir\"") + +# But should succeed in impure mode. +(! nix build -o $TEST_ROOT/result flake2#bar --impure) +nix build -o $TEST_ROOT/result flake2#bar --impure --no-write-lock-file + +# Building a local flake with an unlocked dependency should fail with --no-update-lock-file. +nix build -o $TEST_ROOT/result $flake2Dir#bar --no-update-lock-file 2>&1 | grep 'requires lock file changes' + +# But it should succeed without that flag. +nix build -o $TEST_ROOT/result $flake2Dir#bar --no-write-lock-file +nix build -o $TEST_ROOT/result $flake2Dir#bar --no-update-lock-file 2>&1 | grep 'requires lock file changes' +nix build -o $TEST_ROOT/result $flake2Dir#bar --commit-lock-file +[[ -e $flake2Dir/flake.lock ]] +[[ -z $(git -C $flake2Dir diff master) ]] + +# Rerunning the build should not change the lockfile. +nix build -o $TEST_ROOT/result $flake2Dir#bar +[[ -z $(git -C $flake2Dir diff master) ]] + +# Building with a lockfile should not require a fetch of the registry. +nix build -o $TEST_ROOT/result --flake-registry file:///no-registry.json $flake2Dir#bar --refresh +nix build -o $TEST_ROOT/result --no-registries $flake2Dir#bar --refresh + +# Updating the flake should not change the lockfile. +nix flake update $flake2Dir +[[ -z $(git -C $flake2Dir diff master) ]] + +# Now we should be able to build the flake in pure mode. +nix build -o $TEST_ROOT/result flake2#bar + +# Or without a registry. +nix build -o $TEST_ROOT/result --no-registries git+file://$flake2Dir#bar --refresh + +# Test whether indirect dependencies work. +nix build -o $TEST_ROOT/result $flake3Dir#xyzzy +git -C $flake3Dir add flake.lock + +# Add dependency to flake3. +rm $flake3Dir/flake.nix + +cat > $flake3Dir/flake.nix < $flake3Dir/flake.nix < \$out + ''; + }; + }; +} +EOF + +cp ./config.nix $flake3Dir + +git -C $flake3Dir add flake.nix config.nix +git -C $flake3Dir commit -m 'Add nonFlakeInputs' + +# Check whether `nix build` works with a lockfile which is missing a +# nonFlakeInputs. +nix build -o $TEST_ROOT/result $flake3Dir#sth --commit-lock-file + +nix build -o $TEST_ROOT/result flake3#fnord +[[ $(cat $TEST_ROOT/result) = FNORD ]] + +# Check whether flake input fetching is lazy: flake3#sth does not +# depend on flake2, so this shouldn't fail. +rm -rf $TEST_HOME/.cache +clearStore +mv $flake2Dir $flake2Dir.tmp +mv $nonFlakeDir $nonFlakeDir.tmp +nix build -o $TEST_ROOT/result flake3#sth +(! nix build -o $TEST_ROOT/result flake3#xyzzy) +(! nix build -o $TEST_ROOT/result flake3#fnord) +mv $flake2Dir.tmp $flake2Dir +mv $nonFlakeDir.tmp $nonFlakeDir +nix build -o $TEST_ROOT/result flake3#xyzzy flake3#fnord + +# Test doing multiple `lookupFlake`s +nix build -o $TEST_ROOT/result flake4#xyzzy + +# Test 'nix flake update' and --override-flake. +nix flake update $flake3Dir +[[ -z $(git -C $flake3Dir diff master) ]] + +nix flake update $flake3Dir --recreate-lock-file --override-flake flake2 nixpkgs +[[ ! -z $(git -C $flake3Dir diff master) ]] + +# Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore +git -C $flake3Dir checkout -b removeXyzzy +rm $flake3Dir/flake.nix + +cat > $flake3Dir/flake.nix < \$out + ''; + }; + }; +} +EOF +nix flake update $flake3Dir +git -C $flake3Dir add flake.nix flake.lock +git -C $flake3Dir commit -m 'Remove packages.xyzzy' +git -C $flake3Dir checkout master + +# Test whether fuzzy-matching works for registry entries. +(! nix build -o $TEST_ROOT/result flake4/removeXyzzy#xyzzy) +nix build -o $TEST_ROOT/result flake4/removeXyzzy#sth + +# Testing the nix CLI +nix registry add flake1 flake3 +[[ $(nix registry list | wc -l) == 8 ]] +nix registry pin flake1 +[[ $(nix registry list | wc -l) == 8 ]] +nix registry remove flake1 +[[ $(nix registry list | wc -l) == 7 ]] + +# Test 'nix flake init'. +cat > $templatesDir/flake.nix < $templatesDir/trivial/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake5Dir/flake.nix <&1 | grep 'NAR hash mismatch' + +# Test --override-input. +git -C $flake3Dir reset --hard +nix flake update $flake3Dir --override-input flake2/flake1 flake5 -vvvvv +[[ $(jq .nodes.flake1_2.locked.url $flake3Dir/flake.lock) =~ flake5 ]] + +nix flake update $flake3Dir --override-input flake2/flake1 flake1 +[[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) =~ $hash2 ]] + +nix flake update $flake3Dir --override-input flake2/flake1 flake1/master/$hash1 +[[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) =~ $hash1 ]] + +# Test --update-input. +nix flake update $flake3Dir +[[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) = $hash1 ]] + +nix flake update $flake3Dir --update-input flake2/flake1 +[[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) =~ $hash2 ]] + +# Test 'nix flake list-inputs'. +[[ $(nix flake list-inputs $flake3Dir | wc -l) == 5 ]] +nix flake list-inputs $flake3Dir --json | jq . + +# Test circular flake dependencies. +cat > $flakeA/flake.nix < $flakeB/flake.nix < $out/commits/HEAD + ''; + +in + +makeTest ( + +{ + + nodes = + { # Impersonate github.com and api.github.com. + github = + { config, pkgs, ... }: + { networking.firewall.allowedTCPPorts = [ 80 443 ]; + + services.httpd.enable = true; + services.httpd.adminAddr = "foo@example.org"; + services.httpd.extraConfig = '' + ErrorLog syslog:local6 + ''; + services.httpd.virtualHosts."github.com" = + { forceSSL = true; + sslServerKey = "${cert}/server.key"; + sslServerCert = "${cert}/server.crt"; + servedDirs = + [ { urlPath = "/NixOS/flake-registry/raw/master"; + dir = registry; + } + ]; + }; + services.httpd.virtualHosts."api.github.com" = + { forceSSL = true; + sslServerKey = "${cert}/server.key"; + sslServerCert = "${cert}/server.crt"; + servedDirs = + [ { urlPath = "/repos/NixOS/nixpkgs"; + dir = api; + } + ]; + }; + }; + + client = + { config, lib, pkgs, nodes, ... }: + { virtualisation.writableStore = true; + virtualisation.diskSize = 2048; + virtualisation.pathsInNixDB = [ pkgs.hello pkgs.fuse ]; + virtualisation.memorySize = 4096; + nix.binaryCaches = lib.mkForce [ ]; + nix.extraOptions = "experimental-features = nix-command flakes"; + environment.systemPackages = [ pkgs.jq ]; + networking.hosts.${(builtins.head nodes.github.config.networking.interfaces.eth1.ipv4.addresses).address} = + [ "github.com" "api.github.com" "raw.githubusercontent.com" ]; + security.pki.certificateFiles = [ "${cert}/ca.crt" ]; + }; + }; + + testScript = { nodes }: + '' + use POSIX qw(strftime); + + startAll; + + $github->waitForUnit("httpd.service"); + + $client->succeed("curl -v https://github.com/ >&2"); + + $client->succeed("nix registry list | grep nixpkgs"); + + $client->succeed("nix flake info nixpkgs --json | jq -r .revision") eq "${nixpkgs.rev}\n" + or die "revision mismatch"; + + $client->succeed("nix registry pin nixpkgs"); + + $client->succeed("nix flake info nixpkgs --tarball-ttl 0 >&2"); + + # Shut down the web server. The flake should be cached on the client. + $github->succeed("systemctl stop httpd.service"); + + my $date = $client->succeed("nix flake info nixpkgs --json | jq -M .lastModified"); + strftime("%Y%m%d%H%M%S", gmtime($date)) eq "${nixpkgs.lastModifiedDate}" or die "time mismatch"; + + $client->succeed("nix build nixpkgs#hello"); + + # The build shouldn't fail even with --tarball-ttl 0 (the server + # being down should not be a fatal error). + $client->succeed("nix build nixpkgs#fuse --tarball-ttl 0"); + ''; + +}) diff --git a/tests/init.sh b/tests/init.sh index 0c2c0e170..f9ced6b0d 100644 --- a/tests/init.sh +++ b/tests/init.sh @@ -19,6 +19,7 @@ keep-derivations = false sandbox = false experimental-features = nix-command flakes gc-reserved-space = 0 +flake-registry = $TEST_ROOT/registry.json include nix.conf.extra EOF diff --git a/tests/local-store.sh b/tests/local-store.sh new file mode 100644 index 000000000..4ec3d64b0 --- /dev/null +++ b/tests/local-store.sh @@ -0,0 +1,20 @@ +source common.sh + +cd $TEST_ROOT + +echo example > example.txt +mkdir -p ./x + +NIX_STORE_DIR=$TEST_ROOT/x + +CORRECT_PATH=$(nix-store --store ./x --add example.txt) + +PATH1=$(nix path-info --store ./x $CORRECT_PATH) +[ $CORRECT_PATH == $PATH1 ] + +PATH2=$(nix path-info --store "$PWD/x" $CORRECT_PATH) +[ $CORRECT_PATH == $PATH2 ] + +# FIXME we could also test the query parameter version: +# PATH3=$(nix path-info --store "local?store=$PWD/x" $CORRECT_PATH) +# [ $CORRECT_PATH == $PATH3 ] diff --git a/tests/local.mk b/tests/local.mk index f3ac330d8..0f3bfe606 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -6,7 +6,7 @@ nix_tests = \ gc-auto.sh \ referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \ gc-runtime.sh check-refs.sh filter-source.sh \ - remote-store.sh export.sh export-graph.sh \ + local-store.sh remote-store.sh export.sh export-graph.sh \ timeout.sh secure-drv-outputs.sh nix-channel.sh \ multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \ binary-cache.sh nix-profile.sh repair.sh dump-db.sh case-hack.sh \ @@ -31,13 +31,14 @@ nix_tests = \ nix-copy-ssh.sh \ post-hook.sh \ function-trace.sh \ - recursive.sh + recursive.sh \ + flakes.sh # parallel.sh install-tests += $(foreach x, $(nix_tests), tests/$(x)) tests-environment = NIX_REMOTE= $(bash) -e -clean-files += $(d)/common.sh +clean-files += $(d)/common.sh $(d)/config.nix test-deps += tests/common.sh tests/config.nix tests/plugins/libplugintest.$(SO_EXT) diff --git a/tests/nix-copy-closure.nix b/tests/nix-copy-closure.nix index bb5db7410..9c9d119b7 100644 --- a/tests/nix-copy-closure.nix +++ b/tests/nix-copy-closure.nix @@ -1,8 +1,11 @@ # Test ‘nix-copy-closure’. -{ nixpkgs, system, nix }: +{ nixpkgs, system, overlay }: -with import (nixpkgs + "/nixos/lib/testing.nix") { inherit system; }; +with import (nixpkgs + "/nixos/lib/testing.nix") { + inherit system; + extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; +}; makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in { @@ -11,7 +14,6 @@ makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in { { config, lib, pkgs, ... }: { virtualisation.writableStore = true; virtualisation.pathsInNixDB = [ pkgA ]; - nix.package = nix; nix.binaryCaches = lib.mkForce [ ]; }; @@ -20,7 +22,6 @@ makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in { { services.openssh.enable = true; virtualisation.writableStore = true; virtualisation.pathsInNixDB = [ pkgB pkgC ]; - nix.package = nix; }; }; diff --git a/tests/path.nix b/tests/path.nix new file mode 100644 index 000000000..883c3c41b --- /dev/null +++ b/tests/path.nix @@ -0,0 +1,14 @@ +with import ./config.nix; + +mkDerivation { + name = "filter"; + builder = builtins.toFile "builder" "ln -s $input $out"; + input = + builtins.path { + path = ((builtins.getEnv "TEST_ROOT") + "/filterin"); + filter = path: type: + type != "symlink" + && baseNameOf path != "foo" + && !((import ./lang/lib.nix).hasSuffix ".bak" (baseNameOf path)); + }; +} diff --git a/tests/plugins.sh b/tests/plugins.sh index 4b1baeddc..50bfaf7e9 100644 --- a/tests/plugins.sh +++ b/tests/plugins.sh @@ -2,6 +2,6 @@ source common.sh set -o pipefail -res=$(nix eval '(builtins.anotherNull)' --option setting-set true --option plugin-files $PWD/plugins/libplugintest*) +res=$(nix eval --expr builtins.anotherNull --option setting-set true --option plugin-files $PWD/plugins/libplugintest*) [ "$res"x = "nullx" ] diff --git a/tests/pure-eval.sh b/tests/pure-eval.sh index 49c856448..43a765997 100644 --- a/tests/pure-eval.sh +++ b/tests/pure-eval.sh @@ -2,17 +2,17 @@ source common.sh clearStore -nix eval --pure-eval '(assert 1 + 2 == 3; true)' +nix eval --expr 'assert 1 + 2 == 3; true' -[[ $(nix eval '(builtins.readFile ./pure-eval.sh)') =~ clearStore ]] +[[ $(nix eval --impure --expr 'builtins.readFile ./pure-eval.sh') =~ clearStore ]] -(! nix eval --pure-eval '(builtins.readFile ./pure-eval.sh)') +(! nix eval --expr 'builtins.readFile ./pure-eval.sh') -(! nix eval --pure-eval '(builtins.currentTime)') -(! nix eval --pure-eval '(builtins.currentSystem)') +(! nix eval --expr builtins.currentTime) +(! nix eval --expr builtins.currentSystem) (! nix-instantiate --pure-eval ./simple.nix) -[[ $(nix eval "((import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x)") == 123 ]] -(! nix eval --pure-eval "((import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x)") -nix eval --pure-eval "((import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; sha256 = \"$(nix hash-file pure-eval.nix --type sha256)\"; })).x)" +[[ $(nix eval --impure --expr "(import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x") == 123 ]] +(! nix eval --expr "(import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x") +nix eval --expr "(import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; sha256 = \"$(nix hash-file pure-eval.nix --type sha256)\"; })).x" diff --git a/tests/recursive.sh b/tests/recursive.sh index 2d4f83895..cf10d55bf 100644 --- a/tests/recursive.sh +++ b/tests/recursive.sh @@ -9,7 +9,7 @@ rm -f $TEST_ROOT/result export unreachable=$(nix add-to-store ./recursive.sh) -nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L '( +nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --expr ' with import ./config.nix; with import ; mkDerivation { @@ -49,7 +49,7 @@ nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/resu [[ $(nix $opts path-info --all | wc -l) -eq 3 ]] # Build a derivation. - nix $opts build -L '\''( + nix $opts build -L --impure --expr '\'' derivation { name = "inner1"; builder = builtins.getEnv "SHELL"; @@ -57,13 +57,13 @@ nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/resu fnord = builtins.toFile "fnord" "fnord"; args = [ "-c" "echo $fnord blaat > $out" ]; } - )'\'' + '\'' [[ $(nix $opts path-info --json ./result) =~ fnord ]] ln -s $(nix $opts path-info ./result) $out/inner1 '\'\''; - }) + } ' [[ $(cat $TEST_ROOT/result/inner1) =~ blaat ]] diff --git a/tests/remote-builds.nix b/tests/remote-builds.nix index 18d490830..153956619 100644 --- a/tests/remote-builds.nix +++ b/tests/remote-builds.nix @@ -1,8 +1,11 @@ # Test Nix's remote build feature. -{ nixpkgs, system, nix }: +{ nixpkgs, system, overlay }: -with import (nixpkgs + "/nixos/lib/testing.nix") { inherit system; }; +with import (nixpkgs + "/nixos/lib/testing.nix") { + inherit system; + extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; +}; makeTest ( @@ -13,7 +16,6 @@ let { config, pkgs, ... }: { services.openssh.enable = true; virtualisation.writableStore = true; - nix.package = nix; nix.useSandbox = true; }; @@ -59,7 +61,6 @@ in ]; virtualisation.writableStore = true; virtualisation.pathsInNixDB = [ config.system.build.extraUtils ]; - nix.package = nix; nix.binaryCaches = lib.mkForce [ ]; programs.ssh.extraConfig = "ConnectTimeout 30"; }; diff --git a/tests/restricted.sh b/tests/restricted.sh index e02becc60..242b901dd 100644 --- a/tests/restricted.sh +++ b/tests/restricted.sh @@ -17,18 +17,18 @@ nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../src/nix-channel' (! nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in ') nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in ' -I src=. -p=$(nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --restrict-eval --allowed-uris "file://$(pwd)") +p=$(nix eval --raw --expr "builtins.fetchurl file://$(pwd)/restricted.sh" --impure --restrict-eval --allowed-uris "file://$(pwd)") cmp $p restricted.sh -(! nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --restrict-eval) +(! nix eval --raw --expr "builtins.fetchurl file://$(pwd)/restricted.sh" --impure --restrict-eval) -(! nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --restrict-eval --allowed-uris "file://$(pwd)/restricted.sh/") +(! nix eval --raw --expr "builtins.fetchurl file://$(pwd)/restricted.sh" --impure --restrict-eval --allowed-uris "file://$(pwd)/restricted.sh/") -nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --restrict-eval --allowed-uris "file://$(pwd)/restricted.sh" +nix eval --raw --expr "builtins.fetchurl file://$(pwd)/restricted.sh" --impure --restrict-eval --allowed-uris "file://$(pwd)/restricted.sh" -(! nix eval --raw "(builtins.fetchurl https://github.com/NixOS/patchelf/archive/master.tar.gz)" --restrict-eval) -(! nix eval --raw "(builtins.fetchTarball https://github.com/NixOS/patchelf/archive/master.tar.gz)" --restrict-eval) -(! nix eval --raw "(fetchGit git://github.com/NixOS/patchelf.git)" --restrict-eval) +(! nix eval --raw --expr "builtins.fetchurl https://github.com/NixOS/patchelf/archive/master.tar.gz" --impure --restrict-eval) +(! nix eval --raw --expr "builtins.fetchTarball https://github.com/NixOS/patchelf/archive/master.tar.gz" --impure --restrict-eval) +(! nix eval --raw --expr "fetchGit git://github.com/NixOS/patchelf.git" --impure --restrict-eval) ln -sfn $(pwd)/restricted.nix $TEST_ROOT/restricted.nix [[ $(nix-instantiate --eval $TEST_ROOT/restricted.nix) == 3 ]] @@ -37,7 +37,7 @@ ln -sfn $(pwd)/restricted.nix $TEST_ROOT/restricted.nix (! nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I .) nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I $TEST_ROOT -I . -[[ $(nix eval --raw --restrict-eval -I . '(builtins.readFile "${import ./simple.nix}/hello")') == 'Hello World!' ]] +[[ $(nix eval --raw --impure --restrict-eval -I . --expr 'builtins.readFile "${import ./simple.nix}/hello"') == 'Hello World!' ]] # Check whether we can leak symlink information through directory traversal. traverseDir="$(pwd)/restricted-traverse-me" @@ -45,7 +45,7 @@ ln -sfn "$(pwd)/restricted-secret" "$(pwd)/restricted-innocent" mkdir -p "$traverseDir" goUp="..$(echo "$traverseDir" | sed -e 's,[^/]\+,..,g')" output="$(nix eval --raw --restrict-eval -I "$traverseDir" \ - "(builtins.readFile \"$traverseDir/$goUp$(pwd)/restricted-innocent\")" \ + --expr "builtins.readFile \"$traverseDir/$goUp$(pwd)/restricted-innocent\"" \ 2>&1 || :)" echo "$output" | grep "is forbidden" ! echo "$output" | grep -F restricted-secret diff --git a/tests/search.sh b/tests/search.sh index 14da3127b..ee3261687 100644 --- a/tests/search.sh +++ b/tests/search.sh @@ -3,41 +3,23 @@ source common.sh clearStore clearCache -# No packages -(( $(NIX_PATH= nix search -u|wc -l) == 0 )) - -# Haven't updated cache, still nothing -(( $(nix search -f search.nix hello|wc -l) == 0 )) -(( $(nix search -f search.nix |wc -l) == 0 )) - -# Update cache, search should work -(( $(nix search -f search.nix -u hello|wc -l) > 0 )) - -# Use cache -(( $(nix search -f search.nix foo|wc -l) > 0 )) -(( $(nix search foo|wc -l) > 0 )) - -# Test --no-cache works -# No results from cache -(( $(nix search --no-cache foo |wc -l) == 0 )) -# Does find results from file pointed at -(( $(nix search -f search.nix --no-cache foo |wc -l) > 0 )) +(( $(nix search -f search.nix '' hello | wc -l) > 0 )) # Check descriptions are searched -(( $(nix search broken | wc -l) > 0 )) +(( $(nix search -f search.nix '' broken | wc -l) > 0 )) # Check search that matches nothing -(( $(nix search nosuchpackageexists | wc -l) == 0 )) +(( $(nix search -f search.nix '' nosuchpackageexists | wc -l) == 0 )) # Search for multiple arguments -(( $(nix search hello empty | wc -l) == 3 )) +(( $(nix search -f search.nix '' hello empty | wc -l) == 2 )) # Multiple arguments will not exist -(( $(nix search hello broken | wc -l) == 0 )) +(( $(nix search -f search.nix '' hello broken | wc -l) == 0 )) ## Search expressions # Check that empty search string matches all -nix search|grep -q foo -nix search|grep -q bar -nix search|grep -q hello +nix search -f search.nix '' |grep -q foo +nix search -f search.nix '' |grep -q bar +nix search -f search.nix '' |grep -q hello diff --git a/tests/setuid.nix b/tests/setuid.nix index 63d3c05cb..6f2f7d392 100644 --- a/tests/setuid.nix +++ b/tests/setuid.nix @@ -1,15 +1,17 @@ # Verify that Linux builds cannot create setuid or setgid binaries. -{ nixpkgs, system, nix }: +{ nixpkgs, system, overlay }: -with import (nixpkgs + "/nixos/lib/testing.nix") { inherit system; }; +with import (nixpkgs + "/nixos/lib/testing.nix") { + inherit system; + extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; +}; makeTest { machine = { config, lib, pkgs, ... }: { virtualisation.writableStore = true; - nix.package = nix; nix.binaryCaches = lib.mkForce [ ]; nix.nixPath = [ "nixpkgs=${lib.cleanSource pkgs.path}" ]; virtualisation.pathsInNixDB = [ pkgs.stdenv pkgs.pkgsi686Linux.stdenv ]; diff --git a/tests/shell.nix b/tests/shell.nix index 6cb4f082b..6ce59b416 100644 --- a/tests/shell.nix +++ b/tests/shell.nix @@ -24,6 +24,7 @@ let pkgs = rec { VAR_FROM_NIX = "bar"; TEST_inNixShell = if inNixShell then "true" else "false"; inherit stdenv; + outputs = ["dev" "out"]; }; # Used by nix-shell -p diff --git a/tests/tarball.sh b/tests/tarball.sh index b3ec16d40..88a1a07a0 100644 --- a/tests/tarball.sh +++ b/tests/tarball.sh @@ -27,10 +27,13 @@ test_tarball() { nix-build -o $TEST_ROOT/result -E "import (fetchTarball file://$tarball)" - nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree file://$tarball)" - nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })" - nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })" - nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" 2>&1 | grep 'NAR hash mismatch in input' + nix-build -o $TEST_ROOT/result -E "import (fetchTree file://$tarball)" + nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })" + nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })" + nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" 2>&1 | grep 'NAR hash mismatch in input' + + nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" >&2 + nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" 2>&1 | grep 'true' nix-instantiate --eval -E '1 + 2' -I fnord=file://no-such-tarball.tar$ext nix-instantiate --eval -E 'with ; 1 + 2' -I fnord=file://no-such-tarball$ext