Merge branch 'hash-always-has-type' of github.com:obsidiansystems/nix into better-ca-parse-errors

This commit is contained in:
John Ericson 2020-07-16 17:28:52 +00:00
commit cc0d77f8c9
138 changed files with 8552 additions and 2192 deletions

View file

@ -10,15 +10,8 @@ jobs:
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
with:
fetch-depth: 0
- uses: cachix/install-nix-action@v10 - uses: cachix/install-nix-action@v10
- run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings #- run: nix flake check
macos_perf_test: - run: nix-build -A checks.$(if [[ `uname` = Linux ]]; then echo x86_64-linux; else echo x86_64-darwin; fi)
runs-on: macos-latest
steps:
- name: Disable syspolicy assessments
run: |
spctl --status
sudo spctl --master-disable
- uses: actions/checkout@v2
- uses: cachix/install-nix-action@v10
- run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings

View file

@ -11,6 +11,7 @@ makefiles = \
src/resolve-system-dependencies/local.mk \ src/resolve-system-dependencies/local.mk \
scripts/local.mk \ scripts/local.mk \
corepkgs/local.mk \ corepkgs/local.mk \
misc/bash/local.mk \
misc/systemd/local.mk \ misc/systemd/local.mk \
misc/launchd/local.mk \ misc/launchd/local.mk \
misc/upstart/local.mk \ misc/upstart/local.mk \

View file

@ -123,6 +123,7 @@ AC_PATH_PROG(flex, flex, false)
AC_PATH_PROG(bison, bison, false) AC_PATH_PROG(bison, bison, false)
AC_PATH_PROG(dot, dot) AC_PATH_PROG(dot, dot)
AC_PATH_PROG(lsof, lsof, lsof) AC_PATH_PROG(lsof, lsof, lsof)
NEED_PROG(jq, jq)
AC_SUBST(coreutils, [$(dirname $(type -p cat))]) AC_SUBST(coreutils, [$(dirname $(type -p cat))])

3
default.nix Normal file
View file

@ -0,0 +1,3 @@
(import (fetchTarball https://github.com/edolstra/flake-compat/archive/master.tar.gz) {
src = ./.;
}).defaultNix

26
flake.lock Normal file
View file

@ -0,0 +1,26 @@
{
"nodes": {
"nixpkgs": {
"locked": {
"lastModified": 1591633336,
"narHash": "sha256-oVXv4xAnDJB03LvZGbC72vSVlIbbJr8tpjEW5o/Fdek=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "70717a337f7ae4e486ba71a500367cad697e5f09",
"type": "github"
},
"original": {
"id": "nixpkgs",
"ref": "nixos-20.03-small",
"type": "indirect"
}
},
"root": {
"inputs": {
"nixpkgs": "nixpkgs"
}
}
},
"root": "root",
"version": 6
}

443
flake.nix Normal file
View file

@ -0,0 +1,443 @@
{
description = "The purely functional package manager";
inputs.nixpkgs.url = "nixpkgs/nixos-20.03-small";
outputs = { self, nixpkgs }:
let
version = builtins.readFile ./.version + versionSuffix;
versionSuffix =
if officialRelease
then ""
else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified)}_${self.shortRev or "dirty"}";
officialRelease = false;
systems = [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ];
forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system);
# Memoize nixpkgs for different platforms for efficiency.
nixpkgsFor = forAllSystems (system:
import nixpkgs {
inherit system;
overlays = [ self.overlay ];
}
);
commonDeps = pkgs: with pkgs; rec {
# Use "busybox-sandbox-shell" if present,
# if not (legacy) fallback and hope it's sufficient.
sh = pkgs.busybox-sandbox-shell or (busybox.override {
useMusl = true;
enableStatic = true;
enableMinimal = true;
extraConfig = ''
CONFIG_FEATURE_FANCY_ECHO y
CONFIG_FEATURE_SH_MATH y
CONFIG_FEATURE_SH_MATH_64 y
CONFIG_ASH y
CONFIG_ASH_OPTIMIZE_FOR_SIZE y
CONFIG_ASH_ALIAS y
CONFIG_ASH_BASH_COMPAT y
CONFIG_ASH_CMDCMD y
CONFIG_ASH_ECHO y
CONFIG_ASH_GETOPTS y
CONFIG_ASH_INTERNAL_GLOB y
CONFIG_ASH_JOB_CONTROL y
CONFIG_ASH_PRINTF y
CONFIG_ASH_TEST y
'';
});
configureFlags =
lib.optionals stdenv.isLinux [
"--with-sandbox-shell=${sh}/bin/busybox"
];
buildDeps =
[ bison
flex
libxml2
libxslt
docbook5
docbook_xsl_ns
autoconf-archive
autoreconfHook
curl
bzip2 xz brotli zlib editline
openssl pkgconfig sqlite
libarchive
boost
(if lib.versionAtLeast lib.version "20.03pre"
then nlohmann_json
else nlohmann_json.override { multipleHeaders = true; })
nlohmann_json
# Tests
git
mercurial
jq
gmock
]
++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal]
++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
++ lib.optional (stdenv.isLinux || stdenv.isDarwin)
(aws-sdk-cpp.override {
apis = ["s3" "transfer"];
customMemoryManagement = false;
});
propagatedDeps =
[ (boehmgc.override { enableLargeConfig = true; })
];
perlDeps =
[ perl
perlPackages.DBDSQLite
];
};
in {
# A Nixpkgs overlay that overrides the 'nix' and
# 'nix.perl-bindings' packages.
overlay = final: prev: {
nix = with final; with commonDeps pkgs; (stdenv.mkDerivation {
name = "nix-${version}";
src = self;
VERSION_SUFFIX = versionSuffix;
outputs = [ "out" "dev" "doc" ];
buildInputs = buildDeps;
propagatedBuildInputs = propagatedDeps;
preConfigure =
''
# Copy libboost_context so we don't get all of Boost in our closure.
# https://github.com/NixOS/nixpkgs/issues/45462
mkdir -p $out/lib
cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
rm -f $out/lib/*.a
${lib.optionalString stdenv.isLinux ''
chmod u+w $out/lib/*.so.*
patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
''}
'';
configureFlags = configureFlags ++
[ "--sysconfdir=/etc" ];
enableParallelBuilding = true;
makeFlags = "profiledir=$(out)/etc/profile.d";
doCheck = true;
installFlags = "sysconfdir=$(out)/etc";
postInstall = ''
mkdir -p $doc/nix-support
echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
'';
doInstallCheck = true;
installCheckFlags = "sysconfdir=$(out)/etc";
separateDebugInfo = true;
}) // {
perl-bindings = with final; stdenv.mkDerivation {
name = "nix-perl-${version}";
src = self;
buildInputs =
[ autoconf-archive
autoreconfHook
nix
curl
bzip2
xz
pkgconfig
pkgs.perl
boost
]
++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium;
configureFlags = ''
--with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}
--with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix}
'';
enableParallelBuilding = true;
postUnpack = "sourceRoot=$sourceRoot/perl";
};
};
};
hydraJobs = {
# Binary package for various platforms.
build = nixpkgs.lib.genAttrs systems (system: nixpkgsFor.${system}.nix);
# Perl bindings for various platforms.
perlBindings = nixpkgs.lib.genAttrs systems (system: nixpkgsFor.${system}.nix.perl-bindings);
# Binary tarball for various platforms, containing a Nix store
# with the closure of 'nix' package, and the second half of
# the installation script.
binaryTarball = nixpkgs.lib.genAttrs systems (system:
with nixpkgsFor.${system};
let
installerClosureInfo = closureInfo { rootPaths = [ nix cacert ]; };
in
runCommand "nix-binary-tarball-${version}"
{ #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck;
meta.description = "Distribution-independent Nix bootstrap binaries for ${system}";
}
''
cp ${installerClosureInfo}/registration $TMPDIR/reginfo
substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \
--subst-var-by nix ${nix} \
--subst-var-by cacert ${cacert}
substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \
--subst-var-by nix ${nix} \
--subst-var-by cacert ${cacert}
substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \
--subst-var-by nix ${nix} \
--subst-var-by cacert ${cacert}
substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \
--subst-var-by nix ${nix} \
--subst-var-by cacert ${cacert}
if type -p shellcheck; then
# SC1090: Don't worry about not being able to find
# $nix/etc/profile.d/nix.sh
shellcheck --exclude SC1090 $TMPDIR/install
shellcheck $TMPDIR/install-darwin-multi-user.sh
shellcheck $TMPDIR/install-systemd-multi-user.sh
# SC1091: Don't panic about not being able to source
# /etc/profile
# SC2002: Ignore "useless cat" "error", when loading
# .reginfo, as the cat is a much cleaner
# implementation, even though it is "useless"
# SC2116: Allow ROOT_HOME=$(echo ~root) for resolving
# root's home directory
shellcheck --external-sources \
--exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user
fi
chmod +x $TMPDIR/install
chmod +x $TMPDIR/install-darwin-multi-user.sh
chmod +x $TMPDIR/install-systemd-multi-user.sh
chmod +x $TMPDIR/install-multi-user
dir=nix-${version}-${system}
fn=$out/$dir.tar.xz
mkdir -p $out/nix-support
echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products
tar cvfJ $fn \
--owner=0 --group=0 --mode=u+rw,uga+r \
--absolute-names \
--hard-dereference \
--transform "s,$TMPDIR/install,$dir/install," \
--transform "s,$TMPDIR/reginfo,$dir/.reginfo," \
--transform "s,$NIX_STORE,$dir/store,S" \
$TMPDIR/install $TMPDIR/install-darwin-multi-user.sh \
$TMPDIR/install-systemd-multi-user.sh \
$TMPDIR/install-multi-user $TMPDIR/reginfo \
$(cat ${installerClosureInfo}/store-paths)
'');
# The first half of the installation script. This is uploaded
# to https://nixos.org/nix/install. It downloads the binary
# tarball for the user's system and calls the second half of the
# installation script.
installerScript =
with nixpkgsFor.x86_64-linux;
runCommand "installer-script"
{ buildInputs = [ nix ];
}
''
mkdir -p $out/nix-support
substitute ${./scripts/install.in} $out/install \
${pkgs.lib.concatMapStrings
(system: "--replace '@binaryTarball_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) ")
[ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
} \
--replace '@nixVersion@' ${version}
echo "file installer $out/install" >> $out/nix-support/hydra-build-products
'';
# Line coverage analysis.
coverage =
with nixpkgsFor.x86_64-linux;
with commonDeps pkgs;
releaseTools.coverageAnalysis {
name = "nix-coverage-${version}";
src = self;
enableParallelBuilding = true;
buildInputs = buildDeps ++ propagatedDeps;
dontInstall = false;
doInstallCheck = true;
lcovFilter = [ "*/boost/*" "*-tab.*" ];
# We call `dot', and even though we just use it to
# syntax-check generated dot files, it still requires some
# fonts. So provide those.
FONTCONFIG_FILE = texFunctions.fontsConf;
# To test building without precompiled headers.
makeFlagsArray = [ "PRECOMPILE_HEADERS=0" ];
};
# System tests.
tests.remoteBuilds = import ./tests/remote-builds.nix {
system = "x86_64-linux";
inherit nixpkgs;
inherit (self) overlay;
};
tests.nix-copy-closure = import ./tests/nix-copy-closure.nix {
system = "x86_64-linux";
inherit nixpkgs;
inherit (self) overlay;
};
tests.githubFlakes = (import ./tests/github-flakes.nix rec {
system = "x86_64-linux";
inherit nixpkgs;
inherit (self) overlay;
});
tests.setuid = nixpkgs.lib.genAttrs
["i686-linux" "x86_64-linux"]
(system:
import ./tests/setuid.nix rec {
inherit nixpkgs system;
inherit (self) overlay;
});
# Test whether the binary tarball works in an Ubuntu system.
tests.binaryTarball =
with nixpkgsFor.x86_64-linux;
vmTools.runInLinuxImage (runCommand "nix-binary-tarball-test"
{ diskImage = vmTools.diskImages.ubuntu1204x86_64;
}
''
set -x
useradd -m alice
su - alice -c 'tar xf ${self.hydraJobs.binaryTarball.x86_64-linux}/*.tar.*'
mkdir /dest-nix
mount -o bind /dest-nix /nix # Provide a writable /nix.
chown alice /nix
su - alice -c '_NIX_INSTALLER_TEST=1 ./nix-*/install'
su - alice -c 'nix-store --verify'
su - alice -c 'PAGER= nix-store -qR ${self.hydraJobs.build.x86_64-linux}'
# Check whether 'nix upgrade-nix' works.
cat > /tmp/paths.nix <<EOF
{
x86_64-linux = "${self.hydraJobs.build.x86_64-linux}";
}
EOF
su - alice -c 'nix --experimental-features nix-command upgrade-nix -vvv --nix-store-paths-url file:///tmp/paths.nix'
(! [ -L /home/alice/.profile-1-link ])
su - alice -c 'PAGER= nix-store -qR ${self.hydraJobs.build.x86_64-linux}'
mkdir -p $out/nix-support
touch $out/nix-support/hydra-build-products
umount /nix
'');
/*
# Check whether we can still evaluate all of Nixpkgs.
tests.evalNixpkgs =
import (nixpkgs + "/pkgs/top-level/make-tarball.nix") {
# FIXME: fix pkgs/top-level/make-tarball.nix in NixOS to not require a revCount.
inherit nixpkgs;
pkgs = nixpkgsFor.x86_64-linux;
officialRelease = false;
};
# Check whether we can still evaluate NixOS.
tests.evalNixOS =
with nixpkgsFor.x86_64-linux;
runCommand "eval-nixos" { buildInputs = [ nix ]; }
''
export NIX_STATE_DIR=$TMPDIR
nix-instantiate ${nixpkgs}/nixos/release-combined.nix -A tested --dry-run \
--arg nixpkgs '{ outPath = ${nixpkgs}; revCount = 123; shortRev = "abcdefgh"; }'
touch $out
'';
*/
};
checks = forAllSystems (system: {
binaryTarball = self.hydraJobs.binaryTarball.${system};
perlBindings = self.hydraJobs.perlBindings.${system};
});
packages = forAllSystems (system: {
inherit (nixpkgsFor.${system}) nix;
});
defaultPackage = forAllSystems (system: self.packages.${system}.nix);
devShell = forAllSystems (system:
with nixpkgsFor.${system};
with commonDeps pkgs;
stdenv.mkDerivation {
name = "nix";
buildInputs = buildDeps ++ propagatedDeps ++ perlDeps;
inherit configureFlags;
enableParallelBuilding = true;
installFlags = "sysconfdir=$(out)/etc";
shellHook =
''
export prefix=$(pwd)/inst
configureFlags+=" --prefix=$prefix"
PKG_CONFIG_PATH=$prefix/lib/pkgconfig:$PKG_CONFIG_PATH
PATH=$prefix/bin:$PATH
unset PYTHONPATH
'';
});
};
}

View file

@ -8,7 +8,7 @@ clean-files += Makefile.config
GLOBAL_CXXFLAGS += -Wno-deprecated-declarations GLOBAL_CXXFLAGS += -Wno-deprecated-declarations
$(foreach i, config.h $(call rwildcard, src/lib*, *.hh), \ $(foreach i, config.h $(wildcard src/lib*/*.hh), \
$(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644)))
$(GCH) $(PCH): src/libutil/util.hh config.h $(GCH) $(PCH): src/libutil/util.hh config.h

19
misc/bash/completion.sh Normal file
View file

@ -0,0 +1,19 @@
function _complete_nix {
local -a words
local cword cur
_get_comp_words_by_ref -n ':=&' words cword cur
local have_type
while IFS= read -r line; do
if [[ -z $have_type ]]; then
have_type=1
if [[ $line = filenames ]]; then
compopt -o filenames
fi
else
COMPREPLY+=("$line")
fi
done < <(NIX_GET_COMPLETIONS=$cword "${words[@]}")
__ltrim_colon_completions "$cur"
}
complete -F _complete_nix nix

1
misc/bash/local.mk Normal file
View file

@ -0,0 +1 @@
$(eval $(call install-file-as, $(d)/completion.sh, $(datarootdir)/bash-completion/completions/nix, 0644))

View file

@ -182,7 +182,7 @@ void importPaths(int fd, int dontCheckSigs)
PPCODE: PPCODE:
try { try {
FdSource source(fd); FdSource source(fd);
store()->importPaths(source, nullptr, dontCheckSigs ? NoCheckSigs : CheckSigs); store()->importPaths(source, dontCheckSigs ? NoCheckSigs : CheckSigs);
} catch (Error & e) { } catch (Error & e) {
croak("%s", e.what()); croak("%s", e.what());
} }

View file

@ -1,82 +0,0 @@
{ pkgs }:
with pkgs;
rec {
# Use "busybox-sandbox-shell" if present,
# if not (legacy) fallback and hope it's sufficient.
sh = pkgs.busybox-sandbox-shell or (busybox.override {
useMusl = true;
enableStatic = true;
enableMinimal = true;
extraConfig = ''
CONFIG_FEATURE_FANCY_ECHO y
CONFIG_FEATURE_SH_MATH y
CONFIG_FEATURE_SH_MATH_64 y
CONFIG_ASH y
CONFIG_ASH_OPTIMIZE_FOR_SIZE y
CONFIG_ASH_ALIAS y
CONFIG_ASH_BASH_COMPAT y
CONFIG_ASH_CMDCMD y
CONFIG_ASH_ECHO y
CONFIG_ASH_GETOPTS y
CONFIG_ASH_INTERNAL_GLOB y
CONFIG_ASH_JOB_CONTROL y
CONFIG_ASH_PRINTF y
CONFIG_ASH_TEST y
'';
});
configureFlags =
lib.optionals stdenv.isLinux [
"--with-sandbox-shell=${sh}/bin/busybox"
];
buildDeps =
[ bison
flex
libxml2
libxslt
docbook5
docbook_xsl_ns
autoconf-archive
autoreconfHook
curl
bzip2 xz brotli zlib editline
openssl pkgconfig sqlite
libarchive
boost
nlohmann_json
# Tests
git
mercurial
gmock
]
++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal]
++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
++ lib.optional (stdenv.isLinux || stdenv.isDarwin)
((aws-sdk-cpp.override {
apis = ["s3" "transfer"];
customMemoryManagement = false;
}).overrideDerivation (args: {
/*
patches = args.patches or [] ++ [ (fetchpatch {
url = https://github.com/edolstra/aws-sdk-cpp/commit/3e07e1f1aae41b4c8b340735ff9e8c735f0c063f.patch;
sha256 = "1pij0v449p166f9l29x7ppzk8j7g9k9mp15ilh5qxp29c7fnvxy2";
}) ];
*/
}));
propagatedDeps =
[ (boehmgc.override { enableLargeConfig = true; })
];
perlDeps =
[ perl
perlPackages.DBDSQLite
];
}

View file

@ -1,303 +0,0 @@
{ nix ? builtins.fetchGit ./.
, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-20.03-small.tar.gz
, officialRelease ? false
, systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
}:
let
pkgs = import nixpkgs { system = builtins.currentSystem or "x86_64-linux"; };
version =
builtins.readFile ./.version
+ (if officialRelease then "" else "pre${toString nix.revCount}_${nix.shortRev}");
jobs = rec {
build = pkgs.lib.genAttrs systems (system:
let pkgs = import nixpkgs { inherit system; }; in
with pkgs;
with import ./release-common.nix { inherit pkgs; };
stdenv.mkDerivation {
name = "nix-${version}";
src = nix;
outputs = [ "out" "dev" "doc" ];
buildInputs = buildDeps;
propagatedBuildInputs = propagatedDeps;
preConfigure =
''
# Copy libboost_context so we don't get all of Boost in our closure.
# https://github.com/NixOS/nixpkgs/issues/45462
mkdir -p $out/lib
cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
rm -f $out/lib/*.a
${lib.optionalString stdenv.isLinux ''
chmod u+w $out/lib/*.so.*
patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
''}
(cd perl; autoreconf --install --force --verbose)
'';
configureFlags = configureFlags ++
[ "--sysconfdir=/etc" ];
enableParallelBuilding = true;
makeFlags = "profiledir=$(out)/etc/profile.d";
installFlags = "sysconfdir=$(out)/etc";
postInstall = ''
mkdir -p $doc/nix-support
echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
'';
doCheck = true;
doInstallCheck = true;
installCheckFlags = "sysconfdir=$(out)/etc";
separateDebugInfo = true;
});
perlBindings = pkgs.lib.genAttrs systems (system:
let pkgs = import nixpkgs { inherit system; }; in with pkgs;
releaseTools.nixBuild {
name = "nix-perl-${version}";
src = nix;
buildInputs =
[ autoconf-archive
autoreconfHook
jobs.build.${system}
curl
bzip2
xz
pkgconfig
pkgs.perl
boost
]
++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium;
configureFlags = ''
--with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}
--with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix}
'';
enableParallelBuilding = true;
postUnpack = "sourceRoot=$sourceRoot/perl";
});
binaryTarball = pkgs.lib.genAttrs systems (system:
with import nixpkgs { inherit system; };
let
toplevel = builtins.getAttr system jobs.build;
installerClosureInfo = closureInfo { rootPaths = [ toplevel cacert ]; };
in
runCommand "nix-binary-tarball-${version}"
{ #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck;
meta.description = "Distribution-independent Nix bootstrap binaries for ${system}";
}
''
cp ${installerClosureInfo}/registration $TMPDIR/reginfo
cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh
substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \
--subst-var-by nix ${toplevel} \
--subst-var-by cacert ${cacert}
substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \
--subst-var-by nix ${toplevel} \
--subst-var-by cacert ${cacert}
substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \
--subst-var-by nix ${toplevel} \
--subst-var-by cacert ${cacert}
substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \
--subst-var-by nix ${toplevel} \
--subst-var-by cacert ${cacert}
if type -p shellcheck; then
# SC1090: Don't worry about not being able to find
# $nix/etc/profile.d/nix.sh
shellcheck --exclude SC1090 $TMPDIR/install
shellcheck $TMPDIR/create-darwin-volume.sh
shellcheck $TMPDIR/install-darwin-multi-user.sh
shellcheck $TMPDIR/install-systemd-multi-user.sh
# SC1091: Don't panic about not being able to source
# /etc/profile
# SC2002: Ignore "useless cat" "error", when loading
# .reginfo, as the cat is a much cleaner
# implementation, even though it is "useless"
# SC2116: Allow ROOT_HOME=$(echo ~root) for resolving
# root's home directory
shellcheck --external-sources \
--exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user
fi
chmod +x $TMPDIR/install
chmod +x $TMPDIR/create-darwin-volume.sh
chmod +x $TMPDIR/install-darwin-multi-user.sh
chmod +x $TMPDIR/install-systemd-multi-user.sh
chmod +x $TMPDIR/install-multi-user
dir=nix-${version}-${system}
fn=$out/$dir.tar.xz
mkdir -p $out/nix-support
echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products
tar cvfJ $fn \
--owner=0 --group=0 --mode=u+rw,uga+r \
--absolute-names \
--hard-dereference \
--transform "s,$TMPDIR/install,$dir/install," \
--transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \
--transform "s,$TMPDIR/reginfo,$dir/.reginfo," \
--transform "s,$NIX_STORE,$dir/store,S" \
$TMPDIR/install \
$TMPDIR/create-darwin-volume.sh \
$TMPDIR/install-darwin-multi-user.sh \
$TMPDIR/install-systemd-multi-user.sh \
$TMPDIR/install-multi-user \
$TMPDIR/reginfo \
$(cat ${installerClosureInfo}/store-paths)
'');
coverage =
with pkgs;
with import ./release-common.nix { inherit pkgs; };
releaseTools.coverageAnalysis {
name = "nix-coverage-${version}";
src = nix;
enableParallelBuilding = true;
buildInputs = buildDeps ++ propagatedDeps;
dontInstall = false;
doInstallCheck = true;
lcovFilter = [ "*/boost/*" "*-tab.*" ];
# We call `dot', and even though we just use it to
# syntax-check generated dot files, it still requires some
# fonts. So provide those.
FONTCONFIG_FILE = texFunctions.fontsConf;
# To test building without precompiled headers.
makeFlagsArray = [ "PRECOMPILE_HEADERS=0" ];
};
# System tests.
tests.remoteBuilds = (import ./tests/remote-builds.nix rec {
inherit nixpkgs;
nix = build.x86_64-linux; system = "x86_64-linux";
});
tests.nix-copy-closure = (import ./tests/nix-copy-closure.nix rec {
inherit nixpkgs;
nix = build.x86_64-linux; system = "x86_64-linux";
});
tests.setuid = pkgs.lib.genAttrs
["i686-linux" "x86_64-linux"]
(system:
import ./tests/setuid.nix rec {
inherit nixpkgs;
nix = build.${system}; inherit system;
});
tests.binaryTarball =
with import nixpkgs { system = "x86_64-linux"; };
vmTools.runInLinuxImage (runCommand "nix-binary-tarball-test"
{ diskImage = vmTools.diskImages.ubuntu1204x86_64;
}
''
set -x
useradd -m alice
su - alice -c 'tar xf ${binaryTarball.x86_64-linux}/*.tar.*'
mkdir /dest-nix
mount -o bind /dest-nix /nix # Provide a writable /nix.
chown alice /nix
su - alice -c '_NIX_INSTALLER_TEST=1 ./nix-*/install'
su - alice -c 'nix-store --verify'
su - alice -c 'PAGER= nix-store -qR ${build.x86_64-linux}'
# Check whether 'nix upgrade-nix' works.
cat > /tmp/paths.nix <<EOF
{
x86_64-linux = "${build.x86_64-linux}";
}
EOF
su - alice -c 'nix --experimental-features nix-command upgrade-nix -vvv --nix-store-paths-url file:///tmp/paths.nix'
(! [ -L /home/alice/.profile-1-link ])
su - alice -c 'PAGER= nix-store -qR ${build.x86_64-linux}'
mkdir -p $out/nix-support
touch $out/nix-support/hydra-build-products
umount /nix
''); # */
/*
tests.evalNixpkgs =
import (nixpkgs + "/pkgs/top-level/make-tarball.nix") {
inherit nixpkgs;
inherit pkgs;
nix = build.x86_64-linux;
officialRelease = false;
};
tests.evalNixOS =
pkgs.runCommand "eval-nixos" { buildInputs = [ build.x86_64-linux ]; }
''
export NIX_STATE_DIR=$TMPDIR
nix-instantiate ${nixpkgs}/nixos/release-combined.nix -A tested --dry-run \
--arg nixpkgs '{ outPath = ${nixpkgs}; revCount = 123; shortRev = "abcdefgh"; }'
touch $out
'';
*/
installerScript =
pkgs.runCommand "installer-script"
{ buildInputs = [ build.${builtins.currentSystem or "x86_64-linux"} ]; }
''
mkdir -p $out/nix-support
substitute ${./scripts/install.in} $out/install \
${pkgs.lib.concatMapStrings
(system: "--replace '@binaryTarball_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${binaryTarball.${system}}/*.tar.xz) ")
systems
} \
--replace '@nixVersion@' ${version}
echo "file installer $out/install" >> $out/nix-support/hydra-build-products
'';
};
in jobs

View file

@ -1,25 +0,0 @@
{ useClang ? false }:
with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-20.03-small.tar.gz) {};
with import ./release-common.nix { inherit pkgs; };
(if useClang then clangStdenv else stdenv).mkDerivation {
name = "nix";
buildInputs = buildDeps ++ propagatedDeps ++ perlDeps;
inherit configureFlags;
enableParallelBuilding = true;
installFlags = "sysconfdir=$(out)/etc";
shellHook =
''
export prefix=$(pwd)/inst
configureFlags+=" --prefix=$prefix"
PKG_CONFIG_PATH=$prefix/lib/pkgconfig:$PKG_CONFIG_PATH
PATH=$prefix/bin:$PATH
'';
}

View file

@ -4,6 +4,8 @@
#include "util.hh" #include "util.hh"
#include "eval.hh" #include "eval.hh"
#include "fetchers.hh" #include "fetchers.hh"
#include "registry.hh"
#include "flake/flakeref.hh"
#include "store-api.hh" #include "store-api.hh"
namespace nix { namespace nix {
@ -31,6 +33,27 @@ MixEvalArgs::MixEvalArgs()
.labels = {"path"}, .labels = {"path"},
.handler = {[&](std::string s) { searchPath.push_back(s); }} .handler = {[&](std::string s) { searchPath.push_back(s); }}
}); });
addFlag({
.longName = "impure",
.description = "allow access to mutable paths and repositories",
.handler = {[&]() {
evalSettings.pureEval = false;
}},
});
addFlag({
.longName = "override-flake",
.description = "override a flake registry value",
.labels = {"original-ref", "resolved-ref"},
.handler = {[&](std::string _from, std::string _to) {
auto from = parseFlakeRef(_from, absPath("."));
auto to = parseFlakeRef(_to, absPath("."));
fetchers::Attrs extraAttrs;
if (to.subdir != "") extraAttrs["dir"] = to.subdir;
fetchers::overrideRegistry(from.input, to.input, extraAttrs);
}}
});
} }
Bindings * MixEvalArgs::getAutoArgs(EvalState & state) Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
@ -53,7 +76,7 @@ Path lookupFileArg(EvalState & state, string s)
if (isUri(s)) { if (isUri(s)) {
return state.store->toRealPath( return state.store->toRealPath(
fetchers::downloadTarball( fetchers::downloadTarball(
state.store, resolveUri(s), "source", false).storePath); state.store, resolveUri(s), "source", false).first.storePath);
} else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
Path p = s.substr(1, s.size() - 2); Path p = s.substr(1, s.size() - 2);
return state.findFile(p); return state.findFile(p);

616
src/libexpr/eval-cache.cc Normal file
View file

@ -0,0 +1,616 @@
#include "eval-cache.hh"
#include "sqlite.hh"
#include "eval.hh"
#include "eval-inline.hh"
#include "store-api.hh"
namespace nix::eval_cache {
static const char * schema = R"sql(
create table if not exists Attributes (
parent integer not null,
name text,
type integer not null,
value text,
context text,
primary key (parent, name)
);
)sql";
struct AttrDb
{
std::atomic_bool failed{false};
struct State
{
SQLite db;
SQLiteStmt insertAttribute;
SQLiteStmt insertAttributeWithContext;
SQLiteStmt queryAttribute;
SQLiteStmt queryAttributes;
std::unique_ptr<SQLiteTxn> txn;
};
std::unique_ptr<Sync<State>> _state;
AttrDb(const Hash & fingerprint)
: _state(std::make_unique<Sync<State>>())
{
auto state(_state->lock());
Path cacheDir = getCacheDir() + "/nix/eval-cache-v2";
createDirs(cacheDir);
Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
state->db = SQLite(dbPath);
state->db.isCache();
state->db.exec(schema);
state->insertAttribute.create(state->db,
"insert or replace into Attributes(parent, name, type, value) values (?, ?, ?, ?)");
state->insertAttributeWithContext.create(state->db,
"insert or replace into Attributes(parent, name, type, value, context) values (?, ?, ?, ?, ?)");
state->queryAttribute.create(state->db,
"select rowid, type, value, context from Attributes where parent = ? and name = ?");
state->queryAttributes.create(state->db,
"select name from Attributes where parent = ?");
state->txn = std::make_unique<SQLiteTxn>(state->db);
}
~AttrDb()
{
try {
auto state(_state->lock());
if (!failed)
state->txn->commit();
state->txn.reset();
} catch (...) {
ignoreException();
}
}
template<typename F>
AttrId doSQLite(F && fun)
{
if (failed) return 0;
try {
return fun();
} catch (SQLiteError &) {
ignoreException();
failed = true;
return 0;
}
}
AttrId setAttrs(
AttrKey key,
const std::vector<Symbol> & attrs)
{
return doSQLite([&]()
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::FullAttrs)
(0, false).exec();
AttrId rowId = state->db.getLastInsertedRowId();
assert(rowId);
for (auto & attr : attrs)
state->insertAttribute.use()
(rowId)
(attr)
(AttrType::Placeholder)
(0, false).exec();
return rowId;
});
}
AttrId setString(
AttrKey key,
std::string_view s,
const char * * context = nullptr)
{
return doSQLite([&]()
{
auto state(_state->lock());
if (context) {
std::string ctx;
for (const char * * p = context; *p; ++p) {
if (p != context) ctx.push_back(' ');
ctx.append(*p);
}
state->insertAttributeWithContext.use()
(key.first)
(key.second)
(AttrType::String)
(s)
(ctx).exec();
} else {
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::String)
(s).exec();
}
return state->db.getLastInsertedRowId();
});
}
AttrId setBool(
AttrKey key,
bool b)
{
return doSQLite([&]()
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::Bool)
(b ? 1 : 0).exec();
return state->db.getLastInsertedRowId();
});
}
AttrId setPlaceholder(AttrKey key)
{
return doSQLite([&]()
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::Placeholder)
(0, false).exec();
return state->db.getLastInsertedRowId();
});
}
AttrId setMissing(AttrKey key)
{
return doSQLite([&]()
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::Missing)
(0, false).exec();
return state->db.getLastInsertedRowId();
});
}
AttrId setMisc(AttrKey key)
{
return doSQLite([&]()
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::Misc)
(0, false).exec();
return state->db.getLastInsertedRowId();
});
}
AttrId setFailed(AttrKey key)
{
return doSQLite([&]()
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::Failed)
(0, false).exec();
return state->db.getLastInsertedRowId();
});
}
std::optional<std::pair<AttrId, AttrValue>> getAttr(
AttrKey key,
SymbolTable & symbols)
{
auto state(_state->lock());
auto queryAttribute(state->queryAttribute.use()(key.first)(key.second));
if (!queryAttribute.next()) return {};
auto rowId = (AttrType) queryAttribute.getInt(0);
auto type = (AttrType) queryAttribute.getInt(1);
switch (type) {
case AttrType::Placeholder:
return {{rowId, placeholder_t()}};
case AttrType::FullAttrs: {
// FIXME: expensive, should separate this out.
std::vector<Symbol> attrs;
auto queryAttributes(state->queryAttributes.use()(rowId));
while (queryAttributes.next())
attrs.push_back(symbols.create(queryAttributes.getStr(0)));
return {{rowId, attrs}};
}
case AttrType::String: {
std::vector<std::pair<Path, std::string>> context;
if (!queryAttribute.isNull(3))
for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";"))
context.push_back(decodeContext(s));
return {{rowId, string_t{queryAttribute.getStr(2), context}}};
}
case AttrType::Bool:
return {{rowId, queryAttribute.getInt(2) != 0}};
case AttrType::Missing:
return {{rowId, missing_t()}};
case AttrType::Misc:
return {{rowId, misc_t()}};
case AttrType::Failed:
return {{rowId, failed_t()}};
default:
throw Error("unexpected type in evaluation cache");
}
}
};
static std::shared_ptr<AttrDb> makeAttrDb(const Hash & fingerprint)
{
try {
return std::make_shared<AttrDb>(fingerprint);
} catch (SQLiteError &) {
ignoreException();
return nullptr;
}
}
EvalCache::EvalCache(
std::optional<std::reference_wrapper<const Hash>> useCache,
EvalState & state,
RootLoader rootLoader)
: db(useCache ? makeAttrDb(*useCache) : nullptr)
, state(state)
, rootLoader(rootLoader)
{
}
Value * EvalCache::getRootValue()
{
if (!value) {
debug("getting root value");
value = allocRootValue(rootLoader());
}
return *value;
}
std::shared_ptr<AttrCursor> EvalCache::getRoot()
{
return std::make_shared<AttrCursor>(ref(shared_from_this()), std::nullopt);
}
AttrCursor::AttrCursor(
ref<EvalCache> root,
Parent parent,
Value * value,
std::optional<std::pair<AttrId, AttrValue>> && cachedValue)
: root(root), parent(parent), cachedValue(std::move(cachedValue))
{
if (value)
_value = allocRootValue(value);
}
AttrKey AttrCursor::getKey()
{
if (!parent)
return {0, root->state.sEpsilon};
if (!parent->first->cachedValue) {
parent->first->cachedValue = root->db->getAttr(
parent->first->getKey(), root->state.symbols);
assert(parent->first->cachedValue);
}
return {parent->first->cachedValue->first, parent->second};
}
Value & AttrCursor::getValue()
{
if (!_value) {
if (parent) {
auto & vParent = parent->first->getValue();
root->state.forceAttrs(vParent);
auto attr = vParent.attrs->get(parent->second);
if (!attr)
throw Error("attribute '%s' is unexpectedly missing", getAttrPathStr());
_value = allocRootValue(attr->value);
} else
_value = allocRootValue(root->getRootValue());
}
return **_value;
}
std::vector<Symbol> AttrCursor::getAttrPath() const
{
if (parent) {
auto attrPath = parent->first->getAttrPath();
attrPath.push_back(parent->second);
return attrPath;
} else
return {};
}
std::vector<Symbol> AttrCursor::getAttrPath(Symbol name) const
{
auto attrPath = getAttrPath();
attrPath.push_back(name);
return attrPath;
}
std::string AttrCursor::getAttrPathStr() const
{
return concatStringsSep(".", getAttrPath());
}
std::string AttrCursor::getAttrPathStr(Symbol name) const
{
return concatStringsSep(".", getAttrPath(name));
}
Value & AttrCursor::forceValue()
{
debug("evaluating uncached attribute %s", getAttrPathStr());
auto & v = getValue();
try {
root->state.forceValue(v);
} catch (EvalError &) {
debug("setting '%s' to failed", getAttrPathStr());
if (root->db)
cachedValue = {root->db->setFailed(getKey()), failed_t()};
throw;
}
if (root->db && (!cachedValue || std::get_if<placeholder_t>(&cachedValue->second))) {
if (v.type == tString)
cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context), v.string.s};
else if (v.type == tPath)
cachedValue = {root->db->setString(getKey(), v.path), v.path};
else if (v.type == tBool)
cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean};
else if (v.type == tAttrs)
; // FIXME: do something?
else
cachedValue = {root->db->setMisc(getKey()), misc_t()};
}
return v;
}
std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name)
{
if (root->db) {
if (!cachedValue)
cachedValue = root->db->getAttr(getKey(), root->state.symbols);
if (cachedValue) {
if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) {
for (auto & attr : *attrs)
if (attr == name)
return std::make_shared<AttrCursor>(root, std::make_pair(shared_from_this(), name));
return nullptr;
} else if (std::get_if<placeholder_t>(&cachedValue->second)) {
auto attr = root->db->getAttr({cachedValue->first, name}, root->state.symbols);
if (attr) {
if (std::get_if<missing_t>(&attr->second))
return nullptr;
else if (std::get_if<failed_t>(&attr->second))
throw EvalError("cached failure of attribute '%s'", getAttrPathStr(name));
else
return std::make_shared<AttrCursor>(root,
std::make_pair(shared_from_this(), name), nullptr, std::move(attr));
}
// Incomplete attrset, so need to fall thru and
// evaluate to see whether 'name' exists
} else
return nullptr;
//throw TypeError("'%s' is not an attribute set", getAttrPathStr());
}
}
auto & v = forceValue();
if (v.type != tAttrs)
return nullptr;
//throw TypeError("'%s' is not an attribute set", getAttrPathStr());
auto attr = v.attrs->get(name);
if (!attr) {
if (root->db) {
if (!cachedValue)
cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()};
root->db->setMissing({cachedValue->first, name});
}
return nullptr;
}
std::optional<std::pair<AttrId, AttrValue>> cachedValue2;
if (root->db) {
if (!cachedValue)
cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()};
cachedValue2 = {root->db->setPlaceholder({cachedValue->first, name}), placeholder_t()};
}
return std::make_shared<AttrCursor>(
root, std::make_pair(shared_from_this(), name), attr->value, std::move(cachedValue2));
}
std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(std::string_view name)
{
return maybeGetAttr(root->state.symbols.create(name));
}
std::shared_ptr<AttrCursor> AttrCursor::getAttr(Symbol name)
{
auto p = maybeGetAttr(name);
if (!p)
throw Error("attribute '%s' does not exist", getAttrPathStr(name));
return p;
}
std::shared_ptr<AttrCursor> AttrCursor::getAttr(std::string_view name)
{
return getAttr(root->state.symbols.create(name));
}
std::shared_ptr<AttrCursor> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath)
{
auto res = shared_from_this();
for (auto & attr : attrPath) {
res = res->maybeGetAttr(attr);
if (!res) return {};
}
return res;
}
std::string AttrCursor::getString()
{
if (root->db) {
if (!cachedValue)
cachedValue = root->db->getAttr(getKey(), root->state.symbols);
if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
if (auto s = std::get_if<string_t>(&cachedValue->second)) {
debug("using cached string attribute '%s'", getAttrPathStr());
return s->first;
} else
throw TypeError("'%s' is not a string", getAttrPathStr());
}
}
auto & v = forceValue();
if (v.type != tString && v.type != tPath)
throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type));
return v.type == tString ? v.string.s : v.path;
}
string_t AttrCursor::getStringWithContext()
{
if (root->db) {
if (!cachedValue)
cachedValue = root->db->getAttr(getKey(), root->state.symbols);
if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
if (auto s = std::get_if<string_t>(&cachedValue->second)) {
debug("using cached string attribute '%s'", getAttrPathStr());
return *s;
} else
throw TypeError("'%s' is not a string", getAttrPathStr());
}
}
auto & v = forceValue();
if (v.type == tString)
return {v.string.s, v.getContext()};
else if (v.type == tPath)
return {v.path, {}};
else
throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type));
}
bool AttrCursor::getBool()
{
if (root->db) {
if (!cachedValue)
cachedValue = root->db->getAttr(getKey(), root->state.symbols);
if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
if (auto b = std::get_if<bool>(&cachedValue->second)) {
debug("using cached Boolean attribute '%s'", getAttrPathStr());
return *b;
} else
throw TypeError("'%s' is not a Boolean", getAttrPathStr());
}
}
auto & v = forceValue();
if (v.type != tBool)
throw TypeError("'%s' is not a Boolean", getAttrPathStr());
return v.boolean;
}
std::vector<Symbol> AttrCursor::getAttrs()
{
if (root->db) {
if (!cachedValue)
cachedValue = root->db->getAttr(getKey(), root->state.symbols);
if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) {
debug("using cached attrset attribute '%s'", getAttrPathStr());
return *attrs;
} else
throw TypeError("'%s' is not an attribute set", getAttrPathStr());
}
}
auto & v = forceValue();
if (v.type != tAttrs)
throw TypeError("'%s' is not an attribute set", getAttrPathStr());
std::vector<Symbol> attrs;
for (auto & attr : *getValue().attrs)
attrs.push_back(attr.name);
std::sort(attrs.begin(), attrs.end(), [](const Symbol & a, const Symbol & b) {
return (const string &) a < (const string &) b;
});
if (root->db)
cachedValue = {root->db->setAttrs(getKey(), attrs), attrs};
return attrs;
}
bool AttrCursor::isDerivation()
{
auto aType = maybeGetAttr("type");
return aType && aType->getString() == "derivation";
}
StorePath AttrCursor::forceDerivation()
{
auto aDrvPath = getAttr(root->state.sDrvPath);
auto drvPath = root->state.store->parseStorePath(aDrvPath->getString());
if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) {
/* The eval cache contains 'drvPath', but the actual path has
been garbage-collected. So force it to be regenerated. */
aDrvPath->forceValue();
if (!root->state.store->isValidPath(drvPath))
throw Error("don't know how to recreate store derivation '%s'!",
root->state.store->printStorePath(drvPath));
}
return drvPath;
}
}

121
src/libexpr/eval-cache.hh Normal file
View file

@ -0,0 +1,121 @@
#pragma once
#include "sync.hh"
#include "hash.hh"
#include "eval.hh"
#include <functional>
#include <variant>
namespace nix::eval_cache {
class AttrDb;
class AttrCursor;
class EvalCache : public std::enable_shared_from_this<EvalCache>
{
friend class AttrCursor;
std::shared_ptr<AttrDb> db;
EvalState & state;
typedef std::function<Value *()> RootLoader;
RootLoader rootLoader;
RootValue value;
Value * getRootValue();
public:
EvalCache(
std::optional<std::reference_wrapper<const Hash>> useCache,
EvalState & state,
RootLoader rootLoader);
std::shared_ptr<AttrCursor> getRoot();
};
enum AttrType {
Placeholder = 0,
FullAttrs = 1,
String = 2,
Missing = 3,
Misc = 4,
Failed = 5,
Bool = 6,
};
struct placeholder_t {};
struct missing_t {};
struct misc_t {};
struct failed_t {};
typedef uint64_t AttrId;
typedef std::pair<AttrId, Symbol> AttrKey;
typedef std::pair<std::string, std::vector<std::pair<Path, std::string>>> string_t;
typedef std::variant<
std::vector<Symbol>,
string_t,
placeholder_t,
missing_t,
misc_t,
failed_t,
bool
> AttrValue;
class AttrCursor : public std::enable_shared_from_this<AttrCursor>
{
friend class EvalCache;
ref<EvalCache> root;
typedef std::optional<std::pair<std::shared_ptr<AttrCursor>, Symbol>> Parent;
Parent parent;
RootValue _value;
std::optional<std::pair<AttrId, AttrValue>> cachedValue;
AttrKey getKey();
Value & getValue();
public:
AttrCursor(
ref<EvalCache> root,
Parent parent,
Value * value = nullptr,
std::optional<std::pair<AttrId, AttrValue>> && cachedValue = {});
std::vector<Symbol> getAttrPath() const;
std::vector<Symbol> getAttrPath(Symbol name) const;
std::string getAttrPathStr() const;
std::string getAttrPathStr(Symbol name) const;
std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name);
std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name);
std::shared_ptr<AttrCursor> getAttr(Symbol name);
std::shared_ptr<AttrCursor> getAttr(std::string_view name);
std::shared_ptr<AttrCursor> findAlongAttrPath(const std::vector<Symbol> & attrPath);
std::string getString();
string_t getStringWithContext();
bool getBool();
std::vector<Symbol> getAttrs();
bool isDerivation();
Value & forceValue();
/* Force creation of the .drv file in the Nix store. */
StorePath forceDerivation();
};
}

View file

@ -199,6 +199,18 @@ string showType(const Value & v)
} }
bool Value::isTrivial() const
{
return
type != tApp
&& type != tPrimOpApp
&& (type != tThunk
|| (dynamic_cast<ExprAttrs *>(thunk.expr)
&& ((ExprAttrs *) thunk.expr)->dynamicAttrs.empty())
|| dynamic_cast<ExprLambda *>(thunk.expr));
}
#if HAVE_BOEHMGC #if HAVE_BOEHMGC
/* Called when the Boehm GC runs out of memory. */ /* Called when the Boehm GC runs out of memory. */
static void * oomHandler(size_t requested) static void * oomHandler(size_t requested)
@ -337,6 +349,9 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
, sOutputHashAlgo(symbols.create("outputHashAlgo")) , sOutputHashAlgo(symbols.create("outputHashAlgo"))
, sOutputHashMode(symbols.create("outputHashMode")) , sOutputHashMode(symbols.create("outputHashMode"))
, sRecurseForDerivations(symbols.create("recurseForDerivations")) , sRecurseForDerivations(symbols.create("recurseForDerivations"))
, sDescription(symbols.create("description"))
, sSelf(symbols.create("self"))
, sEpsilon(symbols.create(""))
, repair(NoRepair) , repair(NoRepair)
, store(store) , store(store)
, baseEnv(allocEnv(128)) , baseEnv(allocEnv(128))
@ -366,7 +381,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
if (store->isInStore(r.second)) { if (store->isInStore(r.second)) {
StorePathSet closure; StorePathSet closure;
store->computeFSClosure(store->parseStorePath(store->toStorePath(r.second)), closure); store->computeFSClosure(store->toStorePath(r.second).first, closure);
for (auto & path : closure) for (auto & path : closure)
allowedPaths->insert(store->printStorePath(path)); allowedPaths->insert(store->printStorePath(path));
} else } else
@ -782,7 +797,7 @@ Value * ExprPath::maybeThunk(EvalState & state, Env & env)
} }
void EvalState::evalFile(const Path & path_, Value & v) void EvalState::evalFile(const Path & path_, Value & v, bool mustBeTrivial)
{ {
auto path = checkSourcePath(path_); auto path = checkSourcePath(path_);
@ -811,6 +826,11 @@ void EvalState::evalFile(const Path & path_, Value & v)
fileParseCache[path2] = e; fileParseCache[path2] = e;
try { try {
// Enforce that 'flake.nix' is a direct attrset, not a
// computation.
if (mustBeTrivial &&
!(dynamic_cast<ExprAttrs *>(e)))
throw Error("file '%s' must be an attribute set", path);
eval(e, v); eval(e, v);
} catch (Error & e) { } catch (Error & e) {
addErrorTrace(e, "while evaluating the file '%1%':", path2); addErrorTrace(e, "while evaluating the file '%1%':", path2);
@ -1586,6 +1606,18 @@ string EvalState::forceString(Value & v, const Pos & pos)
} }
/* Decode a context string !<name>!<path> into a pair <path,
name>. */
std::pair<string, string> decodeContext(std::string_view s)
{
if (s.at(0) == '!') {
size_t index = s.find("!", 1);
return {std::string(s.substr(index + 1)), std::string(s.substr(1, index - 1))};
} else
return {s.at(0) == '/' ? std::string(s) : std::string(s.substr(1)), ""};
}
void copyContext(const Value & v, PathSet & context) void copyContext(const Value & v, PathSet & context)
{ {
if (v.string.context) if (v.string.context)
@ -1594,6 +1626,17 @@ void copyContext(const Value & v, PathSet & context)
} }
std::vector<std::pair<Path, std::string>> Value::getContext()
{
std::vector<std::pair<Path, std::string>> res;
assert(type == tString);
if (string.context)
for (const char * * p = string.context; *p; ++p)
res.push_back(decodeContext(*p));
return res;
}
string EvalState::forceString(Value & v, PathSet & context, const Pos & pos) string EvalState::forceString(Value & v, PathSet & context, const Pos & pos)
{ {
string s = forceString(v, pos); string s = forceString(v, pos);

View file

@ -4,13 +4,13 @@
#include "value.hh" #include "value.hh"
#include "nixexpr.hh" #include "nixexpr.hh"
#include "symbol-table.hh" #include "symbol-table.hh"
#include "hash.hh"
#include "config.hh" #include "config.hh"
#include <regex> #include <regex>
#include <map> #include <map>
#include <optional> #include <optional>
#include <unordered_map> #include <unordered_map>
#include <mutex>
namespace nix { namespace nix {
@ -75,7 +75,8 @@ public:
sFile, sLine, sColumn, sFunctor, sToString, sFile, sLine, sColumn, sFunctor, sToString,
sRight, sWrong, sStructuredAttrs, sBuilder, sArgs, sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
sOutputHash, sOutputHashAlgo, sOutputHashMode, sOutputHash, sOutputHashAlgo, sOutputHashMode,
sRecurseForDerivations; sRecurseForDerivations,
sDescription, sSelf, sEpsilon;
Symbol sDerivationNix; Symbol sDerivationNix;
/* If set, force copying files to the Nix store even if they /* If set, force copying files to the Nix store even if they
@ -90,6 +91,7 @@ public:
const ref<Store> store; const ref<Store> store;
private: private:
SrcToStore srcToStore; SrcToStore srcToStore;
@ -152,8 +154,9 @@ public:
Expr * parseStdin(); Expr * parseStdin();
/* Evaluate an expression read from the given file to normal /* Evaluate an expression read from the given file to normal
form. */ form. Optionally enforce that the top-level expression is
void evalFile(const Path & path, Value & v); trivial (i.e. doesn't require arbitrary computation). */
void evalFile(const Path & path, Value & v, bool mustBeTrivial = false);
void resetFileCache(); void resetFileCache();
@ -330,7 +333,7 @@ string showType(const Value & v);
/* Decode a context string !<name>!<path> into a pair <path, /* Decode a context string !<name>!<path> into a pair <path,
name>. */ name>. */
std::pair<string, string> decodeContext(const string & s); std::pair<string, string> decodeContext(std::string_view s);
/* If `path' refers to a directory, then append "/default.nix". */ /* If `path' refers to a directory, then append "/default.nix". */
Path resolveExprPath(Path path); Path resolveExprPath(Path path);

View file

@ -0,0 +1,56 @@
lockFileStr: rootSrc: rootSubdir:
let
lockFile = builtins.fromJSON lockFileStr;
allNodes =
builtins.mapAttrs
(key: node:
let
sourceInfo =
if key == lockFile.root
then rootSrc
else fetchTree (node.info or {} // removeAttrs node.locked ["dir"]);
subdir = if key == lockFile.root then rootSubdir else node.locked.dir or "";
flake = import (sourceInfo + (if subdir != "" then "/" else "") + subdir + "/flake.nix");
inputs = builtins.mapAttrs
(inputName: inputSpec: allNodes.${resolveInput inputSpec})
(node.inputs or {});
# Resolve a input spec into a node name. An input spec is
# either a node name, or a 'follows' path from the root
# node.
resolveInput = inputSpec:
if builtins.isList inputSpec
then getInputByPath lockFile.root inputSpec
else inputSpec;
# Follow an input path (e.g. ["dwarffs" "nixpkgs"]) from the
# root node, returning the final node.
getInputByPath = nodeName: path:
if path == []
then nodeName
else
getInputByPath
# Since this could be a 'follows' input, call resolveInput.
(resolveInput lockFile.nodes.${nodeName}.inputs.${builtins.head path})
(builtins.tail path);
outputs = flake.outputs (inputs // { self = result; });
result = outputs // sourceInfo // { inherit inputs; inherit outputs; inherit sourceInfo; };
in
if node.flake or true then
assert builtins.isFunction flake.outputs;
result
else
sourceInfo
)
lockFile.nodes;
in allNodes.${lockFile.root}

609
src/libexpr/flake/flake.cc Normal file
View file

@ -0,0 +1,609 @@
#include "flake.hh"
#include "lockfile.hh"
#include "primops.hh"
#include "eval-inline.hh"
#include "store-api.hh"
#include "fetchers.hh"
#include "finally.hh"
namespace nix {
using namespace flake;
namespace flake {
typedef std::pair<Tree, FlakeRef> FetchedFlake;
typedef std::vector<std::pair<FlakeRef, FetchedFlake>> FlakeCache;
static std::optional<FetchedFlake> lookupInFlakeCache(
const FlakeCache & flakeCache,
const FlakeRef & flakeRef)
{
// FIXME: inefficient.
for (auto & i : flakeCache) {
if (flakeRef == i.first) {
debug("mapping '%s' to previously seen input '%s' -> '%s",
flakeRef, i.first, i.second.second);
return i.second;
}
}
return std::nullopt;
}
static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree(
EvalState & state,
const FlakeRef & originalRef,
bool allowLookup,
FlakeCache & flakeCache)
{
auto fetched = lookupInFlakeCache(flakeCache, originalRef);
FlakeRef resolvedRef = originalRef;
if (!fetched) {
if (originalRef.input.isDirect()) {
fetched.emplace(originalRef.fetchTree(state.store));
} else {
if (allowLookup) {
resolvedRef = originalRef.resolve(state.store);
auto fetchedResolved = lookupInFlakeCache(flakeCache, originalRef);
if (!fetchedResolved) fetchedResolved.emplace(resolvedRef.fetchTree(state.store));
flakeCache.push_back({resolvedRef, fetchedResolved.value()});
fetched.emplace(fetchedResolved.value());
}
else {
throw Error("'%s' is an indirect flake reference, but registry lookups are not allowed", originalRef);
}
}
flakeCache.push_back({originalRef, fetched.value()});
}
auto [tree, lockedRef] = fetched.value();
debug("got tree '%s' from '%s'",
state.store->printStorePath(tree.storePath), lockedRef);
if (state.allowedPaths)
state.allowedPaths->insert(tree.actualPath);
assert(!originalRef.input.getNarHash() || tree.storePath == originalRef.input.computeStorePath(*state.store));
return {std::move(tree), resolvedRef, lockedRef};
}
static void expectType(EvalState & state, ValueType type,
Value & value, const Pos & pos)
{
if (value.type == tThunk && value.isTrivial())
state.forceValue(value, pos);
if (value.type != type)
throw Error("expected %s but got %s at %s",
showType(type), showType(value.type), pos);
}
static std::map<FlakeId, FlakeInput> parseFlakeInputs(
EvalState & state, Value * value, const Pos & pos);
static FlakeInput parseFlakeInput(EvalState & state,
const std::string & inputName, Value * value, const Pos & pos)
{
expectType(state, tAttrs, *value, pos);
FlakeInput input;
auto sInputs = state.symbols.create("inputs");
auto sUrl = state.symbols.create("url");
auto sFlake = state.symbols.create("flake");
auto sFollows = state.symbols.create("follows");
fetchers::Attrs attrs;
std::optional<std::string> url;
for (nix::Attr attr : *(value->attrs)) {
try {
if (attr.name == sUrl) {
expectType(state, tString, *attr.value, *attr.pos);
url = attr.value->string.s;
attrs.emplace("url", *url);
} else if (attr.name == sFlake) {
expectType(state, tBool, *attr.value, *attr.pos);
input.isFlake = attr.value->boolean;
} else if (attr.name == sInputs) {
input.overrides = parseFlakeInputs(state, attr.value, *attr.pos);
} else if (attr.name == sFollows) {
expectType(state, tString, *attr.value, *attr.pos);
input.follows = parseInputPath(attr.value->string.s);
} else {
state.forceValue(*attr.value);
if (attr.value->type == tString)
attrs.emplace(attr.name, attr.value->string.s);
else
throw TypeError("flake input attribute '%s' is %s while a string is expected",
attr.name, showType(*attr.value));
}
} catch (Error & e) {
e.addTrace(*attr.pos, hintfmt("in flake attribute '%s'", attr.name));
throw;
}
}
if (attrs.count("type"))
try {
input.ref = FlakeRef::fromAttrs(attrs);
} catch (Error & e) {
e.addTrace(pos, hintfmt("in flake input"));
throw;
}
else {
attrs.erase("url");
if (!attrs.empty())
throw Error("unexpected flake input attribute '%s', at %s", attrs.begin()->first, pos);
if (url)
input.ref = parseFlakeRef(*url, {}, true);
}
if (!input.follows && !input.ref)
input.ref = FlakeRef::fromAttrs({{"type", "indirect"}, {"id", inputName}});
return input;
}
static std::map<FlakeId, FlakeInput> parseFlakeInputs(
EvalState & state, Value * value, const Pos & pos)
{
std::map<FlakeId, FlakeInput> inputs;
expectType(state, tAttrs, *value, pos);
for (nix::Attr & inputAttr : *(*value).attrs) {
inputs.emplace(inputAttr.name,
parseFlakeInput(state,
inputAttr.name,
inputAttr.value,
*inputAttr.pos));
}
return inputs;
}
static Flake getFlake(
EvalState & state,
const FlakeRef & originalRef,
bool allowLookup,
FlakeCache & flakeCache)
{
auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree(
state, originalRef, allowLookup, flakeCache);
// Guard against symlink attacks.
auto flakeFile = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir + "/flake.nix");
if (!isInDir(flakeFile, sourceInfo.actualPath))
throw Error("'flake.nix' file of flake '%s' escapes from '%s'",
lockedRef, state.store->printStorePath(sourceInfo.storePath));
Flake flake {
.originalRef = originalRef,
.resolvedRef = resolvedRef,
.lockedRef = lockedRef,
.sourceInfo = std::make_shared<fetchers::Tree>(std::move(sourceInfo))
};
if (!pathExists(flakeFile))
throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir);
Value vInfo;
state.evalFile(flakeFile, vInfo, true); // FIXME: symlink attack
expectType(state, tAttrs, vInfo, Pos(foFile, state.symbols.create(flakeFile), 0, 0));
auto sEdition = state.symbols.create("edition"); // FIXME: remove soon
if (vInfo.attrs->get(sEdition))
warn("flake '%s' has deprecated attribute 'edition'", lockedRef);
if (auto description = vInfo.attrs->get(state.sDescription)) {
expectType(state, tString, *description->value, *description->pos);
flake.description = description->value->string.s;
}
auto sInputs = state.symbols.create("inputs");
if (auto inputs = vInfo.attrs->get(sInputs))
flake.inputs = parseFlakeInputs(state, inputs->value, *inputs->pos);
auto sOutputs = state.symbols.create("outputs");
if (auto outputs = vInfo.attrs->get(sOutputs)) {
expectType(state, tLambda, *outputs->value, *outputs->pos);
flake.vOutputs = allocRootValue(outputs->value);
if ((*flake.vOutputs)->lambda.fun->matchAttrs) {
for (auto & formal : (*flake.vOutputs)->lambda.fun->formals->formals) {
if (formal.name != state.sSelf)
flake.inputs.emplace(formal.name, FlakeInput {
.ref = parseFlakeRef(formal.name)
});
}
}
} else
throw Error("flake '%s' lacks attribute 'outputs'", lockedRef);
for (auto & attr : *vInfo.attrs) {
if (attr.name != sEdition &&
attr.name != state.sDescription &&
attr.name != sInputs &&
attr.name != sOutputs)
throw Error("flake '%s' has an unsupported attribute '%s', at %s",
lockedRef, attr.name, *attr.pos);
}
return flake;
}
Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup)
{
FlakeCache flakeCache;
return getFlake(state, originalRef, allowLookup, flakeCache);
}
/* Compute an in-memory lock file for the specified top-level flake,
and optionally write it to file, it the flake is writable. */
LockedFlake lockFlake(
EvalState & state,
const FlakeRef & topRef,
const LockFlags & lockFlags)
{
settings.requireExperimentalFeature("flakes");
FlakeCache flakeCache;
auto flake = getFlake(state, topRef, lockFlags.useRegistries, flakeCache);
// FIXME: symlink attack
auto oldLockFile = LockFile::read(
flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock");
debug("old lock file: %s", oldLockFile);
// FIXME: check whether all overrides are used.
std::map<InputPath, FlakeInput> overrides;
std::set<InputPath> overridesUsed, updatesUsed;
for (auto & i : lockFlags.inputOverrides)
overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second });
LockFile newLockFile;
std::vector<FlakeRef> parents;
std::function<void(
const FlakeInputs & flakeInputs,
std::shared_ptr<Node> node,
const InputPath & inputPathPrefix,
std::shared_ptr<const Node> oldNode)>
computeLocks;
computeLocks = [&](
const FlakeInputs & flakeInputs,
std::shared_ptr<Node> node,
const InputPath & inputPathPrefix,
std::shared_ptr<const Node> oldNode)
{
debug("computing lock file node '%s'", printInputPath(inputPathPrefix));
/* Get the overrides (i.e. attributes of the form
'inputs.nixops.inputs.nixpkgs.url = ...'). */
// FIXME: check this
for (auto & [id, input] : flake.inputs) {
for (auto & [idOverride, inputOverride] : input.overrides) {
auto inputPath(inputPathPrefix);
inputPath.push_back(id);
inputPath.push_back(idOverride);
overrides.insert_or_assign(inputPath, inputOverride);
}
}
/* Go over the flake inputs, resolve/fetch them if
necessary (i.e. if they're new or the flakeref changed
from what's in the lock file). */
for (auto & [id, input2] : flakeInputs) {
auto inputPath(inputPathPrefix);
inputPath.push_back(id);
auto inputPathS = printInputPath(inputPath);
debug("computing input '%s'", inputPathS);
/* Do we have an override for this input from one of the
ancestors? */
auto i = overrides.find(inputPath);
bool hasOverride = i != overrides.end();
if (hasOverride) overridesUsed.insert(inputPath);
auto & input = hasOverride ? i->second : input2;
/* Resolve 'follows' later (since it may refer to an input
path we haven't processed yet. */
if (input.follows) {
InputPath target;
if (hasOverride || input.absolute)
/* 'follows' from an override is relative to the
root of the graph. */
target = *input.follows;
else {
/* Otherwise, it's relative to the current flake. */
target = inputPathPrefix;
for (auto & i : *input.follows) target.push_back(i);
}
debug("input '%s' follows '%s'", inputPathS, printInputPath(target));
node->inputs.insert_or_assign(id, target);
continue;
}
assert(input.ref);
/* Do we have an entry in the existing lock file? And we
don't have a --update-input flag for this input? */
std::shared_ptr<LockedNode> oldLock;
updatesUsed.insert(inputPath);
if (oldNode && !lockFlags.inputUpdates.count(inputPath))
if (auto oldLock2 = get(oldNode->inputs, id))
if (auto oldLock3 = std::get_if<0>(&*oldLock2))
oldLock = *oldLock3;
if (oldLock
&& oldLock->originalRef == *input.ref
&& !hasOverride)
{
debug("keeping existing input '%s'", inputPathS);
/* Copy the input from the old lock since its flakeref
didn't change and there is no override from a
higher level flake. */
auto childNode = std::make_shared<LockedNode>(
oldLock->lockedRef, oldLock->originalRef, oldLock->isFlake);
node->inputs.insert_or_assign(id, childNode);
/* If we have an --update-input flag for an input
of this input, then we must fetch the flake to
to update it. */
auto lb = lockFlags.inputUpdates.lower_bound(inputPath);
auto hasChildUpdate =
lb != lockFlags.inputUpdates.end()
&& lb->size() > inputPath.size()
&& std::equal(inputPath.begin(), inputPath.end(), lb->begin());
if (hasChildUpdate) {
auto inputFlake = getFlake(
state, oldLock->lockedRef, false, flakeCache);
computeLocks(inputFlake.inputs, childNode, inputPath, oldLock);
} else {
/* No need to fetch this flake, we can be
lazy. However there may be new overrides on the
inputs of this flake, so we need to check
those. */
FlakeInputs fakeInputs;
for (auto & i : oldLock->inputs) {
if (auto lockedNode = std::get_if<0>(&i.second)) {
fakeInputs.emplace(i.first, FlakeInput {
.ref = (*lockedNode)->originalRef,
.isFlake = (*lockedNode)->isFlake,
});
} else if (auto follows = std::get_if<1>(&i.second)) {
fakeInputs.emplace(i.first, FlakeInput {
.follows = *follows,
.absolute = true
});
}
}
computeLocks(fakeInputs, childNode, inputPath, oldLock);
}
} else {
/* We need to create a new lock file entry. So fetch
this input. */
debug("creating new input '%s'", inputPathS);
if (!lockFlags.allowMutable && !input.ref->input.isImmutable())
throw Error("cannot update flake input '%s' in pure mode", inputPathS);
if (input.isFlake) {
auto inputFlake = getFlake(state, *input.ref, lockFlags.useRegistries, flakeCache);
/* Note: in case of an --override-input, we use
the *original* ref (input2.ref) for the
"original" field, rather than the
override. This ensures that the override isn't
nuked the next time we update the lock
file. That is, overrides are sticky unless you
use --no-write-lock-file. */
auto childNode = std::make_shared<LockedNode>(
inputFlake.lockedRef, input2.ref ? *input2.ref : *input.ref);
node->inputs.insert_or_assign(id, childNode);
/* Guard against circular flake imports. */
for (auto & parent : parents)
if (parent == *input.ref)
throw Error("found circular import of flake '%s'", parent);
parents.push_back(*input.ref);
Finally cleanup([&]() { parents.pop_back(); });
/* Recursively process the inputs of this
flake. Also, unless we already have this flake
in the top-level lock file, use this flake's
own lock file. */
computeLocks(
inputFlake.inputs, childNode, inputPath,
oldLock
? std::dynamic_pointer_cast<const Node>(oldLock)
: LockFile::read(
inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root);
}
else {
auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree(
state, *input.ref, lockFlags.useRegistries, flakeCache);
node->inputs.insert_or_assign(id,
std::make_shared<LockedNode>(lockedRef, *input.ref, false));
}
}
}
};
computeLocks(
flake.inputs, newLockFile.root, {},
lockFlags.recreateLockFile ? nullptr : oldLockFile.root);
for (auto & i : lockFlags.inputOverrides)
if (!overridesUsed.count(i.first))
warn("the flag '--override-input %s %s' does not match any input",
printInputPath(i.first), i.second);
for (auto & i : lockFlags.inputUpdates)
if (!updatesUsed.count(i))
warn("the flag '--update-input %s' does not match any input", printInputPath(i));
/* Check 'follows' inputs. */
newLockFile.check();
debug("new lock file: %s", newLockFile);
/* Check whether we need to / can write the new lock file. */
if (!(newLockFile == oldLockFile)) {
auto diff = LockFile::diff(oldLockFile, newLockFile);
if (lockFlags.writeLockFile) {
if (auto sourcePath = topRef.input.getSourcePath()) {
if (!newLockFile.isImmutable()) {
if (settings.warnDirty)
warn("will not write lock file of flake '%s' because it has a mutable input", topRef);
} else {
if (!lockFlags.updateLockFile)
throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef);
auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock";
auto path = *sourcePath + "/" + relPath;
bool lockFileExists = pathExists(path);
if (lockFileExists) {
auto s = chomp(diff);
if (s.empty())
warn("updating lock file '%s'", path);
else
warn("updating lock file '%s':\n%s", path, s);
} else
warn("creating lock file '%s'", path);
newLockFile.write(path);
topRef.input.markChangedFile(
(topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock",
lockFlags.commitLockFile
? std::optional<std::string>(fmt("%s: %s\n\nFlake input changes:\n\n%s",
relPath, lockFileExists ? "Update" : "Add", diff))
: std::nullopt);
/* Rewriting the lockfile changed the top-level
repo, so we should re-read it. FIXME: we could
also just clear the 'rev' field... */
auto prevLockedRef = flake.lockedRef;
FlakeCache dummyCache;
flake = getFlake(state, topRef, lockFlags.useRegistries, dummyCache);
if (lockFlags.commitLockFile &&
flake.lockedRef.input.getRev() &&
prevLockedRef.input.getRev() != flake.lockedRef.input.getRev())
warn("committed new revision '%s'", flake.lockedRef.input.getRev()->gitRev());
/* Make sure that we picked up the change,
i.e. the tree should usually be dirty
now. Corner case: we could have reverted from a
dirty to a clean tree! */
if (flake.lockedRef.input == prevLockedRef.input
&& !flake.lockedRef.input.isImmutable())
throw Error("'%s' did not change after I updated its 'flake.lock' file; is 'flake.lock' under version control?", flake.originalRef);
}
} else
throw Error("cannot write modified lock file of flake '%s' (use '--no-write-lock-file' to ignore)", topRef);
} else
warn("not writing modified lock file of flake '%s':\n%s", topRef, chomp(diff));
}
return LockedFlake { .flake = std::move(flake), .lockFile = std::move(newLockFile) };
}
void callFlake(EvalState & state,
const LockedFlake & lockedFlake,
Value & vRes)
{
auto vLocks = state.allocValue();
auto vRootSrc = state.allocValue();
auto vRootSubdir = state.allocValue();
auto vTmp1 = state.allocValue();
auto vTmp2 = state.allocValue();
mkString(*vLocks, lockedFlake.lockFile.to_string());
emitTreeAttrs(state, *lockedFlake.flake.sourceInfo, lockedFlake.flake.lockedRef.input, *vRootSrc);
mkString(*vRootSubdir, lockedFlake.flake.lockedRef.subdir);
static RootValue vCallFlake = nullptr;
if (!vCallFlake) {
vCallFlake = allocRootValue(state.allocValue());
state.eval(state.parseExprFromString(
#include "call-flake.nix.gen.hh"
, "/"), **vCallFlake);
}
state.callFunction(**vCallFlake, *vLocks, *vTmp1, noPos);
state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos);
state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos);
}
static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
auto flakeRefS = state.forceStringNoCtx(*args[0], pos);
auto flakeRef = parseFlakeRef(flakeRefS, {}, true);
if (evalSettings.pureEval && !flakeRef.input.isImmutable())
throw Error("cannot call 'getFlake' on mutable flake reference '%s', at %s (use --impure to override)", flakeRefS, pos);
callFlake(state,
lockFlake(state, flakeRef,
LockFlags {
.updateLockFile = false,
.useRegistries = !evalSettings.pureEval,
.allowMutable = !evalSettings.pureEval,
}),
v);
}
static RegisterPrimOp r2("__getFlake", 1, prim_getFlake, "flakes");
}
Fingerprint LockedFlake::getFingerprint() const
{
// FIXME: as an optimization, if the flake contains a lock file
// and we haven't changed it, then it's sufficient to use
// flake.sourceInfo.storePath for the fingerprint.
return hashString(htSHA256,
fmt("%s;%d;%d;%s",
flake.sourceInfo->storePath.to_string(),
flake.lockedRef.input.getRevCount().value_or(0),
flake.lockedRef.input.getLastModified().value_or(0),
lockFile));
}
Flake::~Flake() { }
}

111
src/libexpr/flake/flake.hh Normal file
View file

@ -0,0 +1,111 @@
#pragma once
#include "types.hh"
#include "flakeref.hh"
#include "lockfile.hh"
#include "value.hh"
namespace nix {
class EvalState;
namespace fetchers { struct Tree; }
namespace flake {
struct FlakeInput;
typedef std::map<FlakeId, FlakeInput> FlakeInputs;
struct FlakeInput
{
std::optional<FlakeRef> ref;
bool isFlake = true;
std::optional<InputPath> follows;
bool absolute = false; // whether 'follows' is relative to the flake root
FlakeInputs overrides;
};
struct Flake
{
FlakeRef originalRef;
FlakeRef resolvedRef;
FlakeRef lockedRef;
std::optional<std::string> description;
std::shared_ptr<const fetchers::Tree> sourceInfo;
FlakeInputs inputs;
RootValue vOutputs;
~Flake();
};
Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool allowLookup);
/* Fingerprint of a locked flake; used as a cache key. */
typedef Hash Fingerprint;
struct LockedFlake
{
Flake flake;
LockFile lockFile;
Fingerprint getFingerprint() const;
};
struct LockFlags
{
/* Whether to ignore the existing lock file, creating a new one
from scratch. */
bool recreateLockFile = false;
/* Whether to update the lock file at all. If set to false, if any
change to the lock file is needed (e.g. when an input has been
added to flake.nix), you get a fatal error. */
bool updateLockFile = true;
/* Whether to write the lock file to disk. If set to true, if the
any changes to the lock file are needed and the flake is not
writable (i.e. is not a local Git working tree or similar), you
get a fatal error. If set to false, Nix will use the modified
lock file in memory only, without writing it to disk. */
bool writeLockFile = true;
/* Whether to use the registries to lookup indirect flake
references like 'nixpkgs'. */
bool useRegistries = true;
/* Whether mutable flake references (i.e. those without a Git
revision or similar) without a corresponding lock are
allowed. Mutable flake references with a lock are always
allowed. */
bool allowMutable = true;
/* Whether to commit changes to flake.lock. */
bool commitLockFile = false;
/* Flake inputs to be overriden. */
std::map<InputPath, FlakeRef> inputOverrides;
/* Flake inputs to be updated. This means that any existing lock
for those inputs will be ignored. */
std::set<InputPath> inputUpdates;
};
LockedFlake lockFlake(
EvalState & state,
const FlakeRef & flakeRef,
const LockFlags & lockFlags);
void callFlake(
EvalState & state,
const LockedFlake & lockedFlake,
Value & v);
}
void emitTreeAttrs(
EvalState & state,
const fetchers::Tree & tree,
const fetchers::Input & input,
Value & v);
}

View file

@ -0,0 +1,199 @@
#include "flakeref.hh"
#include "store-api.hh"
#include "url.hh"
#include "fetchers.hh"
#include "registry.hh"
namespace nix {
#if 0
// 'dir' path elements cannot start with a '.'. We also reject
// potentially dangerous characters like ';'.
const static std::string subDirElemRegex = "(?:[a-zA-Z0-9_-]+[a-zA-Z0-9._-]*)";
const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRegex + ")*";
#endif
std::string FlakeRef::to_string() const
{
auto url = input.toURL();
if (subdir != "")
url.query.insert_or_assign("dir", subdir);
return url.to_string();
}
fetchers::Attrs FlakeRef::toAttrs() const
{
auto attrs = input.toAttrs();
if (subdir != "")
attrs.emplace("dir", subdir);
return attrs;
}
std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef)
{
str << flakeRef.to_string();
return str;
}
bool FlakeRef::operator ==(const FlakeRef & other) const
{
return input == other.input && subdir == other.subdir;
}
FlakeRef FlakeRef::resolve(ref<Store> store) const
{
auto [input2, extraAttrs] = lookupInRegistries(store, input);
return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir));
}
FlakeRef parseFlakeRef(
const std::string & url, const std::optional<Path> & baseDir, bool allowMissing)
{
auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing);
if (fragment != "")
throw Error("unexpected fragment '%s' in flake reference '%s'", fragment, url);
return flakeRef;
}
std::optional<FlakeRef> maybeParseFlakeRef(
const std::string & url, const std::optional<Path> & baseDir)
{
try {
return parseFlakeRef(url, baseDir);
} catch (Error &) {
return {};
}
}
std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
const std::string & url, const std::optional<Path> & baseDir, bool allowMissing)
{
using namespace fetchers;
static std::string fnRegex = "[0-9a-zA-Z-._~!$&'\"()*+,;=]+";
static std::regex pathUrlRegex(
"(/?" + fnRegex + "(?:/" + fnRegex + ")*/?)"
+ "(?:\\?(" + queryRegex + "))?"
+ "(?:#(" + queryRegex + "))?",
std::regex::ECMAScript);
static std::regex flakeRegex(
"((" + flakeIdRegexS + ")(?:/(?:" + refAndOrRevRegex + "))?)"
+ "(?:#(" + queryRegex + "))?",
std::regex::ECMAScript);
std::smatch match;
/* Check if 'url' is a flake ID. This is an abbreviated syntax for
'flake:<flake-id>?ref=<ref>&rev=<rev>'. */
if (std::regex_match(url, match, flakeRegex)) {
auto parsedURL = ParsedURL{
.url = url,
.base = "flake:" + std::string(match[1]),
.scheme = "flake",
.authority = "",
.path = match[1],
};
return std::make_pair(
FlakeRef(Input::fromURL(parsedURL), ""),
percentDecode(std::string(match[6])));
}
/* Check if 'url' is a path (either absolute or relative to
'baseDir'). If so, search upward to the root of the repo
(i.e. the directory containing .git). */
else if (std::regex_match(url, match, pathUrlRegex)) {
std::string path = match[1];
if (!baseDir && !hasPrefix(path, "/"))
throw BadURL("flake reference '%s' is not an absolute path", url);
path = absPath(path, baseDir, true);
if (!S_ISDIR(lstat(path).st_mode))
throw BadURL("path '%s' is not a flake (because it's not a directory)", path);
if (!allowMissing && !pathExists(path + "/flake.nix"))
throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path);
auto fragment = percentDecode(std::string(match[3]));
auto flakeRoot = path;
std::string subdir;
while (flakeRoot != "/") {
if (pathExists(flakeRoot + "/.git")) {
auto base = std::string("git+file://") + flakeRoot;
auto parsedURL = ParsedURL{
.url = base, // FIXME
.base = base,
.scheme = "git+file",
.authority = "",
.path = flakeRoot,
.query = decodeQuery(match[2]),
};
if (subdir != "") {
if (parsedURL.query.count("dir"))
throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url);
parsedURL.query.insert_or_assign("dir", subdir);
}
if (pathExists(flakeRoot + "/.git/shallow"))
parsedURL.query.insert_or_assign("shallow", "1");
return std::make_pair(
FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")),
fragment);
}
subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir);
flakeRoot = dirOf(flakeRoot);
}
fetchers::Attrs attrs;
attrs.insert_or_assign("type", "path");
attrs.insert_or_assign("path", path);
return std::make_pair(FlakeRef(Input::fromAttrs(std::move(attrs)), ""), fragment);
}
else {
auto parsedURL = parseURL(url);
std::string fragment;
std::swap(fragment, parsedURL.fragment);
return std::make_pair(
FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")),
fragment);
}
}
std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment(
const std::string & url, const std::optional<Path> & baseDir)
{
try {
return parseFlakeRefWithFragment(url, baseDir);
} catch (Error & e) {
return {};
}
}
FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs)
{
auto attrs2(attrs);
attrs2.erase("dir");
return FlakeRef(
fetchers::Input::fromAttrs(std::move(attrs2)),
fetchers::maybeGetStrAttr(attrs, "dir").value_or(""));
}
std::pair<fetchers::Tree, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const
{
auto [tree, lockedInput] = input.fetch(store);
return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)};
}
}

View file

@ -0,0 +1,53 @@
#pragma once
#include "types.hh"
#include "hash.hh"
#include "fetchers.hh"
#include <variant>
namespace nix {
class Store;
typedef std::string FlakeId;
struct FlakeRef
{
fetchers::Input input;
Path subdir;
bool operator==(const FlakeRef & other) const;
FlakeRef(fetchers::Input && input, const Path & subdir)
: input(std::move(input)), subdir(subdir)
{ }
// FIXME: change to operator <<.
std::string to_string() const;
fetchers::Attrs toAttrs() const;
FlakeRef resolve(ref<Store> store) const;
static FlakeRef fromAttrs(const fetchers::Attrs & attrs);
std::pair<fetchers::Tree, FlakeRef> fetchTree(ref<Store> store) const;
};
std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef);
FlakeRef parseFlakeRef(
const std::string & url, const std::optional<Path> & baseDir = {}, bool allowMissing = false);
std::optional<FlakeRef> maybeParseFlake(
const std::string & url, const std::optional<Path> & baseDir = {});
std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
const std::string & url, const std::optional<Path> & baseDir = {}, bool allowMissing = false);
std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment(
const std::string & url, const std::optional<Path> & baseDir = {});
}

View file

@ -0,0 +1,338 @@
#include "lockfile.hh"
#include "store-api.hh"
#include <nlohmann/json.hpp>
namespace nix::flake {
FlakeRef getFlakeRef(
const nlohmann::json & json,
const char * attr,
const char * info)
{
auto i = json.find(attr);
if (i != json.end()) {
auto attrs = jsonToAttrs(*i);
// FIXME: remove when we drop support for version 5.
if (info) {
auto j = json.find(info);
if (j != json.end()) {
for (auto k : jsonToAttrs(*j))
attrs.insert_or_assign(k.first, k.second);
}
}
return FlakeRef::fromAttrs(attrs);
}
throw Error("attribute '%s' missing in lock file", attr);
}
LockedNode::LockedNode(const nlohmann::json & json)
: lockedRef(getFlakeRef(json, "locked", "info"))
, originalRef(getFlakeRef(json, "original", nullptr))
, isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true)
{
if (!lockedRef.input.isImmutable())
throw Error("lockfile contains mutable lock '%s'", attrsToJson(lockedRef.input.toAttrs()));
}
StorePath LockedNode::computeStorePath(Store & store) const
{
return lockedRef.input.computeStorePath(store);
}
std::shared_ptr<Node> LockFile::findInput(const InputPath & path)
{
auto pos = root;
if (!pos) return {};
for (auto & elem : path) {
if (auto i = get(pos->inputs, elem)) {
if (auto node = std::get_if<0>(&*i))
pos = *node;
else if (auto follows = std::get_if<1>(&*i)) {
pos = findInput(*follows);
if (!pos) return {};
}
} else
return {};
}
return pos;
}
LockFile::LockFile(const nlohmann::json & json, const Path & path)
{
auto version = json.value("version", 0);
if (version < 5 || version > 7)
throw Error("lock file '%s' has unsupported version %d", path, version);
std::unordered_map<std::string, std::shared_ptr<Node>> nodeMap;
std::function<void(Node & node, const nlohmann::json & jsonNode)> getInputs;
getInputs = [&](Node & node, const nlohmann::json & jsonNode)
{
if (jsonNode.find("inputs") == jsonNode.end()) return;
for (auto & i : jsonNode["inputs"].items()) {
if (i.value().is_array()) {
InputPath path;
for (auto & j : i.value())
path.push_back(j);
node.inputs.insert_or_assign(i.key(), path);
} else {
std::string inputKey = i.value();
auto k = nodeMap.find(inputKey);
if (k == nodeMap.end()) {
auto jsonNode2 = json["nodes"][inputKey];
auto input = std::make_shared<LockedNode>(jsonNode2);
k = nodeMap.insert_or_assign(inputKey, input).first;
getInputs(*input, jsonNode2);
}
if (auto child = std::dynamic_pointer_cast<LockedNode>(k->second))
node.inputs.insert_or_assign(i.key(), child);
else
// FIXME: replace by follows node
throw Error("lock file contains cycle to root node");
}
}
};
std::string rootKey = json["root"];
nodeMap.insert_or_assign(rootKey, root);
getInputs(*root, json["nodes"][rootKey]);
// FIXME: check that there are no cycles in version >= 7. Cycles
// between inputs are only possible using 'follows' indirections.
// Once we drop support for version <= 6, we can simplify the code
// a bit since we don't need to worry about cycles.
}
nlohmann::json LockFile::toJson() const
{
nlohmann::json nodes;
std::unordered_map<std::shared_ptr<const Node>, std::string> nodeKeys;
std::unordered_set<std::string> keys;
std::function<std::string(const std::string & key, std::shared_ptr<const Node> node)> dumpNode;
dumpNode = [&](std::string key, std::shared_ptr<const Node> node) -> std::string
{
auto k = nodeKeys.find(node);
if (k != nodeKeys.end())
return k->second;
if (!keys.insert(key).second) {
for (int n = 2; ; ++n) {
auto k = fmt("%s_%d", key, n);
if (keys.insert(k).second) {
key = k;
break;
}
}
}
nodeKeys.insert_or_assign(node, key);
auto n = nlohmann::json::object();
if (!node->inputs.empty()) {
auto inputs = nlohmann::json::object();
for (auto & i : node->inputs) {
if (auto child = std::get_if<0>(&i.second)) {
inputs[i.first] = dumpNode(i.first, *child);
} else if (auto follows = std::get_if<1>(&i.second)) {
auto arr = nlohmann::json::array();
for (auto & x : *follows)
arr.push_back(x);
inputs[i.first] = std::move(arr);
}
}
n["inputs"] = std::move(inputs);
}
if (auto lockedNode = std::dynamic_pointer_cast<const LockedNode>(node)) {
n["original"] = fetchers::attrsToJson(lockedNode->originalRef.toAttrs());
n["locked"] = fetchers::attrsToJson(lockedNode->lockedRef.toAttrs());
if (!lockedNode->isFlake) n["flake"] = false;
}
nodes[key] = std::move(n);
return key;
};
nlohmann::json json;
json["version"] = 7;
json["root"] = dumpNode("root", root);
json["nodes"] = std::move(nodes);
return json;
}
std::string LockFile::to_string() const
{
return toJson().dump(2);
}
LockFile LockFile::read(const Path & path)
{
if (!pathExists(path)) return LockFile();
return LockFile(nlohmann::json::parse(readFile(path)), path);
}
std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile)
{
stream << lockFile.toJson().dump(2);
return stream;
}
void LockFile::write(const Path & path) const
{
createDirs(dirOf(path));
writeFile(path, fmt("%s\n", *this));
}
bool LockFile::isImmutable() const
{
std::unordered_set<std::shared_ptr<const Node>> nodes;
std::function<void(std::shared_ptr<const Node> node)> visit;
visit = [&](std::shared_ptr<const Node> node)
{
if (!nodes.insert(node).second) return;
for (auto & i : node->inputs)
if (auto child = std::get_if<0>(&i.second))
visit(*child);
};
visit(root);
for (auto & i : nodes) {
if (i == root) continue;
auto lockedNode = std::dynamic_pointer_cast<const LockedNode>(i);
if (lockedNode && !lockedNode->lockedRef.input.isImmutable()) return false;
}
return true;
}
bool LockFile::operator ==(const LockFile & other) const
{
// FIXME: slow
return toJson() == other.toJson();
}
InputPath parseInputPath(std::string_view s)
{
InputPath path;
for (auto & elem : tokenizeString<std::vector<std::string>>(s, "/")) {
if (!std::regex_match(elem, flakeIdRegex))
throw UsageError("invalid flake input path element '%s'", elem);
path.push_back(elem);
}
return path;
}
std::map<InputPath, Node::Edge> LockFile::getAllInputs() const
{
std::unordered_set<std::shared_ptr<Node>> done;
std::map<InputPath, Node::Edge> res;
std::function<void(const InputPath & prefix, std::shared_ptr<Node> node)> recurse;
recurse = [&](const InputPath & prefix, std::shared_ptr<Node> node)
{
if (!done.insert(node).second) return;
for (auto &[id, input] : node->inputs) {
auto inputPath(prefix);
inputPath.push_back(id);
res.emplace(inputPath, input);
if (auto child = std::get_if<0>(&input))
recurse(inputPath, *child);
}
};
recurse({}, root);
return res;
}
std::ostream & operator <<(std::ostream & stream, const Node::Edge & edge)
{
if (auto node = std::get_if<0>(&edge))
stream << "'" << (*node)->lockedRef << "'";
else if (auto follows = std::get_if<1>(&edge))
stream << fmt("follows '%s'", printInputPath(*follows));
return stream;
}
static bool equals(const Node::Edge & e1, const Node::Edge & e2)
{
if (auto n1 = std::get_if<0>(&e1))
if (auto n2 = std::get_if<0>(&e2))
return (*n1)->lockedRef == (*n2)->lockedRef;
if (auto f1 = std::get_if<1>(&e1))
if (auto f2 = std::get_if<1>(&e2))
return *f1 == *f2;
return false;
}
std::string LockFile::diff(const LockFile & oldLocks, const LockFile & newLocks)
{
auto oldFlat = oldLocks.getAllInputs();
auto newFlat = newLocks.getAllInputs();
auto i = oldFlat.begin();
auto j = newFlat.begin();
std::string res;
while (i != oldFlat.end() || j != newFlat.end()) {
if (j != newFlat.end() && (i == oldFlat.end() || i->first > j->first)) {
res += fmt("* Added '%s': %s\n", printInputPath(j->first), j->second);
++j;
} else if (i != oldFlat.end() && (j == newFlat.end() || i->first < j->first)) {
res += fmt("* Removed '%s'\n", printInputPath(i->first));
++i;
} else {
if (!equals(i->second, j->second)) {
res += fmt("* Updated '%s': %s -> %s\n",
printInputPath(i->first),
i->second,
j->second);
}
++i;
++j;
}
}
return res;
}
void LockFile::check()
{
auto inputs = getAllInputs();
for (auto & [inputPath, input] : inputs) {
if (auto follows = std::get_if<1>(&input)) {
if (!follows->empty() && !get(inputs, *follows))
throw Error("input '%s' follows a non-existent input '%s'",
printInputPath(inputPath),
printInputPath(*follows));
}
}
}
void check();
std::string printInputPath(const InputPath & path)
{
return concatStringsSep("/", path);
}
}

View file

@ -0,0 +1,85 @@
#pragma once
#include "flakeref.hh"
#include <nlohmann/json_fwd.hpp>
namespace nix {
class Store;
struct StorePath;
}
namespace nix::flake {
using namespace fetchers;
typedef std::vector<FlakeId> InputPath;
struct LockedNode;
/* A node in the lock file. It has outgoing edges to other nodes (its
inputs). Only the root node has this type; all other nodes have
type LockedNode. */
struct Node : std::enable_shared_from_this<Node>
{
typedef std::variant<std::shared_ptr<LockedNode>, InputPath> Edge;
std::map<FlakeId, Edge> inputs;
virtual ~Node() { }
};
/* A non-root node in the lock file. */
struct LockedNode : Node
{
FlakeRef lockedRef, originalRef;
bool isFlake = true;
LockedNode(
const FlakeRef & lockedRef,
const FlakeRef & originalRef,
bool isFlake = true)
: lockedRef(lockedRef), originalRef(originalRef), isFlake(isFlake)
{ }
LockedNode(const nlohmann::json & json);
StorePath computeStorePath(Store & store) const;
};
struct LockFile
{
std::shared_ptr<Node> root = std::make_shared<Node>();
LockFile() {};
LockFile(const nlohmann::json & json, const Path & path);
nlohmann::json toJson() const;
std::string to_string() const;
static LockFile read(const Path & path);
void write(const Path & path) const;
bool isImmutable() const;
bool operator ==(const LockFile & other) const;
std::shared_ptr<Node> findInput(const InputPath & path);
std::map<InputPath, Node::Edge> getAllInputs() const;
static std::string diff(const LockFile & oldLocks, const LockFile & newLocks);
/* Check that every 'follows' input target exists. */
void check();
};
std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile);
InputPath parseInputPath(std::string_view s);
std::string printInputPath(const InputPath & path);
}

View file

@ -4,7 +4,12 @@ libexpr_NAME = libnixexpr
libexpr_DIR := $(d) libexpr_DIR := $(d)
libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc libexpr_SOURCES := \
$(wildcard $(d)/*.cc) \
$(wildcard $(d)/primops/*.cc) \
$(wildcard $(d)/flake/*.cc) \
$(d)/lexer-tab.cc \
$(d)/parser-tab.cc
libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr
@ -34,4 +39,9 @@ dist-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer
$(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644)) $(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644))
$(foreach i, $(wildcard src/libexpr/flake/*.hh), \
$(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644)))
$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh
$(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh

View file

@ -719,7 +719,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
if (isUri(elem.second)) { if (isUri(elem.second)) {
try { try {
res = { true, store->toRealPath(fetchers::downloadTarball( res = { true, store->toRealPath(fetchers::downloadTarball(
store, resolveUri(elem.second), "source", false).storePath) }; store, resolveUri(elem.second), "source", false).first.storePath) };
} catch (FileTransferError & e) { } catch (FileTransferError & e) {
logWarning({ logWarning({
.name = "Entry download", .name = "Entry download",

View file

@ -30,18 +30,6 @@ namespace nix {
*************************************************************/ *************************************************************/
/* Decode a context string !<name>!<path> into a pair <path,
name>. */
std::pair<string, string> decodeContext(const string & s)
{
if (s.at(0) == '!') {
size_t index = s.find("!", 1);
return std::pair<string, string>(string(s, index + 1), string(s, 1, index - 1));
} else
return std::pair<string, string>(s.at(0) == '/' ? s : string(s, 1), "");
}
InvalidPathError::InvalidPathError(const Path & path) : InvalidPathError::InvalidPathError(const Path & path) :
EvalError("path '%s' is not valid", path), path(path) {} EvalError("path '%s' is not valid", path), path(path) {}
@ -883,10 +871,10 @@ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, V
.hint = hintfmt("path '%1%' is not in the Nix store", path), .hint = hintfmt("path '%1%' is not in the Nix store", path),
.errPos = pos .errPos = pos
}); });
Path path2 = state.store->toStorePath(path); auto path2 = state.store->toStorePath(path).first;
if (!settings.readOnlyMode) if (!settings.readOnlyMode)
state.store->ensurePath(state.store->parseStorePath(path2)); state.store->ensurePath(path2);
context.insert(path2); context.insert(state.store->printStorePath(path2));
mkString(v, path, context); mkString(v, path, context);
} }

View file

@ -62,23 +62,23 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url); attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
if (ref) attrs.insert_or_assign("ref", *ref); if (ref) attrs.insert_or_assign("ref", *ref);
if (rev) attrs.insert_or_assign("rev", rev->gitRev()); if (rev) attrs.insert_or_assign("rev", rev->gitRev());
if (fetchSubmodules) attrs.insert_or_assign("submodules", true); if (fetchSubmodules) attrs.insert_or_assign("submodules", fetchers::Explicit<bool>{true});
auto input = fetchers::inputFromAttrs(attrs); auto input = fetchers::Input::fromAttrs(std::move(attrs));
// FIXME: use name? // FIXME: use name?
auto [tree, input2] = input->fetchTree(state.store); auto [tree, input2] = input.fetch(state.store);
state.mkAttrs(v, 8); state.mkAttrs(v, 8);
auto storePath = state.store->printStorePath(tree.storePath); auto storePath = state.store->printStorePath(tree.storePath);
mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
// Backward compatibility: set 'rev' to // Backward compatibility: set 'rev' to
// 0000000000000000000000000000000000000000 for a dirty tree. // 0000000000000000000000000000000000000000 for a dirty tree.
auto rev2 = input2->getRev().value_or(Hash(htSHA1)); auto rev2 = input2.getRev().value_or(Hash(htSHA1));
mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev()); mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev());
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev2.gitShortRev()); mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev2.gitShortRev());
// Backward compatibility: set 'revCount' to 0 for a dirty tree. // Backward compatibility: set 'revCount' to 0 for a dirty tree.
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), mkInt(*state.allocAttr(v, state.symbols.create("revCount")),
tree.info.revCount.value_or(0)); input2.getRevCount().value_or(0));
mkBool(*state.allocAttr(v, state.symbols.create("submodules")), fetchSubmodules); mkBool(*state.allocAttr(v, state.symbols.create("submodules")), fetchSubmodules);
v.attrs->sort(); v.attrs->sort();

View file

@ -65,23 +65,23 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url); attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
if (ref) attrs.insert_or_assign("ref", *ref); if (ref) attrs.insert_or_assign("ref", *ref);
if (rev) attrs.insert_or_assign("rev", rev->gitRev()); if (rev) attrs.insert_or_assign("rev", rev->gitRev());
auto input = fetchers::inputFromAttrs(attrs); auto input = fetchers::Input::fromAttrs(std::move(attrs));
// FIXME: use name // FIXME: use name
auto [tree, input2] = input->fetchTree(state.store); auto [tree, input2] = input.fetch(state.store);
state.mkAttrs(v, 8); state.mkAttrs(v, 8);
auto storePath = state.store->printStorePath(tree.storePath); auto storePath = state.store->printStorePath(tree.storePath);
mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
if (input2->getRef()) if (input2.getRef())
mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2->getRef()); mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2.getRef());
// Backward compatibility: set 'rev' to // Backward compatibility: set 'rev' to
// 0000000000000000000000000000000000000000 for a dirty tree. // 0000000000000000000000000000000000000000 for a dirty tree.
auto rev2 = input2->getRev().value_or(Hash(htSHA1)); auto rev2 = input2.getRev().value_or(Hash(htSHA1));
mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev()); mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev());
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(rev2.gitRev(), 0, 12)); mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(rev2.gitRev(), 0, 12));
if (tree.info.revCount) if (auto revCount = input2.getRevCount())
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount); mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
v.attrs->sort(); v.attrs->sort();
if (state.allowedPaths) if (state.allowedPaths)

View file

@ -3,6 +3,7 @@
#include "store-api.hh" #include "store-api.hh"
#include "fetchers.hh" #include "fetchers.hh"
#include "filetransfer.hh" #include "filetransfer.hh"
#include "registry.hh"
#include <ctime> #include <ctime>
#include <iomanip> #include <iomanip>
@ -12,30 +13,37 @@ namespace nix {
void emitTreeAttrs( void emitTreeAttrs(
EvalState & state, EvalState & state,
const fetchers::Tree & tree, const fetchers::Tree & tree,
std::shared_ptr<const fetchers::Input> input, const fetchers::Input & input,
Value & v) Value & v)
{ {
assert(input.isImmutable());
state.mkAttrs(v, 8); state.mkAttrs(v, 8);
auto storePath = state.store->printStorePath(tree.storePath); auto storePath = state.store->printStorePath(tree.storePath);
mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
assert(tree.info.narHash); // FIXME: support arbitrary input attributes.
mkString(*state.allocAttr(v, state.symbols.create("narHash")),
tree.info.narHash->to_string(SRI, true));
if (input->getRev()) { auto narHash = input.getNarHash();
mkString(*state.allocAttr(v, state.symbols.create("rev")), input->getRev()->gitRev()); assert(narHash);
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), input->getRev()->gitShortRev()); mkString(*state.allocAttr(v, state.symbols.create("narHash")),
narHash->to_string(SRI, true));
if (auto rev = input.getRev()) {
mkString(*state.allocAttr(v, state.symbols.create("rev")), rev->gitRev());
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev->gitShortRev());
} }
if (tree.info.revCount) if (auto revCount = input.getRevCount())
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount); mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
if (tree.info.lastModified) if (auto lastModified = input.getLastModified()) {
mkString(*state.allocAttr(v, state.symbols.create("lastModified")), mkInt(*state.allocAttr(v, state.symbols.create("lastModified")), *lastModified);
fmt("%s", std::put_time(std::gmtime(&*tree.info.lastModified), "%Y%m%d%H%M%S"))); mkString(*state.allocAttr(v, state.symbols.create("lastModifiedDate")),
fmt("%s", std::put_time(std::gmtime(&*lastModified), "%Y%m%d%H%M%S")));
}
v.attrs->sort(); v.attrs->sort();
} }
@ -44,7 +52,7 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
{ {
settings.requireExperimentalFeature("flakes"); settings.requireExperimentalFeature("flakes");
std::shared_ptr<const fetchers::Input> input; fetchers::Input input;
PathSet context; PathSet context;
state.forceValue(*args[0]); state.forceValue(*args[0]);
@ -59,9 +67,11 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
if (attr.value->type == tString) if (attr.value->type == tString)
attrs.emplace(attr.name, attr.value->string.s); attrs.emplace(attr.name, attr.value->string.s);
else if (attr.value->type == tBool) else if (attr.value->type == tBool)
attrs.emplace(attr.name, attr.value->boolean); attrs.emplace(attr.name, fetchers::Explicit<bool>{attr.value->boolean});
else if (attr.value->type == tInt)
attrs.emplace(attr.name, attr.value->integer);
else else
throw TypeError("fetchTree argument '%s' is %s while a string or Boolean is expected", throw TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected",
attr.name, showType(*attr.value)); attr.name, showType(*attr.value));
} }
@ -71,15 +81,17 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
.errPos = pos .errPos = pos
}); });
input = fetchers::inputFromAttrs(attrs); input = fetchers::Input::fromAttrs(std::move(attrs));
} else } else
input = fetchers::inputFromURL(state.coerceToString(pos, *args[0], context, false, false)); input = fetchers::Input::fromURL(state.coerceToString(pos, *args[0], context, false, false));
if (evalSettings.pureEval && !input->isImmutable()) if (!evalSettings.pureEval && !input.isDirect())
throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input"); input = lookupInRegistries(state.store, input).first;
// FIXME: use fetchOrSubstituteTree if (evalSettings.pureEval && !input.isImmutable())
auto [tree, input2] = input->fetchTree(state.store); throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input, at %s", pos);
auto [tree, input2] = input.fetch(state.store);
if (state.allowedPaths) if (state.allowedPaths)
state.allowedPaths->insert(tree.actualPath); state.allowedPaths->insert(tree.actualPath);
@ -136,7 +148,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
auto storePath = auto storePath =
unpack unpack
? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).first.storePath
: fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath; : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath;
auto path = state.store->toRealPath(storePath); auto path = state.store->toRealPath(storePath);

View file

@ -28,6 +28,12 @@ public:
return s == s2.s; return s == s2.s;
} }
// FIXME: remove
bool operator == (std::string_view s2) const
{
return s->compare(s2) == 0;
}
bool operator != (const Symbol & s2) const bool operator != (const Symbol & s2) const
{ {
return s != s2.s; return s != s2.s;
@ -68,9 +74,10 @@ private:
Symbols symbols; Symbols symbols;
public: public:
Symbol create(const string & s) Symbol create(std::string_view s)
{ {
std::pair<Symbols::iterator, bool> res = symbols.insert(s); // FIXME: avoid allocation if 's' already exists in the symbol table.
std::pair<Symbols::iterator, bool> res = symbols.emplace(std::string(s));
return Symbol(&*res.first); return Symbol(&*res.first);
} }

View file

@ -166,6 +166,13 @@ struct Value
{ {
return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size; return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size;
} }
/* Check whether forcing this value requires a trivial amount of
computation. In particular, function applications are
non-trivial. */
bool isTrivial() const;
std::vector<std::pair<Path, std::string>> getContext();
}; };

View file

@ -27,7 +27,7 @@ nlohmann::json attrsToJson(const Attrs & attrs)
{ {
nlohmann::json json; nlohmann::json json;
for (auto & attr : attrs) { for (auto & attr : attrs) {
if (auto v = std::get_if<int64_t>(&attr.second)) { if (auto v = std::get_if<uint64_t>(&attr.second)) {
json[attr.first] = *v; json[attr.first] = *v;
} else if (auto v = std::get_if<std::string>(&attr.second)) { } else if (auto v = std::get_if<std::string>(&attr.second)) {
json[attr.first] = *v; json[attr.first] = *v;
@ -55,16 +55,16 @@ std::string getStrAttr(const Attrs & attrs, const std::string & name)
return *s; return *s;
} }
std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name) std::optional<uint64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name)
{ {
auto i = attrs.find(name); auto i = attrs.find(name);
if (i == attrs.end()) return {}; if (i == attrs.end()) return {};
if (auto v = std::get_if<int64_t>(&i->second)) if (auto v = std::get_if<uint64_t>(&i->second))
return *v; return *v;
throw Error("input attribute '%s' is not an integer", name); throw Error("input attribute '%s' is not an integer", name);
} }
int64_t getIntAttr(const Attrs & attrs, const std::string & name) uint64_t getIntAttr(const Attrs & attrs, const std::string & name)
{ {
auto s = maybeGetIntAttr(attrs, name); auto s = maybeGetIntAttr(attrs, name);
if (!s) if (!s)
@ -76,8 +76,8 @@ std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & na
{ {
auto i = attrs.find(name); auto i = attrs.find(name);
if (i == attrs.end()) return {}; if (i == attrs.end()) return {};
if (auto v = std::get_if<int64_t>(&i->second)) if (auto v = std::get_if<Explicit<bool>>(&i->second))
return *v; return v->t;
throw Error("input attribute '%s' is not a Boolean", name); throw Error("input attribute '%s' is not a Boolean", name);
} }
@ -93,7 +93,7 @@ std::map<std::string, std::string> attrsToQuery(const Attrs & attrs)
{ {
std::map<std::string, std::string> query; std::map<std::string, std::string> query;
for (auto & attr : attrs) { for (auto & attr : attrs) {
if (auto v = std::get_if<int64_t>(&attr.second)) { if (auto v = std::get_if<uint64_t>(&attr.second)) {
query.insert_or_assign(attr.first, fmt("%d", *v)); query.insert_or_assign(attr.first, fmt("%d", *v));
} else if (auto v = std::get_if<std::string>(&attr.second)) { } else if (auto v = std::get_if<std::string>(&attr.second)) {
query.insert_or_assign(attr.first, *v); query.insert_or_assign(attr.first, *v);

View file

@ -13,9 +13,14 @@ namespace nix::fetchers {
template<typename T> template<typename T>
struct Explicit { struct Explicit {
T t; T t;
bool operator ==(const Explicit<T> & other) const
{
return t == other.t;
}
}; };
typedef std::variant<std::string, int64_t, Explicit<bool>> Attr; typedef std::variant<std::string, uint64_t, Explicit<bool>> Attr;
typedef std::map<std::string, Attr> Attrs; typedef std::map<std::string, Attr> Attrs;
Attrs jsonToAttrs(const nlohmann::json & json); Attrs jsonToAttrs(const nlohmann::json & json);
@ -26,9 +31,9 @@ std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::strin
std::string getStrAttr(const Attrs & attrs, const std::string & name); std::string getStrAttr(const Attrs & attrs, const std::string & name);
std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name); std::optional<uint64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name);
int64_t getIntAttr(const Attrs & attrs, const std::string & name); uint64_t getIntAttr(const Attrs & attrs, const std::string & name);
std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & name); std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & name);

View file

@ -5,70 +5,268 @@
namespace nix::fetchers { namespace nix::fetchers {
std::unique_ptr<std::vector<std::unique_ptr<InputScheme>>> inputSchemes = nullptr; std::unique_ptr<std::vector<std::shared_ptr<InputScheme>>> inputSchemes = nullptr;
void registerInputScheme(std::unique_ptr<InputScheme> && inputScheme) void registerInputScheme(std::shared_ptr<InputScheme> && inputScheme)
{ {
if (!inputSchemes) inputSchemes = std::make_unique<std::vector<std::unique_ptr<InputScheme>>>(); if (!inputSchemes) inputSchemes = std::make_unique<std::vector<std::shared_ptr<InputScheme>>>();
inputSchemes->push_back(std::move(inputScheme)); inputSchemes->push_back(std::move(inputScheme));
} }
std::unique_ptr<Input> inputFromURL(const ParsedURL & url) Input Input::fromURL(const std::string & url)
{
return fromURL(parseURL(url));
}
static void fixupInput(Input & input)
{
// Check common attributes.
input.getType();
input.getRef();
if (input.getRev())
input.immutable = true;
input.getRevCount();
input.getLastModified();
if (input.getNarHash())
input.immutable = true;
}
Input Input::fromURL(const ParsedURL & url)
{ {
for (auto & inputScheme : *inputSchemes) { for (auto & inputScheme : *inputSchemes) {
auto res = inputScheme->inputFromURL(url); auto res = inputScheme->inputFromURL(url);
if (res) return res; if (res) {
res->scheme = inputScheme;
fixupInput(*res);
return std::move(*res);
}
} }
throw Error("input '%s' is unsupported", url.url); throw Error("input '%s' is unsupported", url.url);
} }
std::unique_ptr<Input> inputFromURL(const std::string & url) Input Input::fromAttrs(Attrs && attrs)
{ {
return inputFromURL(parseURL(url));
}
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs)
{
auto attrs2(attrs);
attrs2.erase("narHash");
for (auto & inputScheme : *inputSchemes) { for (auto & inputScheme : *inputSchemes) {
auto res = inputScheme->inputFromAttrs(attrs2); auto res = inputScheme->inputFromAttrs(attrs);
if (res) { if (res) {
if (auto narHash = maybeGetStrAttr(attrs, "narHash")) res->scheme = inputScheme;
res->narHash = Hash::parseSRI(*narHash); fixupInput(*res);
return res; return std::move(*res);
} }
} }
throw Error("input '%s' is unsupported", attrsToJson(attrs));
Input input;
input.attrs = attrs;
fixupInput(input);
return input;
}
ParsedURL Input::toURL() const
{
if (!scheme)
throw Error("cannot show unsupported input '%s'", attrsToJson(attrs));
return scheme->toURL(*this);
}
std::string Input::to_string() const
{
return toURL().to_string();
} }
Attrs Input::toAttrs() const Attrs Input::toAttrs() const
{ {
auto attrs = toAttrsInternal();
if (narHash)
attrs.emplace("narHash", narHash->to_string(SRI, true));
attrs.emplace("type", type());
return attrs; return attrs;
} }
std::pair<Tree, std::shared_ptr<const Input>> Input::fetchTree(ref<Store> store) const bool Input::hasAllInfo() const
{ {
auto [tree, input] = fetchTreeInternal(store); return getNarHash() && scheme && scheme->hasAllInfo(*this);
}
bool Input::operator ==(const Input & other) const
{
return attrs == other.attrs;
}
bool Input::contains(const Input & other) const
{
if (*this == other) return true;
auto other2(other);
other2.attrs.erase("ref");
other2.attrs.erase("rev");
if (*this == other2) return true;
return false;
}
std::pair<Tree, Input> Input::fetch(ref<Store> store) const
{
if (!scheme)
throw Error("cannot fetch unsupported input '%s'", attrsToJson(toAttrs()));
/* The tree may already be in the Nix store, or it could be
substituted (which is often faster than fetching from the
original source). So check that. */
if (hasAllInfo()) {
try {
auto storePath = computeStorePath(*store);
store->ensurePath(storePath);
debug("using substituted/cached input '%s' in '%s'",
to_string(), store->printStorePath(storePath));
auto actualPath = store->toRealPath(storePath);
return {fetchers::Tree(std::move(actualPath), std::move(storePath)), *this};
} catch (Error & e) {
debug("substitution of input '%s' failed: %s", to_string(), e.what());
}
}
auto [tree, input] = scheme->fetch(store, *this);
if (tree.actualPath == "") if (tree.actualPath == "")
tree.actualPath = store->toRealPath(tree.storePath); tree.actualPath = store->toRealPath(tree.storePath);
if (!tree.info.narHash) auto narHash = store->queryPathInfo(tree.storePath)->narHash;
tree.info.narHash = store->queryPathInfo(tree.storePath)->narHash; input.attrs.insert_or_assign("narHash", narHash->to_string(SRI, true));
if (input->narHash) if (auto prevNarHash = getNarHash()) {
assert(input->narHash == tree.info.narHash); if (narHash != *prevNarHash)
throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash->to_string(SRI, true));
}
if (narHash && narHash != input->narHash) if (auto prevLastModified = getLastModified()) {
throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", if (input.getLastModified() != prevLastModified)
to_string(), tree.actualPath, narHash->to_string(SRI, true), input->narHash->to_string(SRI, true)); throw Error("'lastModified' attribute mismatch in input '%s', expected %d",
input.to_string(), *prevLastModified);
}
if (auto prevRevCount = getRevCount()) {
if (input.getRevCount() != prevRevCount)
throw Error("'revCount' attribute mismatch in input '%s', expected %d",
input.to_string(), *prevRevCount);
}
input.immutable = true;
assert(input.hasAllInfo());
return {std::move(tree), input}; return {std::move(tree), input};
} }
Input Input::applyOverrides(
std::optional<std::string> ref,
std::optional<Hash> rev) const
{
if (!scheme) return *this;
return scheme->applyOverrides(*this, ref, rev);
}
void Input::clone(const Path & destDir) const
{
assert(scheme);
scheme->clone(*this, destDir);
}
std::optional<Path> Input::getSourcePath() const
{
assert(scheme);
return scheme->getSourcePath(*this);
}
void Input::markChangedFile(
std::string_view file,
std::optional<std::string> commitMsg) const
{
assert(scheme);
return scheme->markChangedFile(*this, file, commitMsg);
}
StorePath Input::computeStorePath(Store & store) const
{
auto narHash = getNarHash();
if (!narHash)
throw Error("cannot compute store path for mutable input '%s'", to_string());
return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, "source");
}
std::string Input::getType() const
{
return getStrAttr(attrs, "type");
}
std::optional<Hash> Input::getNarHash() const
{
if (auto s = maybeGetStrAttr(attrs, "narHash")) {
auto hash = s->empty() ? Hash(htSHA256) : Hash::parseSRI(*s);
if (hash.type != htSHA256)
throw UsageError("narHash must be specified with SRI notation");
return newHashAllowEmpty(*s, htSHA256);
}
return {};
}
std::optional<std::string> Input::getRef() const
{
if (auto s = maybeGetStrAttr(attrs, "ref"))
return *s;
return {};
}
std::optional<Hash> Input::getRev() const
{
if (auto s = maybeGetStrAttr(attrs, "rev"))
return Hash::parseAny(*s, htSHA1);
return {};
}
std::optional<uint64_t> Input::getRevCount() const
{
if (auto n = maybeGetIntAttr(attrs, "revCount"))
return *n;
return {};
}
std::optional<time_t> Input::getLastModified() const
{
if (auto n = maybeGetIntAttr(attrs, "lastModified"))
return *n;
return {};
}
ParsedURL InputScheme::toURL(const Input & input)
{
throw Error("don't know how to convert input '%s' to a URL", attrsToJson(input.attrs));
}
Input InputScheme::applyOverrides(
const Input & input,
std::optional<std::string> ref,
std::optional<Hash> rev)
{
if (ref)
throw Error("don't know how to set branch/tag name of input '%s' to '%s'", input.to_string(), *ref);
if (rev)
throw Error("don't know how to set revision of input '%s' to '%s'", input.to_string(), rev->gitRev());
return input;
}
std::optional<Path> InputScheme::getSourcePath(const Input & input)
{
return {};
}
void InputScheme::markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg)
{
assert(false);
}
void InputScheme::clone(const Input & input, const Path & destDir)
{
throw Error("do not know how to clone input '%s'", input.to_string());
}
} }

View file

@ -3,7 +3,6 @@
#include "types.hh" #include "types.hh"
#include "hash.hh" #include "hash.hh"
#include "path.hh" #include "path.hh"
#include "tree-info.hh"
#include "attrs.hh" #include "attrs.hh"
#include "url.hh" #include "url.hh"
@ -13,73 +12,101 @@ namespace nix { class Store; }
namespace nix::fetchers { namespace nix::fetchers {
struct Input;
struct Tree struct Tree
{ {
Path actualPath; Path actualPath;
StorePath storePath; StorePath storePath;
TreeInfo info; Tree(Path && actualPath, StorePath && storePath) : actualPath(actualPath), storePath(std::move(storePath)) {}
}; };
struct Input : std::enable_shared_from_this<Input> struct InputScheme;
struct Input
{ {
std::optional<Hash> narHash; // FIXME: implement friend class InputScheme;
virtual std::string type() const = 0; std::shared_ptr<InputScheme> scheme; // note: can be null
Attrs attrs;
bool immutable = false;
bool direct = true;
virtual ~Input() { } public:
static Input fromURL(const std::string & url);
virtual bool operator ==(const Input & other) const { return false; } static Input fromURL(const ParsedURL & url);
/* Check whether this is a "direct" input, that is, not static Input fromAttrs(Attrs && attrs);
one that goes through a registry. */
virtual bool isDirect() const { return true; }
/* Check whether this is an "immutable" input, that is, ParsedURL toURL() const;
one that contains a commit hash or content hash. */
virtual bool isImmutable() const { return (bool) narHash; }
virtual bool contains(const Input & other) const { return false; } std::string to_string() const;
virtual std::optional<std::string> getRef() const { return {}; }
virtual std::optional<Hash> getRev() const { return {}; }
virtual ParsedURL toURL() const = 0;
std::string to_string() const
{
return toURL().to_string();
}
Attrs toAttrs() const; Attrs toAttrs() const;
std::pair<Tree, std::shared_ptr<const Input>> fetchTree(ref<Store> store) const; /* Check whether this is a "direct" input, that is, not
one that goes through a registry. */
bool isDirect() const { return direct; }
private: /* Check whether this is an "immutable" input, that is,
one that contains a commit hash or content hash. */
bool isImmutable() const { return immutable; }
virtual std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(ref<Store> store) const = 0; bool hasAllInfo() const;
virtual Attrs toAttrsInternal() const = 0; bool operator ==(const Input & other) const;
bool contains(const Input & other) const;
std::pair<Tree, Input> fetch(ref<Store> store) const;
Input applyOverrides(
std::optional<std::string> ref,
std::optional<Hash> rev) const;
void clone(const Path & destDir) const;
std::optional<Path> getSourcePath() const;
void markChangedFile(
std::string_view file,
std::optional<std::string> commitMsg) const;
StorePath computeStorePath(Store & store) const;
// Convience functions for common attributes.
std::string getType() const;
std::optional<Hash> getNarHash() const;
std::optional<std::string> getRef() const;
std::optional<Hash> getRev() const;
std::optional<uint64_t> getRevCount() const;
std::optional<time_t> getLastModified() const;
}; };
struct InputScheme struct InputScheme
{ {
virtual ~InputScheme() { } virtual std::optional<Input> inputFromURL(const ParsedURL & url) = 0;
virtual std::unique_ptr<Input> inputFromURL(const ParsedURL & url) = 0; virtual std::optional<Input> inputFromAttrs(const Attrs & attrs) = 0;
virtual std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) = 0; virtual ParsedURL toURL(const Input & input);
virtual bool hasAllInfo(const Input & input) = 0;
virtual Input applyOverrides(
const Input & input,
std::optional<std::string> ref,
std::optional<Hash> rev);
virtual void clone(const Input & input, const Path & destDir);
virtual std::optional<Path> getSourcePath(const Input & input);
virtual void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg);
virtual std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) = 0;
}; };
std::unique_ptr<Input> inputFromURL(const ParsedURL & url); void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);
std::unique_ptr<Input> inputFromURL(const std::string & url);
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs);
void registerInputScheme(std::unique_ptr<InputScheme> && fetcher);
struct DownloadFileResult struct DownloadFileResult
{ {
@ -94,7 +121,7 @@ DownloadFileResult downloadFile(
const std::string & name, const std::string & name,
bool immutable); bool immutable);
Tree downloadTarball( std::pair<Tree, time_t> downloadTarball(
ref<Store> store, ref<Store> store,
const std::string & url, const std::string & url,
const std::string & name, const std::string & name,

View file

@ -22,80 +22,152 @@ static bool isNotDotGitDirectory(const Path & path)
return not std::regex_match(path, gitDirRegex); return not std::regex_match(path, gitDirRegex);
} }
struct GitInput : Input struct GitInputScheme : InputScheme
{ {
ParsedURL url; std::optional<Input> inputFromURL(const ParsedURL & url) override
std::optional<std::string> ref;
std::optional<Hash> rev;
bool shallow = false;
bool submodules = false;
GitInput(const ParsedURL & url) : url(url)
{ }
std::string type() const override { return "git"; }
bool operator ==(const Input & other) const override
{ {
auto other2 = dynamic_cast<const GitInput *>(&other); if (url.scheme != "git" &&
return url.scheme != "git+http" &&
other2 url.scheme != "git+https" &&
&& url == other2->url url.scheme != "git+ssh" &&
&& rev == other2->rev url.scheme != "git+file") return {};
&& ref == other2->ref;
}
bool isImmutable() const override auto url2(url);
{ if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4);
return (bool) rev || narHash; url2.query.clear();
}
std::optional<std::string> getRef() const override { return ref; }
std::optional<Hash> getRev() const override { return rev; }
ParsedURL toURL() const override
{
ParsedURL url2(url);
if (url2.scheme != "git") url2.scheme = "git+" + url2.scheme;
if (rev) url2.query.insert_or_assign("rev", rev->gitRev());
if (ref) url2.query.insert_or_assign("ref", *ref);
if (shallow) url2.query.insert_or_assign("shallow", "1");
return url2;
}
Attrs toAttrsInternal() const override
{
Attrs attrs; Attrs attrs;
attrs.emplace("url", url.to_string()); attrs.emplace("type", "git");
if (ref)
attrs.emplace("ref", *ref); for (auto &[name, value] : url.query) {
if (rev) if (name == "rev" || name == "ref")
attrs.emplace("rev", rev->gitRev()); attrs.emplace(name, value);
if (shallow) else if (name == "shallow")
attrs.emplace("shallow", true); attrs.emplace(name, Explicit<bool> { value == "1" });
if (submodules) else
attrs.emplace("submodules", true); url2.query.emplace(name, value);
return attrs; }
attrs.emplace("url", url2.to_string());
return inputFromAttrs(attrs);
} }
std::pair<bool, std::string> getActualUrl() const std::optional<Input> inputFromAttrs(const Attrs & attrs) override
{
if (maybeGetStrAttr(attrs, "type") != "git") return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash")
throw Error("unsupported Git input attribute '%s'", name);
parseURL(getStrAttr(attrs, "url"));
maybeGetBoolAttr(attrs, "shallow");
maybeGetBoolAttr(attrs, "submodules");
if (auto ref = maybeGetStrAttr(attrs, "ref")) {
if (std::regex_search(*ref, badGitRefRegex))
throw BadURL("invalid Git branch/tag name '%s'", *ref);
}
Input input;
input.attrs = attrs;
return input;
}
ParsedURL toURL(const Input & input) override
{
auto url = parseURL(getStrAttr(input.attrs, "url"));
if (url.scheme != "git") url.scheme = "git+" + url.scheme;
if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev());
if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref);
if (maybeGetBoolAttr(input.attrs, "shallow").value_or(false))
url.query.insert_or_assign("shallow", "1");
return url;
}
bool hasAllInfo(const Input & input) override
{
bool maybeDirty = !input.getRef();
bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
return
maybeGetIntAttr(input.attrs, "lastModified")
&& (shallow || maybeDirty || maybeGetIntAttr(input.attrs, "revCount"));
}
Input applyOverrides(
const Input & input,
std::optional<std::string> ref,
std::optional<Hash> rev) override
{
auto res(input);
if (rev) res.attrs.insert_or_assign("rev", rev->gitRev());
if (ref) res.attrs.insert_or_assign("ref", *ref);
if (!res.getRef() && res.getRev())
throw Error("Git input '%s' has a commit hash but no branch/tag name", res.to_string());
return res;
}
void clone(const Input & input, const Path & destDir) override
{
auto [isLocal, actualUrl] = getActualUrl(input);
Strings args = {"clone"};
args.push_back(actualUrl);
if (auto ref = input.getRef()) {
args.push_back("--branch");
args.push_back(*ref);
}
if (input.getRev()) throw Error("cloning a specific revision is not implemented");
args.push_back(destDir);
runProgram("git", true, args);
}
std::optional<Path> getSourcePath(const Input & input) override
{
auto url = parseURL(getStrAttr(input.attrs, "url"));
if (url.scheme == "file" && !input.getRef() && !input.getRev())
return url.path;
return {};
}
void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override
{
auto sourcePath = getSourcePath(input);
assert(sourcePath);
runProgram("git", true,
{ "-C", *sourcePath, "add", "--force", "--intent-to-add", "--", std::string(file) });
if (commitMsg)
runProgram("git", true,
{ "-C", *sourcePath, "commit", std::string(file), "-m", *commitMsg });
}
std::pair<bool, std::string> getActualUrl(const Input & input) const
{ {
// Don't clone file:// URIs (but otherwise treat them the // Don't clone file:// URIs (but otherwise treat them the
// same as remote URIs, i.e. don't use the working tree or // same as remote URIs, i.e. don't use the working tree or
// HEAD). // HEAD).
static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing
auto url = parseURL(getStrAttr(input.attrs, "url"));
bool isLocal = url.scheme == "file" && !forceHttp; bool isLocal = url.scheme == "file" && !forceHttp;
return {isLocal, isLocal ? url.path : url.base}; return {isLocal, isLocal ? url.path : url.base};
} }
std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
{ {
auto name = "source"; auto name = "source";
auto input = std::make_shared<GitInput>(*this); Input input(_input);
assert(!rev || rev->type == htSHA1); bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
std::string cacheType = "git"; std::string cacheType = "git";
if (shallow) cacheType += "-shallow"; if (shallow) cacheType += "-shallow";
@ -106,39 +178,35 @@ struct GitInput : Input
return Attrs({ return Attrs({
{"type", cacheType}, {"type", cacheType},
{"name", name}, {"name", name},
{"rev", input->rev->gitRev()}, {"rev", input.getRev()->gitRev()},
}); });
}; };
auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
-> std::pair<Tree, std::shared_ptr<const Input>> -> std::pair<Tree, Input>
{ {
assert(input->rev); assert(input.getRev());
assert(!rev || rev == input->rev); assert(!_input.getRev() || _input.getRev() == input.getRev());
if (!shallow)
input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount"));
input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified"));
return { return {
Tree { Tree(store->toRealPath(storePath), std::move(storePath)),
.actualPath = store->toRealPath(storePath),
.storePath = std::move(storePath),
.info = TreeInfo {
.revCount = shallow ? std::nullopt : std::optional(getIntAttr(infoAttrs, "revCount")),
.lastModified = getIntAttr(infoAttrs, "lastModified"),
},
},
input input
}; };
}; };
if (rev) { if (input.getRev()) {
if (auto res = getCache()->lookup(store, getImmutableAttrs())) if (auto res = getCache()->lookup(store, getImmutableAttrs()))
return makeResult(res->first, std::move(res->second)); return makeResult(res->first, std::move(res->second));
} }
auto [isLocal, actualUrl_] = getActualUrl(); auto [isLocal, actualUrl_] = getActualUrl(input);
auto actualUrl = actualUrl_; // work around clang bug auto actualUrl = actualUrl_; // work around clang bug
// If this is a local directory and no ref or revision is // If this is a local directory and no ref or revision is
// given, then allow the use of an unclean working tree. // given, then allow the use of an unclean working tree.
if (!input->ref && !input->rev && isLocal) { if (!input.getRef() && !input.getRev() && isLocal) {
bool clean = false; bool clean = false;
/* Check whether this repo has any commits. There are /* Check whether this repo has any commits. There are
@ -197,35 +265,35 @@ struct GitInput : Input
auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter); auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
auto tree = Tree { // FIXME: maybe we should use the timestamp of the last
.actualPath = store->printStorePath(storePath), // modified dirty file?
.storePath = std::move(storePath), input.attrs.insert_or_assign(
.info = TreeInfo { "lastModified",
// FIXME: maybe we should use the timestamp of the last haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0);
// modified dirty file?
.lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0,
}
};
return {std::move(tree), input}; return {
Tree(store->printStorePath(storePath), std::move(storePath)),
input
};
} }
} }
if (!input->ref) input->ref = isLocal ? readHead(actualUrl) : "master"; if (!input.getRef()) input.attrs.insert_or_assign("ref", isLocal ? readHead(actualUrl) : "master");
Attrs mutableAttrs({ Attrs mutableAttrs({
{"type", cacheType}, {"type", cacheType},
{"name", name}, {"name", name},
{"url", actualUrl}, {"url", actualUrl},
{"ref", *input->ref}, {"ref", *input.getRef()},
}); });
Path repoDir; Path repoDir;
if (isLocal) { if (isLocal) {
if (!input->rev) if (!input.getRev())
input->rev = Hash::parseAny(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input->ref })), htSHA1); input.attrs.insert_or_assign("rev",
Hash::parseAny(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input.getRef() })), htSHA1).gitRev());
repoDir = actualUrl; repoDir = actualUrl;
@ -233,8 +301,8 @@ struct GitInput : Input
if (auto res = getCache()->lookup(store, mutableAttrs)) { if (auto res = getCache()->lookup(store, mutableAttrs)) {
auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1); auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1);
if (!rev || rev == rev2) { if (!input.getRev() || input.getRev() == rev2) {
input->rev = rev2; input.attrs.insert_or_assign("rev", rev2.gitRev());
return makeResult(res->first, std::move(res->second)); return makeResult(res->first, std::move(res->second));
} }
} }
@ -248,18 +316,18 @@ struct GitInput : Input
} }
Path localRefFile = Path localRefFile =
input->ref->compare(0, 5, "refs/") == 0 input.getRef()->compare(0, 5, "refs/") == 0
? cacheDir + "/" + *input->ref ? cacheDir + "/" + *input.getRef()
: cacheDir + "/refs/heads/" + *input->ref; : cacheDir + "/refs/heads/" + *input.getRef();
bool doFetch; bool doFetch;
time_t now = time(0); time_t now = time(0);
/* If a rev was specified, we need to fetch if it's not in the /* If a rev was specified, we need to fetch if it's not in the
repo. */ repo. */
if (input->rev) { if (input.getRev()) {
try { try {
runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input->rev->gitRev() }); runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input.getRev()->gitRev() });
doFetch = false; doFetch = false;
} catch (ExecError & e) { } catch (ExecError & e) {
if (WIFEXITED(e.status)) { if (WIFEXITED(e.status)) {
@ -282,9 +350,10 @@ struct GitInput : Input
// FIXME: git stderr messes up our progress indicator, so // FIXME: git stderr messes up our progress indicator, so
// we're using --quiet for now. Should process its stderr. // we're using --quiet for now. Should process its stderr.
try { try {
auto fetchRef = input->ref->compare(0, 5, "refs/") == 0 auto ref = input.getRef();
? *input->ref auto fetchRef = ref->compare(0, 5, "refs/") == 0
: "refs/heads/" + *input->ref; ? *ref
: "refs/heads/" + *ref;
runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }); runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) });
} catch (Error & e) { } catch (Error & e) {
if (!pathExists(localRefFile)) throw; if (!pathExists(localRefFile)) throw;
@ -300,8 +369,8 @@ struct GitInput : Input
utimes(localRefFile.c_str(), times); utimes(localRefFile.c_str(), times);
} }
if (!input->rev) if (!input.getRev())
input->rev = Hash::parseAny(chomp(readFile(localRefFile)), htSHA1); input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), htSHA1).gitRev());
} }
bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true"; bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true";
@ -311,7 +380,7 @@ struct GitInput : Input
// FIXME: check whether rev is an ancestor of ref. // FIXME: check whether rev is an ancestor of ref.
printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl); printTalkative("using revision %s of repo '%s'", input.getRev()->gitRev(), actualUrl);
/* Now that we know the ref, check again whether we have it in /* Now that we know the ref, check again whether we have it in
the store. */ the store. */
@ -333,7 +402,7 @@ struct GitInput : Input
runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force",
"--update-head-ok", "--", repoDir, "refs/*:refs/*" }); "--update-head-ok", "--", repoDir, "refs/*:refs/*" });
runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input->rev->gitRev() }); runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() });
runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", actualUrl }); runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", actualUrl });
runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" });
@ -342,7 +411,7 @@ struct GitInput : Input
// FIXME: should pipe this, or find some better way to extract a // FIXME: should pipe this, or find some better way to extract a
// revision. // revision.
auto source = sinkToSource([&](Sink & sink) { auto source = sinkToSource([&](Sink & sink) {
RunOptions gitOptions("git", { "-C", repoDir, "archive", input->rev->gitRev() }); RunOptions gitOptions("git", { "-C", repoDir, "archive", input.getRev()->gitRev() });
gitOptions.standardOut = &sink; gitOptions.standardOut = &sink;
runProgram2(gitOptions); runProgram2(gitOptions);
}); });
@ -352,18 +421,18 @@ struct GitInput : Input
auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter); auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter);
auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() })); auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input.getRev()->gitRev() }));
Attrs infoAttrs({ Attrs infoAttrs({
{"rev", input->rev->gitRev()}, {"rev", input.getRev()->gitRev()},
{"lastModified", lastModified}, {"lastModified", lastModified},
}); });
if (!shallow) if (!shallow)
infoAttrs.insert_or_assign("revCount", infoAttrs.insert_or_assign("revCount",
std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() }))); std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input.getRev()->gitRev() })));
if (!this->rev) if (!_input.getRev())
getCache()->add( getCache()->add(
store, store,
mutableAttrs, mutableAttrs,
@ -382,60 +451,6 @@ struct GitInput : Input
} }
}; };
struct GitInputScheme : InputScheme
{
std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
{
if (url.scheme != "git" &&
url.scheme != "git+http" &&
url.scheme != "git+https" &&
url.scheme != "git+ssh" &&
url.scheme != "git+file") return nullptr;
auto url2(url);
if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4);
url2.query.clear();
Attrs attrs;
attrs.emplace("type", "git");
for (auto &[name, value] : url.query) {
if (name == "rev" || name == "ref")
attrs.emplace(name, value);
else
url2.query.emplace(name, value);
}
attrs.emplace("url", url2.to_string());
return inputFromAttrs(attrs);
}
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
{
if (maybeGetStrAttr(attrs, "type") != "git") return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules")
throw Error("unsupported Git input attribute '%s'", name);
auto input = std::make_unique<GitInput>(parseURL(getStrAttr(attrs, "url")));
if (auto ref = maybeGetStrAttr(attrs, "ref")) {
if (std::regex_search(*ref, badGitRefRegex))
throw BadURL("invalid Git branch/tag name '%s'", *ref);
input->ref = *ref;
}
if (auto rev = maybeGetStrAttr(attrs, "rev"))
input->rev = Hash::parseAny(*rev, htSHA1);
input->shallow = maybeGetBoolAttr(attrs, "shallow").value_or(false);
input->submodules = maybeGetBoolAttr(attrs, "submodules").value_or(false);
return input;
}
};
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitInputScheme>()); }); static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitInputScheme>()); });
} }

View file

@ -8,81 +8,142 @@
namespace nix::fetchers { namespace nix::fetchers {
std::regex ownerRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); // A github or gitlab url
std::regex repoRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); const static std::string urlRegexS = "[a-zA-Z0-9.]*"; // FIXME: check
std::regex urlRegex(urlRegexS, std::regex::ECMAScript);
struct GitHubInput : Input struct GitArchiveInputScheme : InputScheme
{ {
std::string owner; virtual std::string type() = 0;
std::string repo;
std::optional<std::string> ref;
std::optional<Hash> rev;
std::string type() const override { return "github"; } std::optional<Input> inputFromURL(const ParsedURL & url) override
bool operator ==(const Input & other) const override
{ {
auto other2 = dynamic_cast<const GitHubInput *>(&other); if (url.scheme != type()) return {};
return
other2 auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
&& owner == other2->owner
&& repo == other2->repo std::optional<Hash> rev;
&& rev == other2->rev std::optional<std::string> ref;
&& ref == other2->ref; std::optional<std::string> host_url;
if (path.size() == 2) {
} else if (path.size() == 3) {
if (std::regex_match(path[2], revRegex))
rev = Hash::parseAny(path[2], htSHA1);
else if (std::regex_match(path[2], refRegex))
ref = path[2];
else
throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]);
} else
throw BadURL("URL '%s' is invalid", url.url);
for (auto &[name, value] : url.query) {
if (name == "rev") {
if (rev)
throw BadURL("URL '%s' contains multiple commit hashes", url.url);
rev = Hash::parseAny(value, htSHA1);
}
else if (name == "ref") {
if (!std::regex_match(value, refRegex))
throw BadURL("URL '%s' contains an invalid branch/tag name", url.url);
if (ref)
throw BadURL("URL '%s' contains multiple branch/tag names", url.url);
ref = value;
}
else if (name == "url") {
if (!std::regex_match(value, urlRegex))
throw BadURL("URL '%s' contains an invalid instance url", url.url);
host_url = value;
}
// FIXME: barf on unsupported attributes
}
if (ref && rev)
throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url.url, *ref, rev->gitRev());
Input input;
input.attrs.insert_or_assign("type", type());
input.attrs.insert_or_assign("owner", path[0]);
input.attrs.insert_or_assign("repo", path[1]);
if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
if (ref) input.attrs.insert_or_assign("ref", *ref);
if (host_url) input.attrs.insert_or_assign("url", *host_url);
return input;
} }
bool isImmutable() const override std::optional<Input> inputFromAttrs(const Attrs & attrs) override
{ {
return (bool) rev || narHash; if (maybeGetStrAttr(attrs, "type") != type()) return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified")
throw Error("unsupported input attribute '%s'", name);
getStrAttr(attrs, "owner");
getStrAttr(attrs, "repo");
Input input;
input.attrs = attrs;
return input;
} }
std::optional<std::string> getRef() const override { return ref; } ParsedURL toURL(const Input & input) override
std::optional<Hash> getRev() const override { return rev; }
ParsedURL toURL() const override
{ {
auto owner = getStrAttr(input.attrs, "owner");
auto repo = getStrAttr(input.attrs, "repo");
auto ref = input.getRef();
auto rev = input.getRev();
auto path = owner + "/" + repo; auto path = owner + "/" + repo;
assert(!(ref && rev)); assert(!(ref && rev));
if (ref) path += "/" + *ref; if (ref) path += "/" + *ref;
if (rev) path += "/" + rev->to_string(Base16, false); if (rev) path += "/" + rev->to_string(Base16, false);
return ParsedURL { return ParsedURL {
.scheme = "github", .scheme = type(),
.path = path, .path = path,
}; };
} }
Attrs toAttrsInternal() const override bool hasAllInfo(const Input & input) override
{ {
Attrs attrs; return input.getRev() && maybeGetIntAttr(input.attrs, "lastModified");
attrs.emplace("owner", owner);
attrs.emplace("repo", repo);
if (ref)
attrs.emplace("ref", *ref);
if (rev)
attrs.emplace("rev", rev->gitRev());
return attrs;
} }
std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override Input applyOverrides(
const Input & _input,
std::optional<std::string> ref,
std::optional<Hash> rev) override
{ {
auto rev = this->rev; auto input(_input);
auto ref = this->ref.value_or("master"); if (rev && ref)
throw BadURL("cannot apply both a commit hash (%s) and a branch/tag name ('%s') to input '%s'",
if (!rev) { rev->gitRev(), *ref, input.to_string());
auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s", if (rev) {
owner, repo, ref); input.attrs.insert_or_assign("rev", rev->gitRev());
auto json = nlohmann::json::parse( input.attrs.erase("ref");
readFile(
store->toRealPath(
downloadFile(store, url, "source", false).storePath)));
rev = Hash::parseAny(std::string { json["sha"] }, htSHA1);
debug("HEAD revision for '%s' is %s", url, rev->gitRev());
} }
if (ref) {
input.attrs.insert_or_assign("ref", *ref);
input.attrs.erase("rev");
}
return input;
}
auto input = std::make_shared<GitHubInput>(*this); virtual Hash getRevFromRef(nix::ref<Store> store, const Input & input) const = 0;
input->ref = {};
input->rev = *rev; virtual std::string getDownloadUrl(const Input & input) const = 0;
std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
{
Input input(_input);
if (!maybeGetStrAttr(input.attrs, "ref")) input.attrs.insert_or_assign("ref", "HEAD");
auto rev = input.getRev();
if (!rev) rev = getRevFromRef(store, input);
input.attrs.erase("ref");
input.attrs.insert_or_assign("rev", rev->gitRev());
Attrs immutableAttrs({ Attrs immutableAttrs({
{"type", "git-tarball"}, {"type", "git-tarball"},
@ -90,36 +151,25 @@ struct GitHubInput : Input
}); });
if (auto res = getCache()->lookup(store, immutableAttrs)) { if (auto res = getCache()->lookup(store, immutableAttrs)) {
input.attrs.insert_or_assign("lastModified", getIntAttr(res->first, "lastModified"));
return { return {
Tree{ Tree(store->toRealPath(res->second), std::move(res->second)),
.actualPath = store->toRealPath(res->second),
.storePath = std::move(res->second),
.info = TreeInfo {
.lastModified = getIntAttr(res->first, "lastModified"),
},
},
input input
}; };
} }
// FIXME: use regular /archive URLs instead? api.github.com auto url = getDownloadUrl(input);
// might have stricter rate limits.
auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s", auto [tree, lastModified] = downloadTarball(store, url, "source", true);
owner, repo, rev->to_string(Base16, false));
std::string accessToken = settings.githubAccessToken.get(); input.attrs.insert_or_assign("lastModified", lastModified);
if (accessToken != "")
url += "?access_token=" + accessToken;
auto tree = downloadTarball(store, url, "source", true);
getCache()->add( getCache()->add(
store, store,
immutableAttrs, immutableAttrs,
{ {
{"rev", rev->gitRev()}, {"rev", rev->gitRev()},
{"lastModified", *tree.info.lastModified} {"lastModified", lastModified}
}, },
tree.storePath, tree.storePath,
true); true);
@ -128,68 +178,96 @@ struct GitHubInput : Input
} }
}; };
struct GitHubInputScheme : InputScheme struct GitHubInputScheme : GitArchiveInputScheme
{ {
std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override std::string type() override { return "github"; }
Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
{ {
if (url.scheme != "github") return nullptr; auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check
auto path = tokenizeString<std::vector<std::string>>(url.path, "/"); host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
auto input = std::make_unique<GitHubInput>(); auto json = nlohmann::json::parse(
readFile(
if (path.size() == 2) { store->toRealPath(
} else if (path.size() == 3) { downloadFile(store, url, "source", false).storePath)));
if (std::regex_match(path[2], revRegex)) auto rev = Hash::parseAny(std::string { json["sha"] }, htSHA1);
input->rev = Hash::parseAny(path[2], htSHA1); debug("HEAD revision for '%s' is %s", url, rev.gitRev());
else if (std::regex_match(path[2], refRegex)) return rev;
input->ref = path[2];
else
throw BadURL("in GitHub URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]);
} else
throw BadURL("GitHub URL '%s' is invalid", url.url);
for (auto &[name, value] : url.query) {
if (name == "rev") {
if (input->rev)
throw BadURL("GitHub URL '%s' contains multiple commit hashes", url.url);
input->rev = Hash::parseAny(value, htSHA1);
}
else if (name == "ref") {
if (!std::regex_match(value, refRegex))
throw BadURL("GitHub URL '%s' contains an invalid branch/tag name", url.url);
if (input->ref)
throw BadURL("GitHub URL '%s' contains multiple branch/tag names", url.url);
input->ref = value;
}
}
if (input->ref && input->rev)
throw BadURL("GitHub URL '%s' contains both a commit hash and a branch/tag name", url.url);
input->owner = path[0];
input->repo = path[1];
return input;
} }
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override std::string getDownloadUrl(const Input & input) const override
{ {
if (maybeGetStrAttr(attrs, "type") != "github") return {}; // FIXME: use regular /archive URLs instead? api.github.com
// might have stricter rate limits.
auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
auto url = fmt("https://api.%s/repos/%s/%s/tarball/%s", // FIXME: check if this is correct for self hosted instances
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
input.getRev()->to_string(Base16, false));
for (auto & [name, value] : attrs) std::string accessToken = settings.githubAccessToken.get();
if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev") if (accessToken != "")
throw Error("unsupported GitHub input attribute '%s'", name); url += "?access_token=" + accessToken;
auto input = std::make_unique<GitHubInput>(); return url;
input->owner = getStrAttr(attrs, "owner"); }
input->repo = getStrAttr(attrs, "repo");
input->ref = maybeGetStrAttr(attrs, "ref"); void clone(const Input & input, const Path & destDir) override
if (auto rev = maybeGetStrAttr(attrs, "rev")) {
input->rev = Hash::parseAny(*rev, htSHA1); auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
return input; Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
.applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
.clone(destDir);
}
};
struct GitLabInputScheme : GitArchiveInputScheme
{
std::string type() override { return "gitlab"; }
Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
{
auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s",
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
auto json = nlohmann::json::parse(
readFile(
store->toRealPath(
downloadFile(store, url, "source", false).storePath)));
auto rev = Hash::parseAny(std::string(json[0]["id"]), htSHA1);
debug("HEAD revision for '%s' is %s", url, rev.gitRev());
return rev;
}
std::string getDownloadUrl(const Input & input) const override
{
// FIXME: This endpoint has a rate limit threshold of 5 requests per minute
auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s",
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
input.getRev()->to_string(Base16, false));
/* # FIXME: add privat token auth (`curl --header "PRIVATE-TOKEN: <your_access_token>"`)
std::string accessToken = settings.githubAccessToken.get();
if (accessToken != "")
url += "?access_token=" + accessToken;*/
return url;
}
void clone(const Input & input, const Path & destDir) override
{
auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
// FIXME: get username somewhere
Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
.applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
.clone(destDir);
} }
}; };
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitHubInputScheme>()); }); static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitHubInputScheme>()); });
static auto r2 = OnStartup([] { registerInputScheme(std::make_unique<GitLabInputScheme>()); });
} }

104
src/libfetchers/indirect.cc Normal file
View file

@ -0,0 +1,104 @@
#include "fetchers.hh"
namespace nix::fetchers {
std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
struct IndirectInputScheme : InputScheme
{
std::optional<Input> inputFromURL(const ParsedURL & url) override
{
if (url.scheme != "flake") return {};
auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
std::optional<Hash> rev;
std::optional<std::string> ref;
if (path.size() == 1) {
} else if (path.size() == 2) {
if (std::regex_match(path[1], revRegex))
rev = Hash::parseAny(path[1], htSHA1);
else if (std::regex_match(path[1], refRegex))
ref = path[1];
else
throw BadURL("in flake URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[1]);
} else if (path.size() == 3) {
if (!std::regex_match(path[1], refRegex))
throw BadURL("in flake URL '%s', '%s' is not a branch/tag name", url.url, path[1]);
ref = path[1];
if (!std::regex_match(path[2], revRegex))
throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]);
rev = Hash::parseAny(path[2], htSHA1);
} else
throw BadURL("GitHub URL '%s' is invalid", url.url);
std::string id = path[0];
if (!std::regex_match(id, flakeRegex))
throw BadURL("'%s' is not a valid flake ID", id);
// FIXME: forbid query params?
Input input;
input.direct = false;
input.attrs.insert_or_assign("type", "indirect");
input.attrs.insert_or_assign("id", id);
if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
if (ref) input.attrs.insert_or_assign("ref", *ref);
return input;
}
std::optional<Input> inputFromAttrs(const Attrs & attrs) override
{
if (maybeGetStrAttr(attrs, "type") != "indirect") return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "id" && name != "ref" && name != "rev" && name != "narHash")
throw Error("unsupported indirect input attribute '%s'", name);
auto id = getStrAttr(attrs, "id");
if (!std::regex_match(id, flakeRegex))
throw BadURL("'%s' is not a valid flake ID", id);
Input input;
input.direct = false;
input.attrs = attrs;
return input;
}
ParsedURL toURL(const Input & input) override
{
ParsedURL url;
url.scheme = "flake";
url.path = getStrAttr(input.attrs, "id");
if (auto ref = input.getRef()) { url.path += '/'; url.path += *ref; };
if (auto rev = input.getRev()) { url.path += '/'; url.path += rev->gitRev(); };
return url;
}
bool hasAllInfo(const Input & input) override
{
return false;
}
Input applyOverrides(
const Input & _input,
std::optional<std::string> ref,
std::optional<Hash> rev) override
{
auto input(_input);
if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
if (ref) input.attrs.insert_or_assign("ref", *ref);
return input;
}
std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
{
throw Error("indirect input '%s' cannot be fetched directly", input.to_string());
}
};
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<IndirectInputScheme>()); });
}

View file

@ -10,76 +10,124 @@ using namespace std::string_literals;
namespace nix::fetchers { namespace nix::fetchers {
struct MercurialInput : Input struct MercurialInputScheme : InputScheme
{ {
ParsedURL url; std::optional<Input> inputFromURL(const ParsedURL & url) override
std::optional<std::string> ref;
std::optional<Hash> rev;
MercurialInput(const ParsedURL & url) : url(url)
{ }
std::string type() const override { return "hg"; }
bool operator ==(const Input & other) const override
{ {
auto other2 = dynamic_cast<const MercurialInput *>(&other); if (url.scheme != "hg+http" &&
return url.scheme != "hg+https" &&
other2 url.scheme != "hg+ssh" &&
&& url == other2->url url.scheme != "hg+file") return {};
&& rev == other2->rev
&& ref == other2->ref; auto url2(url);
url2.scheme = std::string(url2.scheme, 3);
url2.query.clear();
Attrs attrs;
attrs.emplace("type", "hg");
for (auto &[name, value] : url.query) {
if (name == "rev" || name == "ref")
attrs.emplace(name, value);
else
url2.query.emplace(name, value);
}
attrs.emplace("url", url2.to_string());
return inputFromAttrs(attrs);
} }
bool isImmutable() const override std::optional<Input> inputFromAttrs(const Attrs & attrs) override
{ {
return (bool) rev || narHash; if (maybeGetStrAttr(attrs, "type") != "hg") return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash")
throw Error("unsupported Mercurial input attribute '%s'", name);
parseURL(getStrAttr(attrs, "url"));
if (auto ref = maybeGetStrAttr(attrs, "ref")) {
if (!std::regex_match(*ref, refRegex))
throw BadURL("invalid Mercurial branch/tag name '%s'", *ref);
}
Input input;
input.attrs = attrs;
return input;
} }
std::optional<std::string> getRef() const override { return ref; } ParsedURL toURL(const Input & input) override
std::optional<Hash> getRev() const override { return rev; }
ParsedURL toURL() const override
{ {
ParsedURL url2(url); auto url = parseURL(getStrAttr(input.attrs, "url"));
url2.scheme = "hg+" + url2.scheme; url.scheme = "hg+" + url.scheme;
if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev());
if (ref) url2.query.insert_or_assign("ref", *ref); if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref);
return url; return url;
} }
Attrs toAttrsInternal() const override bool hasAllInfo(const Input & input) override
{ {
Attrs attrs; // FIXME: ugly, need to distinguish between dirty and clean
attrs.emplace("url", url.to_string()); // default trees.
if (ref) return input.getRef() == "default" || maybeGetIntAttr(input.attrs, "revCount");
attrs.emplace("ref", *ref);
if (rev)
attrs.emplace("rev", rev->gitRev());
return attrs;
} }
std::pair<bool, std::string> getActualUrl() const Input applyOverrides(
const Input & input,
std::optional<std::string> ref,
std::optional<Hash> rev) override
{ {
auto res(input);
if (rev) res.attrs.insert_or_assign("rev", rev->gitRev());
if (ref) res.attrs.insert_or_assign("ref", *ref);
return res;
}
std::optional<Path> getSourcePath(const Input & input) override
{
auto url = parseURL(getStrAttr(input.attrs, "url"));
if (url.scheme == "file" && !input.getRef() && !input.getRev())
return url.path;
return {};
}
void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override
{
auto sourcePath = getSourcePath(input);
assert(sourcePath);
// FIXME: shut up if file is already tracked.
runProgram("hg", true,
{ "add", *sourcePath + "/" + std::string(file) });
if (commitMsg)
runProgram("hg", true,
{ "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg });
}
std::pair<bool, std::string> getActualUrl(const Input & input) const
{
auto url = parseURL(getStrAttr(input.attrs, "url"));
bool isLocal = url.scheme == "file"; bool isLocal = url.scheme == "file";
return {isLocal, isLocal ? url.path : url.base}; return {isLocal, isLocal ? url.path : url.base};
} }
std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
{ {
auto name = "source"; auto name = "source";
auto input = std::make_shared<MercurialInput>(*this); Input input(_input);
auto [isLocal, actualUrl_] = getActualUrl(); auto [isLocal, actualUrl_] = getActualUrl(input);
auto actualUrl = actualUrl_; // work around clang bug auto actualUrl = actualUrl_; // work around clang bug
// FIXME: return lastModified. // FIXME: return lastModified.
// FIXME: don't clone local repositories. // FIXME: don't clone local repositories.
if (!input->ref && !input->rev && isLocal && pathExists(actualUrl + "/.hg")) { if (!input.getRef() && !input.getRev() && isLocal && pathExists(actualUrl + "/.hg")) {
bool clean = runProgram("hg", true, { "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == ""; bool clean = runProgram("hg", true, { "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == "";
@ -94,7 +142,7 @@ struct MercurialInput : Input
if (settings.warnDirty) if (settings.warnDirty)
warn("Mercurial tree '%s' is unclean", actualUrl); warn("Mercurial tree '%s' is unclean", actualUrl);
input->ref = chomp(runProgram("hg", true, { "branch", "-R", actualUrl })); input.attrs.insert_or_assign("ref", chomp(runProgram("hg", true, { "branch", "-R", actualUrl })));
auto files = tokenizeString<std::set<std::string>>( auto files = tokenizeString<std::set<std::string>>(
runProgram("hg", true, { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s); runProgram("hg", true, { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
@ -116,60 +164,54 @@ struct MercurialInput : Input
auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter); auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
return {Tree { return {
.actualPath = store->printStorePath(storePath), Tree(store->printStorePath(storePath), std::move(storePath)),
.storePath = std::move(storePath), input
}, input}; };
} }
} }
if (!input->ref) input->ref = "default"; if (!input.getRef()) input.attrs.insert_or_assign("ref", "default");
auto getImmutableAttrs = [&]() auto getImmutableAttrs = [&]()
{ {
return Attrs({ return Attrs({
{"type", "hg"}, {"type", "hg"},
{"name", name}, {"name", name},
{"rev", input->rev->gitRev()}, {"rev", input.getRev()->gitRev()},
}); });
}; };
auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
-> std::pair<Tree, std::shared_ptr<const Input>> -> std::pair<Tree, Input>
{ {
assert(input->rev); assert(input.getRev());
assert(!rev || rev == input->rev); assert(!_input.getRev() || _input.getRev() == input.getRev());
input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount"));
return { return {
Tree{ Tree(store->toRealPath(storePath), std::move(storePath)),
.actualPath = store->toRealPath(storePath),
.storePath = std::move(storePath),
.info = TreeInfo {
.revCount = getIntAttr(infoAttrs, "revCount"),
},
},
input input
}; };
}; };
if (input->rev) { if (input.getRev()) {
if (auto res = getCache()->lookup(store, getImmutableAttrs())) if (auto res = getCache()->lookup(store, getImmutableAttrs()))
return makeResult(res->first, std::move(res->second)); return makeResult(res->first, std::move(res->second));
} }
assert(input->rev || input->ref); auto revOrRef = input.getRev() ? input.getRev()->gitRev() : *input.getRef();
auto revOrRef = input->rev ? input->rev->gitRev() : *input->ref;
Attrs mutableAttrs({ Attrs mutableAttrs({
{"type", "hg"}, {"type", "hg"},
{"name", name}, {"name", name},
{"url", actualUrl}, {"url", actualUrl},
{"ref", *input->ref}, {"ref", *input.getRef()},
}); });
if (auto res = getCache()->lookup(store, mutableAttrs)) { if (auto res = getCache()->lookup(store, mutableAttrs)) {
auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1); auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1);
if (!rev || rev == rev2) { if (!input.getRev() || input.getRev() == rev2) {
input->rev = rev2; input.attrs.insert_or_assign("rev", rev2.gitRev());
return makeResult(res->first, std::move(res->second)); return makeResult(res->first, std::move(res->second));
} }
} }
@ -178,10 +220,10 @@ struct MercurialInput : Input
/* If this is a commit hash that we already have, we don't /* If this is a commit hash that we already have, we don't
have to pull again. */ have to pull again. */
if (!(input->rev if (!(input.getRev()
&& pathExists(cacheDir) && pathExists(cacheDir)
&& runProgram( && runProgram(
RunOptions("hg", { "log", "-R", cacheDir, "-r", input->rev->gitRev(), "--template", "1" }) RunOptions("hg", { "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" })
.killStderr(true)).second == "1")) .killStderr(true)).second == "1"))
{ {
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl)); Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl));
@ -210,9 +252,9 @@ struct MercurialInput : Input
runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" })); runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" }));
assert(tokens.size() == 3); assert(tokens.size() == 3);
input->rev = Hash::parseAny(tokens[0], htSHA1); input.attrs.insert_or_assign("rev", Hash::parseAny(tokens[0], htSHA1).gitRev());
auto revCount = std::stoull(tokens[1]); auto revCount = std::stoull(tokens[1]);
input->ref = tokens[2]; input.attrs.insert_or_assign("ref", tokens[2]);
if (auto res = getCache()->lookup(store, getImmutableAttrs())) if (auto res = getCache()->lookup(store, getImmutableAttrs()))
return makeResult(res->first, std::move(res->second)); return makeResult(res->first, std::move(res->second));
@ -220,18 +262,18 @@ struct MercurialInput : Input
Path tmpDir = createTempDir(); Path tmpDir = createTempDir();
AutoDelete delTmpDir(tmpDir, true); AutoDelete delTmpDir(tmpDir, true);
runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input->rev->gitRev(), tmpDir }); runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input.getRev()->gitRev(), tmpDir });
deletePath(tmpDir + "/.hg_archival.txt"); deletePath(tmpDir + "/.hg_archival.txt");
auto storePath = store->addToStore(name, tmpDir); auto storePath = store->addToStore(name, tmpDir);
Attrs infoAttrs({ Attrs infoAttrs({
{"rev", input->rev->gitRev()}, {"rev", input.getRev()->gitRev()},
{"revCount", (int64_t) revCount}, {"revCount", (int64_t) revCount},
}); });
if (!this->rev) if (!_input.getRev())
getCache()->add( getCache()->add(
store, store,
mutableAttrs, mutableAttrs,
@ -250,54 +292,6 @@ struct MercurialInput : Input
} }
}; };
struct MercurialInputScheme : InputScheme
{
std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
{
if (url.scheme != "hg+http" &&
url.scheme != "hg+https" &&
url.scheme != "hg+ssh" &&
url.scheme != "hg+file") return nullptr;
auto url2(url);
url2.scheme = std::string(url2.scheme, 3);
url2.query.clear();
Attrs attrs;
attrs.emplace("type", "hg");
for (auto &[name, value] : url.query) {
if (name == "rev" || name == "ref")
attrs.emplace(name, value);
else
url2.query.emplace(name, value);
}
attrs.emplace("url", url2.to_string());
return inputFromAttrs(attrs);
}
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
{
if (maybeGetStrAttr(attrs, "type") != "hg") return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "url" && name != "ref" && name != "rev")
throw Error("unsupported Mercurial input attribute '%s'", name);
auto input = std::make_unique<MercurialInput>(parseURL(getStrAttr(attrs, "url")));
if (auto ref = maybeGetStrAttr(attrs, "ref")) {
if (!std::regex_match(*ref, refRegex))
throw BadURL("invalid Mercurial branch/tag name '%s'", *ref);
input->ref = *ref;
}
if (auto rev = maybeGetStrAttr(attrs, "rev"))
input->rev = Hash::parseAny(*rev, htSHA1);
return input;
}
};
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<MercurialInputScheme>()); }); static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<MercurialInputScheme>()); });
} }

View file

@ -3,65 +3,86 @@
namespace nix::fetchers { namespace nix::fetchers {
struct PathInput : Input struct PathInputScheme : InputScheme
{ {
Path path; std::optional<Input> inputFromURL(const ParsedURL & url) override
/* Allow the user to pass in "fake" tree info attributes. This is
useful for making a pinned tree work the same as the repository
from which is exported
(e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */
std::optional<Hash> rev;
std::optional<uint64_t> revCount;
std::optional<time_t> lastModified;
std::string type() const override { return "path"; }
std::optional<Hash> getRev() const override { return rev; }
bool operator ==(const Input & other) const override
{ {
auto other2 = dynamic_cast<const PathInput *>(&other); if (url.scheme != "path") return {};
return
other2 if (url.authority && *url.authority != "")
&& path == other2->path throw Error("path URL '%s' should not have an authority ('%s')", url.url, *url.authority);
&& rev == other2->rev
&& revCount == other2->revCount Input input;
&& lastModified == other2->lastModified; input.attrs.insert_or_assign("type", "path");
input.attrs.insert_or_assign("path", url.path);
for (auto & [name, value] : url.query)
if (name == "rev" || name == "narHash")
input.attrs.insert_or_assign(name, value);
else if (name == "revCount" || name == "lastModified") {
uint64_t n;
if (!string2Int(value, n))
throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
input.attrs.insert_or_assign(name, n);
}
else
throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name);
return input;
} }
bool isImmutable() const override std::optional<Input> inputFromAttrs(const Attrs & attrs) override
{ {
return (bool) narHash; if (maybeGetStrAttr(attrs, "type") != "path") return {};
getStrAttr(attrs, "path");
for (auto & [name, value] : attrs)
/* Allow the user to pass in "fake" tree info
attributes. This is useful for making a pinned tree
work the same as the repository from which is exported
(e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */
if (name == "type" || name == "rev" || name == "revCount" || name == "lastModified" || name == "narHash" || name == "path")
// checked in Input::fromAttrs
;
else
throw Error("unsupported path input attribute '%s'", name);
Input input;
input.attrs = attrs;
return input;
} }
ParsedURL toURL() const override ParsedURL toURL(const Input & input) override
{ {
auto query = attrsToQuery(toAttrsInternal()); auto query = attrsToQuery(input.attrs);
query.erase("path"); query.erase("path");
query.erase("type");
return ParsedURL { return ParsedURL {
.scheme = "path", .scheme = "path",
.path = path, .path = getStrAttr(input.attrs, "path"),
.query = query, .query = query,
}; };
} }
Attrs toAttrsInternal() const override bool hasAllInfo(const Input & input) override
{ {
Attrs attrs; return true;
attrs.emplace("path", path);
if (rev)
attrs.emplace("rev", rev->gitRev());
if (revCount)
attrs.emplace("revCount", *revCount);
if (lastModified)
attrs.emplace("lastModified", *lastModified);
return attrs;
} }
std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override std::optional<Path> getSourcePath(const Input & input) override
{ {
auto input = std::make_shared<PathInput>(*this); return getStrAttr(input.attrs, "path");
}
void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override
{
// nothing to do
}
std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
{
auto path = getStrAttr(input.attrs, "path");
// FIXME: check whether access to 'path' is allowed. // FIXME: check whether access to 'path' is allowed.
@ -74,72 +95,10 @@ struct PathInput : Input
// FIXME: try to substitute storePath. // FIXME: try to substitute storePath.
storePath = store->addToStore("source", path); storePath = store->addToStore("source", path);
return return {
{ Tree(store->toRealPath(*storePath), std::move(*storePath)),
Tree { input
.actualPath = store->toRealPath(*storePath), };
.storePath = std::move(*storePath),
.info = TreeInfo {
.revCount = revCount,
.lastModified = lastModified
}
},
input
};
}
};
struct PathInputScheme : InputScheme
{
std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
{
if (url.scheme != "path") return nullptr;
auto input = std::make_unique<PathInput>();
input->path = url.path;
for (auto & [name, value] : url.query)
if (name == "rev")
input->rev = Hash::parseAny(value, htSHA1);
else if (name == "revCount") {
uint64_t revCount;
if (!string2Int(value, revCount))
throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
input->revCount = revCount;
}
else if (name == "lastModified") {
time_t lastModified;
if (!string2Int(value, lastModified))
throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
input->lastModified = lastModified;
}
else
throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name);
return input;
}
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
{
if (maybeGetStrAttr(attrs, "type") != "path") return {};
auto input = std::make_unique<PathInput>();
input->path = getStrAttr(attrs, "path");
for (auto & [name, value] : attrs)
if (name == "rev")
input->rev = Hash::parseAny(getStrAttr(attrs, "rev"), htSHA1);
else if (name == "revCount")
input->revCount = getIntAttr(attrs, "revCount");
else if (name == "lastModified")
input->lastModified = getIntAttr(attrs, "lastModified");
else if (name == "type" || name == "path")
;
else
throw Error("unsupported path input attribute '%s'", name);
return input;
} }
}; };

212
src/libfetchers/registry.cc Normal file
View file

@ -0,0 +1,212 @@
#include "registry.hh"
#include "fetchers.hh"
#include "util.hh"
#include "globals.hh"
#include "store-api.hh"
#include <nlohmann/json.hpp>
namespace nix::fetchers {
std::shared_ptr<Registry> Registry::read(
const Path & path, RegistryType type)
{
auto registry = std::make_shared<Registry>(type);
if (!pathExists(path))
return std::make_shared<Registry>(type);
try {
auto json = nlohmann::json::parse(readFile(path));
auto version = json.value("version", 0);
if (version == 2) {
for (auto & i : json["flakes"]) {
auto toAttrs = jsonToAttrs(i["to"]);
Attrs extraAttrs;
auto j = toAttrs.find("dir");
if (j != toAttrs.end()) {
extraAttrs.insert(*j);
toAttrs.erase(j);
}
auto exact = i.find("exact");
registry->entries.push_back(
Entry {
.from = Input::fromAttrs(jsonToAttrs(i["from"])),
.to = Input::fromAttrs(std::move(toAttrs)),
.extraAttrs = extraAttrs,
.exact = exact != i.end() && exact.value()
});
}
}
else
throw Error("flake registry '%s' has unsupported version %d", path, version);
} catch (nlohmann::json::exception & e) {
warn("cannot parse flake registry '%s': %s", path, e.what());
} catch (Error & e) {
warn("cannot read flake registry '%s': %s", path, e.what());
}
return registry;
}
void Registry::write(const Path & path)
{
nlohmann::json arr;
for (auto & entry : entries) {
nlohmann::json obj;
obj["from"] = attrsToJson(entry.from.toAttrs());
obj["to"] = attrsToJson(entry.to.toAttrs());
if (!entry.extraAttrs.empty())
obj["to"].update(attrsToJson(entry.extraAttrs));
if (entry.exact)
obj["exact"] = true;
arr.emplace_back(std::move(obj));
}
nlohmann::json json;
json["version"] = 2;
json["flakes"] = std::move(arr);
createDirs(dirOf(path));
writeFile(path, json.dump(2));
}
void Registry::add(
const Input & from,
const Input & to,
const Attrs & extraAttrs)
{
entries.emplace_back(
Entry {
.from = from,
.to = to,
.extraAttrs = extraAttrs
});
}
void Registry::remove(const Input & input)
{
// FIXME: use C++20 std::erase.
for (auto i = entries.begin(); i != entries.end(); )
if (i->from == input)
i = entries.erase(i);
else
++i;
}
static Path getSystemRegistryPath()
{
return settings.nixConfDir + "/registry.json";
}
static std::shared_ptr<Registry> getSystemRegistry()
{
static auto systemRegistry =
Registry::read(getSystemRegistryPath(), Registry::System);
return systemRegistry;
}
Path getUserRegistryPath()
{
return getHome() + "/.config/nix/registry.json";
}
std::shared_ptr<Registry> getUserRegistry()
{
static auto userRegistry =
Registry::read(getUserRegistryPath(), Registry::User);
return userRegistry;
}
static std::shared_ptr<Registry> flagRegistry =
std::make_shared<Registry>(Registry::Flag);
std::shared_ptr<Registry> getFlagRegistry()
{
return flagRegistry;
}
void overrideRegistry(
const Input & from,
const Input & to,
const Attrs & extraAttrs)
{
flagRegistry->add(from, to, extraAttrs);
}
static std::shared_ptr<Registry> getGlobalRegistry(ref<Store> store)
{
static auto reg = [&]() {
auto path = settings.flakeRegistry.get();
if (!hasPrefix(path, "/")) {
auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath;
if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json", true);
path = store->toRealPath(storePath);
}
return Registry::read(path, Registry::Global);
}();
return reg;
}
Registries getRegistries(ref<Store> store)
{
Registries registries;
registries.push_back(getFlagRegistry());
registries.push_back(getUserRegistry());
registries.push_back(getSystemRegistry());
registries.push_back(getGlobalRegistry(store));
return registries;
}
std::pair<Input, Attrs> lookupInRegistries(
ref<Store> store,
const Input & _input)
{
Attrs extraAttrs;
int n = 0;
Input input(_input);
restart:
n++;
if (n > 100) throw Error("cycle detected in flake registry for '%s'", input.to_string());
for (auto & registry : getRegistries(store)) {
// FIXME: O(n)
for (auto & entry : registry->entries) {
if (entry.exact) {
if (entry.from == input) {
input = entry.to;
extraAttrs = entry.extraAttrs;
goto restart;
}
} else {
if (entry.from.contains(input)) {
input = entry.to.applyOverrides(
!entry.from.getRef() && input.getRef() ? input.getRef() : std::optional<std::string>(),
!entry.from.getRev() && input.getRev() ? input.getRev() : std::optional<Hash>());
extraAttrs = entry.extraAttrs;
goto restart;
}
}
}
}
if (!input.isDirect())
throw Error("cannot find flake '%s' in the flake registries", input.to_string());
debug("looked up '%s' -> '%s'", _input.to_string(), input.to_string());
return {input, extraAttrs};
}
}

View file

@ -0,0 +1,64 @@
#pragma once
#include "types.hh"
#include "fetchers.hh"
namespace nix { class Store; }
namespace nix::fetchers {
struct Registry
{
enum RegistryType {
Flag = 0,
User = 1,
System = 2,
Global = 3,
};
RegistryType type;
struct Entry
{
Input from, to;
Attrs extraAttrs;
bool exact = false;
};
std::vector<Entry> entries;
Registry(RegistryType type)
: type(type)
{ }
static std::shared_ptr<Registry> read(
const Path & path, RegistryType type);
void write(const Path & path);
void add(
const Input & from,
const Input & to,
const Attrs & extraAttrs);
void remove(const Input & input);
};
typedef std::vector<std::shared_ptr<Registry>> Registries;
std::shared_ptr<Registry> getUserRegistry();
Path getUserRegistryPath();
Registries getRegistries(ref<Store> store);
void overrideRegistry(
const Input & from,
const Input & to,
const Attrs & extraAttrs);
std::pair<Input, Attrs> lookupInRegistries(
ref<Store> store,
const Input & input);
}

View file

@ -105,7 +105,7 @@ DownloadFileResult downloadFile(
}; };
} }
Tree downloadTarball( std::pair<Tree, time_t> downloadTarball(
ref<Store> store, ref<Store> store,
const std::string & url, const std::string & url,
const std::string & name, const std::string & name,
@ -120,12 +120,9 @@ Tree downloadTarball(
auto cached = getCache()->lookupExpired(store, inAttrs); auto cached = getCache()->lookupExpired(store, inAttrs);
if (cached && !cached->expired) if (cached && !cached->expired)
return Tree { return {
.actualPath = store->toRealPath(cached->storePath), Tree(store->toRealPath(cached->storePath), std::move(cached->storePath)),
.storePath = std::move(cached->storePath), getIntAttr(cached->infoAttrs, "lastModified")
.info = TreeInfo {
.lastModified = getIntAttr(cached->infoAttrs, "lastModified"),
},
}; };
auto res = downloadFile(store, url, name, immutable); auto res = downloadFile(store, url, name, immutable);
@ -160,115 +157,72 @@ Tree downloadTarball(
*unpackedStorePath, *unpackedStorePath,
immutable); immutable);
return Tree { return {
.actualPath = store->toRealPath(*unpackedStorePath), Tree(store->toRealPath(*unpackedStorePath), std::move(*unpackedStorePath)),
.storePath = std::move(*unpackedStorePath), lastModified,
.info = TreeInfo {
.lastModified = lastModified,
},
}; };
} }
struct TarballInput : Input
{
ParsedURL url;
std::optional<Hash> hash;
TarballInput(const ParsedURL & url) : url(url)
{ }
std::string type() const override { return "tarball"; }
bool operator ==(const Input & other) const override
{
auto other2 = dynamic_cast<const TarballInput *>(&other);
return
other2
&& to_string() == other2->to_string()
&& hash == other2->hash;
}
bool isImmutable() const override
{
return hash || narHash;
}
ParsedURL toURL() const override
{
auto url2(url);
// NAR hashes are preferred over file hashes since tar/zip files
// don't have a canonical representation.
if (narHash)
url2.query.insert_or_assign("narHash", narHash->to_string(SRI, true));
else if (hash)
url2.query.insert_or_assign("hash", hash->to_string(SRI, true));
return url2;
}
Attrs toAttrsInternal() const override
{
Attrs attrs;
attrs.emplace("url", url.to_string());
if (hash)
attrs.emplace("hash", hash->to_string(SRI, true));
return attrs;
}
std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
{
auto tree = downloadTarball(store, url.to_string(), "source", false);
auto input = std::make_shared<TarballInput>(*this);
input->narHash = store->queryPathInfo(tree.storePath)->narHash;
return {std::move(tree), input};
}
};
struct TarballInputScheme : InputScheme struct TarballInputScheme : InputScheme
{ {
std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override std::optional<Input> inputFromURL(const ParsedURL & url) override
{ {
if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return nullptr; if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return {};
if (!hasSuffix(url.path, ".zip") if (!hasSuffix(url.path, ".zip")
&& !hasSuffix(url.path, ".tar") && !hasSuffix(url.path, ".tar")
&& !hasSuffix(url.path, ".tar.gz") && !hasSuffix(url.path, ".tar.gz")
&& !hasSuffix(url.path, ".tar.xz") && !hasSuffix(url.path, ".tar.xz")
&& !hasSuffix(url.path, ".tar.bz2")) && !hasSuffix(url.path, ".tar.bz2"))
return nullptr; return {};
auto input = std::make_unique<TarballInput>(url);
auto hash = input->url.query.find("hash");
if (hash != input->url.query.end()) {
input->hash = Hash::parseSRI(hash->second);
input->url.query.erase(hash);
}
auto narHash = input->url.query.find("narHash");
if (narHash != input->url.query.end()) {
input->narHash = Hash::parseSRI(narHash->second);
input->url.query.erase(narHash);
}
Input input;
input.attrs.insert_or_assign("type", "tarball");
input.attrs.insert_or_assign("url", url.to_string());
auto narHash = url.query.find("narHash");
if (narHash != url.query.end())
input.attrs.insert_or_assign("narHash", narHash->second);
return input; return input;
} }
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override std::optional<Input> inputFromAttrs(const Attrs & attrs) override
{ {
if (maybeGetStrAttr(attrs, "type") != "tarball") return {}; if (maybeGetStrAttr(attrs, "type") != "tarball") return {};
for (auto & [name, value] : attrs) for (auto & [name, value] : attrs)
if (name != "type" && name != "url" && name != "hash") if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash")
throw Error("unsupported tarball input attribute '%s'", name); throw Error("unsupported tarball input attribute '%s'", name);
auto input = std::make_unique<TarballInput>(parseURL(getStrAttr(attrs, "url"))); Input input;
if (auto hash = maybeGetStrAttr(attrs, "hash")) input.attrs = attrs;
input->hash = newHashAllowEmpty(*hash, {}); //input.immutable = (bool) maybeGetStrAttr(input.attrs, "hash");
return input; return input;
} }
ParsedURL toURL(const Input & input) override
{
auto url = parseURL(getStrAttr(input.attrs, "url"));
// NAR hashes are preferred over file hashes since tar/zip files
// don't have a canonical representation.
if (auto narHash = input.getNarHash())
url.query.insert_or_assign("narHash", narHash->to_string(SRI, true));
/*
else if (auto hash = maybeGetStrAttr(input.attrs, "hash"))
url.query.insert_or_assign("hash", Hash(*hash).to_string(SRI, true));
*/
return url;
}
bool hasAllInfo(const Input & input) override
{
return true;
}
std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
{
auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false).first;
return {std::move(tree), input};
}
}; };
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<TarballInputScheme>()); }); static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<TarballInputScheme>()); });

View file

@ -1,14 +0,0 @@
#include "tree-info.hh"
#include "store-api.hh"
#include <nlohmann/json.hpp>
namespace nix::fetchers {
StorePath TreeInfo::computeStorePath(Store & store) const
{
assert(narHash);
return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, "source");
}
}

View file

@ -1,29 +0,0 @@
#pragma once
#include "path.hh"
#include "hash.hh"
#include <nlohmann/json_fwd.hpp>
namespace nix { class Store; }
namespace nix::fetchers {
struct TreeInfo
{
std::optional<Hash> narHash;
std::optional<uint64_t> revCount;
std::optional<time_t> lastModified;
bool operator ==(const TreeInfo & other) const
{
return
narHash == other.narHash
&& revCount == other.revCount
&& lastModified == other.lastModified;
}
StorePath computeStorePath(Store & store) const;
};
}

View file

@ -34,9 +34,19 @@ MixCommonArgs::MixCommonArgs(const string & programName)
try { try {
globalConfig.set(name, value); globalConfig.set(name, value);
} catch (UsageError & e) { } catch (UsageError & e) {
warn(e.what()); if (!completions)
warn(e.what());
} }
}}, }},
.completer = [](size_t index, std::string_view prefix) {
if (index == 0) {
std::map<std::string, Config::SettingInfo> settings;
globalConfig.getSettings(settings);
for (auto & s : settings)
if (hasPrefix(s.first, prefix))
completions->insert(s.first);
}
}
}); });
addFlag({ addFlag({

View file

@ -1,12 +1,13 @@
#include "loggers.hh" #include "loggers.hh"
#include "progress-bar.hh" #include "progress-bar.hh"
#include "util.hh"
namespace nix { namespace nix {
LogFormat defaultLogFormat = LogFormat::raw; LogFormat defaultLogFormat = LogFormat::raw;
LogFormat parseLogFormat(const std::string & logFormatStr) { LogFormat parseLogFormat(const std::string & logFormatStr) {
if (logFormatStr == "raw") if (logFormatStr == "raw" || getEnv("NIX_GET_COMPLETIONS"))
return LogFormat::raw; return LogFormat::raw;
else if (logFormatStr == "raw-with-logs") else if (logFormatStr == "raw-with-logs")
return LogFormat::rawWithLogs; return LogFormat::rawWithLogs;

View file

@ -15,6 +15,7 @@
#include <chrono> #include <chrono>
#include <future> #include <future>
#include <regex> #include <regex>
#include <fstream>
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
@ -57,6 +58,13 @@ void BinaryCacheStore::init()
} }
} }
void BinaryCacheStore::upsertFile(const std::string & path,
std::string && data,
const std::string & mimeType)
{
upsertFile(path, std::make_shared<std::stringstream>(std::move(data)), mimeType);
}
void BinaryCacheStore::getFile(const std::string & path, void BinaryCacheStore::getFile(const std::string & path,
Callback<std::shared_ptr<std::string>> callback) noexcept Callback<std::shared_ptr<std::string>> callback) noexcept
{ {
@ -113,13 +121,74 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo)); diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo));
} }
void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource, AutoCloseFD openFile(const Path & path)
RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
{ {
// FIXME: See if we can use the original source to reduce memory usage. auto fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
auto nar = make_ref<std::string>(narSource.drain()); if (!fd)
throw SysError("opening file '%1%'", path);
return fd;
}
if (!repair && isValidPath(info.path)) return; struct FileSource : FdSource
{
AutoCloseFD fd2;
FileSource(const Path & path)
: fd2(openFile(path))
{
fd = fd2.get();
}
};
void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair, CheckSigsFlag checkSigs)
{
assert(info.narHash && info.narSize);
if (!repair && isValidPath(info.path)) {
// FIXME: copyNAR -> null sink
narSource.drain();
return;
}
auto [fdTemp, fnTemp] = createTempFile();
auto now1 = std::chrono::steady_clock::now();
/* Read the NAR simultaneously into a CompressionSink+FileSink (to
write the compressed NAR to disk), into a HashSink (to get the
NAR hash), and into a NarAccessor (to get the NAR listing). */
HashSink fileHashSink(htSHA256);
std::shared_ptr<FSAccessor> narAccessor;
{
FdSink fileSink(fdTemp.get());
TeeSink teeSink(fileSink, fileHashSink);
auto compressionSink = makeCompressionSink(compression, teeSink);
TeeSource teeSource(narSource, *compressionSink);
narAccessor = makeNarAccessor(teeSource);
compressionSink->finish();
}
auto now2 = std::chrono::steady_clock::now();
auto narInfo = make_ref<NarInfo>(info);
narInfo->narSize = info.narSize;
narInfo->narHash = info.narHash;
narInfo->compression = compression;
auto [fileHash, fileSize] = fileHashSink.finish();
narInfo->fileHash = fileHash;
narInfo->fileSize = fileSize;
narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar"
+ (compression == "xz" ? ".xz" :
compression == "bzip2" ? ".bz2" :
compression == "br" ? ".br" :
"");
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache",
printStorePath(narInfo->path), info.narSize,
((1.0 - (double) fileSize / info.narSize) * 100.0),
duration);
/* Verify that all references are valid. This may do some .narinfo /* Verify that all references are valid. This may do some .narinfo
reads, but typically they'll already be cached. */ reads, but typically they'll already be cached. */
@ -132,23 +201,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
printStorePath(info.path), printStorePath(ref)); printStorePath(info.path), printStorePath(ref));
} }
assert(nar->compare(0, narMagic.size(), narMagic) == 0);
auto narInfo = make_ref<NarInfo>(info);
narInfo->narSize = nar->size();
narInfo->narHash = hashString(htSHA256, *nar);
if (info.narHash && info.narHash != narInfo->narHash)
throw Error("refusing to copy corrupted path '%1%' to binary cache", printStorePath(info.path));
auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor);
auto narAccessor = makeNarAccessor(nar);
if (accessor_)
accessor_->addToCache(printStorePath(info.path), *nar, narAccessor);
/* Optionally write a JSON file containing a listing of the /* Optionally write a JSON file containing a listing of the
contents of the NAR. */ contents of the NAR. */
if (writeNARListing) { if (writeNARListing) {
@ -160,33 +212,13 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
{ {
auto res = jsonRoot.placeholder("root"); auto res = jsonRoot.placeholder("root");
listNar(res, narAccessor, "", true); listNar(res, ref<FSAccessor>(narAccessor), "", true);
} }
} }
upsertFile(std::string(info.path.to_string()) + ".ls", jsonOut.str(), "application/json"); upsertFile(std::string(info.path.to_string()) + ".ls", jsonOut.str(), "application/json");
} }
/* Compress the NAR. */
narInfo->compression = compression;
auto now1 = std::chrono::steady_clock::now();
auto narCompressed = compress(compression, *nar, parallelCompression);
auto now2 = std::chrono::steady_clock::now();
narInfo->fileHash = hashString(htSHA256, *narCompressed);
narInfo->fileSize = narCompressed->size();
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache",
printStorePath(narInfo->path), narInfo->narSize,
((1.0 - (double) narCompressed->size() / nar->size()) * 100.0),
duration);
narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar"
+ (compression == "xz" ? ".xz" :
compression == "bzip2" ? ".bz2" :
compression == "br" ? ".br" :
"");
/* Optionally maintain an index of DWARF debug info files /* Optionally maintain an index of DWARF debug info files
consisting of JSON files named 'debuginfo/<build-id>' that consisting of JSON files named 'debuginfo/<build-id>' that
specify the NAR file and member containing the debug info. */ specify the NAR file and member containing the debug info. */
@ -247,12 +279,14 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
/* Atomically write the NAR file. */ /* Atomically write the NAR file. */
if (repair || !fileExists(narInfo->url)) { if (repair || !fileExists(narInfo->url)) {
stats.narWrite++; stats.narWrite++;
upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar"); upsertFile(narInfo->url,
std::make_shared<std::fstream>(fnTemp, std::ios_base::in),
"application/x-nix-nar");
} else } else
stats.narWriteAverted++; stats.narWriteAverted++;
stats.narWriteBytes += nar->size(); stats.narWriteBytes += info.narSize;
stats.narWriteCompressedBytes += narCompressed->size(); stats.narWriteCompressedBytes += fileSize;
stats.narWriteCompressionTimeMs += duration; stats.narWriteCompressionTimeMs += duration;
/* Atomically write the NAR info file.*/ /* Atomically write the NAR info file.*/
@ -351,7 +385,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
ValidPathInfo info(makeFixedOutputPath(method, *h, name)); ValidPathInfo info(makeFixedOutputPath(method, *h, name));
auto source = StringSource { *sink.s }; auto source = StringSource { *sink.s };
addToStore(info, source, repair, CheckSigs, nullptr); addToStore(info, source, repair, CheckSigs);
return std::move(info.path); return std::move(info.path);
} }
@ -366,7 +400,7 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
StringSink sink; StringSink sink;
dumpString(s, sink); dumpString(s, sink);
auto source = StringSource { *sink.s }; auto source = StringSource { *sink.s };
addToStore(info, source, repair, CheckSigs, nullptr); addToStore(info, source, repair, CheckSigs);
} }
return std::move(info.path); return std::move(info.path);

View file

@ -36,9 +36,13 @@ public:
virtual bool fileExists(const std::string & path) = 0; virtual bool fileExists(const std::string & path) = 0;
virtual void upsertFile(const std::string & path, virtual void upsertFile(const std::string & path,
const std::string & data, std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType) = 0; const std::string & mimeType) = 0;
void upsertFile(const std::string & path,
std::string && data,
const std::string & mimeType);
/* Note: subclasses must implement at least one of the two /* Note: subclasses must implement at least one of the two
following getFile() methods. */ following getFile() methods. */
@ -75,8 +79,7 @@ public:
{ unsupported("queryPathFromHashPart"); } { unsupported("queryPathFromHashPart"); }
void addToStore(const ValidPathInfo & info, Source & narSource, void addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair, CheckSigsFlag checkSigs, RepairFlag repair, CheckSigsFlag checkSigs) override;
std::shared_ptr<FSAccessor> accessor) override;
StorePath addToStore(const string & name, const Path & srcPath, StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method, HashType hashAlgo, FileIngestionMethod method, HashType hashAlgo,

View file

@ -2044,7 +2044,7 @@ void DerivationGoal::startBuilder()
auto storePathS = *i++; auto storePathS = *i++;
if (!worker.store.isInStore(storePathS)) if (!worker.store.isInStore(storePathS))
throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS); throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS);
auto storePath = worker.store.parseStorePath(worker.store.toStorePath(storePathS)); auto storePath = worker.store.toStorePath(storePathS).first;
/* Write closure info to <fileName>. */ /* Write closure info to <fileName>. */
writeFile(tmpDir + "/" + fileName, writeFile(tmpDir + "/" + fileName,
@ -2083,7 +2083,7 @@ void DerivationGoal::startBuilder()
for (auto & i : dirsInChroot) for (auto & i : dirsInChroot)
try { try {
if (worker.store.isInStore(i.second.source)) if (worker.store.isInStore(i.second.source))
worker.store.computeFSClosure(worker.store.parseStorePath(worker.store.toStorePath(i.second.source)), closure); worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure);
} catch (InvalidPath & e) { } catch (InvalidPath & e) {
} catch (Error & e) { } catch (Error & e) {
throw Error("while processing 'sandbox-paths': %s", e.what()); throw Error("while processing 'sandbox-paths': %s", e.what());
@ -2768,10 +2768,9 @@ struct RestrictedStore : public LocalFSStore
{ throw Error("addToStore"); } { throw Error("addToStore"); }
void addToStore(const ValidPathInfo & info, Source & narSource, void addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override
std::shared_ptr<FSAccessor> accessor = 0) override
{ {
next->addToStore(info, narSource, repair, checkSigs, accessor); next->addToStore(info, narSource, repair, checkSigs);
goal.addDependency(info.path); goal.addDependency(info.path);
} }

View file

@ -9,7 +9,7 @@ struct Package {
Path path; Path path;
bool active; bool active;
int priority; int priority;
Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {} Package(const Path & path, bool active, int priority) : path{path}, active{active}, priority{priority} {}
}; };
typedef std::vector<Package> Packages; typedef std::vector<Package> Packages;

View file

@ -391,7 +391,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
} }
HashType hashAlgo = parseHashType(s); HashType hashAlgo = parseHashType(s);
TeeSource savedNAR(from); StringSink savedNAR;
TeeSource savedNARSource(from, savedNAR);
RetrieveRegularNARSink savedRegular; RetrieveRegularNARSink savedRegular;
if (method == FileIngestionMethod::Recursive) { if (method == FileIngestionMethod::Recursive) {
@ -399,7 +400,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
a string so that we can pass it to a string so that we can pass it to
addToStoreFromDump(). */ addToStoreFromDump(). */
ParseSink sink; /* null sink; just parse the NAR */ ParseSink sink; /* null sink; just parse the NAR */
parseDump(sink, savedNAR); parseDump(sink, savedNARSource);
} else } else
parseDump(savedRegular, from); parseDump(savedRegular, from);
@ -407,7 +408,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
if (!savedRegular.regular) throw Error("regular file expected"); if (!savedRegular.regular) throw Error("regular file expected");
auto path = store->addToStoreFromDump( auto path = store->addToStoreFromDump(
method == FileIngestionMethod::Recursive ? *savedNAR.data : savedRegular.s, method == FileIngestionMethod::Recursive ? *savedNAR.s : savedRegular.s,
baseName, baseName,
method, method,
hashAlgo); hashAlgo);
@ -442,7 +443,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopImportPaths: { case wopImportPaths: {
logger->startWork(); logger->startWork();
TunnelSource source(from, to); TunnelSource source(from, to);
auto paths = store->importPaths(source, nullptr, auto paths = store->importPaths(source,
trusted ? NoCheckSigs : CheckSigs); trusted ? NoCheckSigs : CheckSigs);
logger->stopWork(); logger->stopWork();
Strings paths2; Strings paths2;
@ -731,9 +732,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
if (GET_PROTOCOL_MINOR(clientVersion) >= 21) if (GET_PROTOCOL_MINOR(clientVersion) >= 21)
source = std::make_unique<TunnelSource>(from, to); source = std::make_unique<TunnelSource>(from, to);
else { else {
TeeSink tee(from); TeeParseSink tee(from);
parseDump(tee, tee.source); parseDump(tee, tee.source);
saved = std::move(*tee.source.data); saved = std::move(*tee.saved.s);
source = std::make_unique<StringSource>(saved); source = std::make_unique<StringSource>(saved);
} }
@ -741,7 +742,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
// FIXME: race if addToStore doesn't read source? // FIXME: race if addToStore doesn't read source?
store->addToStore(info, *source, (RepairFlag) repair, store->addToStore(info, *source, (RepairFlag) repair,
dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr); dontCheckSigs ? NoCheckSigs : CheckSigs);
logger->stopWork(); logger->stopWork();
break; break;

View file

@ -4,7 +4,6 @@
#include "util.hh" #include "util.hh"
#include "worker-protocol.hh" #include "worker-protocol.hh"
#include "fs-accessor.hh" #include "fs-accessor.hh"
#include "istringstream_nocopy.hh"
namespace nix { namespace nix {
@ -101,7 +100,7 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
} }
static DerivationOutput parseDerivationOutput(const Store & store, istringstream_nocopy & str) static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str)
{ {
expect(str, ","); auto path = store.parseStorePath(parsePath(str)); expect(str, ","); auto path = store.parseStorePath(parsePath(str));
expect(str, ","); auto hashAlgo = parseString(str); expect(str, ","); auto hashAlgo = parseString(str);
@ -129,10 +128,10 @@ static DerivationOutput parseDerivationOutput(const Store & store, istringstream
} }
static Derivation parseDerivation(const Store & store, const string & s) static Derivation parseDerivation(const Store & store, std::string && s)
{ {
Derivation drv; Derivation drv;
istringstream_nocopy str(s); std::istringstream str(std::move(s));
expect(str, "Derive(["); expect(str, "Derive([");
/* Parse the list of outputs. */ /* Parse the list of outputs. */

View file

@ -7,24 +7,6 @@
namespace nix { namespace nix {
struct HashAndWriteSink : Sink
{
Sink & writeSink;
HashSink hashSink;
HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256)
{
}
virtual void operator () (const unsigned char * data, size_t len)
{
writeSink(data, len);
hashSink(data, len);
}
Hash currentHash()
{
return hashSink.currentHash().first;
}
};
void Store::exportPaths(const StorePathSet & paths, Sink & sink) void Store::exportPaths(const StorePathSet & paths, Sink & sink)
{ {
auto sorted = topoSortPaths(paths); auto sorted = topoSortPaths(paths);
@ -47,28 +29,29 @@ void Store::exportPath(const StorePath & path, Sink & sink)
{ {
auto info = queryPathInfo(path); auto info = queryPathInfo(path);
HashAndWriteSink hashAndWriteSink(sink); HashSink hashSink(htSHA256);
TeeSink teeSink(sink, hashSink);
narFromPath(path, hashAndWriteSink); narFromPath(path, teeSink);
/* Refuse to export paths that have changed. This prevents /* Refuse to export paths that have changed. This prevents
filesystem corruption from spreading to other machines. filesystem corruption from spreading to other machines.
Don't complain if the stored hash is zero (unknown). */ Don't complain if the stored hash is zero (unknown). */
Hash hash = hashAndWriteSink.currentHash(); Hash hash = hashSink.currentHash().first;
if (hash != info->narHash && info->narHash != Hash(info->narHash->type)) if (hash != info->narHash && info->narHash != Hash(info->narHash->type))
throw Error("hash of path '%s' has changed from '%s' to '%s'!", throw Error("hash of path '%s' has changed from '%s' to '%s'!",
printStorePath(path), info->narHash->to_string(Base32, true), hash.to_string(Base32, true)); printStorePath(path), info->narHash->to_string(Base32, true), hash.to_string(Base32, true));
hashAndWriteSink teeSink
<< exportMagic << exportMagic
<< printStorePath(path); << printStorePath(path);
writeStorePaths(*this, hashAndWriteSink, info->references); writeStorePaths(*this, teeSink, info->references);
hashAndWriteSink teeSink
<< (info->deriver ? printStorePath(*info->deriver) : "") << (info->deriver ? printStorePath(*info->deriver) : "")
<< 0; << 0;
} }
StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, CheckSigsFlag checkSigs) StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
{ {
StorePaths res; StorePaths res;
while (true) { while (true) {
@ -77,7 +60,7 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> acces
if (n != 1) throw Error("input doesn't look like something created by 'nix-store --export'"); if (n != 1) throw Error("input doesn't look like something created by 'nix-store --export'");
/* Extract the NAR from the source. */ /* Extract the NAR from the source. */
TeeSink tee(source); TeeParseSink tee(source);
parseDump(tee, tee.source); parseDump(tee, tee.source);
uint32_t magic = readInt(source); uint32_t magic = readInt(source);
@ -94,16 +77,16 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> acces
if (deriver != "") if (deriver != "")
info.deriver = parseStorePath(deriver); info.deriver = parseStorePath(deriver);
info.narHash = hashString(htSHA256, *tee.source.data); info.narHash = hashString(htSHA256, *tee.saved.s);
info.narSize = tee.source.data->size(); info.narSize = tee.saved.s->size();
// Ignore optional legacy signature. // Ignore optional legacy signature.
if (readInt(source) == 1) if (readInt(source) == 1)
readString(source); readString(source);
// Can't use underlying source, which would have been exhausted // Can't use underlying source, which would have been exhausted
auto source = StringSource { *tee.source.data }; auto source = StringSource { *tee.saved.s };
addToStore(info, source, NoRepair, checkSigs, accessor); addToStore(info, source, NoRepair, checkSigs);
res.push_back(info.path); res.push_back(info.path);
} }

View file

@ -22,6 +22,7 @@
#include <queue> #include <queue>
#include <random> #include <random>
#include <thread> #include <thread>
#include <regex>
using namespace std::string_literals; using namespace std::string_literals;
@ -56,7 +57,7 @@ struct curlFileTransfer : public FileTransfer
Callback<FileTransferResult> callback; Callback<FileTransferResult> callback;
CURL * req = 0; CURL * req = 0;
bool active = false; // whether the handle has been added to the multi object bool active = false; // whether the handle has been added to the multi object
std::string status; std::string statusMsg;
unsigned int attempt = 0; unsigned int attempt = 0;
@ -175,12 +176,13 @@ struct curlFileTransfer : public FileTransfer
size_t realSize = size * nmemb; size_t realSize = size * nmemb;
std::string line((char *) contents, realSize); std::string line((char *) contents, realSize);
printMsg(lvlVomit, format("got header for '%s': %s") % request.uri % trim(line)); printMsg(lvlVomit, format("got header for '%s': %s") % request.uri % trim(line));
if (line.compare(0, 5, "HTTP/") == 0) { // new response starts static std::regex statusLine("HTTP/[^ ]+ +[0-9]+(.*)", std::regex::extended | std::regex::icase);
std::smatch match;
if (std::regex_match(line, match, statusLine)) {
result.etag = ""; result.etag = "";
auto ss = tokenizeString<vector<string>>(line, " ");
status = ss.size() >= 2 ? ss[1] : "";
result.data = std::make_shared<std::string>(); result.data = std::make_shared<std::string>();
result.bodySize = 0; result.bodySize = 0;
statusMsg = trim(match[1]);
acceptRanges = false; acceptRanges = false;
encoding = ""; encoding = "";
} else { } else {
@ -194,7 +196,9 @@ struct curlFileTransfer : public FileTransfer
the expected ETag on a 200 response, then shut the expected ETag on a 200 response, then shut
down the connection because we already have the down the connection because we already have the
data. */ data. */
if (result.etag == request.expectedETag && status == "200") { long httpStatus = 0;
curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus);
if (result.etag == request.expectedETag && httpStatus == 200) {
debug(format("shutting down on 200 HTTP response with expected ETag")); debug(format("shutting down on 200 HTTP response with expected ETag"));
return 0; return 0;
} }
@ -413,8 +417,8 @@ struct curlFileTransfer : public FileTransfer
? FileTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) ? FileTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri))
: httpStatus != 0 : httpStatus != 0
? FileTransferError(err, ? FileTransferError(err,
fmt("unable to %s '%s': HTTP error %d", fmt("unable to %s '%s': HTTP error %d ('%s')",
request.verb(), request.uri, httpStatus) request.verb(), request.uri, httpStatus, statusMsg)
+ (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) + (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code)))
) )
: FileTransferError(err, : FileTransferError(err,

View file

@ -262,11 +262,13 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
{ {
auto foundRoot = [&](const Path & path, const Path & target) { auto foundRoot = [&](const Path & path, const Path & target) {
auto storePath = maybeParseStorePath(toStorePath(target)); try {
if (storePath && isValidPath(*storePath)) auto storePath = toStorePath(target).first;
roots[std::move(*storePath)].emplace(path); if (isValidPath(storePath))
else roots[std::move(storePath)].emplace(path);
printInfo("skipping invalid root from '%1%' to '%2%'", path, target); else
printInfo("skipping invalid root from '%1%' to '%2%'", path, target);
} catch (BadStorePath &) { }
}; };
try { try {
@ -472,15 +474,15 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
for (auto & [target, links] : unchecked) { for (auto & [target, links] : unchecked) {
if (!isInStore(target)) continue; if (!isInStore(target)) continue;
Path pathS = toStorePath(target); try {
if (!isStorePath(pathS)) continue; auto path = toStorePath(target).first;
auto path = parseStorePath(pathS); if (!isValidPath(path)) continue;
if (!isValidPath(path)) continue; debug("got additional root '%1%'", printStorePath(path));
debug("got additional root '%1%'", pathS); if (censor)
if (censor) roots[path].insert(censored);
roots[path].insert(censored); else
else roots[path].insert(links.begin(), links.end());
roots[path].insert(links.begin(), links.end()); } catch (BadStorePath &) { }
} }
} }

View file

@ -368,6 +368,9 @@ public:
Setting<size_t> narBufferSize{this, 32 * 1024 * 1024, "nar-buffer-size", Setting<size_t> narBufferSize{this, 32 * 1024 * 1024, "nar-buffer-size",
"Maximum size of NARs before spilling them to disk."}; "Maximum size of NARs before spilling them to disk."};
Setting<std::string> flakeRegistry{this, "https://github.com/NixOS/flake-registry/raw/master/flake-registry.json", "flake-registry",
"Path or URI of the global flake registry."};
}; };

View file

@ -100,11 +100,11 @@ protected:
} }
void upsertFile(const std::string & path, void upsertFile(const std::string & path,
const std::string & data, std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType) override const std::string & mimeType) override
{ {
auto req = FileTransferRequest(cacheUri + "/" + path); auto req = FileTransferRequest(cacheUri + "/" + path);
req.data = std::make_shared<string>(data); // FIXME: inefficient req.data = std::make_shared<string>(StreamToSourceAdapter(istream).drain());
req.mimeType = mimeType; req.mimeType = mimeType;
try { try {
getFileTransfer()->upload(req); getFileTransfer()->upload(req);

View file

@ -126,8 +126,7 @@ struct LegacySSHStore : public Store
} }
void addToStore(const ValidPathInfo & info, Source & source, void addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs, RepairFlag repair, CheckSigsFlag checkSigs) override
std::shared_ptr<FSAccessor> accessor) override
{ {
debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host); debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host);

View file

@ -31,8 +31,18 @@ protected:
bool fileExists(const std::string & path) override; bool fileExists(const std::string & path) override;
void upsertFile(const std::string & path, void upsertFile(const std::string & path,
const std::string & data, std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType) override; const std::string & mimeType) override
{
auto path2 = binaryCacheDir + "/" + path;
Path tmp = path2 + ".tmp." + std::to_string(getpid());
AutoDelete del(tmp, false);
StreamToSourceAdapter source(istream);
writeFile(tmp, source);
if (rename(tmp.c_str(), path2.c_str()))
throw SysError("renaming '%1%' to '%2%'", tmp, path2);
del.cancel();
}
void getFile(const std::string & path, Sink & sink) override void getFile(const std::string & path, Sink & sink) override
{ {
@ -52,7 +62,9 @@ protected:
if (entry.name.size() != 40 || if (entry.name.size() != 40 ||
!hasSuffix(entry.name, ".narinfo")) !hasSuffix(entry.name, ".narinfo"))
continue; continue;
paths.insert(parseStorePath(storeDir + "/" + entry.name.substr(0, entry.name.size() - 8))); paths.insert(parseStorePath(
storeDir + "/" + entry.name.substr(0, entry.name.size() - 8)
+ "-" + MissingName));
} }
return paths; return paths;
@ -68,28 +80,11 @@ void LocalBinaryCacheStore::init()
BinaryCacheStore::init(); BinaryCacheStore::init();
} }
static void atomicWrite(const Path & path, const std::string & s)
{
Path tmp = path + ".tmp." + std::to_string(getpid());
AutoDelete del(tmp, false);
writeFile(tmp, s);
if (rename(tmp.c_str(), path.c_str()))
throw SysError("renaming '%1%' to '%2%'", tmp, path);
del.cancel();
}
bool LocalBinaryCacheStore::fileExists(const std::string & path) bool LocalBinaryCacheStore::fileExists(const std::string & path)
{ {
return pathExists(binaryCacheDir + "/" + path); return pathExists(binaryCacheDir + "/" + path);
} }
void LocalBinaryCacheStore::upsertFile(const std::string & path,
const std::string & data,
const std::string & mimeType)
{
atomicWrite(binaryCacheDir + "/" + path, data);
}
static RegisterStoreImplementation regStore([]( static RegisterStoreImplementation regStore([](
const std::string & uri, const Store::Params & params) const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store> -> std::shared_ptr<Store>

View file

@ -20,9 +20,9 @@ struct LocalStoreAccessor : public FSAccessor
Path toRealPath(const Path & path) Path toRealPath(const Path & path)
{ {
Path storePath = store->toStorePath(path); auto storePath = store->toStorePath(path).first;
if (!store->isValidPath(store->parseStorePath(storePath))) if (!store->isValidPath(storePath))
throw InvalidPath("path '%1%' is not a valid store path", storePath); throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath));
return store->getRealStoreDir() + std::string(path, store->storeDir.size()); return store->getRealStoreDir() + std::string(path, store->storeDir.size());
} }

View file

@ -594,7 +594,7 @@ uint64_t LocalStore::addValidPath(State & state,
(concatStringsSep(" ", info.sigs), !info.sigs.empty()) (concatStringsSep(" ", info.sigs), !info.sigs.empty())
(renderContentAddress(info.ca), (bool) info.ca) (renderContentAddress(info.ca), (bool) info.ca)
.exec(); .exec();
uint64_t id = sqlite3_last_insert_rowid(state.db); uint64_t id = state.db.getLastInsertedRowId();
/* If this is a derivation, then store the derivation outputs in /* If this is a derivation, then store the derivation outputs in
the database. This is useful for the garbage collector: it can the database. This is useful for the garbage collector: it can
@ -962,7 +962,7 @@ const PublicKeys & LocalStore::getPublicKeys()
void LocalStore::addToStore(const ValidPathInfo & info, Source & source, void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) RepairFlag repair, CheckSigsFlag checkSigs)
{ {
if (!info.narHash) if (!info.narHash)
throw Error("cannot add path '%s' because it lacks a hash", printStorePath(info.path)); throw Error("cannot add path '%s' because it lacks a hash", printStorePath(info.path));
@ -1097,13 +1097,16 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
{ {
Path srcPath(absPath(_srcPath)); Path srcPath(absPath(_srcPath));
if (method != FileIngestionMethod::Recursive)
return addToStoreFromDump(readFile(srcPath), name, method, hashAlgo, repair);
/* For computing the NAR hash. */ /* For computing the NAR hash. */
auto sha256Sink = std::make_unique<HashSink>(htSHA256); auto sha256Sink = std::make_unique<HashSink>(htSHA256);
/* For computing the store path. In recursive SHA-256 mode, this /* For computing the store path. In recursive SHA-256 mode, this
is the same as the NAR hash, so no need to do it again. */ is the same as the NAR hash, so no need to do it again. */
std::unique_ptr<HashSink> hashSink = std::unique_ptr<HashSink> hashSink =
method == FileIngestionMethod::Recursive && hashAlgo == htSHA256 hashAlgo == htSHA256
? nullptr ? nullptr
: std::make_unique<HashSink>(hashAlgo); : std::make_unique<HashSink>(hashAlgo);
@ -1136,10 +1139,7 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
if (!inMemory) sink(buf, len); if (!inMemory) sink(buf, len);
}); });
if (method == FileIngestionMethod::Recursive) dumpPath(srcPath, sink2, filter);
dumpPath(srcPath, sink2, filter);
else
readFile(srcPath, sink2);
}); });
std::unique_ptr<AutoDelete> delTempDir; std::unique_ptr<AutoDelete> delTempDir;
@ -1155,10 +1155,7 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
delTempDir = std::make_unique<AutoDelete>(tempDir); delTempDir = std::make_unique<AutoDelete>(tempDir);
tempPath = tempDir + "/x"; tempPath = tempDir + "/x";
if (method == FileIngestionMethod::Recursive) restorePath(tempPath, *source);
restorePath(tempPath, *source);
else
writeFile(tempPath, *source);
} catch (EndOfFile &) { } catch (EndOfFile &) {
if (!inMemory) throw; if (!inMemory) throw;
@ -1191,10 +1188,7 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
if (inMemory) { if (inMemory) {
/* Restore from the NAR in memory. */ /* Restore from the NAR in memory. */
StringSource source(nar); StringSource source(nar);
if (method == FileIngestionMethod::Recursive) restorePath(realPath, source);
restorePath(realPath, source);
else
writeFile(realPath, source);
} else { } else {
/* Move the temporary path we restored above. */ /* Move the temporary path we restored above. */
if (rename(tempPath.c_str(), realPath.c_str())) if (rename(tempPath.c_str(), realPath.c_str()))

View file

@ -143,8 +143,7 @@ public:
SubstitutablePathInfos & infos) override; SubstitutablePathInfos & infos) override;
void addToStore(const ValidPathInfo & info, Source & source, void addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs, RepairFlag repair, CheckSigsFlag checkSigs) override;
std::shared_ptr<FSAccessor> accessor) override;
StorePath addToStore(const string & name, const Path & srcPath, StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method, HashType hashAlgo, FileIngestionMethod method, HashType hashAlgo,

View file

@ -61,3 +61,6 @@ $(d)/build.cc:
clean-files += $(d)/schema.sql.gen.hh clean-files += $(d)/schema.sql.gen.hh
$(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644)) $(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644))
$(foreach i, $(wildcard src/libstore/builtins/*.hh), \
$(eval $(call install-file-in, $(i), $(includedir)/nix/builtins, 0644)))

View file

@ -18,7 +18,7 @@ struct NarMember
/* If this is a regular file, position of the contents of this /* If this is a regular file, position of the contents of this
file in the NAR. */ file in the NAR. */
size_t start = 0, size = 0; uint64_t start = 0, size = 0;
std::string target; std::string target;
@ -34,17 +34,19 @@ struct NarAccessor : public FSAccessor
NarMember root; NarMember root;
struct NarIndexer : ParseSink, StringSource struct NarIndexer : ParseSink, Source
{ {
NarAccessor & acc; NarAccessor & acc;
Source & source;
std::stack<NarMember *> parents; std::stack<NarMember *> parents;
std::string currentStart;
bool isExec = false; bool isExec = false;
NarIndexer(NarAccessor & acc, const std::string & nar) uint64_t pos = 0;
: StringSource(nar), acc(acc)
NarIndexer(NarAccessor & acc, Source & source)
: acc(acc), source(source)
{ } { }
void createMember(const Path & path, NarMember member) { void createMember(const Path & path, NarMember member) {
@ -79,31 +81,38 @@ struct NarAccessor : public FSAccessor
void preallocateContents(unsigned long long size) override void preallocateContents(unsigned long long size) override
{ {
currentStart = string(s, pos, 16); assert(size <= std::numeric_limits<uint64_t>::max());
assert(size <= std::numeric_limits<size_t>::max()); parents.top()->size = (uint64_t) size;
parents.top()->size = (size_t)size;
parents.top()->start = pos; parents.top()->start = pos;
} }
void receiveContents(unsigned char * data, unsigned int len) override void receiveContents(unsigned char * data, unsigned int len) override
{ { }
// Sanity check
if (!currentStart.empty()) {
assert(len < 16 || currentStart == string((char *) data, 16));
currentStart.clear();
}
}
void createSymlink(const Path & path, const string & target) override void createSymlink(const Path & path, const string & target) override
{ {
createMember(path, createMember(path,
NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target}); NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target});
} }
size_t read(unsigned char * data, size_t len) override
{
auto n = source.read(data, len);
pos += n;
return n;
}
}; };
NarAccessor(ref<const std::string> nar) : nar(nar) NarAccessor(ref<const std::string> nar) : nar(nar)
{ {
NarIndexer indexer(*this, *nar); StringSource source(*nar);
NarIndexer indexer(*this, source);
parseDump(indexer, indexer);
}
NarAccessor(Source & source)
{
NarIndexer indexer(*this, source);
parseDump(indexer, indexer); parseDump(indexer, indexer);
} }
@ -219,6 +228,11 @@ ref<FSAccessor> makeNarAccessor(ref<const std::string> nar)
return make_ref<NarAccessor>(nar); return make_ref<NarAccessor>(nar);
} }
ref<FSAccessor> makeNarAccessor(Source & source)
{
return make_ref<NarAccessor>(source);
}
ref<FSAccessor> makeLazyNarAccessor(const std::string & listing, ref<FSAccessor> makeLazyNarAccessor(const std::string & listing,
GetNarBytes getNarBytes) GetNarBytes getNarBytes)
{ {

View file

@ -6,10 +6,14 @@
namespace nix { namespace nix {
struct Source;
/* Return an object that provides access to the contents of a NAR /* Return an object that provides access to the contents of a NAR
file. */ file. */
ref<FSAccessor> makeNarAccessor(ref<const std::string> nar); ref<FSAccessor> makeNarAccessor(ref<const std::string> nar);
ref<FSAccessor> makeNarAccessor(Source & source);
/* Create a NAR accessor from a NAR listing (in the format produced by /* Create a NAR accessor from a NAR listing (in the format produced by
listNar()). The callback getNarBytes(offset, length) is used by the listNar()). The callback getNarBytes(offset, length) is used by the
readFile() method of the accessor to get the contents of files readFile() method of the accessor to get the contents of files

View file

@ -2,8 +2,6 @@
namespace nix { namespace nix {
MakeError(BadStorePath, Error);
static void checkName(std::string_view path, std::string_view name) static void checkName(std::string_view path, std::string_view name)
{ {
if (name.empty()) if (name.empty())

View file

@ -16,26 +16,26 @@ RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path & cacheDir)
createDirs(cacheDir); createDirs(cacheDir);
} }
Path RemoteFSAccessor::makeCacheFile(const Path & storePath, const std::string & ext) Path RemoteFSAccessor::makeCacheFile(std::string_view hashPart, const std::string & ext)
{ {
assert(cacheDir != ""); assert(cacheDir != "");
return fmt("%s/%s.%s", cacheDir, store->parseStorePath(storePath).hashPart(), ext); return fmt("%s/%s.%s", cacheDir, hashPart, ext);
} }
void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & nar, void RemoteFSAccessor::addToCache(std::string_view hashPart, const std::string & nar,
ref<FSAccessor> narAccessor) ref<FSAccessor> narAccessor)
{ {
nars.emplace(storePath, narAccessor); nars.emplace(hashPart, narAccessor);
if (cacheDir != "") { if (cacheDir != "") {
try { try {
std::ostringstream str; std::ostringstream str;
JSONPlaceholder jsonRoot(str); JSONPlaceholder jsonRoot(str);
listNar(jsonRoot, narAccessor, "", true); listNar(jsonRoot, narAccessor, "", true);
writeFile(makeCacheFile(storePath, "ls"), str.str()); writeFile(makeCacheFile(hashPart, "ls"), str.str());
/* FIXME: do this asynchronously. */ /* FIXME: do this asynchronously. */
writeFile(makeCacheFile(storePath, "nar"), nar); writeFile(makeCacheFile(hashPart, "nar"), nar);
} catch (...) { } catch (...) {
ignoreException(); ignoreException();
@ -47,23 +47,22 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
{ {
auto path = canonPath(path_); auto path = canonPath(path_);
auto storePath = store->toStorePath(path); auto [storePath, restPath] = store->toStorePath(path);
std::string restPath = std::string(path, storePath.size());
if (!store->isValidPath(store->parseStorePath(storePath))) if (!store->isValidPath(storePath))
throw InvalidPath("path '%1%' is not a valid store path", storePath); throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath));
auto i = nars.find(storePath); auto i = nars.find(std::string(storePath.hashPart()));
if (i != nars.end()) return {i->second, restPath}; if (i != nars.end()) return {i->second, restPath};
StringSink sink; StringSink sink;
std::string listing; std::string listing;
Path cacheFile; Path cacheFile;
if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath, "nar"))) { if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) {
try { try {
listing = nix::readFile(makeCacheFile(storePath, "ls")); listing = nix::readFile(makeCacheFile(storePath.hashPart(), "ls"));
auto narAccessor = makeLazyNarAccessor(listing, auto narAccessor = makeLazyNarAccessor(listing,
[cacheFile](uint64_t offset, uint64_t length) { [cacheFile](uint64_t offset, uint64_t length) {
@ -81,7 +80,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
return buf; return buf;
}); });
nars.emplace(storePath, narAccessor); nars.emplace(storePath.hashPart(), narAccessor);
return {narAccessor, restPath}; return {narAccessor, restPath};
} catch (SysError &) { } } catch (SysError &) { }
@ -90,15 +89,15 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
*sink.s = nix::readFile(cacheFile); *sink.s = nix::readFile(cacheFile);
auto narAccessor = makeNarAccessor(sink.s); auto narAccessor = makeNarAccessor(sink.s);
nars.emplace(storePath, narAccessor); nars.emplace(storePath.hashPart(), narAccessor);
return {narAccessor, restPath}; return {narAccessor, restPath};
} catch (SysError &) { } } catch (SysError &) { }
} }
store->narFromPath(store->parseStorePath(storePath), sink); store->narFromPath(storePath, sink);
auto narAccessor = makeNarAccessor(sink.s); auto narAccessor = makeNarAccessor(sink.s);
addToCache(storePath, *sink.s, narAccessor); addToCache(storePath.hashPart(), *sink.s, narAccessor);
return {narAccessor, restPath}; return {narAccessor, restPath};
} }

View file

@ -10,7 +10,7 @@ class RemoteFSAccessor : public FSAccessor
{ {
ref<Store> store; ref<Store> store;
std::map<Path, ref<FSAccessor>> nars; std::map<std::string, ref<FSAccessor>> nars;
Path cacheDir; Path cacheDir;
@ -18,9 +18,9 @@ class RemoteFSAccessor : public FSAccessor
friend class BinaryCacheStore; friend class BinaryCacheStore;
Path makeCacheFile(const Path & storePath, const std::string & ext); Path makeCacheFile(std::string_view hashPart, const std::string & ext);
void addToCache(const Path & storePath, const std::string & nar, void addToCache(std::string_view hashPart, const std::string & nar,
ref<FSAccessor> narAccessor); ref<FSAccessor> narAccessor);
public: public:

View file

@ -466,7 +466,7 @@ std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string &
void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) RepairFlag repair, CheckSigsFlag checkSigs)
{ {
auto conn(getConnection()); auto conn(getConnection());

View file

@ -60,8 +60,7 @@ public:
SubstitutablePathInfos & infos) override; SubstitutablePathInfos & infos) override;
void addToStore(const ValidPathInfo & info, Source & nar, void addToStore(const ValidPathInfo & info, Source & nar,
RepairFlag repair, CheckSigsFlag checkSigs, RepairFlag repair, CheckSigsFlag checkSigs) override;
std::shared_ptr<FSAccessor> accessor) override;
StorePath addToStore(const string & name, const Path & srcPath, StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,

View file

@ -7,7 +7,6 @@
#include "globals.hh" #include "globals.hh"
#include "compression.hh" #include "compression.hh"
#include "filetransfer.hh" #include "filetransfer.hh"
#include "istringstream_nocopy.hh"
#include <aws/core/Aws.h> #include <aws/core/Aws.h>
#include <aws/core/VersionConfig.h> #include <aws/core/VersionConfig.h>
@ -262,12 +261,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
std::shared_ptr<TransferManager> transferManager; std::shared_ptr<TransferManager> transferManager;
std::once_flag transferManagerCreated; std::once_flag transferManagerCreated;
void uploadFile(const std::string & path, const std::string & data, void uploadFile(const std::string & path,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType, const std::string & mimeType,
const std::string & contentEncoding) const std::string & contentEncoding)
{ {
auto stream = std::make_shared<istringstream_nocopy>(data);
auto maxThreads = std::thread::hardware_concurrency(); auto maxThreads = std::thread::hardware_concurrency();
static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor> static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor>
@ -307,7 +305,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
std::shared_ptr<TransferHandle> transferHandle = std::shared_ptr<TransferHandle> transferHandle =
transferManager->UploadFile( transferManager->UploadFile(
stream, bucketName, path, mimeType, istream, bucketName, path, mimeType,
Aws::Map<Aws::String, Aws::String>(), Aws::Map<Aws::String, Aws::String>(),
nullptr /*, contentEncoding */); nullptr /*, contentEncoding */);
@ -333,9 +331,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
if (contentEncoding != "") if (contentEncoding != "")
request.SetContentEncoding(contentEncoding); request.SetContentEncoding(contentEncoding);
auto stream = std::make_shared<istringstream_nocopy>(data); request.SetBody(istream);
request.SetBody(stream);
auto result = checkAws(fmt("AWS error uploading '%s'", path), auto result = checkAws(fmt("AWS error uploading '%s'", path),
s3Helper.client->PutObject(request)); s3Helper.client->PutObject(request));
@ -347,25 +343,34 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1) std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1)
.count(); .count();
printInfo(format("uploaded 's3://%1%/%2%' (%3% bytes) in %4% ms") % auto size = istream->tellg();
bucketName % path % data.size() % duration);
printInfo("uploaded 's3://%s/%s' (%d bytes) in %d ms",
bucketName, path, size, duration);
stats.putTimeMs += duration; stats.putTimeMs += duration;
stats.putBytes += data.size(); stats.putBytes += size;
stats.put++; stats.put++;
} }
void upsertFile(const std::string & path, const std::string & data, void upsertFile(const std::string & path,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType) override const std::string & mimeType) override
{ {
auto compress = [&](std::string compression)
{
auto compressed = nix::compress(compression, StreamToSourceAdapter(istream).drain());
return std::make_shared<std::stringstream>(std::move(*compressed));
};
if (narinfoCompression != "" && hasSuffix(path, ".narinfo")) if (narinfoCompression != "" && hasSuffix(path, ".narinfo"))
uploadFile(path, *compress(narinfoCompression, data), mimeType, narinfoCompression); uploadFile(path, compress(narinfoCompression), mimeType, narinfoCompression);
else if (lsCompression != "" && hasSuffix(path, ".ls")) else if (lsCompression != "" && hasSuffix(path, ".ls"))
uploadFile(path, *compress(lsCompression, data), mimeType, lsCompression); uploadFile(path, compress(lsCompression), mimeType, lsCompression);
else if (logCompression != "" && hasPrefix(path, "log/")) else if (logCompression != "" && hasPrefix(path, "log/"))
uploadFile(path, *compress(logCompression, data), mimeType, logCompression); uploadFile(path, compress(logCompression), mimeType, logCompression);
else else
uploadFile(path, data, mimeType, ""); uploadFile(path, istream, mimeType, "");
} }
void getFile(const std::string & path, Sink & sink) override void getFile(const std::string & path, Sink & sink) override
@ -410,7 +415,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
for (auto object : contents) { for (auto object : contents) {
auto & key = object.GetKey(); auto & key = object.GetKey();
if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue; if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue;
paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-unknown")); paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-" + MissingName));
} }
marker = res.GetNextMarker(); marker = res.GetNextMarker();

View file

@ -61,6 +61,11 @@ void SQLite::exec(const std::string & stmt)
}); });
} }
uint64_t SQLite::getLastInsertedRowId()
{
return sqlite3_last_insert_rowid(db);
}
void SQLiteStmt::create(sqlite3 * db, const string & sql) void SQLiteStmt::create(sqlite3 * db, const string & sql)
{ {
checkInterrupt(); checkInterrupt();
@ -95,10 +100,10 @@ SQLiteStmt::Use::~Use()
sqlite3_reset(stmt); sqlite3_reset(stmt);
} }
SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool notNull) SQLiteStmt::Use & SQLiteStmt::Use::operator () (std::string_view value, bool notNull)
{ {
if (notNull) { if (notNull) {
if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) if (sqlite3_bind_text(stmt, curArg++, value.data(), -1, SQLITE_TRANSIENT) != SQLITE_OK)
throwSQLiteError(stmt.db, "binding argument"); throwSQLiteError(stmt.db, "binding argument");
} else } else
bind(); bind();

View file

@ -26,6 +26,8 @@ struct SQLite
void isCache(); void isCache();
void exec(const std::string & stmt); void exec(const std::string & stmt);
uint64_t getLastInsertedRowId();
}; };
/* RAII wrapper to create and destroy SQLite prepared statements. */ /* RAII wrapper to create and destroy SQLite prepared statements. */
@ -54,7 +56,7 @@ struct SQLiteStmt
~Use(); ~Use();
/* Bind the next parameter. */ /* Bind the next parameter. */
Use & operator () (const std::string & value, bool notNull = true); Use & operator () (std::string_view value, bool notNull = true);
Use & operator () (const unsigned char * data, size_t len, bool notNull = true); Use & operator () (const unsigned char * data, size_t len, bool notNull = true);
Use & operator () (int64_t value, bool notNull = true); Use & operator () (int64_t value, bool notNull = true);
Use & bind(); // null Use & bind(); // null

View file

@ -21,15 +21,15 @@ bool Store::isInStore(const Path & path) const
} }
Path Store::toStorePath(const Path & path) const std::pair<StorePath, Path> Store::toStorePath(const Path & path) const
{ {
if (!isInStore(path)) if (!isInStore(path))
throw Error("path '%1%' is not in the Nix store", path); throw Error("path '%1%' is not in the Nix store", path);
Path::size_type slash = path.find('/', storeDir.size() + 1); Path::size_type slash = path.find('/', storeDir.size() + 1);
if (slash == Path::npos) if (slash == Path::npos)
return path; return {parseStorePath(path), ""};
else else
return Path(path, 0, slash); return {parseStorePath(std::string_view(path).substr(0, slash)), path.substr(slash)};
} }
@ -42,14 +42,14 @@ Path Store::followLinksToStore(std::string_view _path) const
path = absPath(target, dirOf(path)); path = absPath(target, dirOf(path));
} }
if (!isInStore(path)) if (!isInStore(path))
throw NotInStore("path '%1%' is not in the Nix store", path); throw BadStorePath("path '%1%' is not in the Nix store", path);
return path; return path;
} }
StorePath Store::followLinksToStorePath(std::string_view path) const StorePath Store::followLinksToStorePath(std::string_view path) const
{ {
return parseStorePath(toStorePath(followLinksToStore(path))); return toStorePath(followLinksToStore(path)).first;
} }
@ -351,6 +351,14 @@ ref<const ValidPathInfo> Store::queryPathInfo(const StorePath & storePath)
} }
static bool goodStorePath(const StorePath & expected, const StorePath & actual)
{
return
expected.hashPart() == actual.hashPart()
&& (expected.name() == Store::MissingName || expected.name() == actual.name());
}
void Store::queryPathInfo(const StorePath & storePath, void Store::queryPathInfo(const StorePath & storePath,
Callback<ref<const ValidPathInfo>> callback) noexcept Callback<ref<const ValidPathInfo>> callback) noexcept
{ {
@ -378,7 +386,7 @@ void Store::queryPathInfo(const StorePath & storePath,
state_->pathInfoCache.upsert(hashPart, state_->pathInfoCache.upsert(hashPart,
res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second }); res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second });
if (res.first == NarInfoDiskCache::oInvalid || if (res.first == NarInfoDiskCache::oInvalid ||
res.second->path != storePath) !goodStorePath(storePath, res.second->path))
throw InvalidPath("path '%s' is not valid", printStorePath(storePath)); throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
} }
return callback(ref<const ValidPathInfo>(res.second)); return callback(ref<const ValidPathInfo>(res.second));
@ -390,7 +398,7 @@ void Store::queryPathInfo(const StorePath & storePath,
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
queryPathInfoUncached(storePath, queryPathInfoUncached(storePath,
{[this, storePath{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) { {[this, storePathS{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) {
try { try {
auto info = fut.get(); auto info = fut.get();
@ -403,9 +411,11 @@ void Store::queryPathInfo(const StorePath & storePath,
state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info }); state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info });
} }
if (!info || info->path != parseStorePath(storePath)) { auto storePath = parseStorePath(storePathS);
if (!info || !goodStorePath(storePath, info->path)) {
stats.narInfoMissing++; stats.narInfoMissing++;
throw InvalidPath("path '%s' is not valid", storePath); throw InvalidPath("path '%s' is not valid", storePathS);
} }
(*callbackPtr)(ref<const ValidPathInfo>(info)); (*callbackPtr)(ref<const ValidPathInfo>(info));
@ -550,7 +560,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store
if (!narInfo->url.empty()) if (!narInfo->url.empty())
jsonPath.attr("url", narInfo->url); jsonPath.attr("url", narInfo->url);
if (narInfo->fileHash) if (narInfo->fileHash)
jsonPath.attr("downloadHash", narInfo->fileHash->to_string(Base32, true)); jsonPath.attr("downloadHash", narInfo->fileHash->to_string(hashBase, true));
if (narInfo->fileSize) if (narInfo->fileSize)
jsonPath.attr("downloadSize", narInfo->fileSize); jsonPath.attr("downloadSize", narInfo->fileSize);
if (showClosureSize) if (showClosureSize)

View file

@ -31,7 +31,7 @@ MakeError(InvalidPath, Error);
MakeError(Unsupported, Error); MakeError(Unsupported, Error);
MakeError(SubstituteGone, Error); MakeError(SubstituteGone, Error);
MakeError(SubstituterDisabled, Error); MakeError(SubstituterDisabled, Error);
MakeError(NotInStore, Error); MakeError(BadStorePath, Error);
class FSAccessor; class FSAccessor;
@ -318,9 +318,9 @@ public:
the Nix store. */ the Nix store. */
bool isStorePath(std::string_view path) const; bool isStorePath(std::string_view path) const;
/* Chop off the parts after the top-level store name, e.g., /* Split a path like /nix/store/<hash>-<name>/<bla> into
/nix/store/abcd-foo/bar => /nix/store/abcd-foo. */ /nix/store/<hash>-<name> and /<bla>. */
Path toStorePath(const Path & path) const; std::pair<StorePath, Path> toStorePath(const Path & path) const;
/* Follow symlinks until we end up with a path in the Nix store. */ /* Follow symlinks until we end up with a path in the Nix store. */
Path followLinksToStore(std::string_view path) const; Path followLinksToStore(std::string_view path) const;
@ -385,13 +385,16 @@ public:
SubstituteFlag maybeSubstitute = NoSubstitute); SubstituteFlag maybeSubstitute = NoSubstitute);
/* Query the set of all valid paths. Note that for some store /* Query the set of all valid paths. Note that for some store
backends, the name part of store paths may be omitted backends, the name part of store paths may be replaced by 'x'
(i.e. you'll get /nix/store/<hash> rather than (i.e. you'll get /nix/store/<hash>-x rather than
/nix/store/<hash>-<name>). Use queryPathInfo() to obtain the /nix/store/<hash>-<name>). Use queryPathInfo() to obtain the
full store path. */ full store path. FIXME: should return a set of
std::variant<StorePath, HashPart> to get rid of this hack. */
virtual StorePathSet queryAllValidPaths() virtual StorePathSet queryAllValidPaths()
{ unsupported("queryAllValidPaths"); } { unsupported("queryAllValidPaths"); }
constexpr static const char * MissingName = "x";
/* Query information about a valid path. It is permitted to omit /* Query information about a valid path. It is permitted to omit
the name part of the store path. */ the name part of the store path. */
ref<const ValidPathInfo> queryPathInfo(const StorePath & path); ref<const ValidPathInfo> queryPathInfo(const StorePath & path);
@ -440,8 +443,7 @@ public:
/* Import a path into the store. */ /* Import a path into the store. */
virtual void addToStore(const ValidPathInfo & info, Source & narSource, virtual void addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) = 0;
std::shared_ptr<FSAccessor> accessor = 0) = 0;
/* Copy the contents of a path to the store and register the /* Copy the contents of a path to the store and register the
validity the resulting path. The resulting path is returned. validity the resulting path. The resulting path is returned.
@ -624,8 +626,7 @@ public:
the Nix store. Optionally, the contents of the NARs are the Nix store. Optionally, the contents of the NARs are
preloaded into the specified FS accessor to speed up subsequent preloaded into the specified FS accessor to speed up subsequent
access. */ access. */
StorePaths importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, StorePaths importPaths(Source & source, CheckSigsFlag checkSigs = CheckSigs);
CheckSigsFlag checkSigs = CheckSigs);
struct Stats struct Stats
{ {

View file

@ -11,7 +11,7 @@ namespace nix {
#define ANSI_GREEN "\e[32;1m" #define ANSI_GREEN "\e[32;1m"
#define ANSI_YELLOW "\e[33;1m" #define ANSI_YELLOW "\e[33;1m"
#define ANSI_BLUE "\e[34;1m" #define ANSI_BLUE "\e[34;1m"
#define ANSI_MAGENTA "\e[35m;1m" #define ANSI_MAGENTA "\e[35;1m"
#define ANSI_CYAN "\e[36m;1m" #define ANSI_CYAN "\e[36;1m"
} }

View file

@ -63,11 +63,12 @@ struct ParseSink
virtual void createSymlink(const Path & path, const string & target) { }; virtual void createSymlink(const Path & path, const string & target) { };
}; };
struct TeeSink : ParseSink struct TeeParseSink : ParseSink
{ {
StringSink saved;
TeeSource source; TeeSource source;
TeeSink(Source & source) : source(source) { } TeeParseSink(Source & source) : source(source, saved) { }
}; };
void parseDump(ParseSink & sink, Source & source); void parseDump(ParseSink & sink, Source & source);

View file

@ -1,6 +1,8 @@
#include "args.hh" #include "args.hh"
#include "hash.hh" #include "hash.hh"
#include <glob.h>
namespace nix { namespace nix {
void Args::addFlag(Flag && flag_) void Args::addFlag(Flag && flag_)
@ -13,6 +15,20 @@ void Args::addFlag(Flag && flag_)
if (flag->shortName) shortFlags[flag->shortName] = flag; if (flag->shortName) shortFlags[flag->shortName] = flag;
} }
bool pathCompletions = false;
std::shared_ptr<std::set<std::string>> completions;
std::string completionMarker = "___COMPLETE___";
std::optional<std::string> needsCompletion(std::string_view s)
{
if (!completions) return {};
auto i = s.find(completionMarker);
if (i != std::string::npos)
return std::string(s.begin(), i);
return {};
}
void Args::parseCmdline(const Strings & _cmdline) void Args::parseCmdline(const Strings & _cmdline)
{ {
Strings pendingArgs; Strings pendingArgs;
@ -20,6 +36,14 @@ void Args::parseCmdline(const Strings & _cmdline)
Strings cmdline(_cmdline); Strings cmdline(_cmdline);
if (auto s = getEnv("NIX_GET_COMPLETIONS")) {
size_t n = std::stoi(*s);
assert(n > 0 && n <= cmdline.size());
*std::next(cmdline.begin(), n - 1) += completionMarker;
completions = std::make_shared<decltype(completions)::element_type>();
verbosity = lvlError;
}
for (auto pos = cmdline.begin(); pos != cmdline.end(); ) { for (auto pos = cmdline.begin(); pos != cmdline.end(); ) {
auto arg = *pos; auto arg = *pos;
@ -63,7 +87,7 @@ void Args::printHelp(const string & programName, std::ostream & out)
for (auto & exp : expectedArgs) { for (auto & exp : expectedArgs) {
std::cout << renderLabels({exp.label}); std::cout << renderLabels({exp.label});
// FIXME: handle arity > 1 // FIXME: handle arity > 1
if (exp.arity == 0) std::cout << "..."; if (exp.handler.arity == ArityAny) std::cout << "...";
if (exp.optional) std::cout << "?"; if (exp.optional) std::cout << "?";
} }
std::cout << "\n"; std::cout << "\n";
@ -99,18 +123,32 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
auto process = [&](const std::string & name, const Flag & flag) -> bool { auto process = [&](const std::string & name, const Flag & flag) -> bool {
++pos; ++pos;
std::vector<std::string> args; std::vector<std::string> args;
bool anyCompleted = false;
for (size_t n = 0 ; n < flag.handler.arity; ++n) { for (size_t n = 0 ; n < flag.handler.arity; ++n) {
if (pos == end) { if (pos == end) {
if (flag.handler.arity == ArityAny) break; if (flag.handler.arity == ArityAny) break;
throw UsageError("flag '%s' requires %d argument(s)", name, flag.handler.arity); throw UsageError("flag '%s' requires %d argument(s)", name, flag.handler.arity);
} }
if (flag.completer)
if (auto prefix = needsCompletion(*pos)) {
anyCompleted = true;
flag.completer(n, *prefix);
}
args.push_back(*pos++); args.push_back(*pos++);
} }
flag.handler.fun(std::move(args)); if (!anyCompleted)
flag.handler.fun(std::move(args));
return true; return true;
}; };
if (string(*pos, 0, 2) == "--") { if (string(*pos, 0, 2) == "--") {
if (auto prefix = needsCompletion(*pos)) {
for (auto & [name, flag] : longFlags) {
if (!hiddenCategories.count(flag->category)
&& hasPrefix(name, std::string(*prefix, 2)))
completions->insert("--" + name);
}
}
auto i = longFlags.find(string(*pos, 2)); auto i = longFlags.find(string(*pos, 2));
if (i == longFlags.end()) return false; if (i == longFlags.end()) return false;
return process("--" + i->first, *i->second); return process("--" + i->first, *i->second);
@ -123,6 +161,14 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
return process(std::string("-") + c, *i->second); return process(std::string("-") + c, *i->second);
} }
if (auto prefix = needsCompletion(*pos)) {
if (prefix == "-") {
completions->insert("--");
for (auto & [flag, _] : shortFlags)
completions->insert(std::string("-") + flag);
}
}
return false; return false;
} }
@ -138,12 +184,17 @@ bool Args::processArgs(const Strings & args, bool finish)
bool res = false; bool res = false;
if ((exp.arity == 0 && finish) || if ((exp.handler.arity == ArityAny && finish) ||
(exp.arity > 0 && args.size() == exp.arity)) (exp.handler.arity != ArityAny && args.size() == exp.handler.arity))
{ {
std::vector<std::string> ss; std::vector<std::string> ss;
for (auto & s : args) ss.push_back(s); for (const auto &[n, s] : enumerate(args)) {
exp.handler(std::move(ss)); ss.push_back(s);
if (exp.completer)
if (auto prefix = needsCompletion(s))
exp.completer(n, *prefix);
}
exp.handler.fun(ss);
expectedArgs.pop_front(); expectedArgs.pop_front();
res = true; res = true;
} }
@ -154,6 +205,13 @@ bool Args::processArgs(const Strings & args, bool finish)
return res; return res;
} }
static void hashTypeCompleter(size_t index, std::string_view prefix)
{
for (auto & type : hashTypes)
if (hasPrefix(type, prefix))
completions->insert(type);
}
Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht) Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht)
{ {
return Flag { return Flag {
@ -162,7 +220,8 @@ Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht)
.labels = {"hash-algo"}, .labels = {"hash-algo"},
.handler = {[ht](std::string s) { .handler = {[ht](std::string s) {
*ht = parseHashType(s); *ht = parseHashType(s);
}} }},
.completer = hashTypeCompleter
}; };
} }
@ -174,10 +233,42 @@ Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional<
.labels = {"hash-algo"}, .labels = {"hash-algo"},
.handler = {[oht](std::string s) { .handler = {[oht](std::string s) {
*oht = std::optional<HashType> { parseHashType(s) }; *oht = std::optional<HashType> { parseHashType(s) };
}} }},
.completer = hashTypeCompleter
}; };
} }
static void completePath(std::string_view prefix, bool onlyDirs)
{
pathCompletions = true;
glob_t globbuf;
int flags = GLOB_NOESCAPE | GLOB_TILDE;
#ifdef GLOB_ONLYDIR
if (onlyDirs)
flags |= GLOB_ONLYDIR;
#endif
if (glob((std::string(prefix) + "*").c_str(), flags, nullptr, &globbuf) == 0) {
for (size_t i = 0; i < globbuf.gl_pathc; ++i) {
if (onlyDirs) {
auto st = lstat(globbuf.gl_pathv[i]);
if (!S_ISDIR(st.st_mode)) continue;
}
completions->insert(globbuf.gl_pathv[i]);
}
globfree(&globbuf);
}
}
void completePath(size_t, std::string_view prefix)
{
completePath(prefix, false);
}
void completeDir(size_t, std::string_view prefix)
{
completePath(prefix, true);
}
Strings argvToStrings(int argc, char * * argv) Strings argvToStrings(int argc, char * * argv)
{ {
Strings args; Strings args;
@ -225,18 +316,26 @@ void Command::printHelp(const string & programName, std::ostream & out)
MultiCommand::MultiCommand(const Commands & commands) MultiCommand::MultiCommand(const Commands & commands)
: commands(commands) : commands(commands)
{ {
expectedArgs.push_back(ExpectedArg{"command", 1, true, [=](std::vector<std::string> ss) { expectArgs({
assert(!command); .label = "command",
auto cmd = ss[0]; .optional = true,
if (auto alias = get(deprecatedAliases, cmd)) { .handler = {[=](std::string s) {
warn("'%s' is a deprecated alias for '%s'", cmd, *alias); assert(!command);
cmd = *alias; if (auto alias = get(deprecatedAliases, s)) {
} warn("'%s' is a deprecated alias for '%s'", s, *alias);
auto i = commands.find(cmd); s = *alias;
if (i == commands.end()) }
throw UsageError("'%s' is not a recognised command", cmd); if (auto prefix = needsCompletion(s)) {
command = {cmd, i->second()}; for (auto & [name, command] : commands)
}}); if (hasPrefix(name, *prefix))
completions->insert(name);
}
auto i = commands.find(s);
if (i == commands.end())
throw UsageError("'%s' is not a recognised command", s);
command = {s, i->second()};
}}
});
categories[Command::catDefault] = "Available commands"; categories[Command::catDefault] = "Available commands";
} }

View file

@ -8,8 +8,6 @@
namespace nix { namespace nix {
MakeError(UsageError, Error);
enum HashType : char; enum HashType : char;
class Args class Args
@ -28,61 +26,67 @@ protected:
static const size_t ArityAny = std::numeric_limits<size_t>::max(); static const size_t ArityAny = std::numeric_limits<size_t>::max();
struct Handler
{
std::function<void(std::vector<std::string>)> fun;
size_t arity;
Handler() {}
Handler(std::function<void(std::vector<std::string>)> && fun)
: fun(std::move(fun))
, arity(ArityAny)
{ }
Handler(std::function<void()> && handler)
: fun([handler{std::move(handler)}](std::vector<std::string>) { handler(); })
, arity(0)
{ }
Handler(std::function<void(std::string)> && handler)
: fun([handler{std::move(handler)}](std::vector<std::string> ss) {
handler(std::move(ss[0]));
})
, arity(1)
{ }
Handler(std::function<void(std::string, std::string)> && handler)
: fun([handler{std::move(handler)}](std::vector<std::string> ss) {
handler(std::move(ss[0]), std::move(ss[1]));
})
, arity(2)
{ }
Handler(std::vector<std::string> * dest)
: fun([=](std::vector<std::string> ss) { *dest = ss; })
, arity(ArityAny)
{ }
template<class T>
Handler(T * dest)
: fun([=](std::vector<std::string> ss) { *dest = ss[0]; })
, arity(1)
{ }
template<class T>
Handler(T * dest, const T & val)
: fun([=](std::vector<std::string> ss) { *dest = val; })
, arity(0)
{ }
};
/* Flags. */ /* Flags. */
struct Flag struct Flag
{ {
typedef std::shared_ptr<Flag> ptr; typedef std::shared_ptr<Flag> ptr;
struct Handler
{
std::function<void(std::vector<std::string>)> fun;
size_t arity;
Handler() {}
Handler(std::function<void(std::vector<std::string>)> && fun)
: fun(std::move(fun))
, arity(ArityAny)
{ }
Handler(std::function<void()> && handler)
: fun([handler{std::move(handler)}](std::vector<std::string>) { handler(); })
, arity(0)
{ }
Handler(std::function<void(std::string)> && handler)
: fun([handler{std::move(handler)}](std::vector<std::string> ss) {
handler(std::move(ss[0]));
})
, arity(1)
{ }
Handler(std::function<void(std::string, std::string)> && handler)
: fun([handler{std::move(handler)}](std::vector<std::string> ss) {
handler(std::move(ss[0]), std::move(ss[1]));
})
, arity(2)
{ }
template<class T>
Handler(T * dest)
: fun([=](std::vector<std::string> ss) { *dest = ss[0]; })
, arity(1)
{ }
template<class T>
Handler(T * dest, const T & val)
: fun([=](std::vector<std::string> ss) { *dest = val; })
, arity(0)
{ }
};
std::string longName; std::string longName;
char shortName = 0; char shortName = 0;
std::string description; std::string description;
std::string category; std::string category;
Strings labels; Strings labels;
Handler handler; Handler handler;
std::function<void(size_t, std::string_view)> completer;
static Flag mkHashTypeFlag(std::string && longName, HashType * ht); static Flag mkHashTypeFlag(std::string && longName, HashType * ht);
static Flag mkHashTypeOptFlag(std::string && longName, std::optional<HashType> * oht); static Flag mkHashTypeOptFlag(std::string && longName, std::optional<HashType> * oht);
@ -99,9 +103,9 @@ protected:
struct ExpectedArg struct ExpectedArg
{ {
std::string label; std::string label;
size_t arity; // 0 = any bool optional = false;
bool optional; Handler handler;
std::function<void(std::vector<std::string>)> handler; std::function<void(size_t, std::string_view)> completer;
}; };
std::list<ExpectedArg> expectedArgs; std::list<ExpectedArg> expectedArgs;
@ -175,20 +179,28 @@ public:
}); });
} }
void expectArgs(ExpectedArg && arg)
{
expectedArgs.emplace_back(std::move(arg));
}
/* Expect a string argument. */ /* Expect a string argument. */
void expectArg(const std::string & label, string * dest, bool optional = false) void expectArg(const std::string & label, string * dest, bool optional = false)
{ {
expectedArgs.push_back(ExpectedArg{label, 1, optional, [=](std::vector<std::string> ss) { expectArgs({
*dest = ss[0]; .label = label,
}}); .optional = true,
.handler = {dest}
});
} }
/* Expect 0 or more arguments. */ /* Expect 0 or more arguments. */
void expectArgs(const std::string & label, std::vector<std::string> * dest) void expectArgs(const std::string & label, std::vector<std::string> * dest)
{ {
expectedArgs.push_back(ExpectedArg{label, 0, false, [=](std::vector<std::string> ss) { expectArgs({
*dest = std::move(ss); .label = label,
}}); .handler = {dest}
});
} }
friend class MultiCommand; friend class MultiCommand;
@ -259,4 +271,13 @@ typedef std::vector<std::pair<std::string, std::string>> Table2;
void printTable(std::ostream & out, const Table2 & table); void printTable(std::ostream & out, const Table2 & table);
extern std::shared_ptr<std::set<std::string>> completions;
extern bool pathCompletions;
std::optional<std::string> needsCompletion(std::string_view s);
void completePath(size_t, std::string_view prefix);
void completeDir(size_t, std::string_view prefix);
} }

View file

@ -191,6 +191,7 @@ public:
} }
MakeError(Error, BaseError); MakeError(Error, BaseError);
MakeError(UsageError, Error);
class SysError : public Error class SysError : public Error
{ {

View file

@ -9,7 +9,6 @@
#include "archive.hh" #include "archive.hh"
#include "parser.hh" #include "parser.hh"
#include "util.hh" #include "util.hh"
#include "istringstream_nocopy.hh"
#include <sys/types.h> #include <sys/types.h>
#include <sys/stat.h> #include <sys/stat.h>
@ -17,6 +16,7 @@
namespace nix { namespace nix {
static size_t regularHashSize(HashType type) { static size_t regularHashSize(HashType type) {
switch (type) { switch (type) {
case htMD5: return md5HashSize; case htMD5: return md5HashSize;
@ -27,6 +27,10 @@ static size_t regularHashSize(HashType type) {
abort(); abort();
} }
std::set<std::string> hashTypes = { "md5", "sha1", "sha256", "sha512" };
Hash::Hash(HashType type) : type(type) Hash::Hash(HashType type) : type(type)
{ {
hashSize = regularHashSize(type); hashSize = regularHashSize(type);

View file

@ -18,6 +18,8 @@ const int sha1HashSize = 20;
const int sha256HashSize = 32; const int sha256HashSize = 32;
const int sha512HashSize = 64; const int sha512HashSize = 64;
extern std::set<std::string> hashTypes;
extern const string base32Chars; extern const string base32Chars;
enum Base : int { Base64, Base32, Base16, SRI }; enum Base : int { Base64, Base32, Base16, SRI };
@ -124,6 +126,7 @@ Hash compressHash(const Hash & hash, unsigned int newSize);
/* Parse a string representing a hash type. */ /* Parse a string representing a hash type. */
HashType parseHashType(std::string_view s); HashType parseHashType(std::string_view s);
/* Will return nothing on parse error */ /* Will return nothing on parse error */
std::optional<HashType> parseHashTypeOpt(std::string_view s); std::optional<HashType> parseHashTypeOpt(std::string_view s);

View file

@ -1,92 +0,0 @@
/* This file provides a variant of std::istringstream that doesn't
copy its string argument. This is useful for large strings. The
caller must ensure that the string object is not destroyed while
it's referenced by this object. */
#pragma once
#include <string>
#include <iostream>
template <class CharT, class Traits = std::char_traits<CharT>, class Allocator = std::allocator<CharT>>
class basic_istringbuf_nocopy : public std::basic_streambuf<CharT, Traits>
{
public:
typedef std::basic_string<CharT, Traits, Allocator> string_type;
typedef typename std::basic_streambuf<CharT, Traits>::off_type off_type;
typedef typename std::basic_streambuf<CharT, Traits>::pos_type pos_type;
typedef typename std::basic_streambuf<CharT, Traits>::int_type int_type;
typedef typename std::basic_streambuf<CharT, Traits>::traits_type traits_type;
private:
const string_type & s;
off_type off;
public:
basic_istringbuf_nocopy(const string_type & s) : s{s}, off{0}
{
}
private:
pos_type seekoff(off_type off, std::ios_base::seekdir dir, std::ios_base::openmode which)
{
if (which & std::ios_base::in) {
this->off = dir == std::ios_base::beg
? off
: (dir == std::ios_base::end
? s.size() + off
: this->off + off);
}
return pos_type(this->off);
}
pos_type seekpos(pos_type pos, std::ios_base::openmode which)
{
return seekoff(pos, std::ios_base::beg, which);
}
std::streamsize showmanyc()
{
return s.size() - off;
}
int_type underflow()
{
if (typename string_type::size_type(off) == s.size())
return traits_type::eof();
return traits_type::to_int_type(s[off]);
}
int_type uflow()
{
if (typename string_type::size_type(off) == s.size())
return traits_type::eof();
return traits_type::to_int_type(s[off++]);
}
int_type pbackfail(int_type ch)
{
if (off == 0 || (ch != traits_type::eof() && ch != s[off - 1]))
return traits_type::eof();
return traits_type::to_int_type(s[--off]);
}
};
template <class CharT, class Traits = std::char_traits<CharT>, class Allocator = std::allocator<CharT>>
class basic_istringstream_nocopy : public std::basic_iostream<CharT, Traits>
{
typedef basic_istringbuf_nocopy<CharT, Traits, Allocator> buf_type;
buf_type buf;
public:
basic_istringstream_nocopy(const typename buf_type::string_type & s) :
std::basic_iostream<CharT, Traits>(&buf), buf(s) {};
};
typedef basic_istringstream_nocopy<char> istringstream_nocopy;

View file

@ -166,17 +166,30 @@ struct StringSource : Source
}; };
/* Adapter class of a Source that saves all data read to `s'. */ /* A sink that writes all incoming data to two other sinks. */
struct TeeSink : Sink
{
Sink & sink1, & sink2;
TeeSink(Sink & sink1, Sink & sink2) : sink1(sink1), sink2(sink2) { }
virtual void operator () (const unsigned char * data, size_t len)
{
sink1(data, len);
sink2(data, len);
}
};
/* Adapter class of a Source that saves all data read to a sink. */
struct TeeSource : Source struct TeeSource : Source
{ {
Source & orig; Source & orig;
ref<std::string> data; Sink & sink;
TeeSource(Source & orig) TeeSource(Source & orig, Sink & sink)
: orig(orig), data(make_ref<std::string>()) { } : orig(orig), sink(sink) { }
size_t read(unsigned char * data, size_t len) size_t read(unsigned char * data, size_t len)
{ {
size_t n = orig.read(data, len); size_t n = orig.read(data, len);
this->data->append((const char *) data, n); sink(data, len);
return n; return n;
} }
}; };
@ -336,4 +349,27 @@ Source & operator >> (Source & in, bool & b)
} }
/* An adapter that converts a std::basic_istream into a source. */
struct StreamToSourceAdapter : Source
{
std::shared_ptr<std::basic_istream<char>> istream;
StreamToSourceAdapter(std::shared_ptr<std::basic_istream<char>> istream)
: istream(istream)
{ }
size_t read(unsigned char * data, size_t len) override
{
if (!istream->read((char *) data, len)) {
if (istream->eof()) {
if (istream->gcount() == 0)
throw EndOfFile("end of file");
} else
throw Error("I/O error in StreamToSourceAdapter");
}
return istream->gcount();
}
};
} }

View file

@ -23,6 +23,7 @@
#include <sys/types.h> #include <sys/types.h>
#include <sys/socket.h> #include <sys/socket.h>
#include <sys/wait.h> #include <sys/wait.h>
#include <sys/time.h>
#include <sys/un.h> #include <sys/un.h>
#include <unistd.h> #include <unistd.h>
@ -79,7 +80,7 @@ void replaceEnv(std::map<std::string, std::string> newEnv)
} }
Path absPath(Path path, std::optional<Path> dir) Path absPath(Path path, std::optional<Path> dir, bool resolveSymlinks)
{ {
if (path[0] != '/') { if (path[0] != '/') {
if (!dir) { if (!dir) {
@ -100,7 +101,7 @@ Path absPath(Path path, std::optional<Path> dir)
} }
path = *dir + "/" + path; path = *dir + "/" + path;
} }
return canonPath(path); return canonPath(path, resolveSymlinks);
} }
@ -345,7 +346,6 @@ void writeFile(const Path & path, Source & source, mode_t mode)
} }
} }
string readLine(int fd) string readLine(int fd)
{ {
string s; string s;
@ -581,20 +581,31 @@ Paths createDirs(const Path & path)
} }
void createSymlink(const Path & target, const Path & link) void createSymlink(const Path & target, const Path & link,
std::optional<time_t> mtime)
{ {
if (symlink(target.c_str(), link.c_str())) if (symlink(target.c_str(), link.c_str()))
throw SysError("creating symlink from '%1%' to '%2%'", link, target); throw SysError("creating symlink from '%1%' to '%2%'", link, target);
if (mtime) {
struct timeval times[2];
times[0].tv_sec = *mtime;
times[0].tv_usec = 0;
times[1].tv_sec = *mtime;
times[1].tv_usec = 0;
if (lutimes(link.c_str(), times))
throw SysError("setting time of symlink '%s'", link);
}
} }
void replaceSymlink(const Path & target, const Path & link) void replaceSymlink(const Path & target, const Path & link,
std::optional<time_t> mtime)
{ {
for (unsigned int n = 0; true; n++) { for (unsigned int n = 0; true; n++) {
Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link)));
try { try {
createSymlink(target, tmp); createSymlink(target, tmp, mtime);
} catch (SysError & e) { } catch (SysError & e) {
if (e.errNo == EEXIST) continue; if (e.errNo == EEXIST) continue;
throw; throw;
@ -1006,12 +1017,14 @@ std::vector<char *> stringsToCharPtrs(const Strings & ss)
return res; return res;
} }
// Output = "standard out" output stream
string runProgram(Path program, bool searchPath, const Strings & args, string runProgram(Path program, bool searchPath, const Strings & args,
const std::optional<std::string> & input) const std::optional<std::string> & input)
{ {
RunOptions opts(program, args); RunOptions opts(program, args);
opts.searchPath = searchPath; opts.searchPath = searchPath;
// This allows you to refer to a program with a pathname relative to the
// PATH variable.
opts.input = input; opts.input = input;
auto res = runProgram(opts); auto res = runProgram(opts);
@ -1022,6 +1035,7 @@ string runProgram(Path program, bool searchPath, const Strings & args,
return res.second; return res.second;
} }
// Output = error code + "standard out" output stream
std::pair<int, std::string> runProgram(const RunOptions & options_) std::pair<int, std::string> runProgram(const RunOptions & options_)
{ {
RunOptions options(options_); RunOptions options(options_);
@ -1094,6 +1108,8 @@ void runProgram2(const RunOptions & options)
if (options.searchPath) if (options.searchPath)
execvp(options.program.c_str(), stringsToCharPtrs(args_).data()); execvp(options.program.c_str(), stringsToCharPtrs(args_).data());
// This allows you to refer to a program with a pathname relative
// to the PATH variable.
else else
execv(options.program.c_str(), stringsToCharPtrs(args_).data()); execv(options.program.c_str(), stringsToCharPtrs(args_).data());

View file

@ -49,7 +49,9 @@ void clearEnv();
/* Return an absolutized path, resolving paths relative to the /* Return an absolutized path, resolving paths relative to the
specified directory, or the current directory otherwise. The path specified directory, or the current directory otherwise. The path
is also canonicalised. */ is also canonicalised. */
Path absPath(Path path, std::optional<Path> dir = {}); Path absPath(Path path,
std::optional<Path> dir = {},
bool resolveSymlinks = false);
/* Canonicalise a path by removing all `.' or `..' components and /* Canonicalise a path by removing all `.' or `..' components and
double or trailing slashes. Optionally resolves all symlink double or trailing slashes. Optionally resolves all symlink
@ -147,10 +149,12 @@ Path getDataDir();
Paths createDirs(const Path & path); Paths createDirs(const Path & path);
/* Create a symlink. */ /* Create a symlink. */
void createSymlink(const Path & target, const Path & link); void createSymlink(const Path & target, const Path & link,
std::optional<time_t> mtime = {});
/* Atomically create or replace a symlink. */ /* Atomically create or replace a symlink. */
void replaceSymlink(const Path & target, const Path & link); void replaceSymlink(const Path & target, const Path & link,
std::optional<time_t> mtime = {});
/* Wrappers arount read()/write() that read/write exactly the /* Wrappers arount read()/write() that read/write exactly the

View file

@ -671,7 +671,7 @@ static void opImport(Strings opFlags, Strings opArgs)
if (!opArgs.empty()) throw UsageError("no arguments expected"); if (!opArgs.empty()) throw UsageError("no arguments expected");
FdSource source(STDIN_FILENO); FdSource source(STDIN_FILENO);
auto paths = store->importPaths(source, nullptr, NoCheckSigs); auto paths = store->importPaths(source, NoCheckSigs);
for (auto & i : paths) for (auto & i : paths)
cout << fmt("%s\n", store->printStorePath(i)) << std::flush; cout << fmt("%s\n", store->printStorePath(i)) << std::flush;
@ -880,7 +880,7 @@ static void opServe(Strings opFlags, Strings opArgs)
case cmdImportPaths: { case cmdImportPaths: {
if (!writeAllowed) throw Error("importing paths is not allowed"); if (!writeAllowed) throw Error("importing paths is not allowed");
store->importPaths(in, nullptr, NoCheckSigs); // FIXME: should we skip sig checking? store->importPaths(in, NoCheckSigs); // FIXME: should we skip sig checking?
out << 1; // indicate success out << 1; // indicate success
break; break;
} }

46
src/nix/app.cc Normal file
View file

@ -0,0 +1,46 @@
#include "installables.hh"
#include "store-api.hh"
#include "eval-inline.hh"
#include "eval-cache.hh"
#include "names.hh"
namespace nix {
App Installable::toApp(EvalState & state)
{
auto [cursor, attrPath] = getCursor(state, true);
auto type = cursor->getAttr("type")->getString();
if (type == "app") {
auto [program, context] = cursor->getAttr("program")->getStringWithContext();
if (!state.store->isInStore(program))
throw Error("app program '%s' is not in the Nix store", program);
std::vector<StorePathWithOutputs> context2;
for (auto & [path, name] : context)
context2.push_back({state.store->parseStorePath(path), {name}});
return App {
.context = std::move(context2),
.program = program,
};
}
else if (type == "derivation") {
auto drvPath = cursor->forceDerivation();
auto outPath = cursor->getAttr(state.sOutPath)->getString();
auto outputName = cursor->getAttr(state.sOutputName)->getString();
auto name = cursor->getAttr(state.sName)->getString();
return App {
.context = { { drvPath, {outputName} } },
.program = outPath + "/bin/" + DrvName(name).name,
};
}
else
throw Error("attribute '%s' has unsupported type '%s'", attrPath, type);
}
}

View file

@ -1,3 +1,4 @@
#include "eval.hh"
#include "command.hh" #include "command.hh"
#include "common-args.hh" #include "common-args.hh"
#include "shared.hh" #include "shared.hh"
@ -17,6 +18,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
.description = "path of the symlink to the build result", .description = "path of the symlink to the build result",
.labels = {"path"}, .labels = {"path"},
.handler = {&outLink}, .handler = {&outLink},
.completer = completePath
}); });
addFlag({ addFlag({
@ -44,14 +46,14 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
}, },
Example{ Example{
"To make a profile point at GNU Hello:", "To make a profile point at GNU Hello:",
"nix build --profile /tmp/profile nixpkgs.hello" "nix build --profile /tmp/profile nixpkgs#hello"
}, },
}; };
} }
void run(ref<Store> store) override void run(ref<Store> store) override
{ {
auto buildables = build(store, dryRun ? DryRun : Build, installables); auto buildables = build(store, dryRun ? Realise::Nothing : Realise::Outputs, installables);
if (dryRun) return; if (dryRun) return;

View file

@ -25,7 +25,11 @@ struct CmdCatStore : StoreCommand, MixCat
{ {
CmdCatStore() CmdCatStore()
{ {
expectArg("path", &path); expectArgs({
.label = "path",
.handler = {&path},
.completer = completePath
});
} }
std::string description() override std::string description() override
@ -47,7 +51,11 @@ struct CmdCatNar : StoreCommand, MixCat
CmdCatNar() CmdCatNar()
{ {
expectArg("nar", &narPath); expectArgs({
.label = "nar",
.handler = {&narPath},
.completer = completePath
});
expectArg("path", &path); expectArg("path", &path);
} }

View file

@ -63,7 +63,7 @@ void StorePathsCommand::run(ref<Store> store)
} }
else { else {
for (auto & p : toStorePaths(store, realiseMode, installables)) for (auto & p : toStorePaths(store, realiseMode, operateOn, installables))
storePaths.push_back(p); storePaths.push_back(p);
if (recursive) { if (recursive) {
@ -80,7 +80,7 @@ void StorePathsCommand::run(ref<Store> store)
void StorePathCommand::run(ref<Store> store) void StorePathCommand::run(ref<Store> store)
{ {
auto storePaths = toStorePaths(store, NoBuild, installables); auto storePaths = toStorePaths(store, Realise::Nothing, operateOn, installables);
if (storePaths.size() != 1) if (storePaths.size() != 1)
throw UsageError("this command requires exactly one store path"); throw UsageError("this command requires exactly one store path");
@ -108,6 +108,7 @@ MixProfile::MixProfile()
.description = "profile to update", .description = "profile to update",
.labels = {"path"}, .labels = {"path"},
.handler = {&profile}, .handler = {&profile},
.completer = completePath
}); });
} }

View file

@ -4,12 +4,18 @@
#include "args.hh" #include "args.hh"
#include "common-eval-args.hh" #include "common-eval-args.hh"
#include "path.hh" #include "path.hh"
#include "eval.hh" #include "flake/lockfile.hh"
#include <optional>
namespace nix { namespace nix {
extern std::string programPath; extern std::string programPath;
class EvalState;
struct Pos;
class Store;
static constexpr Command::Category catSecondary = 100; static constexpr Command::Category catSecondary = 100;
static constexpr Command::Category catUtility = 101; static constexpr Command::Category catUtility = 101;
static constexpr Command::Category catNixInstallation = 102; static constexpr Command::Category catNixInstallation = 102;
@ -27,28 +33,64 @@ private:
std::shared_ptr<Store> _store; std::shared_ptr<Store> _store;
}; };
struct SourceExprCommand : virtual StoreCommand, MixEvalArgs struct EvalCommand : virtual StoreCommand, MixEvalArgs
{ {
Path file; ref<EvalState> getEvalState();
std::shared_ptr<EvalState> evalState;
};
struct MixFlakeOptions : virtual Args, EvalCommand
{
flake::LockFlags lockFlags;
MixFlakeOptions();
virtual std::optional<FlakeRef> getFlakeRefForCompletion()
{ return {}; }
};
/* How to handle derivations in commands that operate on store paths. */
enum class OperateOn {
/* Operate on the output path. */
Output,
/* Operate on the .drv path. */
Derivation
};
struct SourceExprCommand : virtual Args, MixFlakeOptions
{
std::optional<Path> file;
std::optional<std::string> expr;
// FIXME: move this; not all commands (e.g. 'nix run') use it.
OperateOn operateOn = OperateOn::Output;
SourceExprCommand(); SourceExprCommand();
/* Return a value representing the Nix expression from which we std::vector<std::shared_ptr<Installable>> parseInstallables(
are installing. This is either the file specified by --file, ref<Store> store, std::vector<std::string> ss);
or an attribute set constructed from $NIX_PATH, e.g. { nixpkgs
= import ...; bla = import ...; }. */
Value * getSourceExpr(EvalState & state);
ref<EvalState> getEvalState(); std::shared_ptr<Installable> parseInstallable(
ref<Store> store, const std::string & installable);
private: virtual Strings getDefaultFlakeAttrPaths();
std::shared_ptr<EvalState> evalState; virtual Strings getDefaultFlakeAttrPathPrefixes();
RootValue vSourceExpr; void completeInstallable(std::string_view prefix);
}; };
enum RealiseMode { Build, NoBuild, DryRun }; enum class Realise {
/* Build the derivation. Postcondition: the
derivation outputs exist. */
Outputs,
/* Don't build the derivation. Postcondition: the store derivation
exists. */
Derivation,
/* Evaluate in dry-run mode. Postcondition: nothing. */
Nothing
};
/* A command that operates on a list of "installables", which can be /* A command that operates on a list of "installables", which can be
store paths, attribute paths, Nix expressions, etc. */ store paths, attribute paths, Nix expressions, etc. */
@ -56,15 +98,14 @@ struct InstallablesCommand : virtual Args, SourceExprCommand
{ {
std::vector<std::shared_ptr<Installable>> installables; std::vector<std::shared_ptr<Installable>> installables;
InstallablesCommand() InstallablesCommand();
{
expectArgs("installables", &_installables);
}
void prepare() override; void prepare() override;
virtual bool useDefaultInstallables() { return true; } virtual bool useDefaultInstallables() { return true; }
std::optional<FlakeRef> getFlakeRefForCompletion() override;
private: private:
std::vector<std::string> _installables; std::vector<std::string> _installables;
@ -75,16 +116,18 @@ struct InstallableCommand : virtual Args, SourceExprCommand
{ {
std::shared_ptr<Installable> installable; std::shared_ptr<Installable> installable;
InstallableCommand() InstallableCommand();
{
expectArg("installable", &_installable);
}
void prepare() override; void prepare() override;
std::optional<FlakeRef> getFlakeRefForCompletion() override
{
return parseFlakeRef(_installable, absPath("."));
}
private: private:
std::string _installable; std::string _installable{"."};
}; };
/* A command that operates on zero or more store paths. */ /* A command that operates on zero or more store paths. */
@ -97,7 +140,7 @@ private:
protected: protected:
RealiseMode realiseMode = NoBuild; Realise realiseMode = Realise::Derivation;
public: public:
@ -141,17 +184,15 @@ static RegisterCommand registerCommand(const std::string & name)
return RegisterCommand(name, [](){ return make_ref<T>(); }); return RegisterCommand(name, [](){ return make_ref<T>(); });
} }
std::shared_ptr<Installable> parseInstallable( Buildables build(ref<Store> store, Realise mode,
SourceExprCommand & cmd, ref<Store> store, const std::string & installable,
bool useDefaultInstallables);
Buildables build(ref<Store> store, RealiseMode mode,
std::vector<std::shared_ptr<Installable>> installables); std::vector<std::shared_ptr<Installable>> installables);
std::set<StorePath> toStorePaths(ref<Store> store, RealiseMode mode, std::set<StorePath> toStorePaths(ref<Store> store,
Realise mode, OperateOn operateOn,
std::vector<std::shared_ptr<Installable>> installables); std::vector<std::shared_ptr<Installable>> installables);
StorePath toStorePath(ref<Store> store, RealiseMode mode, StorePath toStorePath(ref<Store> store,
Realise mode, OperateOn operateOn,
std::shared_ptr<Installable> installable); std::shared_ptr<Installable> installable);
std::set<StorePath> toDerivations(ref<Store> store, std::set<StorePath> toDerivations(ref<Store> store,
@ -194,4 +235,13 @@ struct MixEnvironment : virtual Args {
void setEnviron(); void setEnviron();
}; };
void completeFlakeRef(ref<Store> store, std::string_view prefix);
void completeFlakeRefWithFragment(
ref<EvalState> evalState,
flake::LockFlags lockFlags,
Strings attrPathPrefixes,
const Strings & defaultFlakeAttrPaths,
std::string_view prefix);
} }

View file

@ -45,6 +45,8 @@ struct CmdCopy : StorePathsCommand
.description = "whether to try substitutes on the destination store (only supported by SSH)", .description = "whether to try substitutes on the destination store (only supported by SSH)",
.handler = {&substitute, Substitute}, .handler = {&substitute, Substitute},
}); });
realiseMode = Realise::Outputs;
} }
std::string description() override std::string description() override
@ -70,11 +72,11 @@ struct CmdCopy : StorePathsCommand
#ifdef ENABLE_S3 #ifdef ENABLE_S3
Example{ Example{
"To copy Hello to an S3 binary cache:", "To copy Hello to an S3 binary cache:",
"nix copy --to s3://my-bucket?region=eu-west-1 nixpkgs.hello" "nix copy --to s3://my-bucket?region=eu-west-1 nixpkgs#hello"
}, },
Example{ Example{
"To copy Hello to an S3-compatible binary cache:", "To copy Hello to an S3-compatible binary cache:",
"nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com nixpkgs.hello" "nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com nixpkgs#hello"
}, },
#endif #endif
}; };
@ -87,11 +89,16 @@ struct CmdCopy : StorePathsCommand
return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri); return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri);
} }
void run(ref<Store> srcStore, StorePaths storePaths) override void run(ref<Store> store) override
{ {
if (srcUri.empty() && dstUri.empty()) if (srcUri.empty() && dstUri.empty())
throw UsageError("you must pass '--from' and/or '--to'"); throw UsageError("you must pass '--from' and/or '--to'");
StorePathsCommand::run(store);
}
void run(ref<Store> srcStore, StorePaths storePaths) override
{
ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri); ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri);
copyPaths(srcStore, dstStore, StorePathSet(storePaths.begin(), storePaths.end()), copyPaths(srcStore, dstStore, StorePathSet(storePaths.begin(), storePaths.end()),

View file

@ -130,7 +130,9 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
drvName += "-env"; drvName += "-env";
for (auto & output : drv.outputs) for (auto & output : drv.outputs)
drv.env.erase(output.first); drv.env.erase(output.first);
drv.outputs = {{"out", DerivationOutput { .path = StorePath::dummy }}};
drv.env["out"] = ""; drv.env["out"] = "";
drv.env["_outputs_saved"] = drv.env["outputs"];
drv.env["outputs"] = "out"; drv.env["outputs"] = "out";
drv.inputSrcs.insert(std::move(getEnvShPath)); drv.inputSrcs.insert(std::move(getEnvShPath));
Hash h = hashDerivationModulo(*store, drv, true); Hash h = hashDerivationModulo(*store, drv, true);
@ -201,6 +203,11 @@ struct Common : InstallableCommand, MixProfile
out << "eval \"$shellHook\"\n"; out << "eval \"$shellHook\"\n";
} }
Strings getDefaultFlakeAttrPaths() override
{
return {"devShell." + settings.thisSystem.get(), "defaultPackage." + settings.thisSystem.get()};
}
StorePath getShellOutPath(ref<Store> store) StorePath getShellOutPath(ref<Store> store)
{ {
auto path = installable->getStorePath(); auto path = installable->getStorePath();
@ -259,11 +266,15 @@ struct CmdDevelop : Common, MixEnvironment
return { return {
Example{ Example{
"To get the build environment of GNU hello:", "To get the build environment of GNU hello:",
"nix develop nixpkgs.hello" "nix develop nixpkgs#hello"
},
Example{
"To get the build environment of the default package of flake in the current directory:",
"nix develop"
}, },
Example{ Example{
"To store the build environment in a profile:", "To store the build environment in a profile:",
"nix develop --profile /tmp/my-shell nixpkgs.hello" "nix develop --profile /tmp/my-shell nixpkgs#hello"
}, },
Example{ Example{
"To use a build environment previously recorded in a profile:", "To use a build environment previously recorded in a profile:",
@ -294,12 +305,28 @@ struct CmdDevelop : Common, MixEnvironment
stopProgressBar(); stopProgressBar();
auto shell = getEnv("SHELL").value_or("bash");
setEnviron(); setEnviron();
// prevent garbage collection until shell exits // prevent garbage collection until shell exits
setenv("NIX_GCROOT", gcroot.data(), 1); setenv("NIX_GCROOT", gcroot.data(), 1);
Path shell = "bash";
try {
auto state = getEvalState();
auto bashInstallable = std::make_shared<InstallableFlake>(
state,
std::move(installable->nixpkgsFlakeRef()),
Strings{"bashInteractive"},
Strings{"legacyPackages." + settings.thisSystem.get() + "."},
lockFlags);
shell = state->store->printStorePath(
toStorePath(state->store, Realise::Outputs, OperateOn::Output, bashInstallable)) + "/bin/bash";
} catch (Error &) {
ignoreException();
}
auto args = Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath}; auto args = Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath};
restoreAffinity(); restoreAffinity();
@ -323,7 +350,7 @@ struct CmdPrintDevEnv : Common
return { return {
Example{ Example{
"To apply the build environment of GNU hello to the current shell:", "To apply the build environment of GNU hello to the current shell:",
". <(nix print-dev-env nixpkgs.hello)" ". <(nix print-dev-env nixpkgs#hello)"
}, },
}; };
} }

134
src/nix/diff-closures.cc Normal file
View file

@ -0,0 +1,134 @@
#include "command.hh"
#include "shared.hh"
#include "store-api.hh"
#include "common-args.hh"
#include "names.hh"
#include <regex>
using namespace nix;
struct Info
{
std::string outputName;
};
// name -> version -> store paths
typedef std::map<std::string, std::map<std::string, std::map<StorePath, Info>>> GroupedPaths;
GroupedPaths getClosureInfo(ref<Store> store, const StorePath & toplevel)
{
StorePathSet closure;
store->computeFSClosure({toplevel}, closure);
GroupedPaths groupedPaths;
for (auto & path : closure) {
/* Strip the output name. Unfortunately this is ambiguous (we
can't distinguish between output names like "bin" and
version suffixes like "unstable"). */
static std::regex regex("(.*)-([a-z]+|lib32|lib64)");
std::smatch match;
std::string name(path.name());
std::string outputName;
if (std::regex_match(name, match, regex)) {
name = match[1];
outputName = match[2];
}
DrvName drvName(name);
groupedPaths[drvName.name][drvName.version].emplace(path, Info { .outputName = outputName });
}
return groupedPaths;
}
std::string showVersions(const std::set<std::string> & versions)
{
if (versions.empty()) return "";
std::set<std::string> versions2;
for (auto & version : versions)
versions2.insert(version.empty() ? "ε" : version);
return concatStringsSep(", ", versions2);
}
struct CmdDiffClosures : SourceExprCommand
{
std::string _before, _after;
CmdDiffClosures()
{
expectArg("before", &_before);
expectArg("after", &_after);
}
std::string description() override
{
return "show what packages and versions were added and removed between two closures";
}
Category category() override { return catSecondary; }
Examples examples() override
{
return {
{
"To show what got added and removed between two versions of the NixOS system profile:",
"nix diff-closures /nix/var/nix/profiles/system-655-link /nix/var/nix/profiles/system-658-link",
},
};
}
void run(ref<Store> store) override
{
auto before = parseInstallable(store, _before);
auto beforePath = toStorePath(store, Realise::Outputs, operateOn, before);
auto after = parseInstallable(store, _after);
auto afterPath = toStorePath(store, Realise::Outputs, operateOn, after);
auto beforeClosure = getClosureInfo(store, beforePath);
auto afterClosure = getClosureInfo(store, afterPath);
std::set<std::string> allNames;
for (auto & [name, _] : beforeClosure) allNames.insert(name);
for (auto & [name, _] : afterClosure) allNames.insert(name);
for (auto & name : allNames) {
auto & beforeVersions = beforeClosure[name];
auto & afterVersions = afterClosure[name];
auto totalSize = [&](const std::map<std::string, std::map<StorePath, Info>> & versions)
{
uint64_t sum = 0;
for (auto & [_, paths] : versions)
for (auto & [path, _] : paths)
sum += store->queryPathInfo(path)->narSize;
return sum;
};
auto beforeSize = totalSize(beforeVersions);
auto afterSize = totalSize(afterVersions);
auto sizeDelta = (int64_t) afterSize - (int64_t) beforeSize;
auto showDelta = abs(sizeDelta) >= 8 * 1024;
std::set<std::string> removed, unchanged;
for (auto & [version, _] : beforeVersions)
if (!afterVersions.count(version)) removed.insert(version); else unchanged.insert(version);
std::set<std::string> added;
for (auto & [version, _] : afterVersions)
if (!beforeVersions.count(version)) added.insert(version);
if (showDelta || !removed.empty() || !added.empty()) {
std::vector<std::string> items;
if (!removed.empty() || !added.empty())
items.push_back(fmt("%s → %s", showVersions(removed), showVersions(added)));
if (showDelta)
items.push_back(fmt("%s%+.1f KiB" ANSI_NORMAL, sizeDelta > 0 ? ANSI_RED : ANSI_GREEN, sizeDelta / 1024.0));
std::cout << fmt("%s: %s\n", name, concatStringsSep(", ", items));
}
}
}
};
static auto r1 = registerCommand<CmdDiffClosures>("diff-closures");

View file

@ -20,7 +20,7 @@ struct CmdEdit : InstallableCommand
return { return {
Example{ Example{
"To open the Nix expression of the GNU Hello package:", "To open the Nix expression of the GNU Hello package:",
"nix edit nixpkgs.hello" "nix edit nixpkgs#hello"
}, },
}; };
} }

View file

@ -12,10 +12,18 @@ using namespace nix;
struct CmdEval : MixJSON, InstallableCommand struct CmdEval : MixJSON, InstallableCommand
{ {
bool raw = false; bool raw = false;
std::optional<std::string> apply;
CmdEval() CmdEval()
{ {
mkFlag(0, "raw", "print strings unquoted", &raw); mkFlag(0, "raw", "print strings unquoted", &raw);
addFlag({
.longName = "apply",
.description = "apply a function to each argument",
.labels = {"expr"},
.handler = {&apply},
});
} }
std::string description() override std::string description() override
@ -26,21 +34,25 @@ struct CmdEval : MixJSON, InstallableCommand
Examples examples() override Examples examples() override
{ {
return { return {
Example{ {
"To evaluate a Nix expression given on the command line:", "To evaluate a Nix expression given on the command line:",
"nix eval '(1 + 2)'" "nix eval --expr '1 + 2'"
}, },
Example{ {
"To evaluate a Nix expression from a file or URI:", "To evaluate a Nix expression from a file or URI:",
"nix eval -f channel:nixos-17.09 hello.name" "nix eval -f ./my-nixpkgs hello.name"
}, },
Example{ {
"To get the current version of Nixpkgs:", "To get the current version of Nixpkgs:",
"nix eval --raw nixpkgs.lib.version" "nix eval --raw nixpkgs#lib.version"
}, },
Example{ {
"To print the store path of the Hello package:", "To print the store path of the Hello package:",
"nix eval --raw nixpkgs.hello" "nix eval --raw nixpkgs#hello"
},
{
"To get a list of checks in the 'nix' flake:",
"nix eval nix#checks.x86_64-linux --apply builtins.attrNames"
}, },
}; };
} }
@ -57,6 +69,14 @@ struct CmdEval : MixJSON, InstallableCommand
auto v = installable->toValue(*state).first; auto v = installable->toValue(*state).first;
PathSet context; PathSet context;
if (apply) {
auto vApply = state->allocValue();
state->eval(state->parseExprFromString(*apply, absPath(".")), *vApply);
auto vRes = state->allocValue();
state->callFunction(*vApply, *v, *vRes, noPos);
v = vRes;
}
if (raw) { if (raw) {
stopProgressBar(); stopProgressBar();
std::cout << state->coerceToString(noPos, *v, context); std::cout << state->coerceToString(noPos, *v, context);

Some files were not shown because too many files have changed in this diff Show more