Merge branch 'no-manifests'

This commit is contained in:
Eelco Dolstra 2012-08-27 14:34:51 -04:00
commit 15e1b2c223
60 changed files with 2411 additions and 1323 deletions

4
.gitignore vendored
View file

@ -35,8 +35,11 @@ Makefile.in
# /doc/manual/ # /doc/manual/
/doc/manual/manual.html /doc/manual/manual.html
/doc/manual/manual.xmli
/doc/manual/manual.pdf
/doc/manual/manual.is-valid /doc/manual/manual.is-valid
/doc/manual/*.1 /doc/manual/*.1
/doc/manual/*.5
/doc/manual/*.8 /doc/manual/*.8
/doc/manual/images /doc/manual/images
/doc/manual/version.txt /doc/manual/version.txt
@ -60,6 +63,7 @@ Makefile.in
/scripts/GeneratePatches.pm /scripts/GeneratePatches.pm
/scripts/download-using-manifests.pl /scripts/download-using-manifests.pl
/scripts/copy-from-other-stores.pl /scripts/copy-from-other-stores.pl
/scripts/download-from-binary-cache.pl
/scripts/find-runtime-roots.pl /scripts/find-runtime-roots.pl
/scripts/build-remote.pl /scripts/build-remote.pl
/scripts/nix-reduce-build /scripts/nix-reduce-build

View file

@ -25,12 +25,12 @@ AC_ARG_WITH(system, AC_HELP_STRING([--with-system=SYSTEM],
case "$host_os" in case "$host_os" in
linux-gnu*) linux-gnu*)
# For backward compatibility, strip the `-gnu' part. # For backward compatibility, strip the `-gnu' part.
system="$machine_name-linux";; system="$machine_name-linux";;
*) *)
# Strip the version number from names such as `gnu0.3', # Strip the version number from names such as `gnu0.3',
# `darwin10.2.0', etc. # `darwin10.2.0', etc.
system="$machine_name-`echo $host_os | "$SED" -e's/@<:@0-9.@:>@*$//g'`";; system="$machine_name-`echo $host_os | "$SED" -e's/@<:@0-9.@:>@*$//g'`";;
esac]) esac])
sys_name=$(uname -s | tr 'A-Z ' 'a-z_') sys_name=$(uname -s | tr 'A-Z ' 'a-z_')
@ -178,6 +178,7 @@ NEED_PROG(perl, perl)
NEED_PROG(sed, sed) NEED_PROG(sed, sed)
NEED_PROG(tar, tar) NEED_PROG(tar, tar)
NEED_PROG(bzip2, bzip2) NEED_PROG(bzip2, bzip2)
NEED_PROG(xz, xz)
AC_PATH_PROG(dot, dot) AC_PATH_PROG(dot, dot)
AC_PATH_PROG(dblatex, dblatex) AC_PATH_PROG(dblatex, dblatex)
AC_PATH_PROG(gzip, gzip) AC_PATH_PROG(gzip, gzip)
@ -266,7 +267,7 @@ if test "$gc" = yes; then
fi fi
# Check for the required Perl dependencies (DBI and DBD::SQLite). # Check for the required Perl dependencies (DBI, DBD::SQLite and WWW::Curl).
perlFlags="-I$perllibdir" perlFlags="-I$perllibdir"
AC_ARG_WITH(dbi, AC_HELP_STRING([--with-dbi=PATH], AC_ARG_WITH(dbi, AC_HELP_STRING([--with-dbi=PATH],
@ -277,6 +278,10 @@ AC_ARG_WITH(dbd-sqlite, AC_HELP_STRING([--with-dbd-sqlite=PATH],
[prefix of the Perl DBD::SQLite library]), [prefix of the Perl DBD::SQLite library]),
perlFlags="$perlFlags -I$withval") perlFlags="$perlFlags -I$withval")
AC_ARG_WITH(www-curl, AC_HELP_STRING([--with-www-curl=PATH],
[prefix of the Perl WWW::Curl library]),
perlFlags="$perlFlags -I$withval")
AC_MSG_CHECKING([whether DBD::SQLite works]) AC_MSG_CHECKING([whether DBD::SQLite works])
if ! $perl $perlFlags -e 'use DBI; use DBD::SQLite;' 2>&5; then if ! $perl $perlFlags -e 'use DBI; use DBD::SQLite;' 2>&5; then
AC_MSG_RESULT(no) AC_MSG_RESULT(no)
@ -284,6 +289,13 @@ if ! $perl $perlFlags -e 'use DBI; use DBD::SQLite;' 2>&5; then
fi fi
AC_MSG_RESULT(yes) AC_MSG_RESULT(yes)
AC_MSG_CHECKING([whether WWW::Curl works])
if ! $perl $perlFlags -e 'use WWW::Curl;' 2>&5; then
AC_MSG_RESULT(no)
AC_MSG_FAILURE([The Perl module WWW::Curl is missing.])
fi
AC_MSG_RESULT(yes)
AC_SUBST(perlFlags) AC_SUBST(perlFlags)
@ -327,6 +339,18 @@ eval dynlib_suffix=$shrext_cmds
AC_SUBST(dynlib_suffix) AC_SUBST(dynlib_suffix)
# Do we have GNU tar?
AC_MSG_CHECKING([if you have GNU tar])
if $tar --version 2> /dev/null | grep -q GNU; then
AC_MSG_RESULT(yes)
tarFlags="--warning=no-timestamp"
else
AC_MSG_RESULT(no)
fi
AC_SUBST(tarFlags)
AM_CONFIG_HEADER([config.h]) AM_CONFIG_HEADER([config.h])
AC_CONFIG_FILES([Makefile AC_CONFIG_FILES([Makefile
src/Makefile src/Makefile

View file

@ -1,6 +1,6 @@
all-local: config.nix all-local: config.nix
files = nar.nix buildenv.nix buildenv.pl unpack-channel.nix unpack-channel.sh derivation.nix fetchurl.nix \ files = nar.nix buildenv.nix buildenv.pl unpack-channel.nix derivation.nix fetchurl.nix \
imported-drv-to-derivation.nix imported-drv-to-derivation.nix
install-exec-local: install-exec-local:

View file

@ -6,8 +6,10 @@ in {
perl = "@perl@"; perl = "@perl@";
shell = "@shell@"; shell = "@shell@";
coreutils = "@coreutils@"; coreutils = "@coreutils@";
bzip2 = fromEnv "NIX_BZIP2" "@bzip2@"; bzip2 = "@bzip2@";
xz = "@xz@";
tar = "@tar@"; tar = "@tar@";
tarFlags = "@tarFlags@";
tr = "@tr@"; tr = "@tr@";
curl = "@curl@"; curl = "@curl@";
nixBinDir = fromEnv "NIX_BIN_DIR" "@bindir@"; nixBinDir = fromEnv "NIX_BIN_DIR" "@bindir@";

View file

@ -6,28 +6,37 @@ let
'' ''
export PATH=${nixBinDir}:${coreutils} export PATH=${nixBinDir}:${coreutils}
if [ $compressionType = "xz" ]; then
ext=xz
compressor="${xz} -9"
else
ext=bz2
compressor="${bzip2}"
fi
echo "packing $storePath..." echo "packing $storePath..."
mkdir $out mkdir $out
dst=$out/tmp.nar.bz2 dst=$out/tmp.nar.$ext
set -o pipefail set -o pipefail
nix-store --dump "$storePath" | ${bzip2} > $dst nix-store --dump "$storePath" | $compressor > $dst
nix-hash --flat --type $hashAlgo --base32 $dst > $out/narbz2-hash hash=$(nix-hash --flat --type $hashAlgo --base32 $dst)
echo -n $hash > $out/nar-compressed-hash
mv $out/tmp.nar.bz2 $out/$(cat $out/narbz2-hash).nar.bz2 mv $dst $out/$hash.nar.$ext
''; '';
in in
{ storePath, hashAlgo }: { storePath, hashAlgo, compressionType }:
derivation { derivation {
name = "nar"; name = "nar";
system = builtins.currentSystem; system = builtins.currentSystem;
builder = shell; builder = shell;
args = [ "-e" builder ]; args = [ "-e" builder ];
inherit storePath hashAlgo; inherit storePath hashAlgo compressionType;
# Don't build in a chroot because Nix's dependencies may not be there. # Don't build in a chroot because Nix's dependencies may not be there.
__noChroot = true; __noChroot = true;

View file

@ -1,12 +1,29 @@
with import <nix/config.nix>; with import <nix/config.nix>;
{ name, channelName, src }: let
builder = builtins.toFile "unpack-channel.sh"
''
mkdir $out
cd $out
${bzip2} -d < $src | ${tar} xf - --warning=no-timestamp
mv * $out/$channelName
if [ -n "$binaryCacheURL" ]; then
mkdir $out/binary-caches
echo -n "$binaryCacheURL" > $out/binary-caches/$channelName
fi
'';
in
{ name, channelName, src, binaryCacheURL ? "" }:
derivation { derivation {
system = builtins.currentSystem; system = builtins.currentSystem;
builder = shell; builder = shell;
args = [ "-e" ./unpack-channel.sh ]; args = [ "-e" builder ];
inherit name channelName src bzip2 tar tr; inherit name channelName src binaryCacheURL;
PATH = "${nixBinDir}:${coreutils}"; PATH = "${nixBinDir}:${coreutils}";
# No point in doing this remotely. # No point in doing this remotely.

View file

@ -1,4 +0,0 @@
mkdir $out
cd $out
$bzip2 -d < $src | $tar xf -
mv * $out/$channelName

View file

@ -30,6 +30,9 @@ gc-keep-derivations = true # Idem
env-keep-derivations = false env-keep-derivations = false
</programlisting> </programlisting>
<para>You can override settings using the <option>--option</option>
flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
<para>The following settings are currently available: <para>The following settings are currently available:
<variablelist> <variablelist>
@ -243,6 +246,16 @@ env-keep-derivations = false
</varlistentry> </varlistentry>
<varlistentry><term><literal>build-fallback</literal></term>
<listitem><para>If set to <literal>true</literal>, Nix will fall
back to building from source if a binary substitute fails. This
is equivalent to the <option>--fallback</option> flag. The
default is <literal>false</literal>.</para></listitem>
</varlistentry>
<varlistentry xml:id="conf-build-chroot-dirs"><term><literal>build-chroot-dirs</literal></term> <varlistentry xml:id="conf-build-chroot-dirs"><term><literal>build-chroot-dirs</literal></term>
<listitem><para>When builds are performed in a chroot environment, <listitem><para>When builds are performed in a chroot environment,
@ -307,6 +320,50 @@ build-use-chroot = /dev /proc /bin</programlisting>
</varlistentry> </varlistentry>
<varlistentry><term><literal>binary-caches</literal></term>
<listitem><para>A list of URLs of binary caches, separated by
whitespace. The default is empty.<!-- The default is
<literal>http://nixos.org/binary-cache</literal>. --></para></listitem>
</varlistentry>
<varlistentry><term><literal>binary-caches-files</literal></term>
<listitem><para>A list of names of files that will be read to
obtain additional binary cache URLs. The default is
<literal>/nix/var/nix/profiles/per-user/root/channels/binary-caches/*</literal>,
which ensures that Nix will use the binary caches corresponding to
the channels installed by root. Do not set this option to read
files created by untrusted users!</para></listitem>
</varlistentry>
<varlistentry><term><literal>trusted-binary-caches</literal></term>
<listitem><para>A list of URLs of binary caches, separated by
whitespace. These are not used by default, but can be enabled by
users of the Nix daemon by specifying <literal>--option
binary-caches <replaceable>urls</replaceable></literal> on the
command line. Daemon users are only allowed to pass a subset of
the URLs listed in <literal>binary-caches</literal> and
<literal>trusted-binary-caches</literal>.</para></listitem>
</varlistentry>
<varlistentry><term><literal>binary-caches-parallel-connections</literal></term>
<listitem><para>The maximum number of parallel HTTP connections
used by the binary cache substituter to get NAR info files. This
number should be high to minimise latency. It defaults to
150.</para></listitem>
</varlistentry>
<varlistentry><term><literal>system</literal></term> <varlistentry><term><literal>system</literal></term>
<listitem><para>This option specifies the canonical Nix system <listitem><para>This option specifies the canonical Nix system

View file

@ -94,6 +94,11 @@
<citerefentry><refentrytitle>nix.conf</refentrytitle><manvolnum>5</manvolnum></citerefentry>.</para> <citerefentry><refentrytitle>nix.conf</refentrytitle><manvolnum>5</manvolnum></citerefentry>.</para>
</listitem> </listitem>
<listitem>
<para>When using the Nix daemon, the <option>-s</option> flag in
<command>nix-env -qa</command> is now much faster.</para>
</listitem>
</itemizedlist> </itemizedlist>
</section> </section>

View file

@ -16,6 +16,7 @@ BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX)
%endif %endif
BuildRequires: perl(DBD::SQLite) BuildRequires: perl(DBD::SQLite)
BuildRequires: perl(DBI) BuildRequires: perl(DBI)
BuildRequires: perl(WWW::Curl)
BuildRequires: perl(ExtUtils::ParseXS) BuildRequires: perl(ExtUtils::ParseXS)
Requires: /usr/bin/perl Requires: /usr/bin/perl
Requires: curl Requires: curl

View file

@ -1,4 +1,4 @@
PERL_MODULES = lib/Nix/Store.pm lib/Nix/Manifest.pm lib/Nix/GeneratePatches.pm lib/Nix/SSH.pm lib/Nix/CopyClosure.pm lib/Nix/Config.pm.in PERL_MODULES = lib/Nix/Store.pm lib/Nix/Manifest.pm lib/Nix/GeneratePatches.pm lib/Nix/SSH.pm lib/Nix/CopyClosure.pm lib/Nix/Config.pm.in lib/Nix/Utils.pm
all: $(PERL_MODULES:.in=) all: $(PERL_MODULES:.in=)

View file

@ -1,27 +1,39 @@
package Nix::Config; package Nix::Config;
$version = "@version@";
$binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@"; $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
$libexecDir = $ENV{"NIX_LIBEXEC_DIR"} || "@libexecdir@"; $libexecDir = $ENV{"NIX_LIBEXEC_DIR"} || "@libexecdir@";
$stateDir = $ENV{"NIX_STATE_DIR"} || "@localstatedir@/nix"; $stateDir = $ENV{"NIX_STATE_DIR"} || "@localstatedir@/nix";
$manifestDir = $ENV{"NIX_MANIFESTS_DIR"} || "@localstatedir@/nix/manifests"; $manifestDir = $ENV{"NIX_MANIFESTS_DIR"} || "@localstatedir@/nix/manifests";
$logDir = $ENV{"NIX_LOG_DIR"} || "@localstatedir@/log/nix"; $logDir = $ENV{"NIX_LOG_DIR"} || "@localstatedir@/log/nix";
$confDir = $ENV{"NIX_CONF_DIR"} || "@sysconfdir@/nix"; $confDir = $ENV{"NIX_CONF_DIR"} || "@sysconfdir@/nix";
$storeDir = $ENV{"NIX_STORE_DIR"} || "@storedir@";
$bzip2 = $ENV{"NIX_BZIP2"} || "@bzip2@"; $bzip2 = "@bzip2@";
$xz = "@xz@";
$curl = "@curl@"; $curl = "@curl@";
$useBindings = "@perlbindings@" eq "yes"; $useBindings = "@perlbindings@" eq "yes";
%config = ();
sub readConfig { sub readConfig {
my %config; if (defined $ENV{'_NIX_OPTIONS'}) {
my $config = "@sysconfdir@/nix/nix.conf"; foreach my $s (split '\n', $ENV{'_NIX_OPTIONS'}) {
my ($n, $v) = split '=', $s, 2;
$config{$n} = $v;
}
return;
}
my $config = "$confDir/nix.conf";
return unless -f $config; return unless -f $config;
open CONFIG, "<$config" or die "cannot open `$config'"; open CONFIG, "<$config" or die "cannot open `$config'";
while (<CONFIG>) { while (<CONFIG>) {
/^\s*([\w|-]+)\s*=\s*(.*)$/ or next; /^\s*([\w|-]+)\s*=\s*(.*)$/ or next;
$config{$1} = $2; $config{$1} = $2;
print "|$1| -> |$2|\n";
} }
close CONFIG; close CONFIG;
} }

View file

@ -19,7 +19,7 @@ void doInit()
{ {
if (!store) { if (!store) {
try { try {
setDefaultsFromEnvironment(); settings.processEnvironment();
store = openStore(); store = openStore();
} catch (Error & e) { } catch (Error & e) {
croak(e.what()); croak(e.what());

19
perl/lib/Nix/Utils.pm Normal file
View file

@ -0,0 +1,19 @@
package Nix::Utils;
$urlRE = "(?: [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*]+ )";
sub checkURL {
my ($url) = @_;
die "invalid URL $url\n" unless $url =~ /^ $urlRE $ /x;
}
sub uniq {
my %seen;
my @res;
foreach my $name (@_) {
next if $seen{$name};
$seen{$name} = 1;
push @res, $name;
}
return @res;
}

View file

@ -29,6 +29,7 @@ let
--with-xml-flags=--nonet --with-xml-flags=--nonet
--with-dbi=${perlPackages.DBI}/${perl.libPrefix} --with-dbi=${perlPackages.DBI}/${perl.libPrefix}
--with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix} --with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}
--with-www-curl=${perlPackages.WWWCurl}/${perl.libPrefix}
''; '';
postUnpack = '' postUnpack = ''
@ -77,6 +78,7 @@ let
--disable-init-state --disable-init-state
--with-dbi=${perlPackages.DBI}/${perl.libPrefix} --with-dbi=${perlPackages.DBI}/${perl.libPrefix}
--with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix} --with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}
--with-www-curl=${perlPackages.WWWCurl}/${perl.libPrefix}
--enable-gc --enable-gc
''; '';
@ -134,6 +136,7 @@ let
--disable-init-state --disable-init-state
--with-dbi=${perlPackages.DBI}/${perl.libPrefix} --with-dbi=${perlPackages.DBI}/${perl.libPrefix}
--with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix} --with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}
--with-www-curl=${perlPackages.WWWCurl}/${perl.libPrefix}
''; '';
dontInstall = false; dontInstall = false;
@ -192,7 +195,7 @@ let
name = "nix-rpm-${diskImage.name}"; name = "nix-rpm-${diskImage.name}";
src = jobs.tarball; src = jobs.tarball;
diskImage = (diskImageFun vmTools.diskImageFuns) diskImage = (diskImageFun vmTools.diskImageFuns)
{ extraPackages = [ "perl-DBD-SQLite" "perl-devel" "sqlite" "sqlite-devel" "bzip2-devel" "emacs" ]; }; { extraPackages = [ "perl-DBD-SQLite" "perl-devel" "sqlite" "sqlite-devel" "bzip2-devel" "emacs" "perl-WWW-Curl" ]; };
memSize = 1024; memSize = 1024;
meta.schedulingPriority = prio; meta.schedulingPriority = prio;
postRPMInstall = "cd /tmp/rpmout/BUILD/nix-* && make installcheck"; postRPMInstall = "cd /tmp/rpmout/BUILD/nix-* && make installcheck";
@ -211,7 +214,7 @@ let
name = "nix-deb"; name = "nix-deb";
src = jobs.tarball; src = jobs.tarball;
diskImage = (diskImageFun vmTools.diskImageFuns) diskImage = (diskImageFun vmTools.diskImageFuns)
{ extraPackages = [ "libdbd-sqlite3-perl" "libsqlite3-dev" "libbz2-dev" ]; }; { extraPackages = [ "libdbd-sqlite3-perl" "libsqlite3-dev" "libbz2-dev" "libwww-curl-perl" ]; };
memSize = 1024; memSize = 1024;
meta.schedulingPriority = prio; meta.schedulingPriority = prio;
configureFlags = "--sysconfdir=/etc"; configureFlags = "--sysconfdir=/etc";

View file

@ -7,17 +7,14 @@ noinst_SCRIPTS = nix-profile.sh \
find-runtime-roots.pl build-remote.pl nix-reduce-build \ find-runtime-roots.pl build-remote.pl nix-reduce-build \
copy-from-other-stores.pl nix-http-export.cgi copy-from-other-stores.pl nix-http-export.cgi
nix-pull nix-push: download-using-manifests.pl install-exec-local: download-using-manifests.pl copy-from-other-stores.pl download-from-binary-cache.pl find-runtime-roots.pl
install-exec-local: download-using-manifests.pl copy-from-other-stores.pl find-runtime-roots.pl
$(INSTALL) -d $(DESTDIR)$(sysconfdir)/profile.d $(INSTALL) -d $(DESTDIR)$(sysconfdir)/profile.d
$(INSTALL_DATA) nix-profile.sh $(DESTDIR)$(sysconfdir)/profile.d/nix.sh $(INSTALL_DATA) nix-profile.sh $(DESTDIR)$(sysconfdir)/profile.d/nix.sh
$(INSTALL) -d $(DESTDIR)$(libexecdir)/nix $(INSTALL) -d $(DESTDIR)$(libexecdir)/nix
$(INSTALL_PROGRAM) find-runtime-roots.pl $(DESTDIR)$(libexecdir)/nix $(INSTALL_PROGRAM) find-runtime-roots.pl $(DESTDIR)$(libexecdir)/nix
$(INSTALL_PROGRAM) build-remote.pl $(DESTDIR)$(libexecdir)/nix $(INSTALL_PROGRAM) build-remote.pl $(DESTDIR)$(libexecdir)/nix
$(INSTALL) -d $(DESTDIR)$(libexecdir)/nix/substituters $(INSTALL) -d $(DESTDIR)$(libexecdir)/nix/substituters
$(INSTALL_PROGRAM) download-using-manifests.pl $(DESTDIR)$(libexecdir)/nix/substituters $(INSTALL_PROGRAM) download-using-manifests.pl copy-from-other-stores.pl download-from-binary-cache.pl $(DESTDIR)$(libexecdir)/nix/substituters
$(INSTALL_PROGRAM) copy-from-other-stores.pl $(DESTDIR)$(libexecdir)/nix/substituters
$(INSTALL) -d $(DESTDIR)$(sysconfdir)/nix $(INSTALL) -d $(DESTDIR)$(sysconfdir)/nix
include ../substitute.mk include ../substitute.mk
@ -29,6 +26,7 @@ EXTRA_DIST = nix-collect-garbage.in \
nix-build.in \ nix-build.in \
download-using-manifests.pl.in \ download-using-manifests.pl.in \
copy-from-other-stores.pl.in \ copy-from-other-stores.pl.in \
download-from-binary-cache.pl.in \
nix-copy-closure.in \ nix-copy-closure.in \
find-runtime-roots.pl.in \ find-runtime-roots.pl.in \
build-remote.pl.in \ build-remote.pl.in \

View file

@ -36,42 +36,45 @@ sub findStorePath {
if ($ARGV[0] eq "--query") { if ($ARGV[0] eq "--query") {
while (<STDIN>) { while (<STDIN>) {
my $cmd = $_; chomp $cmd; chomp;
my ($cmd, @args) = split " ", $_;
if ($cmd eq "have") { if ($cmd eq "have") {
my $storePath = <STDIN>; chomp $storePath; foreach my $storePath (@args) {
print STDOUT (defined findStorePath($storePath) ? "1\n" : "0\n"); print "$storePath\n" if defined findStorePath($storePath);
}
print "\n";
} }
elsif ($cmd eq "info") { elsif ($cmd eq "info") {
my $storePath = <STDIN>; chomp $storePath; foreach my $storePath (@args) {
my ($store, $sourcePath) = findStorePath($storePath); my ($store, $sourcePath) = findStorePath($storePath);
if (!defined $store) { next unless defined $store;
print "0\n";
next; # not an error $ENV{"NIX_DB_DIR"} = "$store/var/nix/db";
my $deriver = `@bindir@/nix-store --query --deriver $storePath`;
die "cannot query deriver of `$storePath'" if $? != 0;
chomp $deriver;
$deriver = "" if $deriver eq "unknown-deriver";
my @references = split "\n",
`@bindir@/nix-store --query --references $storePath`;
die "cannot query references of `$storePath'" if $? != 0;
my $narSize = `@bindir@/nix-store --query --size $storePath`;
die "cannot query size of `$storePath'" if $? != 0;
chomp $narSize;
print "$storePath\n";
print "$deriver\n";
print scalar @references, "\n";
print "$_\n" foreach @references;
print "$narSize\n";
print "$narSize\n";
} }
print "1\n";
$ENV{"NIX_DB_DIR"} = "$store/var/nix/db"; print "\n";
my $deriver = `@bindir@/nix-store --query --deriver $storePath`;
die "cannot query deriver of `$storePath'" if $? != 0;
chomp $deriver;
$deriver = "" if $deriver eq "unknown-deriver";
my @references = split "\n",
`@bindir@/nix-store --query --references $storePath`;
die "cannot query references of `$storePath'" if $? != 0;
my $narSize = `@bindir@/nix-store --query --size $storePath`;
die "cannot query size of `$storePath'" if $? != 0;
chomp $narSize;
print "$deriver\n";
print scalar @references, "\n";
print "$_\n" foreach @references;
print "$narSize\n";
print "$narSize\n";
} }
else { die "unknown command `$cmd'"; } else { die "unknown command `$cmd'"; }
@ -84,9 +87,10 @@ elsif ($ARGV[0] eq "--substitute") {
my $storePath = $ARGV[1]; my $storePath = $ARGV[1];
my ($store, $sourcePath) = findStorePath $storePath; my ($store, $sourcePath) = findStorePath $storePath;
die unless $store; die unless $store;
print "\n*** Copying `$storePath' from `$sourcePath'\n\n"; print STDERR "\n*** Copying `$storePath' from `$sourcePath'\n\n";
system("$binDir/nix-store --dump $sourcePath | $binDir/nix-store --restore $storePath") == 0 system("$binDir/nix-store --dump $sourcePath | $binDir/nix-store --restore $storePath") == 0
or die "cannot copy `$sourcePath' to `$storePath'"; or die "cannot copy `$sourcePath' to `$storePath'";
print "\n"; # no hash to verify
} }

View file

@ -0,0 +1,537 @@
#! @perl@ -w @perlFlags@
use DBI;
use File::Basename;
use IO::Select;
use Nix::Config;
use Nix::Store;
use Nix::Utils;
use WWW::Curl::Easy;
use WWW::Curl::Multi;
use strict;
Nix::Config::readConfig;
my @caches;
my $gotCaches = 0;
my $maxParallelRequests = int($Nix::Config::config{"binary-caches-parallel-connections"} // 150);
$maxParallelRequests = 1 if $maxParallelRequests < 1;
my $debug = ($ENV{"NIX_DEBUG_SUBST"} // "") eq 1;
my ($dbh, $queryCache, $insertNAR, $queryNAR, $insertNARExistence, $queryNARExistence);
my $curlm = WWW::Curl::Multi->new;
my $activeRequests = 0;
my $curlIdCount = 1;
my %requests;
my %scheduled;
my $caBundle = $ENV{"CURL_CA_BUNDLE"} // $ENV{"OPENSSL_X509_CERT_FILE"};
sub addRequest {
my ($storePath, $url, $head) = @_;
my $curl = WWW::Curl::Easy->new;
my $curlId = $curlIdCount++;
$requests{$curlId} = { storePath => $storePath, url => $url, handle => $curl, content => "", type => $head ? "HEAD" : "GET" };
$curl->setopt(CURLOPT_PRIVATE, $curlId);
$curl->setopt(CURLOPT_URL, $url);
$curl->setopt(CURLOPT_WRITEDATA, \$requests{$curlId}->{content});
$curl->setopt(CURLOPT_FOLLOWLOCATION, 1);
$curl->setopt(CURLOPT_CAINFO, $caBundle) if defined $caBundle;
$curl->setopt(CURLOPT_USERAGENT, "Nix/$Nix::Config::version");
$curl->setopt(CURLOPT_NOBODY, 1) if $head;
$curl->setopt(CURLOPT_FAILONERROR, 1);
if ($activeRequests >= $maxParallelRequests) {
$scheduled{$curlId} = 1;
} else {
$curlm->add_handle($curl);
$activeRequests++;
}
return $requests{$curlId};
}
sub processRequests {
while ($activeRequests) {
my ($rfds, $wfds, $efds) = $curlm->fdset();
#print STDERR "R = @{$rfds}, W = @{$wfds}, E = @{$efds}\n";
# Sleep until we can read or write some data.
if (scalar @{$rfds} + scalar @{$wfds} + scalar @{$efds} > 0) {
IO::Select->select(IO::Select->new(@{$rfds}), IO::Select->new(@{$wfds}), IO::Select->new(@{$efds}), 0.1);
}
if ($curlm->perform() != $activeRequests) {
while (my ($id, $result) = $curlm->info_read) {
if ($id) {
my $request = $requests{$id} or die;
my $handle = $request->{handle};
$request->{result} = $result;
$request->{httpStatus} = $handle->getinfo(CURLINFO_RESPONSE_CODE);
print STDERR "$request->{type} on $request->{url} [$request->{result}, $request->{httpStatus}]\n" if $debug;
$activeRequests--;
delete $request->{handle};
if (scalar(keys %scheduled) > 0) {
my $id2 = (keys %scheduled)[0];
$curlm->add_handle($requests{$id2}->{handle});
$activeRequests++;
delete $scheduled{$id2};
}
}
}
}
}
}
sub initCache {
my $dbPath = "$Nix::Config::stateDir/binary-cache-v1.sqlite";
# Open/create the database.
$dbh = DBI->connect("dbi:SQLite:dbname=$dbPath", "", "")
or die "cannot open database `$dbPath'";
$dbh->{RaiseError} = 1;
$dbh->{PrintError} = 0;
$dbh->do("pragma synchronous = off"); # we can always reproduce the cache
$dbh->do("pragma journal_mode = truncate");
# Initialise the database schema, if necessary.
$dbh->do(<<EOF);
create table if not exists BinaryCaches (
id integer primary key autoincrement not null,
url text unique not null,
timestamp integer not null,
storeDir text not null,
wantMassQuery integer not null
);
EOF
$dbh->do(<<EOF);
create table if not exists NARs (
cache integer not null,
storePath text not null,
url text not null,
compression text not null,
fileHash text,
fileSize integer,
narHash text,
narSize integer,
refs text,
deriver text,
system text,
timestamp integer not null,
primary key (cache, storePath),
foreign key (cache) references BinaryCaches(id) on delete cascade
);
EOF
$dbh->do(<<EOF);
create table if not exists NARExistence (
cache integer not null,
storePath text not null,
exist integer not null,
timestamp integer not null,
primary key (cache, storePath),
foreign key (cache) references BinaryCaches(id) on delete cascade
);
EOF
$queryCache = $dbh->prepare("select id, storeDir, wantMassQuery from BinaryCaches where url = ?") or die;
$insertNAR = $dbh->prepare(
"insert or replace into NARs(cache, storePath, url, compression, fileHash, fileSize, narHash, " .
"narSize, refs, deriver, system, timestamp) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") or die;
$queryNAR = $dbh->prepare("select * from NARs where cache = ? and storePath = ?") or die;
$insertNARExistence = $dbh->prepare(
"insert or replace into NARExistence(cache, storePath, exist, timestamp) values (?, ?, ?, ?)") or die;
$queryNARExistence = $dbh->prepare("select exist from NARExistence where cache = ? and storePath = ?") or die;
}
sub getAvailableCaches {
return if $gotCaches;
$gotCaches = 1;
sub strToList {
my ($s) = @_;
return map { s/\/+$//; $_ } split(/ /, $s);
}
my @urls = strToList ($Nix::Config::config{"binary-caches"} // "");
# // ($Nix::Config::storeDir eq "/nix/store" ? "http://nixos.org/binary-cache" : ""));
my $urlsFiles = $Nix::Config::config{"binary-cache-files"}
// "/nix/var/nix/profiles/per-user/root/channels/binary-caches/*";
foreach my $urlFile (glob $urlsFiles) {
next unless -f $urlFile;
open FILE, "<$urlFile" or die "cannot open $urlFile\n";
my $url = <FILE>; chomp $url;
close FILE;
push @urls, strToList($url);
}
# Allow Nix daemon users to override the binary caches to a subset
# of those listed in the config file. Note that untrusted-*
# denotes options passed by the client.
if (defined $Nix::Config::config{"untrusted-binary-caches"}) {
my @untrustedUrls = strToList $Nix::Config::config{"untrusted-binary-caches"};
my @trustedUrls = (@urls, strToList($Nix::Config::config{"trusted-binary-caches"} // ""));
@urls = ();
foreach my $url (@untrustedUrls) {
die "binary cache $url is not trusted (please add it to trusted-binary-caches in $Nix::Config::confDir/nix.conf)\n"
unless grep { $url eq $_ } @trustedUrls > 0;
push @urls, $url;
}
}
foreach my $url (Nix::Utils::uniq @urls) {
# FIXME: not atomic.
$queryCache->execute($url);
my $res = $queryCache->fetchrow_hashref();
if (defined $res) {
next if $res->{storeDir} ne $Nix::Config::storeDir;
push @caches, { id => $res->{id}, url => $url, wantMassQuery => $res->{wantMassQuery} };
next;
}
# Get the cache info file.
my $request = addRequest(undef, $url . "/nix-cache-info");
processRequests;
if ($request->{result} != 0) {
print STDERR "could not download $request->{url} (" .
($request->{result} != 0 ? "Curl error $request->{result}" : "HTTP status $request->{httpStatus}") . ")\n";
next;
}
my $storeDir = "/nix/store";
my $wantMassQuery = 0;
foreach my $line (split "\n", $request->{content}) {
unless ($line =~ /^(.*): (.*)$/) {
print STDERR "bad cache info file $request->{url}\n";
return undef;
}
if ($1 eq "StoreDir") { $storeDir = $2; }
elsif ($1 eq "WantMassQuery") { $wantMassQuery = int($2); }
}
$dbh->do("insert into BinaryCaches(url, timestamp, storeDir, wantMassQuery) values (?, ?, ?, ?)",
{}, $url, time(), $storeDir, $wantMassQuery);
my $id = $dbh->last_insert_id("", "", "", "");
next if $storeDir ne $Nix::Config::storeDir;
push @caches, { id => $id, url => $url, wantMassQuery => $wantMassQuery };
}
}
sub processNARInfo {
my ($storePath, $cache, $request) = @_;
if ($request->{result} != 0) {
if ($request->{result} != 37 && $request->{httpStatus} != 404) {
print STDERR "could not download $request->{url} (" .
($request->{result} != 0 ? "Curl error $request->{result}" : "HTTP status $request->{httpStatus}") . ")\n";
} else {
$insertNARExistence->execute($cache->{id}, basename($storePath), 0, time())
unless $request->{url} =~ /^file:/;
}
return undef;
}
my ($storePath2, $url, $fileHash, $fileSize, $narHash, $narSize, $deriver, $system);
my $compression = "bzip2";
my @refs;
foreach my $line (split "\n", $request->{content}) {
unless ($line =~ /^(.*): (.*)$/) {
print STDERR "bad NAR info file $request->{url}\n";
return undef;
}
if ($1 eq "StorePath") { $storePath2 = $2; }
elsif ($1 eq "URL") { $url = $2; }
elsif ($1 eq "Compression") { $compression = $2; }
elsif ($1 eq "FileHash") { $fileHash = $2; }
elsif ($1 eq "FileSize") { $fileSize = int($2); }
elsif ($1 eq "NarHash") { $narHash = $2; }
elsif ($1 eq "NarSize") { $narSize = int($2); }
elsif ($1 eq "References") { @refs = split / /, $2; }
elsif ($1 eq "Deriver") { $deriver = $2; }
elsif ($1 eq "System") { $system = $2; }
}
return undef if $storePath ne $storePath2;
if ($storePath ne $storePath2 || !defined $url || !defined $narHash) {
print STDERR "bad NAR info file $request->{url}\n";
return undef;
}
# Cache the result.
$insertNAR->execute(
$cache->{id}, basename($storePath), $url, $compression, $fileHash, $fileSize,
$narHash, $narSize, join(" ", @refs), $deriver, $system, time())
unless $request->{url} =~ /^file:/;
return
{ url => $url
, compression => $compression
, fileHash => $fileHash
, fileSize => $fileSize
, narHash => $narHash
, narSize => $narSize
, refs => [ @refs ]
, deriver => $deriver
, system => $system
};
}
sub getCachedInfoFrom {
my ($storePath, $cache) = @_;
$queryNAR->execute($cache->{id}, basename($storePath));
my $res = $queryNAR->fetchrow_hashref();
return undef unless defined $res;
return
{ url => $res->{url}
, compression => $res->{compression}
, fileHash => $res->{fileHash}
, fileSize => $res->{fileSize}
, narHash => $res->{narHash}
, narSize => $res->{narSize}
, refs => [ split " ", $res->{refs} ]
, deriver => $res->{deriver}
} if defined $res;
}
sub negativeHit {
my ($storePath, $cache) = @_;
$queryNARExistence->execute($cache->{id}, basename($storePath));
my $res = $queryNARExistence->fetchrow_hashref();
return defined $res && $res->{exist} == 0;
}
sub positiveHit {
my ($storePath, $cache) = @_;
return 1 if defined getCachedInfoFrom($storePath, $cache);
$queryNARExistence->execute($cache->{id}, basename($storePath));
my $res = $queryNARExistence->fetchrow_hashref();
return defined $res && $res->{exist} == 1;
}
sub printInfo {
my ($storePath, $info) = @_;
print "$storePath\n";
print $info->{deriver} ? "$Nix::Config::storeDir/$info->{deriver}" : "", "\n";
print scalar @{$info->{refs}}, "\n";
print "$Nix::Config::storeDir/$_\n" foreach @{$info->{refs}};
print $info->{fileSize} || 0, "\n";
print $info->{narSize} || 0, "\n";
}
sub infoUrl {
my ($binaryCacheUrl, $storePath) = @_;
my $pathHash = substr(basename($storePath), 0, 32);
my $infoUrl = "$binaryCacheUrl/$pathHash.narinfo";
}
sub printInfoParallel {
my @paths = @_;
# First print all paths for which we have cached info.
my @left;
foreach my $storePath (@paths) {
my $found = 0;
foreach my $cache (@caches) {
my $info = getCachedInfoFrom($storePath, $cache);
if (defined $info) {
printInfo($storePath, $info);
$found = 1;
last;
}
}
push @left, $storePath if !$found;
}
return if scalar @left == 0;
foreach my $cache (@caches) {
my @left2;
%requests = ();
foreach my $storePath (@left) {
if (negativeHit($storePath, $cache)) {
push @left2, $storePath;
next;
}
addRequest($storePath, infoUrl($cache->{url}, $storePath));
}
processRequests;
foreach my $request (values %requests) {
my $info = processNARInfo($request->{storePath}, $cache, $request);
if (defined $info) {
printInfo($request->{storePath}, $info);
} else {
push @left2, $request->{storePath};
}
}
@left = @left2;
}
}
sub printSubstitutablePaths {
my @paths = @_;
# First look for paths that have cached info.
my @left;
foreach my $storePath (@paths) {
my $found = 0;
foreach my $cache (@caches) {
next unless $cache->{wantMassQuery};
if (positiveHit($storePath, $cache)) {
print "$storePath\n";
$found = 1;
last;
}
}
push @left, $storePath if !$found;
}
return if scalar @left == 0;
# For remaining paths, do HEAD requests.
foreach my $cache (@caches) {
next unless $cache->{wantMassQuery};
my @left2;
%requests = ();
foreach my $storePath (@left) {
if (negativeHit($storePath, $cache)) {
push @left2, $storePath;
next;
}
addRequest($storePath, infoUrl($cache->{url}, $storePath), 1);
}
processRequests;
foreach my $request (values %requests) {
if ($request->{result} != 0) {
if ($request->{result} != 37 && $request->{httpStatus} != 404) {
print STDERR "could not check $request->{url} (" .
($request->{result} != 0 ? "Curl error $request->{result}" : "HTTP status $request->{httpStatus}") . ")\n";
} else {
$insertNARExistence->execute($cache->{id}, basename($request->{storePath}), 0, time())
unless $request->{url} =~ /^file:/;
}
push @left2, $request->{storePath};
} else {
$insertNARExistence->execute($cache->{id}, basename($request->{storePath}), 1, time())
unless $request->{url} =~ /^file:/;
print "$request->{storePath}\n";
}
}
@left = @left2;
}
}
sub downloadBinary {
my ($storePath) = @_;
foreach my $cache (@caches) {
my $info = getCachedInfoFrom($storePath, $cache);
unless (defined $info) {
next if negativeHit($storePath, $cache);
my $request = addRequest($storePath, infoUrl($cache->{url}, $storePath));
processRequests;
$info = processNARInfo($storePath, $cache, $request);
}
next unless defined $info;
my $decompressor;
if ($info->{compression} eq "bzip2") { $decompressor = "$Nix::Config::bzip2 -d"; }
elsif ($info->{compression} eq "xz") { $decompressor = "$Nix::Config::xz -d"; }
else {
print STDERR "unknown compression method $info->{compression}\n";
next;
}
my $url = "$cache->{url}/$info->{url}"; # FIXME: handle non-relative URLs
print STDERR "\n*** Downloading $url into $storePath...\n";
Nix::Utils::checkURL $url;
if (system("$Nix::Config::curl --fail --location --insecure '$url' | $decompressor | $Nix::Config::binDir/nix-store --restore $storePath") != 0) {
die "download of `$info->{url}' failed" . ($! ? ": $!" : "") . "\n" unless $? == 0;
next;
}
# Tell Nix about the expected hash so it can verify it.
print "$info->{narHash}\n";
print STDERR "\n";
return;
}
print STDERR "could not download $storePath from any binary cache\n";
}
initCache();
if ($ARGV[0] eq "--query") {
while (<STDIN>) {
getAvailableCaches;
chomp;
my ($cmd, @args) = split " ", $_;
if ($cmd eq "have") {
printSubstitutablePaths(@args);
print "\n";
}
elsif ($cmd eq "info") {
printInfoParallel(@args);
print "\n";
}
else { die "unknown command `$cmd'"; }
flush STDOUT;
}
}
elsif ($ARGV[0] eq "--substitute") {
my $storePath = $ARGV[1] or die;
getAvailableCaches;
downloadBinary($storePath);
}
else {
die;
}

View file

@ -4,6 +4,7 @@ use strict;
use Nix::Config; use Nix::Config;
use Nix::Manifest; use Nix::Manifest;
use Nix::Store; use Nix::Store;
use Nix::Utils;
use POSIX qw(strftime); use POSIX qw(strftime);
use File::Temp qw(tempdir); use File::Temp qw(tempdir);
@ -15,6 +16,9 @@ my $logFile = "$Nix::Config::logDir/downloads";
# estimating the expected download size. # estimating the expected download size.
my $fast = 1; my $fast = 1;
# --insecure is fine because Nix verifies the hash of the result.
my $curl = "$Nix::Config::curl --fail --location --insecure";
# Open the manifest cache and update it if necessary. # Open the manifest cache and update it if necessary.
my $dbh = updateManifestDB(); my $dbh = updateManifestDB();
@ -173,56 +177,54 @@ sub computeSmallestDownload {
if ($ARGV[0] eq "--query") { if ($ARGV[0] eq "--query") {
while (<STDIN>) { while (<STDIN>) {
my $cmd = $_; chomp $cmd; chomp;
my ($cmd, @args) = split " ", $_;
if ($cmd eq "have") { if ($cmd eq "have") {
my $storePath = <STDIN>; chomp $storePath; foreach my $storePath (@args) {
print STDOUT ( print "$storePath\n" if scalar @{$dbh->selectcol_arrayref("select 1 from NARs where storePath = ?", {}, $storePath)} > 0;
scalar @{$dbh->selectcol_arrayref("select 1 from NARs where storePath = ?", {}, $storePath)} > 0 }
? "1\n" : "0\n"); print "\n";
} }
elsif ($cmd eq "info") { elsif ($cmd eq "info") {
my $storePath = <STDIN>; chomp $storePath; foreach my $storePath (@args) {
my $infos = $dbh->selectall_arrayref( my $infos = $dbh->selectall_arrayref(
"select * from NARs where storePath = ?", "select * from NARs where storePath = ?",
{ Slice => {} }, $storePath); { Slice => {} }, $storePath);
my $info; next unless scalar @{$infos} > 0;
if (scalar @{$infos} > 0) { my $info = @{$infos}[0];
$info = @{$infos}[0];
}
else {
print "0\n";
next; # not an error
}
print "1\n"; print "$storePath\n";
print "$info->{deriver}\n"; print "$info->{deriver}\n";
my @references = split " ", $info->{refs}; my @references = split " ", $info->{refs};
print scalar @references, "\n"; print scalar @references, "\n";
print "$_\n" foreach @references; print "$_\n" foreach @references;
my @path = computeSmallestDownload $storePath; my @path = computeSmallestDownload $storePath;
my $downloadSize = 0; my $downloadSize = 0;
while (scalar @path > 0) { while (scalar @path > 0) {
my $edge = pop @path; my $edge = pop @path;
my $u = $edge->{start}; my $u = $edge->{start};
my $v = $edge->{end}; my $v = $edge->{end};
if ($edge->{type} eq "patch") { if ($edge->{type} eq "patch") {
$downloadSize += $edge->{info}->{size} || 0; $downloadSize += $edge->{info}->{size} || 0;
} }
elsif ($edge->{type} eq "narfile") { elsif ($edge->{type} eq "narfile") {
$downloadSize += $edge->{info}->{size} || 0; $downloadSize += $edge->{info}->{size} || 0;
}
} }
print "$downloadSize\n";
my $narSize = $info->{narSize} || 0;
print "$narSize\n";
} }
print "$downloadSize\n"; print "\n";
my $narSize = $info->{narSize} || 0;
print "$narSize\n";
} }
else { die "unknown command `$cmd'"; } else { die "unknown command `$cmd'"; }
@ -273,16 +275,6 @@ $dbh->disconnect;
my $curStep = 1; my $curStep = 1;
my $maxStep = scalar @path; my $maxStep = scalar @path;
sub downloadFile {
my $url = shift;
$ENV{"PRINT_PATH"} = 1;
$ENV{"QUIET"} = 1;
my ($hash, $path) = `$Nix::Config::binDir/nix-prefetch-url '$url'`;
die "download of `$url' failed" . ($! ? ": $!" : "") . "\n" unless $? == 0;
chomp $path;
return $path;
}
my $finalNarHash; my $finalNarHash;
while (scalar @path > 0) { while (scalar @path > 0) {
@ -314,13 +306,16 @@ while (scalar @path > 0) {
# Download the patch. # Download the patch.
print STDERR " downloading patch...\n"; print STDERR " downloading patch...\n";
my $patchPath = downloadFile "$patch->{url}"; my $patchPath = "$tmpDir/patch";
Nix::Utils::checkURL $patch->{url};
system("$curl '$patch->{url}' -o $patchPath") == 0
or die "cannot download patch `$patch->{url}'\n";
# Apply the patch to the NAR archive produced in step 1 (for # Apply the patch to the NAR archive produced in step 1 (for
# the already present path) or a later step (for patch sequences). # the already present path) or a later step (for patch sequences).
print STDERR " applying patch...\n"; print STDERR " applying patch...\n";
system("$Nix::Config::libexecDir/bspatch $tmpNar $tmpNar2 $patchPath") == 0 system("$Nix::Config::libexecDir/bspatch $tmpNar $tmpNar2 $patchPath") == 0
or die "cannot apply patch `$patchPath' to $tmpNar"; or die "cannot apply patch `$patchPath' to $tmpNar\n";
if ($curStep < $maxStep) { if ($curStep < $maxStep) {
# The archive will be used as the base of the next patch. # The archive will be used as the base of the next patch.
@ -330,7 +325,7 @@ while (scalar @path > 0) {
# into the target path. # into the target path.
print STDERR " unpacking patched archive...\n"; print STDERR " unpacking patched archive...\n";
system("$Nix::Config::binDir/nix-store --restore $v < $tmpNar2") == 0 system("$Nix::Config::binDir/nix-store --restore $v < $tmpNar2") == 0
or die "cannot unpack $tmpNar2 into `$v'"; or die "cannot unpack $tmpNar2 into `$v'\n";
} }
$finalNarHash = $patch->{narHash}; $finalNarHash = $patch->{narHash};
@ -343,19 +338,15 @@ while (scalar @path > 0) {
my $size = $narFile->{size} || -1; my $size = $narFile->{size} || -1;
print LOGFILE "$$ narfile $narFile->{url} $size $v\n"; print LOGFILE "$$ narfile $narFile->{url} $size $v\n";
# Download the archive. Nix::Utils::checkURL $narFile->{url};
print STDERR " downloading archive...\n";
my $narFilePath = downloadFile "$narFile->{url}";
if ($curStep < $maxStep) { if ($curStep < $maxStep) {
# The archive will be used a base to a patch. # The archive will be used a base to a patch.
system("$Nix::Config::bzip2 -d < '$narFilePath' > $tmpNar") == 0 system("$curl '$narFile->{url}' | $Nix::Config::bzip2 -d > $tmpNar") == 0
or die "cannot unpack `$narFilePath' into `$v'"; or die "cannot download and unpack `$narFile->{url}' into `$v'\n";
} else { } else {
# Unpack the archive into the target path. # Unpack the archive into the target path.
print STDERR " unpacking archive...\n"; system("$curl '$narFile->{url}' | $Nix::Config::bzip2 -d | $Nix::Config::binDir/nix-store --restore '$v'") == 0
system("$Nix::Config::bzip2 -d < '$narFilePath' | $Nix::Config::binDir/nix-store --restore '$v'") == 0 or die "cannot download and unpack `$narFile->{url}' into `$v'\n";
or die "cannot unpack `$narFilePath' into `$v'";
} }
$finalNarHash = $narFile->{narHash}; $finalNarHash = $narFile->{narHash};
@ -365,21 +356,10 @@ while (scalar @path > 0) {
} }
# Make sure that the hash declared in the manifest matches what we # Tell Nix about the expected hash so it can verify it.
# downloaded and unpacked. die "cannot check integrity of the downloaded path since its hash is not known\n"
unless defined $finalNarHash;
if (defined $finalNarHash) { print "$finalNarHash\n";
my ($hashAlgo, $hash) = parseHash $finalNarHash;
# The hash in the manifest can be either in base-16 or base-32.
# Handle both.
my $hash2 = hashPath($hashAlgo, $hashAlgo eq "sha256" && length($hash) != 64, $targetPath);
die "hash mismatch in downloaded path $targetPath; expected $hash, got $hash2\n"
if $hash ne $hash2;
} else {
die "cannot check integrity of the downloaded path since its hash is not known\n";
}
print STDERR "\n"; print STDERR "\n";

View file

@ -58,6 +58,11 @@ EOF
# '` hack # '` hack
} }
elsif ($arg eq "--version") {
print "nix-build (Nix) $Nix::Config::version\n";
exit 0;
}
elsif ($arg eq "--add-drv-link") { elsif ($arg eq "--add-drv-link") {
$drvLink = "./derivation"; $drvLink = "./derivation";
} }

View file

@ -80,12 +80,6 @@ sub update {
readChannels; readChannels;
# Create the manifests directory if it doesn't exist.
mkdir $manifestDir, 0755 unless -e $manifestDir;
# Do we have write permission to the manifests directory?
die "$0: you do not have write permission to `$manifestDir'!\n" unless -W $manifestDir;
# Download each channel. # Download each channel.
my $exprs = ""; my $exprs = "";
foreach my $name (keys %channels) { foreach my $name (keys %channels) {
@ -102,10 +96,19 @@ sub update {
$headers =~ s/\r//g; $headers =~ s/\r//g;
$url = $1 if $headers =~ /^Location:\s*(.*)\s*$/m; $url = $1 if $headers =~ /^Location:\s*(.*)\s*$/m;
# Pull the channel manifest. # Check if the channel advertises a binary cache.
$ENV{'NIX_ORIG_URL'} = $origUrl; my $binaryCacheURL = `$Nix::Config::curl --silent '$url'/binary-cache-url`;
system("$Nix::Config::binDir/nix-pull", "--skip-wrong-store", "$url/MANIFEST") == 0 my $extraAttrs = "";
or die "cannot pull manifest from `$url'\n"; if ($? == 0 && $binaryCacheURL ne "") {
$extraAttrs .= "binaryCacheURL = \"$binaryCacheURL\"; ";
} else {
# No binary cache, so pull the channel manifest.
mkdir $manifestDir, 0755 unless -e $manifestDir;
die "$0: you do not have write permission to `$manifestDir'!\n" unless -W $manifestDir;
$ENV{'NIX_ORIG_URL'} = $origUrl;
system("$Nix::Config::binDir/nix-pull", "--skip-wrong-store", "$url/MANIFEST") == 0
or die "cannot pull manifest from `$url'\n";
}
# Download the channel tarball. # Download the channel tarball.
my $fullURL = "$url/nixexprs.tar.bz2"; my $fullURL = "$url/nixexprs.tar.bz2";
@ -120,7 +123,7 @@ sub update {
my $cname = $name; my $cname = $name;
$cname .= $1 if basename($url) =~ /(-\d.*)$/; $cname .= $1 if basename($url) =~ /(-\d.*)$/;
$exprs .= "'f: f { name = \"$cname\"; channelName = \"$name\"; src = builtins.storePath \"$path\"; }' "; $exprs .= "'f: f { name = \"$cname\"; channelName = \"$name\"; src = builtins.storePath \"$path\"; $extraAttrs }' ";
} }
# Unpack the channel tarballs into the Nix store and install them # Unpack the channel tarballs into the Nix store and install them
@ -194,6 +197,11 @@ while (scalar @ARGV) {
usageError; usageError;
} }
elsif ($arg eq "--version") {
print "nix-channel (Nix) $Nix::Config::version\n";
exit 0;
}
else { else {
die "unknown argument `$arg'; try `--help'"; die "unknown argument `$arg'; try `--help'";
} }

View file

@ -3,6 +3,7 @@
use strict; use strict;
use File::Temp qw(tempdir); use File::Temp qw(tempdir);
use Nix::Config; use Nix::Config;
use Nix::Utils;
sub usageError { sub usageError {
@ -72,7 +73,7 @@ my $tmpDir = tempdir("nix-install-package.XXXXXX", CLEANUP => 1, TMPDIR => 1)
sub barf { sub barf {
my $msg = shift; my $msg = shift;
print "$msg\n"; print "\nInstallation failed: $msg\n";
<STDIN> if $interactive; <STDIN> if $interactive;
exit 1; exit 1;
} }
@ -92,7 +93,6 @@ open PKGFILE, "<$pkgFile" or barf "cannot open `$pkgFile': $!";
my $contents = <PKGFILE>; my $contents = <PKGFILE>;
close PKGFILE; close PKGFILE;
my $urlRE = "(?: [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\']+ )";
my $nameRE = "(?: [A-Za-z0-9\+\-\.\_\?\=]+ )"; # see checkStoreName() my $nameRE = "(?: [A-Za-z0-9\+\-\.\_\?\=]+ )"; # see checkStoreName()
my $systemRE = "(?: [A-Za-z0-9\+\-\_]+ )"; my $systemRE = "(?: [A-Za-z0-9\+\-\_]+ )";
my $pathRE = "(?: \/ [\/A-Za-z0-9\+\-\.\_\?\=]* )"; my $pathRE = "(?: \/ [\/A-Za-z0-9\+\-\.\_\?\=]* )";
@ -101,7 +101,7 @@ my $pathRE = "(?: \/ [\/A-Za-z0-9\+\-\.\_\?\=]* )";
# store path. We'll let nix-env do that. # store path. We'll let nix-env do that.
$contents =~ $contents =~
/ ^ \s* (\S+) \s+ ($urlRE) \s+ ($nameRE) \s+ ($systemRE) \s+ ($pathRE) \s+ ($pathRE) /x / ^ \s* (\S+) \s+ ($Nix::Utils::urlRE) \s+ ($nameRE) \s+ ($systemRE) \s+ ($pathRE) \s+ ($pathRE) ( \s+ ($Nix::Utils::urlRE) )? /x
or barf "invalid package contents"; or barf "invalid package contents";
my $version = $1; my $version = $1;
my $manifestURL = $2; my $manifestURL = $2;
@ -109,6 +109,7 @@ my $drvName = $3;
my $system = $4; my $system = $4;
my $drvPath = $5; my $drvPath = $5;
my $outPath = $6; my $outPath = $6;
my $binaryCacheURL = $8;
barf "invalid package version `$version'" unless $version eq "NIXPKG1"; barf "invalid package version `$version'" unless $version eq "NIXPKG1";
@ -122,17 +123,25 @@ if ($interactive) {
} }
# Store the manifest in the temporary directory so that we don't if (defined $binaryCacheURL) {
# pollute /nix/var/nix/manifests. This also requires that we don't
# use the Nix daemon (because otherwise download-using-manifests won't
# see our NIX_MANIFESTS_DIRS environment variable).
$ENV{NIX_MANIFESTS_DIR} = $tmpDir;
$ENV{NIX_REMOTE} = "";
push @extraNixEnvArgs, "--option", "binary-caches", $binaryCacheURL;
print "\nPulling manifests...\n"; } else {
system("$Nix::Config::binDir/nix-pull", $manifestURL) == 0
or barf "nix-pull failed: $?"; # Store the manifest in the temporary directory so that we don't
# pollute /nix/var/nix/manifests. This also requires that we
# don't use the Nix daemon (because otherwise
# download-using-manifests won't see our NIX_MANIFESTS_DIRS
# environment variable).
$ENV{NIX_MANIFESTS_DIR} = $tmpDir;
$ENV{NIX_REMOTE} = "";
print "\nPulling manifests...\n";
system("$Nix::Config::binDir/nix-pull", $manifestURL) == 0
or barf "nix-pull failed: $?";
}
print "\nInstalling package...\n"; print "\nInstalling package...\n";

View file

@ -1,77 +1,77 @@
#! @perl@ -w @perlFlags@ #! @perl@ -w @perlFlags@
use strict; use strict;
use File::Basename;
use File::Temp qw(tempdir); use File::Temp qw(tempdir);
use File::Path qw(mkpath);
use File::stat; use File::stat;
use File::Copy;
use Nix::Config; use Nix::Config;
use Nix::Store;
use Nix::Manifest; use Nix::Manifest;
my $hashAlgo = "sha256";
my $tmpDir = tempdir("nix-push.XXXXXX", CLEANUP => 1, TMPDIR => 1) my $tmpDir = tempdir("nix-push.XXXXXX", CLEANUP => 1, TMPDIR => 1)
or die "cannot create a temporary directory"; or die "cannot create a temporary directory";
my $nixExpr = "$tmpDir/create-nars.nix"; my $nixExpr = "$tmpDir/create-nars.nix";
my $manifest = "$tmpDir/MANIFEST";
my $curl = "$Nix::Config::curl --fail --silent";
my $extraCurlFlags = ${ENV{'CURL_FLAGS'}};
$curl = "$curl $extraCurlFlags" if defined $extraCurlFlags;
# Parse the command line. # Parse the command line.
my $localCopy; my $compressionType = "xz";
my $localArchivesDir; my $force = 0;
my $localManifestFile; my $destDir;
my $writeManifest = 0;
my $targetArchivesUrl; my $archivesURL;
my @roots;
my $archivesPutURL;
my $archivesGetURL;
my $manifestPutURL;
sub showSyntax { sub showSyntax {
print STDERR <<EOF print STDERR <<EOF
Usage: nix-push --copy ARCHIVES_DIR MANIFEST_FILE PATHS... Usage: nix-push --dest DIR [--manifest] [--url-prefix URL] PATHS...
or: nix-push ARCHIVES_PUT_URL ARCHIVES_GET_URL MANIFEST_PUT_URL PATHS...
`nix-push' copies or uploads the closure of PATHS to the given `nix-push' packs the closure of PATHS into a set of NAR files stored
destination. in DIR. Optionally generate a manifest.
EOF EOF
; # ` ; # `
exit 1; exit 1;
} }
showSyntax if scalar @ARGV < 1; for (my $n = 0; $n < scalar @ARGV; $n++) {
my $arg = $ARGV[$n];
if ($ARGV[0] eq "--copy") { if ($arg eq "--help") {
showSyntax if scalar @ARGV < 3; showSyntax;
$localCopy = 1; } elsif ($arg eq "--bzip2") {
shift @ARGV; $compressionType = "bzip2";
$localArchivesDir = shift @ARGV; } elsif ($arg eq "--force") {
$localManifestFile = shift @ARGV; $force = 1;
if ($ARGV[0] eq "--target") { } elsif ($arg eq "--dest") {
shift @ARGV; $n++;
$targetArchivesUrl = shift @ARGV; die "$0: `$arg' requires an argument\n" unless $n < scalar @ARGV;
} $destDir = $ARGV[$n];
else { mkpath($destDir, 0, 0755);
$targetArchivesUrl = "file://$localArchivesDir"; } elsif ($arg eq "--manifest") {
$writeManifest = 1;
} elsif ($arg eq "--url-prefix") {
$n++;
die "$0: `$arg' requires an argument\n" unless $n < scalar @ARGV;
$archivesURL = $ARGV[$n];
} elsif (substr($arg, 0, 1) eq "-") {
showSyntax;
} else {
push @roots, $arg;
} }
} }
else {
showSyntax if scalar @ARGV < 3; showSyntax if !defined $destDir;
$localCopy = 0;
$archivesPutURL = shift @ARGV; $archivesURL = "file://$destDir" unless defined $archivesURL;
$archivesGetURL = shift @ARGV;
$manifestPutURL = shift @ARGV;
}
# From the given store paths, determine the set of requisite store # From the given store paths, determine the set of requisite store
# paths, i.e, the paths required to realise them. # paths, i.e, the paths required to realise them.
my %storePaths; my %storePaths;
foreach my $path (@ARGV) { foreach my $path (@roots) {
die unless $path =~ /^\//; die unless $path =~ /^\//;
# Get all paths referenced by the normalisation of the given # Get all paths referenced by the normalisation of the given
@ -92,8 +92,8 @@ foreach my $path (@ARGV) {
my @storePaths = keys %storePaths; my @storePaths = keys %storePaths;
# For each path, create a Nix expression that turns the path into # Create a list of Nix derivations that turn each path into a Nix
# a Nix archive. # archive.
open NIX, ">$nixExpr"; open NIX, ">$nixExpr";
print NIX "["; print NIX "[";
@ -103,7 +103,7 @@ foreach my $storePath (@storePaths) {
# Construct a Nix expression that creates a Nix archive. # Construct a Nix expression that creates a Nix archive.
my $nixexpr = my $nixexpr =
"(import <nix/nar.nix> " . "(import <nix/nar.nix> " .
"{ storePath = builtins.storePath \"$storePath\"; hashAlgo = \"$hashAlgo\"; }) "; "{ storePath = builtins.storePath \"$storePath\"; hashAlgo = \"sha256\"; compressionType = \"$compressionType\"; }) ";
print NIX $nixexpr; print NIX $nixexpr;
} }
@ -112,172 +112,132 @@ print NIX "]";
close NIX; close NIX;
# Instantiate store derivations from the Nix expression. # Build the Nix expression.
my @storeExprs; print STDERR "building compressed archives...\n";
print STDERR "instantiating store derivations...\n"; my @narPaths;
my $pid = open(READ, "$Nix::Config::binDir/nix-instantiate $nixExpr|") my $pid = open(READ, "$Nix::Config::binDir/nix-build $nixExpr -o $tmpDir/result |")
or die "cannot run nix-instantiate"; or die "cannot run nix-build";
while (<READ>) { while (<READ>) {
chomp; chomp;
die unless /^\//; die unless /^\//;
push @storeExprs, $_; push @narPaths, $_;
} }
close READ or die "nix-instantiate failed: $?"; close READ or die "nix-build failed: $?";
# Build the derivations. # Write the cache info file.
print STDERR "creating archives...\n"; my $cacheInfoFile = "$destDir/nix-cache-info";
if (! -e $cacheInfoFile) {
my @narPaths; open FILE, ">$cacheInfoFile" or die "cannot create $cacheInfoFile: $!";
print FILE "StoreDir: $Nix::Config::storeDir\n";
my @tmp = @storeExprs; print FILE "WantMassQuery: 0\n"; # by default, don't hit this cache for "nix-env -qas"
while (scalar @tmp > 0) { close FILE;
my $n = scalar @tmp;
if ($n > 256) { $n = 256 };
my @tmp2 = @tmp[0..$n - 1];
@tmp = @tmp[$n..scalar @tmp - 1];
my $pid = open(READ, "$Nix::Config::binDir/nix-store --realise @tmp2|")
or die "cannot run nix-store";
while (<READ>) {
chomp;
die unless (/^\//);
push @narPaths, "$_";
}
close READ or die "nix-store failed: $?";
} }
# Create the manifest. # Copy the archives and the corresponding NAR info files.
print STDERR "creating manifest...\n"; print STDERR "copying archives...\n";
my $totalNarSize = 0;
my $totalCompressedSize = 0;
my %narFiles; my %narFiles;
my %patches;
my @narArchives;
for (my $n = 0; $n < scalar @storePaths; $n++) { for (my $n = 0; $n < scalar @storePaths; $n++) {
my $storePath = $storePaths[$n]; my $storePath = $storePaths[$n];
my $narDir = $narPaths[$n]; my $narDir = $narPaths[$n];
my $baseName = basename $storePath;
$storePath =~ /\/([^\/]*)$/; # Get info about the store path.
my $basename = $1; my ($deriver, $narHash, $time, $narSize, $refs) = queryPathInfo($storePath, 1);
defined $basename or die;
open HASH, "$narDir/narbz2-hash" or die "cannot open narbz2-hash";
my $narbz2Hash = <HASH>;
chomp $narbz2Hash;
$narbz2Hash =~ /^[0-9a-z]+$/ or die "invalid hash";
close HASH;
my $narName = "$narbz2Hash.nar.bz2";
my $narFile = "$narDir/$narName";
(-f $narFile) or die "narfile for $storePath not found";
push @narArchives, $narFile;
my $narbz2Size = stat($narFile)->size;
my $references = `$Nix::Config::binDir/nix-store --query --references '$storePath'`;
die "cannot query references for `$storePath'" if $? != 0;
$references = join(" ", split(" ", $references));
my $deriver = `$Nix::Config::binDir/nix-store --query --deriver '$storePath'`;
die "cannot query deriver for `$storePath'" if $? != 0;
chomp $deriver;
$deriver = "" if $deriver eq "unknown-deriver";
my $narHash = `$Nix::Config::binDir/nix-store --query --hash '$storePath'`;
die "cannot query hash for `$storePath'" if $? != 0;
chomp $narHash;
# In some exceptional cases (such as VM tests that use the Nix # In some exceptional cases (such as VM tests that use the Nix
# store of the host), the database doesn't contain the hash. So # store of the host), the database doesn't contain the hash. So
# compute it. # compute it.
if ($narHash =~ /^sha256:0*$/) { if ($narHash =~ /^sha256:0*$/) {
$narHash = `$Nix::Config::binDir/nix-hash --type sha256 --base32 '$storePath'`; my $nar = "$tmpDir/nar";
die "cannot hash `$storePath'" if $? != 0; system("$Nix::Config::binDir/nix-store --dump $storePath > $nar") == 0
or die "cannot dump $storePath\n";
$narHash = `$Nix::Config::binDir/nix-hash --type sha256 --base32 --flat $nar`;
die "cannot hash `$nar'" if $? != 0;
chomp $narHash; chomp $narHash;
$narHash = "sha256:$narHash"; $narHash = "sha256:$narHash";
$narSize = stat("$nar")->size;
unlink $nar or die;
} }
my $narSize = `$Nix::Config::binDir/nix-store --query --size '$storePath'`; $totalNarSize += $narSize;
die "cannot query size for `$storePath'" if $? != 0;
chomp $narSize;
my $url; # Get info about the compressed NAR.
if ($localCopy) { open HASH, "$narDir/nar-compressed-hash" or die "cannot open nar-compressed-hash";
$url = "$targetArchivesUrl/$narName"; my $compressedHash = <HASH>;
} else { chomp $compressedHash;
$url = "$archivesGetURL/$narName"; $compressedHash =~ /^[0-9a-z]+$/ or die "invalid hash";
close HASH;
my $narName = "$compressedHash.nar." . ($compressionType eq "xz" ? "xz" : "bz2");
my $narFile = "$narDir/$narName";
(-f $narFile) or die "NAR file for $storePath not found";
my $compressedSize = stat($narFile)->size;
$totalCompressedSize += $compressedSize;
printf STDERR "%s [%.2f MiB, %.1f%%]\n", $storePath,
$compressedSize / (1024 * 1024), $compressedSize / $narSize * 100;
# Copy the compressed NAR.
my $dst = "$destDir/$narName";
if (! -f $dst) {
my $tmp = "$destDir/.tmp.$$.$narName";
copy($narFile, $tmp) or die "cannot copy $narFile to $tmp: $!\n";
rename($tmp, $dst) or die "cannot rename $tmp to $dst: $!\n";
} }
# Write the info file.
my $info;
$info .= "StorePath: $storePath\n";
$info .= "URL: $narName\n";
$info .= "Compression: $compressionType\n";
$info .= "FileHash: sha256:$compressedHash\n";
$info .= "FileSize: $compressedSize\n";
$info .= "NarHash: $narHash\n";
$info .= "NarSize: $narSize\n";
$info .= "References: " . join(" ", map { basename $_ } @{$refs}) . "\n";
if (defined $deriver) {
$info .= "Deriver: " . basename $deriver . "\n";
if (isValidPath($deriver)) {
my $drv = derivationFromPath($deriver);
$info .= "System: $drv->{platform}\n";
}
}
my $pathHash = substr(basename($storePath), 0, 32);
$dst = "$destDir/$pathHash.narinfo";
if ($force || ! -f $dst) {
my $tmp = "$destDir/.tmp.$$.$pathHash.narinfo";
open INFO, ">$tmp" or die;
print INFO "$info" or die;
close INFO or die;
rename($tmp, $dst) or die "cannot rename $tmp to $dst: $!\n";
}
$narFiles{$storePath} = [ $narFiles{$storePath} = [
{ url => $url { url => "$archivesURL/$narName"
, hash => "$hashAlgo:$narbz2Hash" , hash => "sha256:$compressedHash"
, size => $narbz2Size , size => $compressedSize
, narHash => "$narHash" , narHash => "$narHash"
, narSize => $narSize , narSize => $narSize
, references => $references , references => join(" ", @{$refs})
, deriver => $deriver , deriver => $deriver
} }
]; ] if $writeManifest;
} }
writeManifest $manifest, \%narFiles, \%patches; printf STDERR "total compressed size %.2f MiB, %.1f%%\n",
$totalCompressedSize / (1024 * 1024), $totalCompressedSize / $totalNarSize * 100;
sub copyFile { # Optionally write a manifest.
my $src = shift; writeManifest "$destDir/MANIFEST", \%narFiles, \() if $writeManifest;
my $dst = shift;
my $tmp = "$dst.tmp.$$";
system("@coreutils@/cp", $src, $tmp) == 0 or die "cannot copy file";
rename($tmp, $dst) or die "cannot rename file: $!";
}
# Upload/copy the archives.
print STDERR "uploading/copying archives...\n";
sub archiveExists {
my $name = shift;
print STDERR " HEAD on $archivesGetURL/$name\n";
return system("$curl --head $archivesGetURL/$name > /dev/null") == 0;
}
foreach my $narArchive (@narArchives) {
$narArchive =~ /\/([^\/]*)$/;
my $basename = $1;
if ($localCopy) {
# Since nix-push creates $dst atomically, if it exists we
# don't have to copy again.
my $dst = "$localArchivesDir/$basename";
if (! -f "$localArchivesDir/$basename") {
print STDERR " $narArchive\n";
copyFile $narArchive, $dst;
}
}
else {
if (!archiveExists("$basename")) {
print STDERR " $narArchive\n";
system("$curl --show-error --upload-file " .
"'$narArchive' '$archivesPutURL/$basename' > /dev/null") == 0 or
die "curl failed on $narArchive: $?";
}
}
}
# Upload the manifest.
print STDERR "uploading manifest...\n";
if ($localCopy) {
copyFile $manifest, $localManifestFile;
copyFile "$manifest.bz2", "$localManifestFile.bz2";
} else {
system("$curl --show-error --upload-file " .
"'$manifest' '$manifestPutURL' > /dev/null") == 0 or
die "curl failed on $manifest: $?";
system("$curl --show-error --upload-file " .
"'$manifest'.bz2 '$manifestPutURL'.bz2 > /dev/null") == 0 or
die "curl failed on $manifest: $?";
}

View file

@ -181,7 +181,7 @@ EvalState::EvalState()
searchPathInsertionPoint = searchPath.end(); searchPathInsertionPoint = searchPath.end();
Strings paths = tokenizeString(getEnv("NIX_PATH", ""), ":"); Strings paths = tokenizeString(getEnv("NIX_PATH", ""), ":");
foreach (Strings::iterator, i, paths) addToSearchPath(*i); foreach (Strings::iterator, i, paths) addToSearchPath(*i);
addToSearchPath("nix=" + nixDataDir + "/nix/corepkgs"); addToSearchPath("nix=" + settings.nixDataDir + "/nix/corepkgs");
searchPathInsertionPoint = searchPath.begin(); searchPathInsertionPoint = searchPath.begin();
createBaseEnv(); createBaseEnv();
@ -1091,7 +1091,7 @@ string EvalState::coerceToString(Value & v, PathSet & context,
if (srcToStore[path] != "") if (srcToStore[path] != "")
dstPath = srcToStore[path]; dstPath = srcToStore[path];
else { else {
dstPath = readOnlyMode dstPath = settings.readOnlyMode
? computeStorePathForPath(path).first ? computeStorePathForPath(path).first
: store->addToStore(path); : store->addToStore(path);
srcToStore[path] = dstPath; srcToStore[path] = dstPath;

View file

@ -32,6 +32,8 @@ private:
bool metaInfoRead; bool metaInfoRead;
MetaInfo meta; MetaInfo meta;
bool failed; // set if we get an AssertionError
public: public:
string name; string name;
string attrPath; /* path towards the derivation */ string attrPath; /* path towards the derivation */
@ -40,7 +42,7 @@ public:
/* !!! make this private */ /* !!! make this private */
Bindings * attrs; Bindings * attrs;
DrvInfo() : metaInfoRead(false), attrs(0) { }; DrvInfo() : metaInfoRead(false), failed(false), attrs(0) { };
string queryDrvPath(EvalState & state) const; string queryDrvPath(EvalState & state) const;
string queryOutPath(EvalState & state) const; string queryOutPath(EvalState & state) const;
@ -58,6 +60,9 @@ public:
} }
void setMetaInfo(const MetaInfo & meta); void setMetaInfo(const MetaInfo & meta);
void setFailed() { failed = true; };
bool hasFailed() { return failed; };
}; };

View file

@ -51,6 +51,12 @@ static void prim_import(EvalState & state, Value * * args, Value & v)
% path % ctx); % path % ctx);
if (isDerivation(ctx)) if (isDerivation(ctx))
try { try {
/* For performance, prefetch all substitute info. */
PathSet willBuild, willSubstitute, unknown;
unsigned long long downloadSize, narSize;
queryMissing(*store, singleton<PathSet>(ctx),
willBuild, willSubstitute, unknown, downloadSize, narSize);
/* !!! If using a substitute, we only need to fetch /* !!! If using a substitute, we only need to fetch
the selected output of this derivation. */ the selected output of this derivation. */
store->buildPaths(singleton<PathSet>(ctx)); store->buildPaths(singleton<PathSet>(ctx));
@ -617,7 +623,7 @@ static void prim_toFile(EvalState & state, Value * * args, Value & v)
refs.insert(path); refs.insert(path);
} }
Path storePath = readOnlyMode Path storePath = settings.readOnlyMode
? computeStorePathForText(name, contents, refs) ? computeStorePathForText(name, contents, refs)
: store->addTextToStore(name, contents, refs); : store->addTextToStore(name, contents, refs);
@ -681,7 +687,7 @@ static void prim_filterSource(EvalState & state, Value * * args, Value & v)
FilterFromExpr filter(state, *args[0]); FilterFromExpr filter(state, *args[0]);
Path dstPath = readOnlyMode Path dstPath = settings.readOnlyMode
? computeStorePathForPath(path, true, htSHA256, filter).first ? computeStorePathForPath(path, true, htSHA256, filter).first
: store->addToStore(path, true, htSHA256, filter); : store->addToStore(path, true, htSHA256, filter);
@ -1134,7 +1140,7 @@ void EvalState::createBaseEnv()
mkInt(v, time(0)); mkInt(v, time(0));
addConstant("__currentTime", v); addConstant("__currentTime", v);
mkString(v, thisSystem.c_str()); mkString(v, settings.thisSystem.c_str());
addConstant("__currentSystem", v); addConstant("__currentSystem", v);
// Miscellaneous // Miscellaneous

View file

@ -64,7 +64,7 @@ void printMissing(StoreAPI & store, const PathSet & paths)
if (!unknown.empty()) { if (!unknown.empty()) {
printMsg(lvlInfo, format("don't know how to build these paths%1%:") printMsg(lvlInfo, format("don't know how to build these paths%1%:")
% (readOnlyMode ? " (may be caused by read-only store access)" : "")); % (settings.readOnlyMode ? " (may be caused by read-only store access)" : ""));
foreach (PathSet::iterator, i, unknown) foreach (PathSet::iterator, i, unknown)
printMsg(lvlInfo, format(" %1%") % *i); printMsg(lvlInfo, format(" %1%") % *i);
} }
@ -83,11 +83,20 @@ static void setLogType(string lt)
static bool showTrace = false; static bool showTrace = false;
string getArg(const string & opt,
Strings::iterator & i, const Strings::iterator & end)
{
++i;
if (i == end) throw UsageError(format("`%1%' requires an argument") % opt);
return *i;
}
/* Initialize and reorder arguments, then call the actual argument /* Initialize and reorder arguments, then call the actual argument
processor. */ processor. */
static void initAndRun(int argc, char * * argv) static void initAndRun(int argc, char * * argv)
{ {
setDefaultsFromEnvironment(); settings.processEnvironment();
settings.loadConfFile();
/* Catch SIGINT. */ /* Catch SIGINT. */
struct sigaction act; struct sigaction act;
@ -146,20 +155,19 @@ static void initAndRun(int argc, char * * argv)
remaining.clear(); remaining.clear();
/* Process default options. */ /* Process default options. */
int verbosityDelta = 0; int verbosityDelta = lvlInfo;
for (Strings::iterator i = args.begin(); i != args.end(); ++i) { for (Strings::iterator i = args.begin(); i != args.end(); ++i) {
string arg = *i; string arg = *i;
if (arg == "--verbose" || arg == "-v") verbosityDelta++; if (arg == "--verbose" || arg == "-v") verbosityDelta++;
else if (arg == "--quiet") verbosityDelta--; else if (arg == "--quiet") verbosityDelta--;
else if (arg == "--log-type") { else if (arg == "--log-type") {
++i; string s = getArg(arg, i, args.end());
if (i == args.end()) throw UsageError("`--log-type' requires an argument"); setLogType(s);
setLogType(*i);
} }
else if (arg == "--no-build-output" || arg == "-Q") else if (arg == "--no-build-output" || arg == "-Q")
buildVerbosity = lvlVomit; settings.buildVerbosity = lvlVomit;
else if (arg == "--print-build-trace") else if (arg == "--print-build-trace")
printBuildTrace = true; settings.printBuildTrace = true;
else if (arg == "--help") { else if (arg == "--help") {
printHelp(); printHelp();
return; return;
@ -169,23 +177,23 @@ static void initAndRun(int argc, char * * argv)
return; return;
} }
else if (arg == "--keep-failed" || arg == "-K") else if (arg == "--keep-failed" || arg == "-K")
keepFailed = true; settings.keepFailed = true;
else if (arg == "--keep-going" || arg == "-k") else if (arg == "--keep-going" || arg == "-k")
keepGoing = true; settings.keepGoing = true;
else if (arg == "--fallback") else if (arg == "--fallback")
tryFallback = true; settings.set("build-fallback", "true");
else if (arg == "--max-jobs" || arg == "-j") else if (arg == "--max-jobs" || arg == "-j")
maxBuildJobs = getIntArg<unsigned int>(arg, i, args.end()); settings.set("build-max-jobs", getArg(arg, i, args.end()));
else if (arg == "--cores") else if (arg == "--cores")
buildCores = getIntArg<unsigned int>(arg, i, args.end()); settings.set("build-cores", getArg(arg, i, args.end()));
else if (arg == "--readonly-mode") else if (arg == "--readonly-mode")
readOnlyMode = true; settings.readOnlyMode = true;
else if (arg == "--max-silent-time") else if (arg == "--max-silent-time")
maxSilentTime = getIntArg<unsigned int>(arg, i, args.end()); settings.set("build-max-silent-time", getArg(arg, i, args.end()));
else if (arg == "--timeout") else if (arg == "--timeout")
buildTimeout = getIntArg<unsigned int>(arg, i, args.end()); settings.set("build-timeout", getArg(arg, i, args.end()));
else if (arg == "--no-build-hook") else if (arg == "--no-build-hook")
useBuildHook = false; settings.useBuildHook = false;
else if (arg == "--show-trace") else if (arg == "--show-trace")
showTrace = true; showTrace = true;
else if (arg == "--option") { else if (arg == "--option") {
@ -193,14 +201,15 @@ static void initAndRun(int argc, char * * argv)
string name = *i; string name = *i;
++i; if (i == args.end()) throw UsageError("`--option' requires two arguments"); ++i; if (i == args.end()) throw UsageError("`--option' requires two arguments");
string value = *i; string value = *i;
overrideSetting(name, tokenizeString(value)); settings.set(name, value);
} }
else remaining.push_back(arg); else remaining.push_back(arg);
} }
verbosityDelta += queryIntSetting("verbosity", lvlInfo);
verbosity = (Verbosity) (verbosityDelta < 0 ? 0 : verbosityDelta); verbosity = (Verbosity) (verbosityDelta < 0 ? 0 : verbosityDelta);
settings.update();
run(remaining); run(remaining);
/* Close the Nix database. */ /* Close the Nix database. */

View file

@ -94,7 +94,7 @@ typedef map<Path, WeakGoalPtr> WeakGoalMap;
class Goal : public boost::enable_shared_from_this<Goal> class Goal : public boost::enable_shared_from_this<Goal>
{ {
public: public:
typedef enum {ecBusy, ecSuccess, ecFailed} ExitCode; typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters} ExitCode;
protected: protected:
@ -111,6 +111,10 @@ protected:
/* Number of goals we are/were waiting for that have failed. */ /* Number of goals we are/were waiting for that have failed. */
unsigned int nrFailed; unsigned int nrFailed;
/* Number of substitution goals we are/were waiting for that
failed because there are no substituters. */
unsigned int nrNoSubstituters;
/* Name of this goal for debugging purposes. */ /* Name of this goal for debugging purposes. */
string name; string name;
@ -119,7 +123,7 @@ protected:
Goal(Worker & worker) : worker(worker) Goal(Worker & worker) : worker(worker)
{ {
nrFailed = 0; nrFailed = nrNoSubstituters = 0;
exitCode = ecBusy; exitCode = ecBusy;
} }
@ -225,8 +229,6 @@ private:
public: public:
bool cacheFailure;
/* Set if at least one derivation had a BuildError (i.e. permanent /* Set if at least one derivation had a BuildError (i.e. permanent
failure). */ failure). */
bool permanentFailure; bool permanentFailure;
@ -306,9 +308,11 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
trace(format("waitee `%1%' done; %2% left") % trace(format("waitee `%1%' done; %2% left") %
waitee->name % waitees.size()); waitee->name % waitees.size());
if (result == ecFailed) ++nrFailed; if (result == ecFailed || result == ecNoSubstituters) ++nrFailed;
if (waitees.empty() || (result == ecFailed && !keepGoing)) { if (result == ecNoSubstituters) ++nrNoSubstituters;
if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) {
/* If we failed and keepGoing is not set, we remove all /* If we failed and keepGoing is not set, we remove all
remaining waitees. */ remaining waitees. */
@ -330,7 +334,7 @@ void Goal::amDone(ExitCode result)
{ {
trace("done"); trace("done");
assert(exitCode == ecBusy); assert(exitCode == ecBusy);
assert(result == ecSuccess || result == ecFailed); assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters);
exitCode = result; exitCode = result;
foreach (WeakGoals::iterator, i, waiters) { foreach (WeakGoals::iterator, i, waiters) {
GoalPtr goal = i->lock(); GoalPtr goal = i->lock();
@ -365,6 +369,7 @@ void commonChildInit(Pipe & logPipe)
if (dup2(logPipe.writeSide, STDERR_FILENO) == -1) if (dup2(logPipe.writeSide, STDERR_FILENO) == -1)
throw SysError("cannot pipe standard error into log file"); throw SysError("cannot pipe standard error into log file");
logPipe.readSide.close(); logPipe.readSide.close();
logPipe.writeSide.close();
/* Dup stderr to stdout. */ /* Dup stderr to stdout. */
if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1)
@ -459,14 +464,13 @@ void UserLock::acquire()
{ {
assert(uid == 0); assert(uid == 0);
string buildUsersGroup = querySetting("build-users-group", ""); assert(settings.buildUsersGroup != "");
assert(buildUsersGroup != "");
/* Get the members of the build-users-group. */ /* Get the members of the build-users-group. */
struct group * gr = getgrnam(buildUsersGroup.c_str()); struct group * gr = getgrnam(settings.buildUsersGroup.c_str());
if (!gr) if (!gr)
throw Error(format("the group `%1%' specified in `build-users-group' does not exist") throw Error(format("the group `%1%' specified in `build-users-group' does not exist")
% buildUsersGroup); % settings.buildUsersGroup);
gid = gr->gr_gid; gid = gr->gr_gid;
/* Copy the result of getgrnam. */ /* Copy the result of getgrnam. */
@ -478,7 +482,7 @@ void UserLock::acquire()
if (users.empty()) if (users.empty())
throw Error(format("the build users group `%1%' has no members") throw Error(format("the build users group `%1%' has no members")
% buildUsersGroup); % settings.buildUsersGroup);
/* Find a user account that isn't currently in use for another /* Find a user account that isn't currently in use for another
build. */ build. */
@ -488,11 +492,11 @@ void UserLock::acquire()
struct passwd * pw = getpwnam(i->c_str()); struct passwd * pw = getpwnam(i->c_str());
if (!pw) if (!pw)
throw Error(format("the user `%1%' in the group `%2%' does not exist") throw Error(format("the user `%1%' in the group `%2%' does not exist")
% *i % buildUsersGroup); % *i % settings.buildUsersGroup);
createDirs(nixStateDir + "/userpool"); createDirs(settings.nixStateDir + "/userpool");
fnUserLock = (format("%1%/userpool/%2%") % nixStateDir % pw->pw_uid).str(); fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str();
if (lockedPaths.find(fnUserLock) != lockedPaths.end()) if (lockedPaths.find(fnUserLock) != lockedPaths.end())
/* We already have a lock on this one. */ /* We already have a lock on this one. */
@ -512,7 +516,7 @@ void UserLock::acquire()
/* Sanity check... */ /* Sanity check... */
if (uid == getuid() || uid == geteuid()) if (uid == getuid() || uid == geteuid())
throw Error(format("the Nix user should not be a member of `%1%'") throw Error(format("the Nix user should not be a member of `%1%'")
% buildUsersGroup); % settings.buildUsersGroup);
return; return;
} }
@ -520,7 +524,7 @@ void UserLock::acquire()
throw Error(format("all build users are currently in use; " throw Error(format("all build users are currently in use; "
"consider creating additional users and adding them to the `%1%' group") "consider creating additional users and adding them to the `%1%' group")
% buildUsersGroup); % settings.buildUsersGroup);
} }
@ -539,7 +543,7 @@ static void runSetuidHelper(const string & command,
const string & arg) const string & arg)
{ {
Path program = getEnv("NIX_SETUID_HELPER", Path program = getEnv("NIX_SETUID_HELPER",
nixLibexecDir + "/nix-setuid-helper"); settings.nixLibexecDir + "/nix-setuid-helper");
/* Fork. */ /* Fork. */
Pid pid; Pid pid;
@ -594,12 +598,6 @@ bool amPrivileged()
} }
bool haveBuildUsers()
{
return querySetting("build-users-group", "") != "";
}
void getOwnership(const Path & path) void getOwnership(const Path & path)
{ {
runSetuidHelper("get-ownership", path); runSetuidHelper("get-ownership", path);
@ -614,7 +612,7 @@ void deletePathWrapped(const Path & path, unsigned long long & bytesFreed)
} catch (SysError & e) { } catch (SysError & e) {
/* If this failed due to a permission error, then try it with /* If this failed due to a permission error, then try it with
the setuid helper. */ the setuid helper. */
if (haveBuildUsers() && !amPrivileged()) { if (settings.buildUsersGroup != "" && !amPrivileged()) {
getOwnership(path); getOwnership(path);
deletePath(path, bytesFreed); deletePath(path, bytesFreed);
} else } else
@ -692,10 +690,10 @@ HookInstance::HookInstance()
if (dup2(builderOut.writeSide, 4) == -1) if (dup2(builderOut.writeSide, 4) == -1)
throw SysError("dupping builder's stdout/stderr"); throw SysError("dupping builder's stdout/stderr");
/* XXX: Pass `buildTimeout' to the hook? */ /* XXX: Pass `buildTimeout' to the hook? */
execl(buildHook.c_str(), buildHook.c_str(), thisSystem.c_str(), execl(buildHook.c_str(), buildHook.c_str(), settings.thisSystem.c_str(),
(format("%1%") % maxSilentTime).str().c_str(), (format("%1%") % settings.maxSilentTime).str().c_str(),
(format("%1%") % printBuildTrace).str().c_str(), (format("%1%") % settings.printBuildTrace).str().c_str(),
NULL); NULL);
throw SysError(format("executing `%1%'") % buildHook); throw SysError(format("executing `%1%'") % buildHook);
@ -735,6 +733,8 @@ HookInstance::~HookInstance()
typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
class SubstitutionGoal;
class DerivationGoal : public Goal class DerivationGoal : public Goal
{ {
private: private:
@ -933,7 +933,7 @@ void DerivationGoal::init()
{ {
trace("init"); trace("init");
if (readOnlyMode) if (settings.readOnlyMode)
throw Error(format("cannot build derivation `%1%' - no write access to the Nix store") % drvPath); throw Error(format("cannot build derivation `%1%' - no write access to the Nix store") % drvPath);
/* The first thing to do is to make sure that the derivation /* The first thing to do is to make sure that the derivation
@ -985,10 +985,8 @@ void DerivationGoal::haveDerivation()
/* We are first going to try to create the invalid output paths /* We are first going to try to create the invalid output paths
through substitutes. If that doesn't work, we'll build through substitutes. If that doesn't work, we'll build
them. */ them. */
foreach (PathSet::iterator, i, invalidOutputs) if (settings.useSubstitutes)
/* Don't bother creating a substitution goal if there are no foreach (PathSet::iterator, i, invalidOutputs)
substitutes. */
if (queryBoolSetting("build-use-substitutes", true) && worker.store.hasSubstitutes(*i))
addWaitee(worker.makeSubstitutionGoal(*i)); addWaitee(worker.makeSubstitutionGoal(*i));
if (waitees.empty()) /* to prevent hang (no wake-up event) */ if (waitees.empty()) /* to prevent hang (no wake-up event) */
@ -1002,10 +1000,10 @@ void DerivationGoal::outputsSubstituted()
{ {
trace("all outputs substituted (maybe)"); trace("all outputs substituted (maybe)");
if (nrFailed > 0 && !tryFallback) if (nrFailed > 0 && nrFailed > nrNoSubstituters && !settings.tryFallback)
throw Error(format("some substitutes for the outputs of derivation `%1%' failed; try `--fallback'") % drvPath); throw Error(format("some substitutes for the outputs of derivation `%1%' failed; try `--fallback'") % drvPath);
nrFailed = 0; nrFailed = nrNoSubstituters = 0;
if (checkPathValidity(false).size() == 0) { if (checkPathValidity(false).size() == 0) {
amDone(ecSuccess); amDone(ecSuccess);
@ -1101,9 +1099,9 @@ PathSet outputPaths(const DerivationOutputs & outputs)
static bool canBuildLocally(const string & platform) static bool canBuildLocally(const string & platform)
{ {
return platform == thisSystem return platform == settings.thisSystem
#ifdef CAN_DO_LINUX32_BUILDS #ifdef CAN_DO_LINUX32_BUILDS
|| (platform == "i686-linux" && thisSystem == "x86_64-linux") || (platform == "i686-linux" && settings.thisSystem == "x86_64-linux")
#endif #endif
; ;
} }
@ -1205,7 +1203,7 @@ void DerivationGoal::tryToBuild()
derivation prefers to be done locally, do it even if derivation prefers to be done locally, do it even if
maxBuildJobs is 0. */ maxBuildJobs is 0. */
unsigned int curBuilds = worker.getNrLocalBuilds(); unsigned int curBuilds = worker.getNrLocalBuilds();
if (curBuilds >= maxBuildJobs && !(preferLocalBuild && curBuilds == 0)) { if (curBuilds >= settings.maxBuildJobs && !(preferLocalBuild && curBuilds == 0)) {
worker.waitForBuildSlot(shared_from_this()); worker.waitForBuildSlot(shared_from_this());
outputLocks.unlock(); outputLocks.unlock();
return; return;
@ -1220,7 +1218,7 @@ void DerivationGoal::tryToBuild()
printMsg(lvlError, e.msg()); printMsg(lvlError, e.msg());
outputLocks.unlock(); outputLocks.unlock();
buildUser.release(); buildUser.release();
if (printBuildTrace) if (settings.printBuildTrace)
printMsg(lvlError, format("@ build-failed %1% %2% %3% %4%") printMsg(lvlError, format("@ build-failed %1% %2% %3% %4%")
% drvPath % drv.outputs["out"].path % 0 % e.msg()); % drvPath % drv.outputs["out"].path % 0 % e.msg());
worker.permanentFailure = true; worker.permanentFailure = true;
@ -1352,7 +1350,7 @@ void DerivationGoal::buildDone()
bool hookError = hook && bool hookError = hook &&
(!WIFEXITED(status) || WEXITSTATUS(status) != 100); (!WIFEXITED(status) || WEXITSTATUS(status) != 100);
if (printBuildTrace) { if (settings.printBuildTrace) {
if (hook && hookError) if (hook && hookError)
printMsg(lvlError, format("@ hook-failed %1% %2% %3% %4%") printMsg(lvlError, format("@ hook-failed %1% %2% %3% %4%")
% drvPath % drv.outputs["out"].path % status % e.msg()); % drvPath % drv.outputs["out"].path % status % e.msg());
@ -1368,7 +1366,7 @@ void DerivationGoal::buildDone()
able to access the network). Hook errors (like able to access the network). Hook errors (like
communication problems with the remote machine) shouldn't communication problems with the remote machine) shouldn't
be cached either. */ be cached either. */
if (worker.cacheFailure && !hookError && !fixedOutput) if (settings.cacheFailure && !hookError && !fixedOutput)
foreach (DerivationOutputs::iterator, i, drv.outputs) foreach (DerivationOutputs::iterator, i, drv.outputs)
worker.store.registerFailedPath(i->second.path); worker.store.registerFailedPath(i->second.path);
@ -1380,7 +1378,7 @@ void DerivationGoal::buildDone()
/* Release the build user, if applicable. */ /* Release the build user, if applicable. */
buildUser.release(); buildUser.release();
if (printBuildTrace) { if (settings.printBuildTrace) {
printMsg(lvlError, format("@ build-succeeded %1% %2%") printMsg(lvlError, format("@ build-succeeded %1% %2%")
% drvPath % drv.outputs["out"].path); % drvPath % drv.outputs["out"].path);
} }
@ -1391,7 +1389,7 @@ void DerivationGoal::buildDone()
HookReply DerivationGoal::tryBuildHook() HookReply DerivationGoal::tryBuildHook()
{ {
if (!useBuildHook || getEnv("NIX_BUILD_HOOK") == "") return rpDecline; if (!settings.useBuildHook || getEnv("NIX_BUILD_HOOK") == "") return rpDecline;
if (!worker.hook) if (!worker.hook)
worker.hook = boost::shared_ptr<HookInstance>(new HookInstance); worker.hook = boost::shared_ptr<HookInstance>(new HookInstance);
@ -1404,7 +1402,7 @@ HookReply DerivationGoal::tryBuildHook()
/* Send the request to the hook. */ /* Send the request to the hook. */
writeLine(worker.hook->toHook.writeSide, (format("%1% %2% %3% %4%") writeLine(worker.hook->toHook.writeSide, (format("%1% %2% %3% %4%")
% (worker.getNrLocalBuilds() < maxBuildJobs ? "1" : "0") % (worker.getNrLocalBuilds() < settings.maxBuildJobs ? "1" : "0")
% drv.platform % drvPath % concatStringsSep(",", features)).str()); % drv.platform % drvPath % concatStringsSep(",", features)).str());
/* Read the first line of input, which should be a word indicating /* Read the first line of input, which should be a word indicating
@ -1463,7 +1461,7 @@ HookReply DerivationGoal::tryBuildHook()
fds.insert(hook->builderOut.readSide); fds.insert(hook->builderOut.readSide);
worker.childStarted(shared_from_this(), hook->pid, fds, false, false); worker.childStarted(shared_from_this(), hook->pid, fds, false, false);
if (printBuildTrace) if (settings.printBuildTrace)
printMsg(lvlError, format("@ build-started %1% %2% %3% %4%") printMsg(lvlError, format("@ build-started %1% %2% %3% %4%")
% drvPath % drv.outputs["out"].path % drv.platform % logFile); % drvPath % drv.outputs["out"].path % drv.platform % logFile);
@ -1494,7 +1492,7 @@ void DerivationGoal::startBuilder()
if (!canBuildLocally(drv.platform)) if (!canBuildLocally(drv.platform))
throw Error( throw Error(
format("a `%1%' is required to build `%3%', but I am a `%2%'") format("a `%1%' is required to build `%3%', but I am a `%2%'")
% drv.platform % thisSystem % drvPath); % drv.platform % settings.thisSystem % drvPath);
/* Construct the environment passed to the builder. */ /* Construct the environment passed to the builder. */
@ -1515,10 +1513,10 @@ void DerivationGoal::startBuilder()
shouldn't care, but this is useful for purity checking (e.g., shouldn't care, but this is useful for purity checking (e.g.,
the compiler or linker might only want to accept paths to files the compiler or linker might only want to accept paths to files
in the store or in the build directory). */ in the store or in the build directory). */
env["NIX_STORE"] = nixStore; env["NIX_STORE"] = settings.nixStore;
/* The maximum number of cores to utilize for parallel building. */ /* The maximum number of cores to utilize for parallel building. */
env["NIX_BUILD_CORES"] = (format("%d") % buildCores).str(); env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str();
/* Add all bindings specified in the derivation. */ /* Add all bindings specified in the derivation. */
foreach (StringPairs::iterator, i, drv.env) foreach (StringPairs::iterator, i, drv.env)
@ -1610,7 +1608,7 @@ void DerivationGoal::startBuilder()
/* If `build-users-group' is not empty, then we have to build as /* If `build-users-group' is not empty, then we have to build as
one of the members of that group. */ one of the members of that group. */
if (haveBuildUsers()) { if (settings.buildUsersGroup != "") {
buildUser.acquire(); buildUser.acquire();
assert(buildUser.getUID() != 0); assert(buildUser.getUID() != 0);
assert(buildUser.getGID() != 0); assert(buildUser.getGID() != 0);
@ -1632,15 +1630,15 @@ void DerivationGoal::startBuilder()
the builder can create its output but not mess with the the builder can create its output but not mess with the
outputs of other processes). */ outputs of other processes). */
struct stat st; struct stat st;
if (stat(nixStore.c_str(), &st) == -1) if (stat(settings.nixStore.c_str(), &st) == -1)
throw SysError(format("cannot stat `%1%'") % nixStore); throw SysError(format("cannot stat `%1%'") % settings.nixStore);
if (!(st.st_mode & S_ISVTX) || if (!(st.st_mode & S_ISVTX) ||
((st.st_mode & S_IRWXG) != S_IRWXG) || ((st.st_mode & S_IRWXG) != S_IRWXG) ||
(st.st_gid != buildUser.getGID())) (st.st_gid != buildUser.getGID()))
throw Error(format( throw Error(format(
"builder does not have write permission to `%2%'; " "builder does not have write permission to `%2%'; "
"try `chgrp %1% %2%; chmod 1775 %2%'") "try `chgrp %1% %2%; chmod 1775 %2%'")
% buildUser.getGID() % nixStore); % buildUser.getGID() % settings.nixStore);
} }
@ -1649,7 +1647,7 @@ void DerivationGoal::startBuilder()
functions like fetchurl (which needs a proper /etc/resolv.conf) functions like fetchurl (which needs a proper /etc/resolv.conf)
work properly. Purity checking for fixed-output derivations work properly. Purity checking for fixed-output derivations
is somewhat pointless anyway. */ is somewhat pointless anyway. */
useChroot = queryBoolSetting("build-use-chroot", false); useChroot = settings.useChroot;
if (fixedOutput) useChroot = false; if (fixedOutput) useChroot = false;
@ -1689,8 +1687,8 @@ void DerivationGoal::startBuilder()
% (buildUser.enabled() ? buildUser.getUID() : getuid()) % (buildUser.enabled() ? buildUser.getUID() : getuid())
% (buildUser.enabled() ? buildUser.getGID() : getgid())).str()); % (buildUser.enabled() ? buildUser.getGID() : getgid())).str());
/* Declare the build user's group so that programs get a consistent /* Declare the build user's group so that programs get a consistent
view of the system (e.g., "id -gn"). */ view of the system (e.g., "id -gn"). */
writeFile(chrootRootDir + "/etc/group", writeFile(chrootRootDir + "/etc/group",
(format("nixbld:!:%1%:\n") (format("nixbld:!:%1%:\n")
% (buildUser.enabled() ? buildUser.getGID() : getgid())).str()); % (buildUser.enabled() ? buildUser.getGID() : getgid())).str());
@ -1699,16 +1697,8 @@ void DerivationGoal::startBuilder()
writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n"); writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n");
/* Bind-mount a user-configurable set of directories from the /* Bind-mount a user-configurable set of directories from the
host file system. The `/dev/pts' directory must be mounted host file system. */
separately so that newly-created pseudo-terminals show dirsInChroot = settings.dirsInChroot;
up. */
Paths defaultDirs;
defaultDirs.push_back("/dev");
defaultDirs.push_back("/dev/pts");
Paths dirsInChroot_ = querySetting("build-chroot-dirs", defaultDirs);
dirsInChroot.insert(dirsInChroot_.begin(), dirsInChroot_.end());
dirsInChroot.insert(tmpDir); dirsInChroot.insert(tmpDir);
/* Make the closure of the inputs available in the chroot, /* Make the closure of the inputs available in the chroot,
@ -1718,8 +1708,8 @@ void DerivationGoal::startBuilder()
can be bind-mounted). !!! As an extra security can be bind-mounted). !!! As an extra security
precaution, make the fake Nix store only writable by the precaution, make the fake Nix store only writable by the
build user. */ build user. */
createDirs(chrootRootDir + nixStore); createDirs(chrootRootDir + settings.nixStore);
chmod_(chrootRootDir + nixStore, 01777); chmod_(chrootRootDir + settings.nixStore, 01777);
foreach (PathSet::iterator, i, inputPaths) { foreach (PathSet::iterator, i, inputPaths) {
struct stat st; struct stat st;
@ -1819,7 +1809,7 @@ void DerivationGoal::startBuilder()
worker.childStarted(shared_from_this(), pid, worker.childStarted(shared_from_this(), pid,
singleton<set<int> >(builderOut.readSide), true, true); singleton<set<int> >(builderOut.readSide), true, true);
if (printBuildTrace) { if (settings.printBuildTrace) {
printMsg(lvlError, format("@ build-started %1% %2% %3% %4%") printMsg(lvlError, format("@ build-started %1% %2% %3% %4%")
% drvPath % drv.outputs["out"].path % drv.platform % logFile); % drvPath % drv.outputs["out"].path % drv.platform % logFile);
} }
@ -1917,16 +1907,14 @@ void DerivationGoal::initChild()
#ifdef CAN_DO_LINUX32_BUILDS #ifdef CAN_DO_LINUX32_BUILDS
/* Change the personality to 32-bit if we're doing an /* Change the personality to 32-bit if we're doing an
i686-linux build on an x86_64-linux machine. */ i686-linux build on an x86_64-linux machine. */
if (drv.platform == "i686-linux" && thisSystem == "x86_64-linux") { if (drv.platform == "i686-linux" && settings.thisSystem == "x86_64-linux") {
if (personality(0x0008 | 0x8000000 /* == PER_LINUX32_3GB */) == -1) if (personality(0x0008 | 0x8000000 /* == PER_LINUX32_3GB */) == -1)
throw SysError("cannot set i686-linux personality"); throw SysError("cannot set i686-linux personality");
} }
/* Impersonate a Linux 2.6 machine to get some determinism in /* Impersonate a Linux 2.6 machine to get some determinism in
builds that depend on the kernel version. */ builds that depend on the kernel version. */
if ((drv.platform == "i686-linux" || drv.platform == "x86_64-linux") && if ((drv.platform == "i686-linux" || drv.platform == "x86_64-linux") && settings.impersonateLinux26) {
queryBoolSetting("build-impersonate-linux-26", true))
{
int cur = personality(0xffffffff); int cur = personality(0xffffffff);
if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */); if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */);
} }
@ -1968,7 +1956,7 @@ void DerivationGoal::initChild()
} else { } else {
/* Let the setuid helper take care of it. */ /* Let the setuid helper take care of it. */
program = nixLibexecDir + "/nix-setuid-helper"; program = settings.nixLibexecDir + "/nix-setuid-helper";
args.push_back(program.c_str()); args.push_back(program.c_str());
args.push_back("run-builder"); args.push_back("run-builder");
user = buildUser.getUser().c_str(); user = buildUser.getUser().c_str();
@ -2078,12 +2066,12 @@ void DerivationGoal::computeClosure()
} }
/* Get rid of all weird permissions. */ /* Get rid of all weird permissions. */
canonicalisePathMetaData(path); canonicalisePathMetaData(path);
/* For this output path, find the references to other paths /* For this output path, find the references to other paths
contained in it. Compute the SHA-256 NAR hash at the same contained in it. Compute the SHA-256 NAR hash at the same
time. The hash is stored in the database so that we can time. The hash is stored in the database so that we can
verify later on whether nobody has messed with the store. */ verify later on whether nobody has messed with the store. */
HashResult hash; HashResult hash;
PathSet references = scanForReferences(path, allPaths, hash); PathSet references = scanForReferences(path, allPaths, hash);
contentHashes[path] = hash; contentHashes[path] = hash;
@ -2136,13 +2124,13 @@ string drvsLogDir = "drvs";
Path DerivationGoal::openLogFile() Path DerivationGoal::openLogFile()
{ {
if (!queryBoolSetting("build-keep-log", true)) return ""; if (!settings.keepLog) return "";
/* Create a log file. */ /* Create a log file. */
Path dir = (format("%1%/%2%") % nixLogDir % drvsLogDir).str(); Path dir = (format("%1%/%2%") % settings.nixLogDir % drvsLogDir).str();
createDirs(dir); createDirs(dir);
if (queryBoolSetting("build-compress-log", true)) { if (settings.compressLog) {
Path logFileName = (format("%1%/%2%.bz2") % dir % baseNameOf(drvPath)).str(); Path logFileName = (format("%1%/%2%.bz2") % dir % baseNameOf(drvPath)).str();
AutoCloseFD fd = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666); AutoCloseFD fd = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666);
@ -2189,9 +2177,9 @@ void DerivationGoal::closeLogFile()
void DerivationGoal::deleteTmpDir(bool force) void DerivationGoal::deleteTmpDir(bool force)
{ {
if (tmpDir != "") { if (tmpDir != "") {
if (keepFailed && !force) { if (settings.keepFailed && !force) {
printMsg(lvlError, printMsg(lvlError,
format("builder for `%1%' failed; keeping build directory `%2%'") format("builder for `%1%' failed; keeping build directory `%2%'")
% drvPath % tmpDir); % drvPath % tmpDir);
if (buildUser.enabled() && !amPrivileged()) if (buildUser.enabled() && !amPrivileged())
getOwnership(tmpDir); getOwnership(tmpDir);
@ -2209,7 +2197,7 @@ void DerivationGoal::handleChildOutput(int fd, const string & data)
if ((hook && fd == hook->builderOut.readSide) || if ((hook && fd == hook->builderOut.readSide) ||
(!hook && fd == builderOut.readSide)) (!hook && fd == builderOut.readSide))
{ {
if (verbosity >= buildVerbosity) if (verbosity >= settings.buildVerbosity)
writeToStderr((unsigned char *) data.data(), data.size()); writeToStderr((unsigned char *) data.data(), data.size());
if (bzLogFile) { if (bzLogFile) {
int err; int err;
@ -2245,13 +2233,13 @@ PathSet DerivationGoal::checkPathValidity(bool returnValid)
bool DerivationGoal::pathFailed(const Path & path) bool DerivationGoal::pathFailed(const Path & path)
{ {
if (!worker.cacheFailure) return false; if (!settings.cacheFailure) return false;
if (!worker.store.hasPathFailed(path)) return false; if (!worker.store.hasPathFailed(path)) return false;
printMsg(lvlError, format("builder for `%1%' failed previously (cached)") % path); printMsg(lvlError, format("builder for `%1%' failed previously (cached)") % path);
if (printBuildTrace) if (settings.printBuildTrace)
printMsg(lvlError, format("@ build-failed %1% %2% cached") % drvPath % path); printMsg(lvlError, format("@ build-failed %1% %2% cached") % drvPath % path);
worker.permanentFailure = true; worker.permanentFailure = true;
@ -2278,10 +2266,16 @@ private:
/* The current substituter. */ /* The current substituter. */
Path sub; Path sub;
/* Whether any substituter can realise this path */
bool hasSubstitute;
/* Path info returned by the substituter's query info operation. */ /* Path info returned by the substituter's query info operation. */
SubstitutablePathInfo info; SubstitutablePathInfo info;
/* Pipe for the substitute's standard output/error. */ /* Pipe for the substituter's standard output. */
Pipe outPipe;
/* Pipe for the substituter's standard error. */
Pipe logPipe; Pipe logPipe;
/* The process ID of the builder. */ /* The process ID of the builder. */
@ -2319,6 +2313,7 @@ public:
SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker) SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker)
: Goal(worker) : Goal(worker)
, hasSubstitute(false)
{ {
this->storePath = storePath; this->storePath = storePath;
state = &SubstitutionGoal::init; state = &SubstitutionGoal::init;
@ -2365,10 +2360,10 @@ void SubstitutionGoal::init()
return; return;
} }
if (readOnlyMode) if (settings.readOnlyMode)
throw Error(format("cannot substitute path `%1%' - no write access to the Nix store") % storePath); throw Error(format("cannot substitute path `%1%' - no write access to the Nix store") % storePath);
subs = substituters; subs = settings.substituters;
tryNext(); tryNext();
} }
@ -2382,17 +2377,23 @@ void SubstitutionGoal::tryNext()
/* None left. Terminate this goal and let someone else deal /* None left. Terminate this goal and let someone else deal
with it. */ with it. */
debug(format("path `%1%' is required, but there is no substituter that can build it") % storePath); debug(format("path `%1%' is required, but there is no substituter that can build it") % storePath);
amDone(ecFailed); /* Hack: don't indicate failure if there were no substituters.
In that case the calling derivation should just do a
build. */
amDone(hasSubstitute ? ecFailed : ecNoSubstituters);
return; return;
} }
sub = subs.front(); sub = subs.front();
subs.pop_front(); subs.pop_front();
if (!worker.store.querySubstitutablePathInfo(sub, storePath, info)) { SubstitutablePathInfos infos;
tryNext(); PathSet dummy(singleton<PathSet>(storePath));
return; worker.store.querySubstitutablePathInfos(sub, dummy, infos);
} SubstitutablePathInfos::iterator k = infos.find(storePath);
if (k == infos.end()) { tryNext(); return; }
info = k->second;
hasSubstitute = true;
/* To maintain the closure invariant, we first have to realise the /* To maintain the closure invariant, we first have to realise the
paths referenced by this one. */ paths referenced by this one. */
@ -2434,7 +2435,7 @@ void SubstitutionGoal::tryToRun()
is maxBuildJobs == 0 (no local builds allowed), we still allow is maxBuildJobs == 0 (no local builds allowed), we still allow
a substituter to run. This is because substitutions cannot be a substituter to run. This is because substitutions cannot be
distributed to another machine via the build hook. */ distributed to another machine via the build hook. */
if (worker.getNrLocalBuilds() >= (maxBuildJobs == 0 ? 1 : maxBuildJobs)) { if (worker.getNrLocalBuilds() >= (settings.maxBuildJobs == 0 ? 1 : settings.maxBuildJobs)) {
worker.waitForBuildSlot(shared_from_this()); worker.waitForBuildSlot(shared_from_this());
return; return;
} }
@ -2467,6 +2468,7 @@ void SubstitutionGoal::tryToRun()
printMsg(lvlInfo, format("fetching path `%1%'...") % storePath); printMsg(lvlInfo, format("fetching path `%1%'...") % storePath);
outPipe.create();
logPipe.create(); logPipe.create();
/* Remove the (stale) output path if it exists. */ /* Remove the (stale) output path if it exists. */
@ -2483,10 +2485,17 @@ void SubstitutionGoal::tryToRun()
case 0: case 0:
try { /* child */ try { /* child */
logPipe.readSide.close();
commonChildInit(logPipe); commonChildInit(logPipe);
if (dup2(outPipe.writeSide, STDOUT_FILENO) == -1)
throw SysError("cannot dup output pipe into stdout");
outPipe.readSide.close();
outPipe.writeSide.close();
/* Pass configuration options (including those overriden
with --option) to the substituter. */
setenv("_NIX_OPTIONS", settings.pack().c_str(), 1);
/* Fill in the arguments. */ /* Fill in the arguments. */
Strings args; Strings args;
args.push_back(baseNameOf(sub)); args.push_back(baseNameOf(sub));
@ -2507,13 +2516,14 @@ void SubstitutionGoal::tryToRun()
/* parent */ /* parent */
pid.setSeparatePG(true); pid.setSeparatePG(true);
pid.setKillSignal(SIGTERM); pid.setKillSignal(SIGTERM);
outPipe.writeSide.close();
logPipe.writeSide.close(); logPipe.writeSide.close();
worker.childStarted(shared_from_this(), worker.childStarted(shared_from_this(),
pid, singleton<set<int> >(logPipe.readSide), true, true); pid, singleton<set<int> >(logPipe.readSide), true, true);
state = &SubstitutionGoal::finished; state = &SubstitutionGoal::finished;
if (printBuildTrace) { if (settings.printBuildTrace) {
printMsg(lvlError, format("@ substituter-started %1% %2%") printMsg(lvlError, format("@ substituter-started %1% %2%")
% storePath % sub); % storePath % sub);
} }
@ -2535,9 +2545,12 @@ void SubstitutionGoal::finished()
/* Close the read side of the logger pipe. */ /* Close the read side of the logger pipe. */
logPipe.readSide.close(); logPipe.readSide.close();
debug(format("substitute for `%1%' finished") % storePath); /* Get the hash info from stdout. */
string expectedHashStr = statusOk(status) ? readLine(outPipe.readSide) : "";
outPipe.readSide.close();
/* Check the exit status and the build result. */ /* Check the exit status and the build result. */
HashResult hash;
try { try {
if (!statusOk(status)) if (!statusOk(status))
@ -2547,11 +2560,28 @@ void SubstitutionGoal::finished()
if (!pathExists(storePath)) if (!pathExists(storePath))
throw SubstError(format("substitute did not produce path `%1%'") % storePath); throw SubstError(format("substitute did not produce path `%1%'") % storePath);
hash = hashPath(htSHA256, storePath);
/* Verify the expected hash we got from the substituer. */
if (expectedHashStr != "") {
size_t n = expectedHashStr.find(':');
if (n == string::npos)
throw Error(format("bad hash from substituter: %1%") % expectedHashStr);
HashType hashType = parseHashType(string(expectedHashStr, 0, n));
if (hashType == htUnknown)
throw Error(format("unknown hash algorithm in `%1%'") % expectedHashStr);
Hash expectedHash = parseHash16or32(hashType, string(expectedHashStr, n + 1));
Hash actualHash = hashType == htSHA256 ? hash.first : hashPath(hashType, storePath).first;
if (expectedHash != actualHash)
throw SubstError(format("hash mismatch in downloaded path `%1%': expected %2%, got %3%")
% storePath % printHash(expectedHash) % printHash(actualHash));
}
} catch (SubstError & e) { } catch (SubstError & e) {
printMsg(lvlInfo, e.msg()); printMsg(lvlInfo, e.msg());
if (printBuildTrace) { if (settings.printBuildTrace) {
printMsg(lvlError, format("@ substituter-failed %1% %2% %3%") printMsg(lvlError, format("@ substituter-failed %1% %2% %3%")
% storePath % status % e.msg()); % storePath % status % e.msg());
} }
@ -2564,8 +2594,6 @@ void SubstitutionGoal::finished()
canonicalisePathMetaData(storePath); canonicalisePathMetaData(storePath);
HashResult hash = hashPath(htSHA256, storePath);
worker.store.optimisePath(storePath); // FIXME: combine with hashPath() worker.store.optimisePath(storePath); // FIXME: combine with hashPath()
ValidPathInfo info2; ValidPathInfo info2;
@ -2581,7 +2609,7 @@ void SubstitutionGoal::finished()
printMsg(lvlChatty, printMsg(lvlChatty,
format("substitution of path `%1%' succeeded") % storePath); format("substitution of path `%1%' succeeded") % storePath);
if (printBuildTrace) { if (settings.printBuildTrace) {
printMsg(lvlError, format("@ substituter-succeeded %1%") % storePath); printMsg(lvlError, format("@ substituter-succeeded %1%") % storePath);
} }
@ -2592,7 +2620,7 @@ void SubstitutionGoal::finished()
void SubstitutionGoal::handleChildOutput(int fd, const string & data) void SubstitutionGoal::handleChildOutput(int fd, const string & data)
{ {
assert(fd == logPipe.readSide); assert(fd == logPipe.readSide);
if (verbosity >= buildVerbosity) if (verbosity >= settings.buildVerbosity)
writeToStderr((unsigned char *) data.data(), data.size()); writeToStderr((unsigned char *) data.data(), data.size());
/* Don't write substitution output to a log file for now. We /* Don't write substitution output to a log file for now. We
probably should, though. */ probably should, though. */
@ -2620,7 +2648,6 @@ Worker::Worker(LocalStore & store)
working = true; working = true;
nrLocalBuilds = 0; nrLocalBuilds = 0;
lastWokenUp = 0; lastWokenUp = 0;
cacheFailure = queryBoolSetting("build-cache-failure", false);
permanentFailure = false; permanentFailure = false;
} }
@ -2685,7 +2712,7 @@ void Worker::removeGoal(GoalPtr goal)
topGoals.erase(goal); topGoals.erase(goal);
/* If a top-level goal failed, then kill all other goals /* If a top-level goal failed, then kill all other goals
(unless keepGoing was set). */ (unless keepGoing was set). */
if (goal->getExitCode() == Goal::ecFailed && !keepGoing) if (goal->getExitCode() == Goal::ecFailed && !settings.keepGoing)
topGoals.clear(); topGoals.clear();
} }
@ -2757,7 +2784,7 @@ void Worker::childTerminated(pid_t pid, bool wakeSleepers)
void Worker::waitForBuildSlot(GoalPtr goal) void Worker::waitForBuildSlot(GoalPtr goal)
{ {
debug("wait for build slot"); debug("wait for build slot");
if (getNrLocalBuilds() < maxBuildJobs) if (getNrLocalBuilds() < settings.maxBuildJobs)
wakeUp(goal); /* we can do it right away */ wakeUp(goal); /* we can do it right away */
else else
wantingToBuild.insert(goal); wantingToBuild.insert(goal);
@ -2806,7 +2833,7 @@ void Worker::run(const Goals & _topGoals)
if (!children.empty() || !waitingForAWhile.empty()) if (!children.empty() || !waitingForAWhile.empty())
waitForInput(); waitForInput();
else { else {
if (awake.empty() && maxBuildJobs == 0) throw Error( if (awake.empty() && settings.maxBuildJobs == 0) throw Error(
"unable to start any build; either increase `--max-jobs' " "unable to start any build; either increase `--max-jobs' "
"or enable distributed builds"); "or enable distributed builds");
assert(!awake.empty()); assert(!awake.empty());
@ -2816,9 +2843,9 @@ void Worker::run(const Goals & _topGoals)
/* If --keep-going is not set, it's possible that the main goal /* If --keep-going is not set, it's possible that the main goal
exited while some of its subgoals were still active. But if exited while some of its subgoals were still active. But if
--keep-going *is* set, then they must all be finished now. */ --keep-going *is* set, then they must all be finished now. */
assert(!keepGoing || awake.empty()); assert(!settings.keepGoing || awake.empty());
assert(!keepGoing || wantingToBuild.empty()); assert(!settings.keepGoing || wantingToBuild.empty());
assert(!keepGoing || children.empty()); assert(!settings.keepGoing || children.empty());
} }
@ -2838,15 +2865,15 @@ void Worker::waitForInput()
time_t before = time(0); time_t before = time(0);
/* If a global timeout has been set, sleep until it's done. */ /* If a global timeout has been set, sleep until it's done. */
if (buildTimeout != 0) { if (settings.buildTimeout != 0) {
useTimeout = true; useTimeout = true;
if (lastWait == 0 || lastWait > before) lastWait = before; if (lastWait == 0 || lastWait > before) lastWait = before;
timeout.tv_sec = std::max((time_t) 0, lastWait + buildTimeout - before); timeout.tv_sec = std::max((time_t) 0, lastWait + settings.buildTimeout - before);
} }
/* If we're monitoring for silence on stdout/stderr, sleep until /* If we're monitoring for silence on stdout/stderr, sleep until
the first deadline for any child. */ the first deadline for any child. */
if (maxSilentTime != 0) { if (settings.maxSilentTime != 0) {
time_t oldest = 0; time_t oldest = 0;
foreach (Children::iterator, i, children) { foreach (Children::iterator, i, children) {
if (i->second.monitorForSilence) { if (i->second.monitorForSilence) {
@ -2855,10 +2882,10 @@ void Worker::waitForInput()
} }
} }
if (oldest) { if (oldest) {
time_t silenceTimeout = std::max((time_t) 0, oldest + maxSilentTime - before); time_t silenceTimeout = std::max((time_t) 0, oldest + settings.maxSilentTime - before);
timeout.tv_sec = useTimeout timeout.tv_sec = useTimeout
? std::min(silenceTimeout, timeout.tv_sec) ? std::min(silenceTimeout, timeout.tv_sec)
: silenceTimeout; : silenceTimeout;
useTimeout = true; useTimeout = true;
printMsg(lvlVomit, format("sleeping %1% seconds") % timeout.tv_sec); printMsg(lvlVomit, format("sleeping %1% seconds") % timeout.tv_sec);
} }
@ -2866,14 +2893,12 @@ void Worker::waitForInput()
/* If we are polling goals that are waiting for a lock, then wake /* If we are polling goals that are waiting for a lock, then wake
up after a few seconds at most. */ up after a few seconds at most. */
int wakeUpInterval = queryIntSetting("build-poll-interval", 5);
if (!waitingForAWhile.empty()) { if (!waitingForAWhile.empty()) {
useTimeout = true; useTimeout = true;
if (lastWokenUp == 0) if (lastWokenUp == 0)
printMsg(lvlError, "waiting for locks or build slots..."); printMsg(lvlError, "waiting for locks or build slots...");
if (lastWokenUp == 0 || lastWokenUp > before) lastWokenUp = before; if (lastWokenUp == 0 || lastWokenUp > before) lastWokenUp = before;
timeout.tv_sec = std::max((time_t) 0, lastWokenUp + wakeUpInterval - before); timeout.tv_sec = std::max((time_t) 0, (time_t) (lastWokenUp + settings.pollInterval - before));
} else lastWokenUp = 0; } else lastWokenUp = 0;
using namespace std; using namespace std;
@ -2939,27 +2964,27 @@ void Worker::waitForInput()
} }
} }
if (maxSilentTime != 0 && if (settings.maxSilentTime != 0 &&
j->second.monitorForSilence && j->second.monitorForSilence &&
after - j->second.lastOutput >= (time_t) maxSilentTime) after - j->second.lastOutput >= (time_t) settings.maxSilentTime)
{ {
printMsg(lvlError, printMsg(lvlError,
format("%1% timed out after %2% seconds of silence") format("%1% timed out after %2% seconds of silence")
% goal->getName() % maxSilentTime); % goal->getName() % settings.maxSilentTime);
goal->cancel(); goal->cancel();
} }
if (buildTimeout != 0 && if (settings.buildTimeout != 0 &&
after - before >= (time_t) buildTimeout) after - before >= (time_t) settings.buildTimeout)
{ {
printMsg(lvlError, printMsg(lvlError,
format("%1% timed out after %2% seconds of activity") format("%1% timed out after %2% seconds of activity")
% goal->getName() % buildTimeout); % goal->getName() % settings.buildTimeout);
goal->cancel(); goal->cancel();
} }
} }
if (!waitingForAWhile.empty() && lastWokenUp + wakeUpInterval <= after) { if (!waitingForAWhile.empty() && lastWokenUp + settings.pollInterval <= after) {
lastWokenUp = after; lastWokenUp = after;
foreach (WeakGoals::iterator, i, waitingForAWhile) { foreach (WeakGoals::iterator, i, waitingForAWhile) {
GoalPtr goal = i->lock(); GoalPtr goal = i->lock();

View file

@ -38,7 +38,7 @@ Path writeDerivation(StoreAPI & store,
held during a garbage collection). */ held during a garbage collection). */
string suffix = name + drvExtension; string suffix = name + drvExtension;
string contents = unparseDerivation(drv); string contents = unparseDerivation(drv);
return readOnlyMode return settings.readOnlyMode
? computeStorePathForText(suffix, contents, references) ? computeStorePathForText(suffix, contents, references)
: store.addTextToStore(suffix, contents, references); : store.addTextToStore(suffix, contents, references);
} }

View file

@ -34,7 +34,7 @@ static const int defaultGcLevel = 1000;
int LocalStore::openGCLock(LockType lockType) int LocalStore::openGCLock(LockType lockType)
{ {
Path fnGCLock = (format("%1%/%2%") Path fnGCLock = (format("%1%/%2%")
% nixStateDir % gcLockName).str(); % settings.nixStateDir % gcLockName).str();
debug(format("acquiring global GC lock `%1%'") % fnGCLock); debug(format("acquiring global GC lock `%1%'") % fnGCLock);
@ -85,7 +85,7 @@ void LocalStore::addIndirectRoot(const Path & path)
{ {
string hash = printHash32(hashString(htSHA1, path)); string hash = printHash32(hashString(htSHA1, path));
Path realRoot = canonPath((format("%1%/%2%/auto/%3%") Path realRoot = canonPath((format("%1%/%2%/auto/%3%")
% nixStateDir % gcRootsDir % hash).str()); % settings.nixStateDir % gcRootsDir % hash).str());
createSymlink(realRoot, path); createSymlink(realRoot, path);
} }
@ -113,7 +113,7 @@ Path addPermRoot(StoreAPI & store, const Path & _storePath,
else { else {
if (!allowOutsideRootsDir) { if (!allowOutsideRootsDir) {
Path rootsDir = canonPath((format("%1%/%2%") % nixStateDir % gcRootsDir).str()); Path rootsDir = canonPath((format("%1%/%2%") % settings.nixStateDir % gcRootsDir).str());
if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/") if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/")
throw Error(format( throw Error(format(
@ -130,7 +130,7 @@ Path addPermRoot(StoreAPI & store, const Path & _storePath,
Instead of reading all the roots, it would be more efficient to Instead of reading all the roots, it would be more efficient to
check if the root is in a directory in or linked from the check if the root is in a directory in or linked from the
gcroots directory. */ gcroots directory. */
if (queryBoolSetting("gc-check-reachability", false)) { if (settings.checkRootReachability) {
Roots roots = store.findRoots(); Roots roots = store.findRoots();
if (roots.find(gcRoot) == roots.end()) if (roots.find(gcRoot) == roots.end())
printMsg(lvlError, printMsg(lvlError,
@ -160,7 +160,7 @@ void LocalStore::addTempRoot(const Path & path)
if (fdTempRoots == -1) { if (fdTempRoots == -1) {
while (1) { while (1) {
Path dir = (format("%1%/%2%") % nixStateDir % tempRootsDir).str(); Path dir = (format("%1%/%2%") % settings.nixStateDir % tempRootsDir).str();
createDirs(dir); createDirs(dir);
fnTempRoots = (format("%1%/%2%") fnTempRoots = (format("%1%/%2%")
@ -173,7 +173,7 @@ void LocalStore::addTempRoot(const Path & path)
processes with the same pid. */ processes with the same pid. */
unlink(fnTempRoots.c_str()); unlink(fnTempRoots.c_str());
fdTempRoots = openLockFile(fnTempRoots, true); fdTempRoots = openLockFile(fnTempRoots, true);
fdGCLock.close(); fdGCLock.close();
@ -238,10 +238,10 @@ static void readTempRoots(PathSet & tempRoots, FDs & fds)
/* Read the `temproots' directory for per-process temporary root /* Read the `temproots' directory for per-process temporary root
files. */ files. */
Strings tempRootFiles = readDirectory( Strings tempRootFiles = readDirectory(
(format("%1%/%2%") % nixStateDir % tempRootsDir).str()); (format("%1%/%2%") % settings.nixStateDir % tempRootsDir).str());
foreach (Strings::iterator, i, tempRootFiles) { foreach (Strings::iterator, i, tempRootFiles) {
Path path = (format("%1%/%2%/%3%") % nixStateDir % tempRootsDir % *i).str(); Path path = (format("%1%/%2%/%3%") % settings.nixStateDir % tempRootsDir % *i).str();
debug(format("reading temporary root file `%1%'") % path); debug(format("reading temporary root file `%1%'") % path);
FDPtr fd(new AutoCloseFD(open(path.c_str(), O_RDWR, 0666))); FDPtr fd(new AutoCloseFD(open(path.c_str(), O_RDWR, 0666)));
@ -350,7 +350,7 @@ static void findRoots(StoreAPI & store, const Path & path,
static Roots findRoots(StoreAPI & store, bool deleteStale) static Roots findRoots(StoreAPI & store, bool deleteStale)
{ {
Roots roots; Roots roots;
Path rootsDir = canonPath((format("%1%/%2%") % nixStateDir % gcRootsDir).str()); Path rootsDir = canonPath((format("%1%/%2%") % settings.nixStateDir % gcRootsDir).str());
findRoots(store, rootsDir, true, deleteStale, roots); findRoots(store, rootsDir, true, deleteStale, roots);
return roots; return roots;
} }
@ -365,7 +365,7 @@ Roots LocalStore::findRoots()
static void addAdditionalRoots(StoreAPI & store, PathSet & roots) static void addAdditionalRoots(StoreAPI & store, PathSet & roots)
{ {
Path rootFinder = getEnv("NIX_ROOT_FINDER", Path rootFinder = getEnv("NIX_ROOT_FINDER",
nixLibexecDir + "/nix/find-runtime-roots.pl"); settings.nixLibexecDir + "/nix/find-runtime-roots.pl");
if (rootFinder.empty()) return; if (rootFinder.empty()) return;
@ -623,8 +623,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
GCState state(results); GCState state(results);
state.options = options; state.options = options;
state.gcKeepOutputs = queryBoolSetting("gc-keep-outputs", false); state.gcKeepOutputs = settings.gcKeepOutputs;
state.gcKeepDerivations = queryBoolSetting("gc-keep-derivations", true); state.gcKeepDerivations = settings.gcKeepDerivations;
/* Using `--ignore-liveness' with `--delete' can have unintended /* Using `--ignore-liveness' with `--delete' can have unintended
consequences if `gc-keep-outputs' or `gc-keep-derivations' are consequences if `gc-keep-outputs' or `gc-keep-derivations' are
@ -685,8 +685,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
try { try {
AutoCloseDir dir = opendir(nixStore.c_str()); AutoCloseDir dir = opendir(settings.nixStore.c_str());
if (!dir) throw SysError(format("opening directory `%1%'") % nixStore); if (!dir) throw SysError(format("opening directory `%1%'") % settings.nixStore);
/* Read the store and immediately delete all paths that /* Read the store and immediately delete all paths that
aren't valid. When using --max-freed etc., deleting aren't valid. When using --max-freed etc., deleting
@ -700,14 +700,14 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
checkInterrupt(); checkInterrupt();
string name = dirent->d_name; string name = dirent->d_name;
if (name == "." || name == "..") continue; if (name == "." || name == "..") continue;
Path path = nixStore + "/" + name; Path path = settings.nixStore + "/" + name;
if (isValidPath(path)) if (isValidPath(path))
entries.push_back(path); entries.push_back(path);
else else
tryToDelete(state, path); tryToDelete(state, path);
} }
dir.close(); dir.close();
/* Now delete the unreachable valid paths. Randomise the /* Now delete the unreachable valid paths. Randomise the
order in which we delete entries to make the collector order in which we delete entries to make the collector

View file

@ -10,36 +10,63 @@
namespace nix { namespace nix {
string nixStore = "/UNINIT"; Settings settings;
string nixDataDir = "/UNINIT";
string nixLogDir = "/UNINIT";
string nixStateDir = "/UNINIT";
string nixDBPath = "/UNINIT";
string nixConfDir = "/UNINIT";
string nixLibexecDir = "/UNINIT";
string nixBinDir = "/UNINIT";
bool keepFailed = false;
bool keepGoing = false;
bool tryFallback = false;
Verbosity buildVerbosity = lvlError;
unsigned int maxBuildJobs = 1;
unsigned int buildCores = 1;
bool readOnlyMode = false;
string thisSystem = "unset";
time_t maxSilentTime = 0;
time_t buildTimeout = 0;
Paths substituters;
bool useBuildHook = true;
bool printBuildTrace = false;
static bool settingsRead = false; Settings::Settings()
{
keepFailed = false;
keepGoing = false;
tryFallback = false;
buildVerbosity = lvlError;
maxBuildJobs = 1;
buildCores = 1;
readOnlyMode = false;
thisSystem = SYSTEM;
maxSilentTime = 0;
buildTimeout = 0;
useBuildHook = true;
printBuildTrace = false;
reservedSize = 1024 * 1024;
fsyncMetadata = true;
useSQLiteWAL = true;
syncBeforeRegistering = false;
useSubstitutes = true;
useChroot = false;
dirsInChroot.insert("/dev");
dirsInChroot.insert("/dev/pts");
impersonateLinux26 = false;
keepLog = true;
compressLog = true;
cacheFailure = false;
pollInterval = 5;
checkRootReachability = false;
gcKeepOutputs = false;
gcKeepDerivations = true;
autoOptimiseStore = true;
envKeepDerivations = false;
}
static std::map<string, Strings> settings;
/* Overriden settings. */ void Settings::processEnvironment()
std::map<string, Strings> settingsCmdline; {
nixStore = canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR)));
nixDataDir = canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR));
nixLogDir = canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR));
nixStateDir = canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR));
nixDBPath = getEnv("NIX_DB_DIR", nixStateDir + "/db");
nixConfDir = canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR));
nixLibexecDir = canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR));
nixBinDir = canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR));
string subs = getEnv("NIX_SUBSTITUTERS", "default");
if (subs == "default") {
substituters.push_back(nixLibexecDir + "/nix/substituters/copy-from-other-stores.pl");
substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl");
substituters.push_back(nixLibexecDir + "/nix/substituters/download-from-binary-cache.pl");
} else
substituters = tokenizeString(subs, ":");
}
string & at(Strings & ss, unsigned int n) string & at(Strings & ss, unsigned int n)
@ -50,7 +77,7 @@ string & at(Strings & ss, unsigned int n)
} }
static void readSettings() void Settings::loadConfFile()
{ {
Path settingsFile = (format("%1%/%2%") % nixConfDir % "nix.conf").str(); Path settingsFile = (format("%1%/%2%") % nixConfDir % "nix.conf").str();
if (!pathExists(settingsFile)) return; if (!pathExists(settingsFile)) return;
@ -78,94 +105,102 @@ static void readSettings()
Strings::iterator i = tokens.begin(); Strings::iterator i = tokens.begin();
advance(i, 2); advance(i, 2);
settings[name] = Strings(i, tokens.end()); settings[name] = concatStringsSep(" ", Strings(i, tokens.end())); // FIXME: slow
}; };
settings.insert(settingsCmdline.begin(), settingsCmdline.end());
settingsRead = true;
} }
Strings querySetting(const string & name, const Strings & def) void Settings::set(const string & name, const string & value)
{ {
if (!settingsRead) readSettings(); settings[name] = value;
std::map<string, Strings>::iterator i = settings.find(name); overrides[name] = value;
return i == settings.end() ? def : i->second;
} }
string querySetting(const string & name, const string & def) void Settings::update()
{ {
Strings defs; get(tryFallback, "build-fallback");
defs.push_back(def); get(maxBuildJobs, "build-max-jobs");
get(buildCores, "build-cores");
Strings value = querySetting(name, defs); get(thisSystem, "system");
if (value.size() != 1) get(maxSilentTime, "build-max-silent-time");
throw Error(format("configuration option `%1%' should not be a list") % name); get(buildTimeout, "build-timeout");
get(reservedSize, "gc-reserved-space");
return value.front(); get(fsyncMetadata, "fsync-metadata");
get(useSQLiteWAL, "use-sqlite-wal");
get(syncBeforeRegistering, "sync-before-registering");
get(useSubstitutes, "build-use-substitutes");
get(buildUsersGroup, "build-users-group");
get(useChroot, "build-use-chroot");
get(dirsInChroot, "build-chroot-dirs");
get(impersonateLinux26, "build-impersonate-linux-26");
get(keepLog, "build-keep-log");
get(compressLog, "build-compress-log");
get(cacheFailure, "build-cache-failure");
get(pollInterval, "build-poll-interval");
get(checkRootReachability, "gc-check-reachability");
get(gcKeepOutputs, "gc-keep-outputs");
get(gcKeepDerivations, "gc-keep-derivations");
get(autoOptimiseStore, "auto-optimise-store");
get(envKeepDerivations, "env-keep-derivations");
} }
bool queryBoolSetting(const string & name, bool def) void Settings::get(string & res, const string & name)
{ {
string v = querySetting(name, def ? "true" : "false"); SettingsMap::iterator i = settings.find(name);
if (v == "true") return true; if (i == settings.end()) return;
else if (v == "false") return false; res = i->second;
}
void Settings::get(bool & res, const string & name)
{
SettingsMap::iterator i = settings.find(name);
if (i == settings.end()) return;
if (i->second == "true") res = true;
else if (i->second == "false") res = false;
else throw Error(format("configuration option `%1%' should be either `true' or `false', not `%2%'") else throw Error(format("configuration option `%1%' should be either `true' or `false', not `%2%'")
% name % v); % name % i->second);
} }
unsigned int queryIntSetting(const string & name, unsigned int def) void Settings::get(PathSet & res, const string & name)
{ {
int n; SettingsMap::iterator i = settings.find(name);
if (!string2Int(querySetting(name, int2String(def)), n) || n < 0) if (i == settings.end()) return;
res.clear();
Strings ss = tokenizeString(i->second);
res.insert(ss.begin(), ss.end());
}
template<class N> void Settings::get(N & res, const string & name)
{
SettingsMap::iterator i = settings.find(name);
if (i == settings.end()) return;
if (!string2Int(i->second, res))
throw Error(format("configuration setting `%1%' should have an integer value") % name); throw Error(format("configuration setting `%1%' should have an integer value") % name);
return n;
} }
void overrideSetting(const string & name, const Strings & value) string Settings::pack()
{ {
if (settingsRead) settings[name] = value; string s;
settingsCmdline[name] = value; foreach (SettingsMap::iterator, i, settings) {
if (i->first.find('\n') != string::npos ||
i->first.find('=') != string::npos ||
i->second.find('\n') != string::npos)
throw Error("illegal option name/value");
s += i->first; s += '='; s += i->second; s += '\n';
}
return s;
} }
void reloadSettings() Settings::SettingsMap Settings::getOverrides()
{ {
settingsRead = false; return overrides;
settings.clear();
}
void setDefaultsFromEnvironment()
{
/* Setup Nix paths. */
nixStore = canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR)));
nixDataDir = canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR));
nixLogDir = canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR));
nixStateDir = canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR));
nixDBPath = getEnv("NIX_DB_DIR", nixStateDir + "/db");
nixConfDir = canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR));
nixLibexecDir = canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR));
nixBinDir = canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR));
string subs = getEnv("NIX_SUBSTITUTERS", "default");
if (subs == "default") {
substituters.push_back(nixLibexecDir + "/nix/substituters/copy-from-other-stores.pl");
substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl");
} else
substituters = tokenizeString(subs, ":");
/* Get some settings from the configuration file. */
thisSystem = querySetting("system", SYSTEM);
maxBuildJobs = queryIntSetting("build-max-jobs", 1);
buildCores = queryIntSetting("build-cores", 1);
maxSilentTime = queryIntSetting("build-max-silent-time", 0);
buildTimeout = queryIntSetting("build-timeout", 0);
} }

View file

@ -2,118 +2,191 @@
#include "types.hh" #include "types.hh"
#include <map>
namespace nix { namespace nix {
/* Path names. */ struct Settings {
/* nixStore is the directory where we generally store atomic and typedef std::map<string, string> SettingsMap;
derived files. */
extern string nixStore;
extern string nixDataDir; /* !!! fix */ Settings();
/* nixLogDir is the directory where we log various operations. */ void processEnvironment();
extern string nixLogDir;
/* nixStateDir is the directory where state is stored. */ void loadConfFile();
extern string nixStateDir;
/* nixDBPath is the path name of our Berkeley DB environment. */ void set(const string & name, const string & value);
extern string nixDBPath;
/* nixConfDir is the directory where configuration files are void update();
stored. */
extern string nixConfDir;
/* nixLibexecDir is the directory where internal helper programs are string pack();
stored. */
extern string nixLibexecDir;
/* nixBinDir is the directory where the main programs are stored. */ SettingsMap getOverrides();
extern string nixBinDir;
/* The directory where we store sources and derived files. */
Path nixStore;
Path nixDataDir; /* !!! fix */
/* The directory where we log various operations. */
Path nixLogDir;
/* The directory where state is stored. */
Path nixStateDir;
/* The directory where we keep the SQLite database. */
Path nixDBPath;
/* The directory where configuration files are stored. */
Path nixConfDir;
/* The directory where internal helper programs are stored. */
Path nixLibexecDir;
/* The directory where the main programs are stored. */
Path nixBinDir;
/* Whether to keep temporary directories of failed builds. */
bool keepFailed;
/* Whether to keep building subgoals when a sibling (another
subgoal of the same goal) fails. */
bool keepGoing;
/* Whether, if we cannot realise the known closure corresponding
to a derivation, we should try to normalise the derivation
instead. */
bool tryFallback;
/* Verbosity level for build output. */
Verbosity buildVerbosity;
/* Maximum number of parallel build jobs. 0 means unlimited. */
unsigned int maxBuildJobs;
/* Number of CPU cores to utilize in parallel within a build,
i.e. by passing this number to Make via '-j'. 0 means that the
number of actual CPU cores on the local host ought to be
auto-detected. */
unsigned int buildCores;
/* Read-only mode. Don't copy stuff to the store, don't change
the database. */
bool readOnlyMode;
/* The canonical system name, as returned by config.guess. */
string thisSystem;
/* The maximum time in seconds that a builer can go without
producing any output on stdout/stderr before it is killed. 0
means infinity. */
time_t maxSilentTime;
/* The maximum duration in seconds that a builder can run. 0
means infinity. */
time_t buildTimeout;
/* The substituters. There are programs that can somehow realise
a store path without building, e.g., by downloading it or
copying it from a CD. */
Paths substituters;
/* Whether to use build hooks (for distributed builds). Sometimes
users want to disable this from the command-line. */
bool useBuildHook;
/* Whether buildDerivations() should print out lines on stderr in
a fixed format to allow its progress to be monitored. Each
line starts with a "@". The following are defined:
@ build-started <drvpath> <outpath> <system> <logfile>
@ build-failed <drvpath> <outpath> <exitcode> <error text>
@ build-succeeded <drvpath> <outpath>
@ substituter-started <outpath> <substituter>
@ substituter-failed <outpath> <exitcode> <error text>
@ substituter-succeeded <outpath>
Best combined with --no-build-output, otherwise stderr might
conceivably contain lines in this format printed by the
builders. */
bool printBuildTrace;
/* Amount of reserved space for the garbage collector
(/nix/var/nix/db/reserved). */
off_t reservedSize;
/* Whether SQLite should use fsync. */
bool fsyncMetadata;
/* Whether SQLite should use WAL mode. */
bool useSQLiteWAL;
/* Whether to call sync() before registering a path as valid. */
bool syncBeforeRegistering;
/* Whether to use substitutes. */
bool useSubstitutes;
/* The Unix group that contains the build users. */
string buildUsersGroup;
/* Whether to build in chroot. */
bool useChroot;
/* The directories from the host filesystem to be included in the
chroot. */
PathSet dirsInChroot;
/* Whether to impersonate a Linux 2.6 machine on newer kernels. */
bool impersonateLinux26;
/* Whether to store build logs. */
bool keepLog;
/* Whether to compress logs. */
bool compressLog;
/* Whether to cache build failures. */
bool cacheFailure;
/* How often (in seconds) to poll for locks. */
unsigned int pollInterval;
/* Whether to check if new GC roots can in fact be found by the
garbage collector. */
bool checkRootReachability;
/* Whether the garbage collector should keep outputs of live
derivations. */
bool gcKeepOutputs;
/* Whether the garbage collector should keep derivers of live
paths. */
bool gcKeepDerivations;
/* Whether to automatically replace files with identical contents
with hard links. */
bool autoOptimiseStore;
/* Whether to add derivations as a dependency of user environments
(to prevent them from being GCed). */
bool envKeepDerivations;
private:
SettingsMap settings, overrides;
void get(string & res, const string & name);
void get(bool & res, const string & name);
void get(PathSet & res, const string & name);
template<class N> void get(N & res, const string & name);
};
/* Misc. global flags. */ // FIXME: don't use a global variable.
extern Settings settings;
/* Whether to keep temporary directories of failed builds. */
extern bool keepFailed;
/* Whether to keep building subgoals when a sibling (another subgoal
of the same goal) fails. */
extern bool keepGoing;
/* Whether, if we cannot realise the known closure corresponding to a
derivation, we should try to normalise the derivation instead. */
extern bool tryFallback;
/* Verbosity level for build output. */
extern Verbosity buildVerbosity;
/* Maximum number of parallel build jobs. 0 means unlimited. */
extern unsigned int maxBuildJobs;
/* Number of CPU cores to utilize in parallel within a build, i.e. by passing
this number to Make via '-j'. 0 means that the number of actual CPU cores on
the local host ought to be auto-detected. */
extern unsigned int buildCores;
/* Read-only mode. Don't copy stuff to the store, don't change the
database. */
extern bool readOnlyMode;
/* The canonical system name, as returned by config.guess. */
extern string thisSystem;
/* The maximum time in seconds that a builer can go without producing
any output on stdout/stderr before it is killed. 0 means
infinity. */
extern time_t maxSilentTime;
/* The maximum duration in seconds that a builder can run. 0 means
infinity. */
extern time_t buildTimeout;
/* The substituters. There are programs that can somehow realise a
store path without building, e.g., by downloading it or copying it
from a CD. */
extern Paths substituters;
/* Whether to use build hooks (for distributed builds). Sometimes
users want to disable this from the command-line. */
extern bool useBuildHook;
/* Whether buildDerivations() should print out lines on stderr in a
fixed format to allow its progress to be monitored. Each line
starts with a "@". The following are defined:
@ build-started <drvpath> <outpath> <system> <logfile>
@ build-failed <drvpath> <outpath> <exitcode> <error text>
@ build-succeeded <drvpath> <outpath>
@ substituter-started <outpath> <substituter>
@ substituter-failed <outpath> <exitcode> <error text>
@ substituter-succeeded <outpath>
Best combined with --no-build-output, otherwise stderr might
conceivably contain lines in this format printed by the builders.
*/
extern bool printBuildTrace;
Strings querySetting(const string & name, const Strings & def);
string querySetting(const string & name, const string & def);
bool queryBoolSetting(const string & name, bool def);
unsigned int queryIntSetting(const string & name, unsigned int def);
void overrideSetting(const string & name, const Strings & value);
void reloadSettings();
void setDefaultsFromEnvironment();
} }

View file

@ -181,7 +181,7 @@ struct SQLiteTxn
void checkStoreNotSymlink() void checkStoreNotSymlink()
{ {
if (getEnv("NIX_IGNORE_SYMLINK_STORE") == "1") return; if (getEnv("NIX_IGNORE_SYMLINK_STORE") == "1") return;
Path path = nixStore; Path path = settings.nixStore;
struct stat st; struct stat st;
while (path != "/") { while (path != "/") {
if (lstat(path.c_str(), &st)) if (lstat(path.c_str(), &st))
@ -198,23 +198,21 @@ void checkStoreNotSymlink()
LocalStore::LocalStore(bool reserveSpace) LocalStore::LocalStore(bool reserveSpace)
{ {
substitutablePathsLoaded = false; schemaPath = settings.nixDBPath + "/schema";
schemaPath = nixDBPath + "/schema"; if (settings.readOnlyMode) {
if (readOnlyMode) {
openDB(false); openDB(false);
return; return;
} }
/* Create missing state directories if they don't already exist. */ /* Create missing state directories if they don't already exist. */
createDirs(nixStore); createDirs(settings.nixStore);
createDirs(linksDir = nixStore + "/.links"); createDirs(linksDir = settings.nixStore + "/.links");
Path profilesDir = nixStateDir + "/profiles"; Path profilesDir = settings.nixStateDir + "/profiles";
createDirs(nixStateDir + "/profiles"); createDirs(settings.nixStateDir + "/profiles");
createDirs(nixStateDir + "/temproots"); createDirs(settings.nixStateDir + "/temproots");
createDirs(nixDBPath); createDirs(settings.nixDBPath);
Path gcRootsDir = nixStateDir + "/gcroots"; Path gcRootsDir = settings.nixStateDir + "/gcroots";
if (!pathExists(gcRootsDir)) { if (!pathExists(gcRootsDir)) {
createDirs(gcRootsDir); createDirs(gcRootsDir);
if (symlink(profilesDir.c_str(), (gcRootsDir + "/profiles").c_str()) == -1) if (symlink(profilesDir.c_str(), (gcRootsDir + "/profiles").c_str()) == -1)
@ -228,13 +226,12 @@ LocalStore::LocalStore(bool reserveSpace)
needed, we reserve some dummy space that we can free just needed, we reserve some dummy space that we can free just
before doing a garbage collection. */ before doing a garbage collection. */
try { try {
Path reservedPath = nixDBPath + "/reserved"; Path reservedPath = settings.nixDBPath + "/reserved";
if (reserveSpace) { if (reserveSpace) {
int reservedSize = queryIntSetting("gc-reserved-space", 1024 * 1024);
struct stat st; struct stat st;
if (stat(reservedPath.c_str(), &st) == -1 || if (stat(reservedPath.c_str(), &st) == -1 ||
st.st_size != reservedSize) st.st_size != settings.reservedSize)
writeFile(reservedPath, string(reservedSize, 'X')); writeFile(reservedPath, string(settings.reservedSize, 'X'));
} }
else else
deletePath(reservedPath); deletePath(reservedPath);
@ -244,11 +241,11 @@ LocalStore::LocalStore(bool reserveSpace)
/* Acquire the big fat lock in shared mode to make sure that no /* Acquire the big fat lock in shared mode to make sure that no
schema upgrade is in progress. */ schema upgrade is in progress. */
try { try {
Path globalLockPath = nixDBPath + "/big-lock"; Path globalLockPath = settings.nixDBPath + "/big-lock";
globalLock = openLockFile(globalLockPath.c_str(), true); globalLock = openLockFile(globalLockPath.c_str(), true);
} catch (SysError & e) { } catch (SysError & e) {
if (e.errNo != EACCES) throw; if (e.errNo != EACCES) throw;
readOnlyMode = true; settings.readOnlyMode = true;
openDB(false); openDB(false);
return; return;
} }
@ -327,7 +324,7 @@ int LocalStore::getSchema()
void LocalStore::openDB(bool create) void LocalStore::openDB(bool create)
{ {
/* Open the Nix database. */ /* Open the Nix database. */
if (sqlite3_open_v2((nixDBPath + "/db.sqlite").c_str(), &db.db, if (sqlite3_open_v2((settings.nixDBPath + "/db.sqlite").c_str(), &db.db,
SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK)
throw Error("cannot open SQLite database"); throw Error("cannot open SQLite database");
@ -344,13 +341,13 @@ void LocalStore::openDB(bool create)
should be safe enough. If the user asks for it, don't sync at should be safe enough. If the user asks for it, don't sync at
all. This can cause database corruption if the system all. This can cause database corruption if the system
crashes. */ crashes. */
string syncMode = queryBoolSetting("fsync-metadata", true) ? "normal" : "off"; string syncMode = settings.fsyncMetadata ? "normal" : "off";
if (sqlite3_exec(db, ("pragma synchronous = " + syncMode + ";").c_str(), 0, 0, 0) != SQLITE_OK) if (sqlite3_exec(db, ("pragma synchronous = " + syncMode + ";").c_str(), 0, 0, 0) != SQLITE_OK)
throwSQLiteError(db, "setting synchronous mode"); throwSQLiteError(db, "setting synchronous mode");
/* Set the SQLite journal mode. WAL mode is fastest, so it's the /* Set the SQLite journal mode. WAL mode is fastest, so it's the
default. */ default. */
string mode = queryBoolSetting("use-sqlite-wal", true) ? "wal" : "truncate"; string mode = settings.useSQLiteWAL ? "wal" : "truncate";
string prevMode; string prevMode;
{ {
SQLiteStmt stmt; SQLiteStmt stmt;
@ -423,7 +420,7 @@ void canonicalisePathMetaData(const Path & path, bool recurse)
struct stat st; struct stat st;
if (lstat(path.c_str(), &st)) if (lstat(path.c_str(), &st))
throw SysError(format("getting attributes of path `%1%'") % path); throw SysError(format("getting attributes of path `%1%'") % path);
/* Really make sure that the path is of a supported type. This /* Really make sure that the path is of a supported type. This
has already been checked in dumpPath(). */ has already been checked in dumpPath(). */
@ -478,8 +475,8 @@ void canonicalisePathMetaData(const Path & path, bool recurse)
if (recurse && S_ISDIR(st.st_mode)) { if (recurse && S_ISDIR(st.st_mode)) {
Strings names = readDirectory(path); Strings names = readDirectory(path);
foreach (Strings::iterator, i, names) foreach (Strings::iterator, i, names)
canonicalisePathMetaData(path + "/" + *i, true); canonicalisePathMetaData(path + "/" + *i, true);
} }
makeImmutable(path); makeImmutable(path);
@ -494,7 +491,7 @@ void canonicalisePathMetaData(const Path & path)
be a symlink, since we can't change its ownership. */ be a symlink, since we can't change its ownership. */
struct stat st; struct stat st;
if (lstat(path.c_str(), &st)) if (lstat(path.c_str(), &st))
throw SysError(format("getting attributes of path `%1%'") % path); throw SysError(format("getting attributes of path `%1%'") % path);
if (st.st_uid != geteuid()) { if (st.st_uid != geteuid()) {
assert(S_ISLNK(st.st_mode)); assert(S_ISLNK(st.st_mode));
@ -756,7 +753,16 @@ bool LocalStore::isValidPath(const Path & path)
} }
PathSet LocalStore::queryValidPaths() PathSet LocalStore::queryValidPaths(const PathSet & paths)
{
PathSet res;
foreach (PathSet::const_iterator, i, paths)
if (isValidPath(*i)) res.insert(*i);
return res;
}
PathSet LocalStore::queryAllValidPaths()
{ {
SQLiteStmt stmt; SQLiteStmt stmt;
stmt.create(db, "select path from ValidPaths"); stmt.create(db, "select path from ValidPaths");
@ -883,7 +889,7 @@ Path LocalStore::queryPathFromHashPart(const string & hashPart)
SQLiteTxn txn(db); SQLiteTxn txn(db);
Path prefix = nixStore + "/" + hashPart; Path prefix = settings.nixStore + "/" + hashPart;
SQLiteStmtUse use(stmtQueryPathFromHashPart); SQLiteStmtUse use(stmtQueryPathFromHashPart);
stmtQueryPathFromHashPart.bind(prefix); stmtQueryPathFromHashPart.bind(prefix);
@ -903,10 +909,11 @@ void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter &
debug(format("starting substituter program `%1%'") % substituter); debug(format("starting substituter program `%1%'") % substituter);
Pipe toPipe, fromPipe; Pipe toPipe, fromPipe, errorPipe;
toPipe.create(); toPipe.create();
fromPipe.create(); fromPipe.create();
errorPipe.create();
run.pid = fork(); run.pid = fork();
@ -924,12 +931,18 @@ void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter &
written in Perl (i.e. all of them) fail. */ written in Perl (i.e. all of them) fail. */
unsetenv("DYLD_LIBRARY_PATH"); unsetenv("DYLD_LIBRARY_PATH");
/* Pass configuration options (including those overriden
with --option) to the substituter. */
setenv("_NIX_OPTIONS", settings.pack().c_str(), 1);
fromPipe.readSide.close(); fromPipe.readSide.close();
toPipe.writeSide.close(); toPipe.writeSide.close();
if (dup2(toPipe.readSide, STDIN_FILENO) == -1) if (dup2(toPipe.readSide, STDIN_FILENO) == -1)
throw SysError("dupping stdin"); throw SysError("dupping stdin");
if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1) if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1)
throw SysError("dupping stdout"); throw SysError("dupping stdout");
if (dup2(errorPipe.writeSide, STDERR_FILENO) == -1)
throw SysError("dupping stderr");
closeMostFDs(set<int>()); closeMostFDs(set<int>());
execl(substituter.c_str(), substituter.c_str(), "--query", NULL); execl(substituter.c_str(), substituter.c_str(), "--query", NULL);
throw SysError(format("executing `%1%'") % substituter); throw SysError(format("executing `%1%'") % substituter);
@ -943,6 +956,7 @@ void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter &
run.to = toPipe.writeSide.borrow(); run.to = toPipe.writeSide.borrow();
run.from = fromPipe.readSide.borrow(); run.from = fromPipe.readSide.borrow();
run.error = errorPipe.readSide.borrow();
} }
@ -955,50 +969,79 @@ template<class T> T getIntLine(int fd)
} }
bool LocalStore::hasSubstitutes(const Path & path) PathSet LocalStore::querySubstitutablePaths(const PathSet & paths)
{ {
foreach (Paths::iterator, i, substituters) { PathSet res;
foreach (Paths::iterator, i, settings.substituters) {
if (res.size() == paths.size()) break;
RunningSubstituter & run(runningSubstituters[*i]); RunningSubstituter & run(runningSubstituters[*i]);
startSubstituter(*i, run); startSubstituter(*i, run);
writeLine(run.to, "have\n" + path); string s = "have ";
if (getIntLine<int>(run.from)) return true; foreach (PathSet::const_iterator, j, paths)
if (res.find(*j) == res.end()) { s += *j; s += " "; }
writeLine(run.to, s);
while (true) {
/* FIXME: we only read stderr when an error occurs, so
substituters should only write (short) messages to
stderr when they fail. I.e. they shouldn't write debug
output. */
try {
Path path = readLine(run.from);
if (path == "") break;
res.insert(path);
} catch (EndOfFile e) {
throw Error(format("substituter `%1%' failed: %2%") % *i % chomp(drainFD(run.error)));
}
}
} }
return res;
return false;
} }
bool LocalStore::querySubstitutablePathInfo(const Path & substituter, void LocalStore::querySubstitutablePathInfos(const Path & substituter,
const Path & path, SubstitutablePathInfo & info) PathSet & paths, SubstitutablePathInfos & infos)
{ {
RunningSubstituter & run(runningSubstituters[substituter]); RunningSubstituter & run(runningSubstituters[substituter]);
startSubstituter(substituter, run); startSubstituter(substituter, run);
writeLine(run.to, "info\n" + path); string s = "info ";
foreach (PathSet::const_iterator, i, paths)
if (infos.find(*i) == infos.end()) { s += *i; s += " "; }
writeLine(run.to, s);
if (!getIntLine<int>(run.from)) return false; while (true) {
try {
info.deriver = readLine(run.from); Path path = readLine(run.from);
if (info.deriver != "") assertStorePath(info.deriver); if (path == "") break;
int nrRefs = getIntLine<int>(run.from); if (paths.find(path) == paths.end())
while (nrRefs--) { throw Error(format("got unexpected path `%1%' from substituter") % path);
Path p = readLine(run.from); paths.erase(path);
assertStorePath(p); SubstitutablePathInfo & info(infos[path]);
info.references.insert(p); info.deriver = readLine(run.from);
if (info.deriver != "") assertStorePath(info.deriver);
int nrRefs = getIntLine<int>(run.from);
while (nrRefs--) {
Path p = readLine(run.from);
assertStorePath(p);
info.references.insert(p);
}
info.downloadSize = getIntLine<long long>(run.from);
info.narSize = getIntLine<long long>(run.from);
} catch (EndOfFile e) {
throw Error(format("substituter `%1%' failed: %2%") % substituter % chomp(drainFD(run.error)));
}
} }
info.downloadSize = getIntLine<long long>(run.from);
info.narSize = getIntLine<long long>(run.from);
return true;
} }
bool LocalStore::querySubstitutablePathInfo(const Path & path, void LocalStore::querySubstitutablePathInfos(const PathSet & paths,
SubstitutablePathInfo & info) SubstitutablePathInfos & infos)
{ {
foreach (Paths::iterator, i, substituters) PathSet todo = paths;
if (querySubstitutablePathInfo(*i, path, info)) return true; foreach (Paths::iterator, i, settings.substituters) {
return false; if (todo.empty()) break;
querySubstitutablePathInfos(*i, todo, infos);
}
} }
@ -1018,11 +1061,10 @@ void LocalStore::registerValidPath(const ValidPathInfo & info)
void LocalStore::registerValidPaths(const ValidPathInfos & infos) void LocalStore::registerValidPaths(const ValidPathInfos & infos)
{ {
/* sqlite will fsync by default, but the new valid paths may not be fsync-ed. /* SQLite will fsync by default, but the new valid paths may not be fsync-ed.
* So some may want to fsync them before registering the validity, at the * So some may want to fsync them before registering the validity, at the
* expense of some speed of the path registering operation. */ * expense of some speed of the path registering operation. */
if (queryBoolSetting("sync-before-registering", false)) if (settings.syncBeforeRegistering) sync();
sync();
while (1) { while (1) {
try { try {
@ -1266,7 +1308,7 @@ void LocalStore::exportPath(const Path & path, bool sign,
Path hashFile = tmpDir + "/hash"; Path hashFile = tmpDir + "/hash";
writeFile(hashFile, printHash(hash)); writeFile(hashFile, printHash(hash));
Path secretKey = nixConfDir + "/signing-key.sec"; Path secretKey = settings.nixConfDir + "/signing-key.sec";
checkSecrecy(secretKey); checkSecrecy(secretKey);
Strings args; Strings args;
@ -1312,7 +1354,7 @@ Path LocalStore::createTempDirInStore()
/* There is a slight possibility that `tmpDir' gets deleted by /* There is a slight possibility that `tmpDir' gets deleted by
the GC between createTempDir() and addTempRoot(), so repeat the GC between createTempDir() and addTempRoot(), so repeat
until `tmpDir' exists. */ until `tmpDir' exists. */
tmpDir = createTempDir(nixStore); tmpDir = createTempDir(settings.nixStore);
addTempRoot(tmpDir); addTempRoot(tmpDir);
} while (!pathExists(tmpDir)); } while (!pathExists(tmpDir));
return tmpDir; return tmpDir;
@ -1364,7 +1406,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source)
args.push_back("rsautl"); args.push_back("rsautl");
args.push_back("-verify"); args.push_back("-verify");
args.push_back("-inkey"); args.push_back("-inkey");
args.push_back(nixConfDir + "/signing-key.pub"); args.push_back(settings.nixConfDir + "/signing-key.pub");
args.push_back("-pubin"); args.push_back("-pubin");
args.push_back("-in"); args.push_back("-in");
args.push_back(sigFile); args.push_back(sigFile);
@ -1473,13 +1515,13 @@ void LocalStore::verifyStore(bool checkContents)
/* Acquire the global GC lock to prevent a garbage collection. */ /* Acquire the global GC lock to prevent a garbage collection. */
AutoCloseFD fdGCLock = openGCLock(ltWrite); AutoCloseFD fdGCLock = openGCLock(ltWrite);
Paths entries = readDirectory(nixStore); Paths entries = readDirectory(settings.nixStore);
PathSet store(entries.begin(), entries.end()); PathSet store(entries.begin(), entries.end());
/* Check whether all valid paths actually exist. */ /* Check whether all valid paths actually exist. */
printMsg(lvlInfo, "checking path existence..."); printMsg(lvlInfo, "checking path existence...");
PathSet validPaths2 = queryValidPaths(), validPaths, done; PathSet validPaths2 = queryAllValidPaths(), validPaths, done;
foreach (PathSet::iterator, i, validPaths2) foreach (PathSet::iterator, i, validPaths2)
verifyPath(*i, store, done, validPaths); verifyPath(*i, store, done, validPaths);
@ -1583,9 +1625,9 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store,
PathSet LocalStore::queryValidPathsOld() PathSet LocalStore::queryValidPathsOld()
{ {
PathSet paths; PathSet paths;
Strings entries = readDirectory(nixDBPath + "/info"); Strings entries = readDirectory(settings.nixDBPath + "/info");
foreach (Strings::iterator, i, entries) foreach (Strings::iterator, i, entries)
if (i->at(0) != '.') paths.insert(nixStore + "/" + *i); if (i->at(0) != '.') paths.insert(settings.nixStore + "/" + *i);
return paths; return paths;
} }
@ -1597,7 +1639,7 @@ ValidPathInfo LocalStore::queryPathInfoOld(const Path & path)
/* Read the info file. */ /* Read the info file. */
string baseName = baseNameOf(path); string baseName = baseNameOf(path);
Path infoFile = (format("%1%/info/%2%") % nixDBPath % baseName).str(); Path infoFile = (format("%1%/info/%2%") % settings.nixDBPath % baseName).str();
if (!pathExists(infoFile)) if (!pathExists(infoFile))
throw Error(format("path `%1%' is not valid") % path); throw Error(format("path `%1%' is not valid") % path);
string info = readFile(infoFile); string info = readFile(infoFile);

View file

@ -45,7 +45,7 @@ struct OptimiseStats
struct RunningSubstituter struct RunningSubstituter
{ {
Pid pid; Pid pid;
AutoCloseFD to, from; AutoCloseFD to, from, error;
}; };
@ -80,9 +80,6 @@ struct SQLiteStmt
class LocalStore : public StoreAPI class LocalStore : public StoreAPI
{ {
private: private:
bool substitutablePathsLoaded;
PathSet substitutablePaths;
typedef std::map<Path, RunningSubstituter> RunningSubstituters; typedef std::map<Path, RunningSubstituter> RunningSubstituters;
RunningSubstituters runningSubstituters; RunningSubstituters runningSubstituters;
@ -100,7 +97,9 @@ public:
bool isValidPath(const Path & path); bool isValidPath(const Path & path);
PathSet queryValidPaths(); PathSet queryValidPaths(const PathSet & paths);
PathSet queryAllValidPaths();
ValidPathInfo queryPathInfo(const Path & path); ValidPathInfo queryPathInfo(const Path & path);
@ -124,15 +123,13 @@ public:
Path queryPathFromHashPart(const string & hashPart); Path queryPathFromHashPart(const string & hashPart);
PathSet querySubstitutablePaths(); PathSet querySubstitutablePaths(const PathSet & paths);
bool hasSubstitutes(const Path & path); void querySubstitutablePathInfos(const Path & substituter,
PathSet & paths, SubstitutablePathInfos & infos);
bool querySubstitutablePathInfo(const Path & path, void querySubstitutablePathInfos(const PathSet & paths,
SubstitutablePathInfo & info); SubstitutablePathInfos & infos);
bool querySubstitutablePathInfo(const Path & substituter,
const Path & path, SubstitutablePathInfo & info);
Path addToStore(const Path & srcPath, Path addToStore(const Path & srcPath,
bool recursive = true, HashType hashAlgo = htSHA256, bool recursive = true, HashType hashAlgo = htSHA256,
@ -293,9 +290,6 @@ void canonicalisePathMetaData(const Path & path, bool recurse);
MakeError(PathInUse, Error); MakeError(PathInUse, Error);
/* Whether we are in build users mode. */
bool haveBuildUsers();
/* Whether we are root. */ /* Whether we are root. */
bool amPrivileged(); bool amPrivileged();

View file

@ -55,45 +55,95 @@ void queryMissing(StoreAPI & store, const PathSet & targets,
PathSet todo(targets.begin(), targets.end()), done; PathSet todo(targets.begin(), targets.end()), done;
while (!todo.empty()) { /* Getting substitute info has high latency when using the binary
Path p = *(todo.begin()); cache substituter. Thus it's essential to do substitute
todo.erase(p); queries in parallel as much as possible. To accomplish this
if (done.find(p) != done.end()) continue; we do the following:
done.insert(p);
if (isDerivation(p)) { - For all paths still to be processed (todo), we add all
if (!store.isValidPath(p)) { paths for which we need info to the set query. For an
unknown.insert(p); unbuilt derivation this is the output paths; otherwise, it's
continue; the path itself.
- We get info about all paths in query in parallel.
- We process the results and add new items to todo if
necessary. E.g. if a path is substitutable, then we need to
get info on its references.
- Repeat until todo is empty.
*/
while (!todo.empty()) {
PathSet query, todoDrv, todoNonDrv;
foreach (PathSet::iterator, i, todo) {
if (done.find(*i) != done.end()) continue;
done.insert(*i);
if (isDerivation(*i)) {
if (!store.isValidPath(*i)) {
// FIXME: we could try to substitute p.
unknown.insert(*i);
continue;
}
Derivation drv = derivationFromPath(store, *i);
PathSet invalid;
foreach (DerivationOutputs::iterator, j, drv.outputs)
if (!store.isValidPath(j->second.path)) invalid.insert(j->second.path);
if (invalid.empty()) continue;
todoDrv.insert(*i);
if (settings.useSubstitutes) query.insert(invalid.begin(), invalid.end());
} }
Derivation drv = derivationFromPath(store, p);
else {
if (store.isValidPath(*i)) continue;
query.insert(*i);
todoNonDrv.insert(*i);
}
}
todo.clear();
SubstitutablePathInfos infos;
store.querySubstitutablePathInfos(query, infos);
foreach (PathSet::iterator, i, todoDrv) {
// FIXME: cache this
Derivation drv = derivationFromPath(store, *i);
bool mustBuild = false; bool mustBuild = false;
foreach (DerivationOutputs::iterator, i, drv.outputs) if (settings.useSubstitutes) {
if (!store.isValidPath(i->second.path) && foreach (DerivationOutputs::iterator, j, drv.outputs)
!(queryBoolSetting("build-use-substitutes", true) && store.hasSubstitutes(i->second.path))) if (!store.isValidPath(j->second.path) &&
mustBuild = true; infos.find(j->second.path) == infos.end())
mustBuild = true;
} else
mustBuild = true;
if (mustBuild) { if (mustBuild) {
willBuild.insert(p); willBuild.insert(*i);
todo.insert(drv.inputSrcs.begin(), drv.inputSrcs.end()); todo.insert(drv.inputSrcs.begin(), drv.inputSrcs.end());
foreach (DerivationInputs::iterator, i, drv.inputDrvs) foreach (DerivationInputs::iterator, i, drv.inputDrvs)
todo.insert(i->first); todo.insert(i->first);
} else } else
foreach (DerivationOutputs::iterator, i, drv.outputs) foreach (DerivationOutputs::iterator, i, drv.outputs)
todo.insert(i->second.path); todoNonDrv.insert(i->second.path);
} }
else { foreach (PathSet::iterator, i, todoNonDrv) {
if (store.isValidPath(p)) continue; done.insert(*i);
SubstitutablePathInfo info; SubstitutablePathInfos::iterator info = infos.find(*i);
if (store.querySubstitutablePathInfo(p, info)) { if (info != infos.end()) {
willSubstitute.insert(p); willSubstitute.insert(*i);
downloadSize += info.downloadSize; downloadSize += info->second.downloadSize;
narSize += info.narSize; narSize += info->second.narSize;
todo.insert(info.references.begin(), info.references.end()); todo.insert(info->second.references.begin(), info->second.references.end());
} else } else
unknown.insert(p); unknown.insert(*i);
} }
} }
} }

View file

@ -19,7 +19,7 @@ static void makeWritable(const Path & path)
{ {
struct stat st; struct stat st;
if (lstat(path.c_str(), &st)) if (lstat(path.c_str(), &st))
throw SysError(format("getting attributes of path `%1%'") % path); throw SysError(format("getting attributes of path `%1%'") % path);
if (S_ISDIR(st.st_mode) || S_ISREG(st.st_mode)) makeMutable(path); if (S_ISDIR(st.st_mode) || S_ISREG(st.st_mode)) makeMutable(path);
if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1) if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
throw SysError(format("changing writability of `%1%'") % path); throw SysError(format("changing writability of `%1%'") % path);
@ -57,12 +57,12 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
struct stat st; struct stat st;
if (lstat(path.c_str(), &st)) if (lstat(path.c_str(), &st))
throw SysError(format("getting attributes of path `%1%'") % path); throw SysError(format("getting attributes of path `%1%'") % path);
if (S_ISDIR(st.st_mode)) { if (S_ISDIR(st.st_mode)) {
Strings names = readDirectory(path); Strings names = readDirectory(path);
foreach (Strings::iterator, i, names) foreach (Strings::iterator, i, names)
optimisePath_(stats, path + "/" + *i); optimisePath_(stats, path + "/" + *i);
return; return;
} }
@ -113,7 +113,7 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
current file with a hard link to that file. */ current file with a hard link to that file. */
struct stat stLink; struct stat stLink;
if (lstat(linkPath.c_str(), &stLink)) if (lstat(linkPath.c_str(), &stLink))
throw SysError(format("getting attributes of path `%1%'") % linkPath); throw SysError(format("getting attributes of path `%1%'") % linkPath);
stats.sameContents++; stats.sameContents++;
if (st.st_ino == stLink.st_ino) { if (st.st_ino == stLink.st_ino) {
@ -149,7 +149,7 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
MakeImmutable mk1(linkPath); MakeImmutable mk1(linkPath);
Path tempLink = (format("%1%/.tmp-link-%2%-%3%") Path tempLink = (format("%1%/.tmp-link-%2%-%3%")
% nixStore % getpid() % rand()).str(); % settings.nixStore % getpid() % rand()).str();
if (link(linkPath.c_str(), tempLink.c_str()) == -1) { if (link(linkPath.c_str(), tempLink.c_str()) == -1) {
if (errno == EMLINK) { if (errno == EMLINK) {
@ -194,7 +194,7 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
void LocalStore::optimiseStore(OptimiseStats & stats) void LocalStore::optimiseStore(OptimiseStats & stats)
{ {
PathSet paths = queryValidPaths(); PathSet paths = queryAllValidPaths();
foreach (PathSet::iterator, i, paths) { foreach (PathSet::iterator, i, paths) {
addTempRoot(*i); addTempRoot(*i);
@ -207,10 +207,8 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
void LocalStore::optimisePath(const Path & path) void LocalStore::optimisePath(const Path & path)
{ {
if (queryBoolSetting("auto-optimise-store", true)) { OptimiseStats stats;
OptimiseStats stats; if (settings.autoOptimiseStore) optimisePath_(stats, path);
optimisePath_(stats, path);
}
} }

View file

@ -100,7 +100,7 @@ void RemoteStore::forkSlave()
/* Start the worker. */ /* Start the worker. */
Path worker = getEnv("NIX_WORKER"); Path worker = getEnv("NIX_WORKER");
if (worker == "") if (worker == "")
worker = nixBinDir + "/nix-worker"; worker = settings.nixBinDir + "/nix-worker";
child = fork(); child = fork();
@ -142,7 +142,7 @@ void RemoteStore::connectToDaemon()
if (fdSocket == -1) if (fdSocket == -1)
throw SysError("cannot create Unix domain socket"); throw SysError("cannot create Unix domain socket");
string socketPath = nixStateDir + DEFAULT_SOCKET_PATH; string socketPath = settings.nixStateDir + DEFAULT_SOCKET_PATH;
/* Urgh, sockaddr_un allows path names of only 108 characters. So /* Urgh, sockaddr_un allows path names of only 108 characters. So
chdir to the socket directory so that we can pass a relative chdir to the socket directory so that we can pass a relative
@ -184,23 +184,33 @@ RemoteStore::~RemoteStore()
void RemoteStore::setOptions() void RemoteStore::setOptions()
{ {
writeInt(wopSetOptions, to); writeInt(wopSetOptions, to);
writeInt(keepFailed, to);
writeInt(keepGoing, to); writeInt(settings.keepFailed, to);
writeInt(tryFallback, to); writeInt(settings.keepGoing, to);
writeInt(settings.tryFallback, to);
writeInt(verbosity, to); writeInt(verbosity, to);
writeInt(maxBuildJobs, to); writeInt(settings.maxBuildJobs, to);
writeInt(maxSilentTime, to); writeInt(settings.maxSilentTime, to);
if (GET_PROTOCOL_MINOR(daemonVersion) >= 2) if (GET_PROTOCOL_MINOR(daemonVersion) >= 2)
writeInt(useBuildHook, to); writeInt(settings.useBuildHook, to);
if (GET_PROTOCOL_MINOR(daemonVersion) >= 4) { if (GET_PROTOCOL_MINOR(daemonVersion) >= 4) {
writeInt(buildVerbosity, to); writeInt(settings.buildVerbosity, to);
writeInt(logType, to); writeInt(logType, to);
writeInt(printBuildTrace, to); writeInt(settings.printBuildTrace, to);
} }
if (GET_PROTOCOL_MINOR(daemonVersion) >= 6) if (GET_PROTOCOL_MINOR(daemonVersion) >= 6)
writeInt(buildCores, to); writeInt(settings.buildCores, to);
if (GET_PROTOCOL_MINOR(daemonVersion) >= 10) if (GET_PROTOCOL_MINOR(daemonVersion) >= 10)
writeInt(queryBoolSetting("build-use-substitutes", true), to); writeInt(settings.useSubstitutes, to);
if (GET_PROTOCOL_MINOR(daemonVersion) >= 12) {
Settings::SettingsMap overrides = settings.getOverrides();
writeInt(overrides.size(), to);
foreach (Settings::SettingsMap::iterator, i, overrides) {
writeString(i->first, to);
writeString(i->second, to);
}
}
processStderr(); processStderr();
} }
@ -217,42 +227,96 @@ bool RemoteStore::isValidPath(const Path & path)
} }
PathSet RemoteStore::queryValidPaths() PathSet RemoteStore::queryValidPaths(const PathSet & paths)
{ {
openConnection(); openConnection();
writeInt(wopQueryValidPaths, to); if (GET_PROTOCOL_MINOR(daemonVersion) < 12) {
PathSet res;
foreach (PathSet::const_iterator, i, paths)
if (isValidPath(*i)) res.insert(*i);
return res;
} else {
writeInt(wopQueryValidPaths, to);
writeStrings(paths, to);
processStderr();
return readStorePaths<PathSet>(from);
}
}
PathSet RemoteStore::queryAllValidPaths()
{
openConnection();
writeInt(wopQueryAllValidPaths, to);
processStderr(); processStderr();
return readStorePaths<PathSet>(from); return readStorePaths<PathSet>(from);
} }
bool RemoteStore::hasSubstitutes(const Path & path) PathSet RemoteStore::querySubstitutablePaths(const PathSet & paths)
{ {
openConnection(); openConnection();
writeInt(wopHasSubstitutes, to); if (GET_PROTOCOL_MINOR(daemonVersion) < 12) {
writeString(path, to); PathSet res;
processStderr(); foreach (PathSet::const_iterator, i, paths) {
unsigned int reply = readInt(from); writeInt(wopHasSubstitutes, to);
return reply != 0; writeString(*i, to);
processStderr();
if (readInt(from)) res.insert(*i);
}
return res;
} else {
writeInt(wopQuerySubstitutablePaths, to);
writeStrings(paths, to);
processStderr();
return readStorePaths<PathSet>(from);
}
} }
bool RemoteStore::querySubstitutablePathInfo(const Path & path, void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
SubstitutablePathInfo & info) SubstitutablePathInfos & infos)
{ {
if (paths.empty()) return;
openConnection(); openConnection();
if (GET_PROTOCOL_MINOR(daemonVersion) < 3) return false;
writeInt(wopQuerySubstitutablePathInfo, to); if (GET_PROTOCOL_MINOR(daemonVersion) < 3) return;
writeString(path, to);
processStderr(); if (GET_PROTOCOL_MINOR(daemonVersion) < 12) {
unsigned int reply = readInt(from);
if (reply == 0) return false; foreach (PathSet::const_iterator, i, paths) {
info.deriver = readString(from); SubstitutablePathInfo info;
if (info.deriver != "") assertStorePath(info.deriver); writeInt(wopQuerySubstitutablePathInfo, to);
info.references = readStorePaths<PathSet>(from); writeString(*i, to);
info.downloadSize = readLongLong(from); processStderr();
info.narSize = GET_PROTOCOL_MINOR(daemonVersion) >= 7 ? readLongLong(from) : 0; unsigned int reply = readInt(from);
return true; if (reply == 0) continue;
info.deriver = readString(from);
if (info.deriver != "") assertStorePath(info.deriver);
info.references = readStorePaths<PathSet>(from);
info.downloadSize = readLongLong(from);
info.narSize = GET_PROTOCOL_MINOR(daemonVersion) >= 7 ? readLongLong(from) : 0;
infos[*i] = info;
}
} else {
writeInt(wopQuerySubstitutablePathInfos, to);
writeStrings(paths, to);
processStderr();
unsigned int count = readInt(from);
for (unsigned int n = 0; n < count; n++) {
Path path = readStorePath(from);
SubstitutablePathInfo & info(infos[path]);
info.deriver = readString(from);
if (info.deriver != "") assertStorePath(info.deriver);
info.references = readStorePaths<PathSet>(from);
info.downloadSize = readLongLong(from);
info.narSize = readLongLong(from);
}
}
} }

View file

@ -26,7 +26,9 @@ public:
bool isValidPath(const Path & path); bool isValidPath(const Path & path);
PathSet queryValidPaths(); PathSet queryValidPaths(const PathSet & paths);
PathSet queryAllValidPaths();
ValidPathInfo queryPathInfo(const Path & path); ValidPathInfo queryPathInfo(const Path & path);
@ -44,10 +46,10 @@ public:
Path queryPathFromHashPart(const string & hashPart); Path queryPathFromHashPart(const string & hashPart);
bool hasSubstitutes(const Path & path); PathSet querySubstitutablePaths(const PathSet & paths);
bool querySubstitutablePathInfo(const Path & path, void querySubstitutablePathInfos(const PathSet & paths,
SubstitutablePathInfo & info); SubstitutablePathInfos & infos);
Path addToStore(const Path & srcPath, Path addToStore(const Path & srcPath,
bool recursive = true, HashType hashAlgo = htSHA256, bool recursive = true, HashType hashAlgo = htSHA256,

View file

@ -19,16 +19,16 @@ GCOptions::GCOptions()
bool isInStore(const Path & path) bool isInStore(const Path & path)
{ {
return path[0] == '/' return path[0] == '/'
&& string(path, 0, nixStore.size()) == nixStore && string(path, 0, settings.nixStore.size()) == settings.nixStore
&& path.size() >= nixStore.size() + 2 && path.size() >= settings.nixStore.size() + 2
&& path[nixStore.size()] == '/'; && path[settings.nixStore.size()] == '/';
} }
bool isStorePath(const Path & path) bool isStorePath(const Path & path)
{ {
return isInStore(path) return isInStore(path)
&& path.find('/', nixStore.size() + 1) == Path::npos; && path.find('/', settings.nixStore.size() + 1) == Path::npos;
} }
@ -43,7 +43,7 @@ Path toStorePath(const Path & path)
{ {
if (!isInStore(path)) if (!isInStore(path))
throw Error(format("path `%1%' is not in the Nix store") % path); throw Error(format("path `%1%' is not in the Nix store") % path);
Path::size_type slash = path.find('/', nixStore.size() + 1); Path::size_type slash = path.find('/', settings.nixStore.size() + 1);
if (slash == Path::npos) if (slash == Path::npos)
return path; return path;
else else
@ -74,7 +74,7 @@ Path followLinksToStorePath(const Path & path)
string storePathToName(const Path & path) string storePathToName(const Path & path)
{ {
assertStorePath(path); assertStorePath(path);
return string(path, nixStore.size() + 34); return string(path, settings.nixStore.size() + 34);
} }
@ -173,11 +173,11 @@ Path makeStorePath(const string & type,
{ {
/* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
string s = type + ":sha256:" + printHash(hash) + ":" string s = type + ":sha256:" + printHash(hash) + ":"
+ nixStore + ":" + name; + settings.nixStore + ":" + name;
checkStoreName(name); checkStoreName(name);
return nixStore + "/" return settings.nixStore + "/"
+ printHash32(compressHash(hashString(htSHA256, s), 20)) + printHash32(compressHash(hashString(htSHA256, s), 20))
+ "-" + name; + "-" + name;
} }

View file

@ -80,6 +80,8 @@ struct SubstitutablePathInfo
unsigned long long narSize; /* 0 = unknown */ unsigned long long narSize; /* 0 = unknown */
}; };
typedef std::map<Path, SubstitutablePathInfo> SubstitutablePathInfos;
struct ValidPathInfo struct ValidPathInfo
{ {
@ -102,20 +104,23 @@ public:
virtual ~StoreAPI() { } virtual ~StoreAPI() { }
/* Checks whether a path is valid. */ /* Check whether a path is valid. */
virtual bool isValidPath(const Path & path) = 0; virtual bool isValidPath(const Path & path) = 0;
/* Query the set of valid paths. */ /* Query which of the given paths is valid. */
virtual PathSet queryValidPaths() = 0; virtual PathSet queryValidPaths(const PathSet & paths) = 0;
/* Query the set of all valid paths. */
virtual PathSet queryAllValidPaths() = 0;
/* Query information about a valid path. */ /* Query information about a valid path. */
virtual ValidPathInfo queryPathInfo(const Path & path) = 0; virtual ValidPathInfo queryPathInfo(const Path & path) = 0;
/* Queries the hash of a valid path. */ /* Query the hash of a valid path. */
virtual Hash queryPathHash(const Path & path) = 0; virtual Hash queryPathHash(const Path & path) = 0;
/* Queries the set of outgoing FS references for a store path. /* Query the set of outgoing FS references for a store path. The
The result is not cleared. */ result is not cleared. */
virtual void queryReferences(const Path & path, virtual void queryReferences(const Path & path,
PathSet & references) = 0; PathSet & references) = 0;
@ -138,13 +143,14 @@ public:
path, or "" if the path doesn't exist. */ path, or "" if the path doesn't exist. */
virtual Path queryPathFromHashPart(const string & hashPart) = 0; virtual Path queryPathFromHashPart(const string & hashPart) = 0;
/* Query whether a path has substitutes. */ /* Query which of the given paths have substitutes. */
virtual bool hasSubstitutes(const Path & path) = 0; virtual PathSet querySubstitutablePaths(const PathSet & paths) = 0;
/* Query the references, deriver and download size of a /* Query substitute info (i.e. references, derivers and download
substitutable path. */ sizes) of a set of paths. If a path does not have substitute
virtual bool querySubstitutablePathInfo(const Path & path, info, it's omitted from the resulting infos map. */
SubstitutablePathInfo & info) = 0; virtual void querySubstitutablePathInfos(const PathSet & paths,
SubstitutablePathInfos & infos) = 0;
/* Copy the contents of a path to the store and register the /* Copy the contents of a path to the store and register the
validity the resulting path. The resulting path is returned. validity the resulting path. The resulting path is returned.

View file

@ -6,7 +6,7 @@ namespace nix {
#define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_1 0x6e697863
#define WORKER_MAGIC_2 0x6478696f #define WORKER_MAGIC_2 0x6478696f
#define PROTOCOL_VERSION 0x10b #define PROTOCOL_VERSION 0x10c
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
@ -32,13 +32,16 @@ typedef enum {
wopCollectGarbage = 20, wopCollectGarbage = 20,
wopQuerySubstitutablePathInfo = 21, wopQuerySubstitutablePathInfo = 21,
wopQueryDerivationOutputs = 22, wopQueryDerivationOutputs = 22,
wopQueryValidPaths = 23, wopQueryAllValidPaths = 23,
wopQueryFailedPaths = 24, wopQueryFailedPaths = 24,
wopClearFailedPaths = 25, wopClearFailedPaths = 25,
wopQueryPathInfo = 26, wopQueryPathInfo = 26,
wopImportPaths = 27, wopImportPaths = 27,
wopQueryDerivationOutputNames = 28, wopQueryDerivationOutputNames = 28,
wopQueryPathFromHashPart = 29, wopQueryPathFromHashPart = 29,
wopQuerySubstitutablePathInfos = 30,
wopQueryValidPaths = 31,
wopQuerySubstitutablePaths = 32,
} WorkerOp; } WorkerOp;

View file

@ -253,7 +253,7 @@ string readLine(int fd)
if (errno != EINTR) if (errno != EINTR)
throw SysError("reading a line"); throw SysError("reading a line");
} else if (rd == 0) } else if (rd == 0)
throw Error("unexpected EOF reading a line"); throw EndOfFile("unexpected EOF reading a line");
else { else {
if (ch == '\n') return s; if (ch == '\n') return s;
s += ch; s += ch;
@ -1010,6 +1010,13 @@ string concatStringsSep(const string & sep, const Strings & ss)
} }
string chomp(const string & s)
{
size_t i = s.find_last_not_of(" \n\r\t");
return i == string::npos ? "" : string(s, 0, i + 1);
}
string statusToString(int status) string statusToString(int status)
{ {
if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {

View file

@ -294,6 +294,10 @@ Strings tokenizeString(const string & s, const string & separators = " \t\n\r");
string concatStringsSep(const string & sep, const Strings & ss); string concatStringsSep(const string & sep, const Strings & ss);
/* Remove trailing whitespace from a string. */
string chomp(const string & s);
/* Convert the exit status of a child as returned by wait() into an /* Convert the exit status of a child as returned by wait() into an
error string. */ error string. */
string statusToString(int status); string statusToString(int status);

View file

@ -55,7 +55,6 @@ struct Globals
EvalState state; EvalState state;
bool dryRun; bool dryRun;
bool preserveInstalled; bool preserveInstalled;
bool keepDerivations;
string forceName; string forceName;
bool prebuiltOnly; bool prebuiltOnly;
}; };
@ -113,6 +112,11 @@ static void getAllExprs(EvalState & state,
StringSet namesSorted(names.begin(), names.end()); StringSet namesSorted(names.begin(), names.end());
foreach (StringSet::iterator, i, namesSorted) { foreach (StringSet::iterator, i, namesSorted) {
/* Ignore the manifest.nix used by profiles. This is
necessary to prevent it from showing up in channels (which
are implemented using profiles). */
if (*i == "manifest.nix") continue;
Path path2 = path + "/" + *i; Path path2 = path + "/" + *i;
struct stat st; struct stat st;
@ -211,9 +215,12 @@ static int comparePriorities(EvalState & state,
static bool isPrebuilt(EvalState & state, const DrvInfo & elem) static bool isPrebuilt(EvalState & state, const DrvInfo & elem)
{ {
assert(false);
#if 0
return return
store->isValidPath(elem.queryOutPath(state)) || store->isValidPath(elem.queryOutPath(state)) ||
store->hasSubstitutes(elem.queryOutPath(state)); store->hasSubstitutes(elem.queryOutPath(state));
#endif
} }
@ -263,8 +270,8 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems,
if (k != newest.end()) { if (k != newest.end()) {
d = j->first.system == k->second.first.system ? 0 : d = j->first.system == k->second.first.system ? 0 :
j->first.system == thisSystem ? 1 : j->first.system == settings.thisSystem ? 1 :
k->second.first.system == thisSystem ? -1 : 0; k->second.first.system == settings.thisSystem ? -1 : 0;
if (d == 0) if (d == 0)
d = comparePriorities(state, j->first, k->second.first); d = comparePriorities(state, j->first, k->second.first);
if (d == 0) if (d == 0)
@ -495,7 +502,7 @@ static void installDerivations(Globals & globals,
if (globals.dryRun) return; if (globals.dryRun) return;
if (createUserEnv(globals.state, allElems, if (createUserEnv(globals.state, allElems,
profile, globals.keepDerivations, lockToken)) break; profile, settings.envKeepDerivations, lockToken)) break;
} }
} }
@ -602,7 +609,7 @@ static void upgradeDerivations(Globals & globals,
if (globals.dryRun) return; if (globals.dryRun) return;
if (createUserEnv(globals.state, newElems, if (createUserEnv(globals.state, newElems,
globals.profile, globals.keepDerivations, lockToken)) break; globals.profile, settings.envKeepDerivations, lockToken)) break;
} }
} }
@ -669,7 +676,7 @@ static void opSetFlag(Globals & globals,
/* Write the new user environment. */ /* Write the new user environment. */
if (createUserEnv(globals.state, installedElems, if (createUserEnv(globals.state, installedElems,
globals.profile, globals.keepDerivations, lockToken)) break; globals.profile, settings.envKeepDerivations, lockToken)) break;
} }
} }
@ -737,7 +744,7 @@ static void uninstallDerivations(Globals & globals, Strings & selectors,
if (globals.dryRun) return; if (globals.dryRun) return;
if (createUserEnv(globals.state, newElems, if (createUserEnv(globals.state, newElems,
profile, globals.keepDerivations, lockToken)) break; profile, settings.envKeepDerivations, lockToken)) break;
} }
} }
@ -866,7 +873,7 @@ static void opQuery(Globals & globals,
enum { sInstalled, sAvailable } source = sInstalled; enum { sInstalled, sAvailable } source = sInstalled;
readOnlyMode = true; /* makes evaluation a bit faster */ settings.readOnlyMode = true; /* makes evaluation a bit faster */
for (Strings::iterator i = args.begin(); i != args.end(); ) { for (Strings::iterator i = args.begin(); i != args.end(); ) {
string arg = *i++; string arg = *i++;
@ -930,6 +937,22 @@ static void opQuery(Globals & globals,
} }
/* Query which paths have substitutes. */
PathSet validPaths, substitutablePaths;
if (printStatus) {
PathSet paths;
foreach (vector<DrvInfo>::iterator, i, elems2)
try {
paths.insert(i->queryOutPath(globals.state));
} catch (AssertionError & e) {
printMsg(lvlTalkative, format("skipping derivation named `%1%' which gives an assertion failure") % i->name);
i->setFailed();
}
validPaths = store->queryValidPaths(paths);
substitutablePaths = store->querySubstitutablePaths(paths);
}
/* Print the desired columns, or XML output. */ /* Print the desired columns, or XML output. */
Table table; Table table;
std::ostringstream dummy; std::ostringstream dummy;
@ -938,6 +961,8 @@ static void opQuery(Globals & globals,
foreach (vector<DrvInfo>::iterator, i, elems2) { foreach (vector<DrvInfo>::iterator, i, elems2) {
try { try {
if (i->hasFailed()) continue;
startNest(nest, lvlDebug, format("outputting query result `%1%'") % i->attrPath); startNest(nest, lvlDebug, format("outputting query result `%1%'") % i->attrPath);
if (globals.prebuiltOnly && !isPrebuilt(globals.state, *i)) continue; if (globals.prebuiltOnly && !isPrebuilt(globals.state, *i)) continue;
@ -949,9 +974,10 @@ static void opQuery(Globals & globals,
XMLAttrs attrs; XMLAttrs attrs;
if (printStatus) { if (printStatus) {
bool hasSubs = store->hasSubstitutes(i->queryOutPath(globals.state)); Path outPath = i->queryOutPath(globals.state);
bool isInstalled = installed.find(i->queryOutPath(globals.state)) != installed.end(); bool hasSubs = substitutablePaths.find(outPath) != substitutablePaths.end();
bool isValid = store->isValidPath(i->queryOutPath(globals.state)); bool isInstalled = installed.find(outPath) != installed.end();
bool isValid = validPaths.find(outPath) != validPaths.end();
if (xmlOutput) { if (xmlOutput) {
attrs["installed"] = isInstalled ? "1" : "0"; attrs["installed"] = isInstalled ? "1" : "0";
attrs["valid"] = isValid ? "1" : "0"; attrs["valid"] = isValid ? "1" : "0";
@ -1240,9 +1266,6 @@ void run(Strings args)
globals.preserveInstalled = false; globals.preserveInstalled = false;
globals.prebuiltOnly = false; globals.prebuiltOnly = false;
globals.keepDerivations =
queryBoolSetting("env-keep-derivations", false);
for (Strings::iterator i = args.begin(); i != args.end(); ) { for (Strings::iterator i = args.begin(); i != args.end(); ) {
string arg = *i++; string arg = *i++;
@ -1309,7 +1332,7 @@ void run(Strings args)
Path profileLink = getHomeDir() + "/.nix-profile"; Path profileLink = getHomeDir() + "/.nix-profile";
globals.profile = pathExists(profileLink) globals.profile = pathExists(profileLink)
? absPath(readLink(profileLink), dirOf(profileLink)) ? absPath(readLink(profileLink), dirOf(profileLink))
: canonPath(nixStateDir + "/profiles/default"); : canonPath(settings.nixStateDir + "/profiles/default");
} }
store = openStore(); store = openStore();

View file

@ -96,11 +96,11 @@ void run(Strings args)
if (arg == "-") if (arg == "-")
readStdin = true; readStdin = true;
else if (arg == "--eval-only") { else if (arg == "--eval-only") {
readOnlyMode = true; settings.readOnlyMode = true;
evalOnly = true; evalOnly = true;
} }
else if (arg == "--parse-only") { else if (arg == "--parse-only") {
readOnlyMode = true; settings.readOnlyMode = true;
parseOnly = evalOnly = true; parseOnly = evalOnly = true;
} }
else if (arg == "--find-file") else if (arg == "--find-file")

View file

@ -432,7 +432,7 @@ static void opReadLog(Strings opFlags, Strings opArgs)
Path path = useDeriver(followLinksToStorePath(*i)); Path path = useDeriver(followLinksToStorePath(*i));
Path logPath = (format("%1%/%2%/%3%") % Path logPath = (format("%1%/%2%/%3%") %
nixLogDir % drvsLogDir % baseNameOf(path)).str(); settings.nixLogDir % drvsLogDir % baseNameOf(path)).str();
Path logBz2Path = logPath + ".bz2"; Path logBz2Path = logPath + ".bz2";
if (pathExists(logPath)) { if (pathExists(logPath)) {
@ -469,7 +469,7 @@ static void opDumpDB(Strings opFlags, Strings opArgs)
if (!opFlags.empty()) throw UsageError("unknown flag"); if (!opFlags.empty()) throw UsageError("unknown flag");
if (!opArgs.empty()) if (!opArgs.empty())
throw UsageError("no arguments expected"); throw UsageError("no arguments expected");
PathSet validPaths = store->queryValidPaths(); PathSet validPaths = store->queryAllValidPaths();
foreach (PathSet::iterator, i, validPaths) foreach (PathSet::iterator, i, validPaths)
cout << store->makeValidityRegistration(singleton<PathSet>(*i), true, true); cout << store->makeValidityRegistration(singleton<PathSet>(*i), true, true);
} }

View file

@ -297,12 +297,30 @@ static void performOp(unsigned int clientVersion,
break; break;
} }
case wopQueryValidPaths: {
PathSet paths = readStorePaths<PathSet>(from);
startWork();
PathSet res = store->queryValidPaths(paths);
stopWork();
writeStrings(res, to);
break;
}
case wopHasSubstitutes: { case wopHasSubstitutes: {
Path path = readStorePath(from); Path path = readStorePath(from);
startWork(); startWork();
bool result = store->hasSubstitutes(path); PathSet res = store->querySubstitutablePaths(singleton<PathSet>(path));
stopWork(); stopWork();
writeInt(result, to); writeInt(res.find(path) != res.end(), to);
break;
}
case wopQuerySubstitutablePaths: {
PathSet paths = readStorePaths<PathSet>(from);
startWork();
PathSet res = store->querySubstitutablePaths(paths);
stopWork();
writeStrings(res, to);
break; break;
} }
@ -509,26 +527,30 @@ static void performOp(unsigned int clientVersion,
} }
case wopSetOptions: { case wopSetOptions: {
keepFailed = readInt(from) != 0; settings.keepFailed = readInt(from) != 0;
keepGoing = readInt(from) != 0; settings.keepGoing = readInt(from) != 0;
tryFallback = readInt(from) != 0; settings.tryFallback = readInt(from) != 0;
verbosity = (Verbosity) readInt(from); verbosity = (Verbosity) readInt(from);
maxBuildJobs = readInt(from); settings.maxBuildJobs = readInt(from);
maxSilentTime = readInt(from); settings.maxSilentTime = readInt(from);
if (GET_PROTOCOL_MINOR(clientVersion) >= 2) if (GET_PROTOCOL_MINOR(clientVersion) >= 2)
useBuildHook = readInt(from) != 0; settings.useBuildHook = readInt(from) != 0;
if (GET_PROTOCOL_MINOR(clientVersion) >= 4) { if (GET_PROTOCOL_MINOR(clientVersion) >= 4) {
buildVerbosity = (Verbosity) readInt(from); settings.buildVerbosity = (Verbosity) readInt(from);
logType = (LogType) readInt(from); logType = (LogType) readInt(from);
printBuildTrace = readInt(from) != 0; settings.printBuildTrace = readInt(from) != 0;
} }
if (GET_PROTOCOL_MINOR(clientVersion) >= 6) if (GET_PROTOCOL_MINOR(clientVersion) >= 6)
buildCores = readInt(from); settings.buildCores = readInt(from);
if (GET_PROTOCOL_MINOR(clientVersion) >= 10) { if (GET_PROTOCOL_MINOR(clientVersion) >= 10)
int x = readInt(from); settings.useSubstitutes = readInt(from) != 0;
Strings ss; if (GET_PROTOCOL_MINOR(clientVersion) >= 12) {
ss.push_back(x == 0 ? "false" : "true"); unsigned int n = readInt(from);
overrideSetting("build-use-substitutes", ss); for (unsigned int i = 0; i < n; i++) {
string name = readString(from);
string value = readString(from);
settings.set("untrusted-" + name, value);
}
} }
startWork(); startWork();
stopWork(); stopWork();
@ -538,23 +560,43 @@ static void performOp(unsigned int clientVersion,
case wopQuerySubstitutablePathInfo: { case wopQuerySubstitutablePathInfo: {
Path path = absPath(readString(from)); Path path = absPath(readString(from));
startWork(); startWork();
SubstitutablePathInfo info; SubstitutablePathInfos infos;
bool res = store->querySubstitutablePathInfo(path, info); store->querySubstitutablePathInfos(singleton<PathSet>(path), infos);
stopWork(); stopWork();
writeInt(res ? 1 : 0, to); SubstitutablePathInfos::iterator i = infos.find(path);
if (res) { if (i == infos.end())
writeString(info.deriver, to); writeInt(0, to);
writeStrings(info.references, to); else {
writeLongLong(info.downloadSize, to); writeInt(1, to);
writeString(i->second.deriver, to);
writeStrings(i->second.references, to);
writeLongLong(i->second.downloadSize, to);
if (GET_PROTOCOL_MINOR(clientVersion) >= 7) if (GET_PROTOCOL_MINOR(clientVersion) >= 7)
writeLongLong(info.narSize, to); writeLongLong(i->second.narSize, to);
} }
break; break;
} }
case wopQueryValidPaths: { case wopQuerySubstitutablePathInfos: {
PathSet paths = readStorePaths<PathSet>(from);
startWork(); startWork();
PathSet paths = store->queryValidPaths(); SubstitutablePathInfos infos;
store->querySubstitutablePathInfos(paths, infos);
stopWork();
writeInt(infos.size(), to);
foreach (SubstitutablePathInfos::iterator, i, infos) {
writeString(i->first, to);
writeString(i->second.deriver, to);
writeStrings(i->second.references, to);
writeLongLong(i->second.downloadSize, to);
writeLongLong(i->second.narSize, to);
}
break;
}
case wopQueryAllValidPaths: {
startWork();
PathSet paths = store->queryAllValidPaths();
stopWork(); stopWork();
writeStrings(paths, to); writeStrings(paths, to);
break; break;
@ -730,7 +772,7 @@ static void daemonLoop()
if (fdSocket == -1) if (fdSocket == -1)
throw SysError("cannot create Unix domain socket"); throw SysError("cannot create Unix domain socket");
string socketPath = nixStateDir + DEFAULT_SOCKET_PATH; string socketPath = settings.nixStateDir + DEFAULT_SOCKET_PATH;
createDirs(dirOf(socketPath)); createDirs(dirOf(socketPath));
@ -781,10 +823,10 @@ static void daemonLoop()
(struct sockaddr *) &remoteAddr, &remoteAddrLen); (struct sockaddr *) &remoteAddr, &remoteAddrLen);
checkInterrupt(); checkInterrupt();
if (remote == -1) { if (remote == -1) {
if (errno == EINTR) if (errno == EINTR)
continue; continue;
else else
throw SysError("accepting connection"); throw SysError("accepting connection");
} }
closeOnExec(remote); closeOnExec(remote);
@ -829,10 +871,6 @@ static void daemonLoop()
strncpy(argvSaved[1], processName.c_str(), strlen(argvSaved[1])); strncpy(argvSaved[1], processName.c_str(), strlen(argvSaved[1]));
} }
/* Since the daemon can be long-running, the
settings may have changed. So force a reload. */
reloadSettings();
/* Handle the connection. */ /* Handle the connection. */
from.fd = remote; from.fd = remote;
to.fd = remote; to.fd = remote;

View file

@ -16,11 +16,13 @@
-e "s^@shell\@^$(bash)^g" \ -e "s^@shell\@^$(bash)^g" \
-e "s^@curl\@^$(curl)^g" \ -e "s^@curl\@^$(curl)^g" \
-e "s^@bzip2\@^$(bzip2)^g" \ -e "s^@bzip2\@^$(bzip2)^g" \
-e "s^@xz\@^$(xz)^g" \
-e "s^@perl\@^$(perl)^g" \ -e "s^@perl\@^$(perl)^g" \
-e "s^@perlFlags\@^$(perlFlags)^g" \ -e "s^@perlFlags\@^$(perlFlags)^g" \
-e "s^@coreutils\@^$(coreutils)^g" \ -e "s^@coreutils\@^$(coreutils)^g" \
-e "s^@sed\@^$(sed)^g" \ -e "s^@sed\@^$(sed)^g" \
-e "s^@tar\@^$(tar)^g" \ -e "s^@tar\@^$(tar)^g" \
-e "s^@tarFlags\@^$(tarFlags)^g" \
-e "s^@gzip\@^$(gzip)^g" \ -e "s^@gzip\@^$(gzip)^g" \
-e "s^@pv\@^$(pv)^g" \ -e "s^@pv\@^$(pv)^g" \
-e "s^@tr\@^$(tr)^g" \ -e "s^@tr\@^$(tr)^g" \

View file

@ -9,7 +9,8 @@ TESTS = init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \
gc-runtime.sh install-package.sh check-refs.sh filter-source.sh \ gc-runtime.sh install-package.sh check-refs.sh filter-source.sh \
remote-store.sh export.sh export-graph.sh negative-caching.sh \ remote-store.sh export.sh export-graph.sh negative-caching.sh \
binary-patching.sh timeout.sh secure-drv-outputs.sh nix-channel.sh \ binary-patching.sh timeout.sh secure-drv-outputs.sh nix-channel.sh \
multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \
binary-cache.sh
XFAIL_TESTS = XFAIL_TESTS =

35
tests/binary-cache.sh Normal file
View file

@ -0,0 +1,35 @@
source common.sh
clearStore
# Create the binary cache.
cacheDir=$TEST_ROOT/binary-cache
rm -rf $cacheDir
outPath=$(nix-build dependencies.nix --no-out-link)
nix-push --dest $cacheDir $outPath
# By default, a binary cache doesn't support "nix-env -qas", but does
# support installation.
clearStore
rm -f $NIX_STATE_DIR/binary-cache*
nix-env --option binary-caches "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "---"
nix-store --option binary-caches "file://$cacheDir" -r $outPath
# But with the right configuration, "nix-env -qas" should also work.
clearStore
rm -f $NIX_STATE_DIR/binary-cache*
echo "WantMassQuery: 1" >> $cacheDir/nix-cache-info
nix-env --option binary-caches "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "--S"
nix-store --option binary-caches "file://$cacheDir" -r $outPath
nix-store --check-validity $outPath
nix-store -qR $outPath | grep input-2

View file

@ -7,14 +7,17 @@ mkdir -p $TEST_ROOT/cache2 $TEST_ROOT/patches
RESULT=$TEST_ROOT/result RESULT=$TEST_ROOT/result
# Build version 1 and 2 of the "foo" package. # Build version 1 and 2 of the "foo" package.
nix-push --copy $TEST_ROOT/cache2 $TEST_ROOT/manifest1 \ nix-push --dest $TEST_ROOT/cache2 --manifest --bzip2 \
$(nix-build -o $RESULT binary-patching.nix --arg version 1) $(nix-build -o $RESULT binary-patching.nix --arg version 1)
mv $TEST_ROOT/cache2/MANIFEST $TEST_ROOT/manifest1
out2=$(nix-build -o $RESULT binary-patching.nix --arg version 2) out2=$(nix-build -o $RESULT binary-patching.nix --arg version 2)
nix-push --copy $TEST_ROOT/cache2 $TEST_ROOT/manifest2 $out2 nix-push --dest $TEST_ROOT/cache2 --manifest --bzip2 $out2
mv $TEST_ROOT/cache2/MANIFEST $TEST_ROOT/manifest2
out3=$(nix-build -o $RESULT binary-patching.nix --arg version 3) out3=$(nix-build -o $RESULT binary-patching.nix --arg version 3)
nix-push --copy $TEST_ROOT/cache2 $TEST_ROOT/manifest3 $out3 nix-push --dest $TEST_ROOT/cache2 --manifest --bzip2 $out3
mv $TEST_ROOT/cache2/MANIFEST $TEST_ROOT/manifest3
rm $RESULT rm $RESULT

View file

@ -16,6 +16,7 @@ export NIX_DB_DIR=$TEST_ROOT/db
export NIX_CONF_DIR=$TEST_ROOT/etc export NIX_CONF_DIR=$TEST_ROOT/etc
export NIX_MANIFESTS_DIR=$TEST_ROOT/var/nix/manifests export NIX_MANIFESTS_DIR=$TEST_ROOT/var/nix/manifests
export SHARED=$TEST_ROOT/shared export SHARED=$TEST_ROOT/shared
export NIX_REMOTE=$NIX_REMOTE_
export PATH=@bindir@:$PATH export PATH=@bindir@:$PATH
@ -79,3 +80,5 @@ fail() {
echo "$1" echo "$1"
exit 1 exit 1
} }
set -x

View file

@ -9,7 +9,7 @@ clearStore
clearProfiles clearProfiles
cat > $TEST_ROOT/foo.nixpkg <<EOF cat > $TEST_ROOT/foo.nixpkg <<EOF
NIXPKG1 file://$TEST_ROOT/manifest simple $system $drvPath $outPath NIXPKG1 file://$TEST_ROOT/cache/MANIFEST simple $system $drvPath $outPath
EOF EOF
nix-install-package --non-interactive -p $profiles/test $TEST_ROOT/foo.nixpkg nix-install-package --non-interactive -p $profiles/test $TEST_ROOT/foo.nixpkg

View file

@ -19,7 +19,7 @@ nix-channel --remove xyzzy
# Create a channel. # Create a channel.
rm -rf $TEST_ROOT/foo rm -rf $TEST_ROOT/foo
mkdir -p $TEST_ROOT/foo mkdir -p $TEST_ROOT/foo
nix-push --copy $TEST_ROOT/foo $TEST_ROOT/foo/MANIFEST $(nix-store -r $(nix-instantiate dependencies.nix)) nix-push --dest $TEST_ROOT/foo --manifest --bzip2 $(nix-store -r $(nix-instantiate dependencies.nix))
rm -rf $TEST_ROOT/nixexprs rm -rf $TEST_ROOT/nixexprs
mkdir -p $TEST_ROOT/nixexprs mkdir -p $TEST_ROOT/nixexprs
cp config.nix dependencies.nix dependencies.builder*.sh $TEST_ROOT/nixexprs/ cp config.nix dependencies.nix dependencies.builder*.sh $TEST_ROOT/nixexprs/

View file

@ -2,7 +2,7 @@ source common.sh
pullCache () { pullCache () {
echo "pulling cache..." echo "pulling cache..."
nix-pull file://$TEST_ROOT/manifest nix-pull file://$TEST_ROOT/cache/MANIFEST
} }
clearStore clearStore

View file

@ -1,5 +1,7 @@
source common.sh source common.sh
clearStore
drvPath=$(nix-instantiate dependencies.nix) drvPath=$(nix-instantiate dependencies.nix)
outPath=$(nix-store -r $drvPath) outPath=$(nix-store -r $drvPath)
@ -7,4 +9,4 @@ echo "pushing $drvPath"
mkdir -p $TEST_ROOT/cache mkdir -p $TEST_ROOT/cache
nix-push --copy $TEST_ROOT/cache $TEST_ROOT/manifest $drvPath nix-push --dest $TEST_ROOT/cache --manifest $drvPath --bzip2

View file

@ -10,6 +10,7 @@ touch $reference
echo "making registration..." echo "making registration..."
set +x
for ((n = 0; n < $max; n++)); do for ((n = 0; n < $max; n++)); do
storePath=$NIX_STORE_DIR/$n storePath=$NIX_STORE_DIR/$n
echo -n > $storePath echo -n > $storePath
@ -19,6 +20,7 @@ for ((n = 0; n < $max; n++)); do
fi fi
echo $storePath; echo; echo 2; echo $reference; echo $ref2 echo $storePath; echo; echo 2; echo $reference; echo $ref2
done > $TEST_ROOT/reg_info done > $TEST_ROOT/reg_info
set -x
echo "registering..." echo "registering..."

View file

@ -3,7 +3,7 @@ source common.sh
echo '*** testing slave mode ***' echo '*** testing slave mode ***'
clearStore clearStore
clearManifests clearManifests
NIX_REMOTE=slave $SHELL ./user-envs.sh NIX_REMOTE_=slave $SHELL ./user-envs.sh
echo '*** testing daemon mode ***' echo '*** testing daemon mode ***'
clearStore clearStore

View file

@ -2,22 +2,25 @@
echo substituter args: $* >&2 echo substituter args: $* >&2
if test $1 = "--query"; then if test $1 = "--query"; then
while read cmd; do while read cmd args; do
echo FOO $cmd >&2 echo "CMD = $cmd, ARGS = $args" >&2
if test "$cmd" = "have"; then if test "$cmd" = "have"; then
read path for path in $args; do
if grep -q "$path" $TEST_ROOT/sub-paths; then read path
echo 1 if grep -q "$path" $TEST_ROOT/sub-paths; then
else echo $path
echo 0 fi
fi done
echo
elif test "$cmd" = "info"; then elif test "$cmd" = "info"; then
read path for path in $args; do
echo 1 echo $path
echo "" # deriver echo "" # deriver
echo 0 # nr of refs echo 0 # nr of refs
echo $((1 * 1024 * 1024)) # download size echo $((1 * 1024 * 1024)) # download size
echo $((2 * 1024 * 1024)) # nar size echo $((2 * 1024 * 1024)) # nar size
done
echo
else else
echo "bad command $cmd" echo "bad command $cmd"
exit 1 exit 1
@ -26,6 +29,7 @@ if test $1 = "--query"; then
elif test $1 = "--substitute"; then elif test $1 = "--substitute"; then
mkdir $2 mkdir $2
echo "Hallo Wereld" > $2/hello echo "Hallo Wereld" > $2/hello
echo # no expected hash
else else
echo "unknown substituter operation" echo "unknown substituter operation"
exit 1 exit 1

View file

@ -2,21 +2,23 @@
echo substituter2 args: $* >&2 echo substituter2 args: $* >&2
if test $1 = "--query"; then if test $1 = "--query"; then
while read cmd; do while read cmd args; do
if test "$cmd" = "have"; then if test "$cmd" = have; then
read path for path in $args; do
if grep -q "$path" $TEST_ROOT/sub-paths; then if grep -q "$path" $TEST_ROOT/sub-paths; then
echo 1 echo $path
else fi
echo 0 done
fi echo
elif test "$cmd" = "info"; then elif test "$cmd" = info; then
read path for path in $args; do
echo 1 echo $path
echo "" # deriver echo "" # deriver
echo 0 # nr of refs echo 0 # nr of refs
echo 0 # download size echo 0 # download size
echo 0 # nar size echo 0 # nar size
done
echo
else else
echo "bad command $cmd" echo "bad command $cmd"
exit 1 exit 1