forked from lix-project/lix
Merge branch 'no-manifests'
This commit is contained in:
commit
15e1b2c223
4
.gitignore
vendored
4
.gitignore
vendored
|
@ -35,8 +35,11 @@ Makefile.in
|
||||||
|
|
||||||
# /doc/manual/
|
# /doc/manual/
|
||||||
/doc/manual/manual.html
|
/doc/manual/manual.html
|
||||||
|
/doc/manual/manual.xmli
|
||||||
|
/doc/manual/manual.pdf
|
||||||
/doc/manual/manual.is-valid
|
/doc/manual/manual.is-valid
|
||||||
/doc/manual/*.1
|
/doc/manual/*.1
|
||||||
|
/doc/manual/*.5
|
||||||
/doc/manual/*.8
|
/doc/manual/*.8
|
||||||
/doc/manual/images
|
/doc/manual/images
|
||||||
/doc/manual/version.txt
|
/doc/manual/version.txt
|
||||||
|
@ -60,6 +63,7 @@ Makefile.in
|
||||||
/scripts/GeneratePatches.pm
|
/scripts/GeneratePatches.pm
|
||||||
/scripts/download-using-manifests.pl
|
/scripts/download-using-manifests.pl
|
||||||
/scripts/copy-from-other-stores.pl
|
/scripts/copy-from-other-stores.pl
|
||||||
|
/scripts/download-from-binary-cache.pl
|
||||||
/scripts/find-runtime-roots.pl
|
/scripts/find-runtime-roots.pl
|
||||||
/scripts/build-remote.pl
|
/scripts/build-remote.pl
|
||||||
/scripts/nix-reduce-build
|
/scripts/nix-reduce-build
|
||||||
|
|
32
configure.ac
32
configure.ac
|
@ -25,12 +25,12 @@ AC_ARG_WITH(system, AC_HELP_STRING([--with-system=SYSTEM],
|
||||||
|
|
||||||
case "$host_os" in
|
case "$host_os" in
|
||||||
linux-gnu*)
|
linux-gnu*)
|
||||||
# For backward compatibility, strip the `-gnu' part.
|
# For backward compatibility, strip the `-gnu' part.
|
||||||
system="$machine_name-linux";;
|
system="$machine_name-linux";;
|
||||||
*)
|
*)
|
||||||
# Strip the version number from names such as `gnu0.3',
|
# Strip the version number from names such as `gnu0.3',
|
||||||
# `darwin10.2.0', etc.
|
# `darwin10.2.0', etc.
|
||||||
system="$machine_name-`echo $host_os | "$SED" -e's/@<:@0-9.@:>@*$//g'`";;
|
system="$machine_name-`echo $host_os | "$SED" -e's/@<:@0-9.@:>@*$//g'`";;
|
||||||
esac])
|
esac])
|
||||||
|
|
||||||
sys_name=$(uname -s | tr 'A-Z ' 'a-z_')
|
sys_name=$(uname -s | tr 'A-Z ' 'a-z_')
|
||||||
|
@ -178,6 +178,7 @@ NEED_PROG(perl, perl)
|
||||||
NEED_PROG(sed, sed)
|
NEED_PROG(sed, sed)
|
||||||
NEED_PROG(tar, tar)
|
NEED_PROG(tar, tar)
|
||||||
NEED_PROG(bzip2, bzip2)
|
NEED_PROG(bzip2, bzip2)
|
||||||
|
NEED_PROG(xz, xz)
|
||||||
AC_PATH_PROG(dot, dot)
|
AC_PATH_PROG(dot, dot)
|
||||||
AC_PATH_PROG(dblatex, dblatex)
|
AC_PATH_PROG(dblatex, dblatex)
|
||||||
AC_PATH_PROG(gzip, gzip)
|
AC_PATH_PROG(gzip, gzip)
|
||||||
|
@ -266,7 +267,7 @@ if test "$gc" = yes; then
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|
||||||
# Check for the required Perl dependencies (DBI and DBD::SQLite).
|
# Check for the required Perl dependencies (DBI, DBD::SQLite and WWW::Curl).
|
||||||
perlFlags="-I$perllibdir"
|
perlFlags="-I$perllibdir"
|
||||||
|
|
||||||
AC_ARG_WITH(dbi, AC_HELP_STRING([--with-dbi=PATH],
|
AC_ARG_WITH(dbi, AC_HELP_STRING([--with-dbi=PATH],
|
||||||
|
@ -277,6 +278,10 @@ AC_ARG_WITH(dbd-sqlite, AC_HELP_STRING([--with-dbd-sqlite=PATH],
|
||||||
[prefix of the Perl DBD::SQLite library]),
|
[prefix of the Perl DBD::SQLite library]),
|
||||||
perlFlags="$perlFlags -I$withval")
|
perlFlags="$perlFlags -I$withval")
|
||||||
|
|
||||||
|
AC_ARG_WITH(www-curl, AC_HELP_STRING([--with-www-curl=PATH],
|
||||||
|
[prefix of the Perl WWW::Curl library]),
|
||||||
|
perlFlags="$perlFlags -I$withval")
|
||||||
|
|
||||||
AC_MSG_CHECKING([whether DBD::SQLite works])
|
AC_MSG_CHECKING([whether DBD::SQLite works])
|
||||||
if ! $perl $perlFlags -e 'use DBI; use DBD::SQLite;' 2>&5; then
|
if ! $perl $perlFlags -e 'use DBI; use DBD::SQLite;' 2>&5; then
|
||||||
AC_MSG_RESULT(no)
|
AC_MSG_RESULT(no)
|
||||||
|
@ -284,6 +289,13 @@ if ! $perl $perlFlags -e 'use DBI; use DBD::SQLite;' 2>&5; then
|
||||||
fi
|
fi
|
||||||
AC_MSG_RESULT(yes)
|
AC_MSG_RESULT(yes)
|
||||||
|
|
||||||
|
AC_MSG_CHECKING([whether WWW::Curl works])
|
||||||
|
if ! $perl $perlFlags -e 'use WWW::Curl;' 2>&5; then
|
||||||
|
AC_MSG_RESULT(no)
|
||||||
|
AC_MSG_FAILURE([The Perl module WWW::Curl is missing.])
|
||||||
|
fi
|
||||||
|
AC_MSG_RESULT(yes)
|
||||||
|
|
||||||
AC_SUBST(perlFlags)
|
AC_SUBST(perlFlags)
|
||||||
|
|
||||||
|
|
||||||
|
@ -327,6 +339,18 @@ eval dynlib_suffix=$shrext_cmds
|
||||||
AC_SUBST(dynlib_suffix)
|
AC_SUBST(dynlib_suffix)
|
||||||
|
|
||||||
|
|
||||||
|
# Do we have GNU tar?
|
||||||
|
AC_MSG_CHECKING([if you have GNU tar])
|
||||||
|
if $tar --version 2> /dev/null | grep -q GNU; then
|
||||||
|
AC_MSG_RESULT(yes)
|
||||||
|
tarFlags="--warning=no-timestamp"
|
||||||
|
else
|
||||||
|
AC_MSG_RESULT(no)
|
||||||
|
fi
|
||||||
|
AC_SUBST(tarFlags)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
AM_CONFIG_HEADER([config.h])
|
AM_CONFIG_HEADER([config.h])
|
||||||
AC_CONFIG_FILES([Makefile
|
AC_CONFIG_FILES([Makefile
|
||||||
src/Makefile
|
src/Makefile
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
all-local: config.nix
|
all-local: config.nix
|
||||||
|
|
||||||
files = nar.nix buildenv.nix buildenv.pl unpack-channel.nix unpack-channel.sh derivation.nix fetchurl.nix \
|
files = nar.nix buildenv.nix buildenv.pl unpack-channel.nix derivation.nix fetchurl.nix \
|
||||||
imported-drv-to-derivation.nix
|
imported-drv-to-derivation.nix
|
||||||
|
|
||||||
install-exec-local:
|
install-exec-local:
|
||||||
|
|
|
@ -6,8 +6,10 @@ in {
|
||||||
perl = "@perl@";
|
perl = "@perl@";
|
||||||
shell = "@shell@";
|
shell = "@shell@";
|
||||||
coreutils = "@coreutils@";
|
coreutils = "@coreutils@";
|
||||||
bzip2 = fromEnv "NIX_BZIP2" "@bzip2@";
|
bzip2 = "@bzip2@";
|
||||||
|
xz = "@xz@";
|
||||||
tar = "@tar@";
|
tar = "@tar@";
|
||||||
|
tarFlags = "@tarFlags@";
|
||||||
tr = "@tr@";
|
tr = "@tr@";
|
||||||
curl = "@curl@";
|
curl = "@curl@";
|
||||||
nixBinDir = fromEnv "NIX_BIN_DIR" "@bindir@";
|
nixBinDir = fromEnv "NIX_BIN_DIR" "@bindir@";
|
||||||
|
|
|
@ -6,28 +6,37 @@ let
|
||||||
''
|
''
|
||||||
export PATH=${nixBinDir}:${coreutils}
|
export PATH=${nixBinDir}:${coreutils}
|
||||||
|
|
||||||
|
if [ $compressionType = "xz" ]; then
|
||||||
|
ext=xz
|
||||||
|
compressor="${xz} -9"
|
||||||
|
else
|
||||||
|
ext=bz2
|
||||||
|
compressor="${bzip2}"
|
||||||
|
fi
|
||||||
|
|
||||||
echo "packing ‘$storePath’..."
|
echo "packing ‘$storePath’..."
|
||||||
mkdir $out
|
mkdir $out
|
||||||
dst=$out/tmp.nar.bz2
|
dst=$out/tmp.nar.$ext
|
||||||
|
|
||||||
set -o pipefail
|
set -o pipefail
|
||||||
nix-store --dump "$storePath" | ${bzip2} > $dst
|
nix-store --dump "$storePath" | $compressor > $dst
|
||||||
|
|
||||||
nix-hash --flat --type $hashAlgo --base32 $dst > $out/narbz2-hash
|
hash=$(nix-hash --flat --type $hashAlgo --base32 $dst)
|
||||||
|
echo -n $hash > $out/nar-compressed-hash
|
||||||
|
|
||||||
mv $out/tmp.nar.bz2 $out/$(cat $out/narbz2-hash).nar.bz2
|
mv $dst $out/$hash.nar.$ext
|
||||||
'';
|
'';
|
||||||
|
|
||||||
in
|
in
|
||||||
|
|
||||||
{ storePath, hashAlgo }:
|
{ storePath, hashAlgo, compressionType }:
|
||||||
|
|
||||||
derivation {
|
derivation {
|
||||||
name = "nar";
|
name = "nar";
|
||||||
system = builtins.currentSystem;
|
system = builtins.currentSystem;
|
||||||
builder = shell;
|
builder = shell;
|
||||||
args = [ "-e" builder ];
|
args = [ "-e" builder ];
|
||||||
inherit storePath hashAlgo;
|
inherit storePath hashAlgo compressionType;
|
||||||
|
|
||||||
# Don't build in a chroot because Nix's dependencies may not be there.
|
# Don't build in a chroot because Nix's dependencies may not be there.
|
||||||
__noChroot = true;
|
__noChroot = true;
|
||||||
|
|
|
@ -1,12 +1,29 @@
|
||||||
with import <nix/config.nix>;
|
with import <nix/config.nix>;
|
||||||
|
|
||||||
{ name, channelName, src }:
|
let
|
||||||
|
|
||||||
|
builder = builtins.toFile "unpack-channel.sh"
|
||||||
|
''
|
||||||
|
mkdir $out
|
||||||
|
cd $out
|
||||||
|
${bzip2} -d < $src | ${tar} xf - --warning=no-timestamp
|
||||||
|
mv * $out/$channelName
|
||||||
|
if [ -n "$binaryCacheURL" ]; then
|
||||||
|
mkdir $out/binary-caches
|
||||||
|
echo -n "$binaryCacheURL" > $out/binary-caches/$channelName
|
||||||
|
fi
|
||||||
|
'';
|
||||||
|
|
||||||
|
in
|
||||||
|
|
||||||
|
{ name, channelName, src, binaryCacheURL ? "" }:
|
||||||
|
|
||||||
derivation {
|
derivation {
|
||||||
system = builtins.currentSystem;
|
system = builtins.currentSystem;
|
||||||
builder = shell;
|
builder = shell;
|
||||||
args = [ "-e" ./unpack-channel.sh ];
|
args = [ "-e" builder ];
|
||||||
inherit name channelName src bzip2 tar tr;
|
inherit name channelName src binaryCacheURL;
|
||||||
|
|
||||||
PATH = "${nixBinDir}:${coreutils}";
|
PATH = "${nixBinDir}:${coreutils}";
|
||||||
|
|
||||||
# No point in doing this remotely.
|
# No point in doing this remotely.
|
||||||
|
|
|
@ -1,4 +0,0 @@
|
||||||
mkdir $out
|
|
||||||
cd $out
|
|
||||||
$bzip2 -d < $src | $tar xf -
|
|
||||||
mv * $out/$channelName
|
|
|
@ -30,6 +30,9 @@ gc-keep-derivations = true # Idem
|
||||||
env-keep-derivations = false
|
env-keep-derivations = false
|
||||||
</programlisting>
|
</programlisting>
|
||||||
|
|
||||||
|
<para>You can override settings using the <option>--option</option>
|
||||||
|
flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
|
||||||
|
|
||||||
<para>The following settings are currently available:
|
<para>The following settings are currently available:
|
||||||
|
|
||||||
<variablelist>
|
<variablelist>
|
||||||
|
@ -243,6 +246,16 @@ env-keep-derivations = false
|
||||||
</varlistentry>
|
</varlistentry>
|
||||||
|
|
||||||
|
|
||||||
|
<varlistentry><term><literal>build-fallback</literal></term>
|
||||||
|
|
||||||
|
<listitem><para>If set to <literal>true</literal>, Nix will fall
|
||||||
|
back to building from source if a binary substitute fails. This
|
||||||
|
is equivalent to the <option>--fallback</option> flag. The
|
||||||
|
default is <literal>false</literal>.</para></listitem>
|
||||||
|
|
||||||
|
</varlistentry>
|
||||||
|
|
||||||
|
|
||||||
<varlistentry xml:id="conf-build-chroot-dirs"><term><literal>build-chroot-dirs</literal></term>
|
<varlistentry xml:id="conf-build-chroot-dirs"><term><literal>build-chroot-dirs</literal></term>
|
||||||
|
|
||||||
<listitem><para>When builds are performed in a chroot environment,
|
<listitem><para>When builds are performed in a chroot environment,
|
||||||
|
@ -307,6 +320,50 @@ build-use-chroot = /dev /proc /bin</programlisting>
|
||||||
</varlistentry>
|
</varlistentry>
|
||||||
|
|
||||||
|
|
||||||
|
<varlistentry><term><literal>binary-caches</literal></term>
|
||||||
|
|
||||||
|
<listitem><para>A list of URLs of binary caches, separated by
|
||||||
|
whitespace. The default is empty.<!-- The default is
|
||||||
|
<literal>http://nixos.org/binary-cache</literal>. --></para></listitem>
|
||||||
|
|
||||||
|
</varlistentry>
|
||||||
|
|
||||||
|
|
||||||
|
<varlistentry><term><literal>binary-caches-files</literal></term>
|
||||||
|
|
||||||
|
<listitem><para>A list of names of files that will be read to
|
||||||
|
obtain additional binary cache URLs. The default is
|
||||||
|
<literal>/nix/var/nix/profiles/per-user/root/channels/binary-caches/*</literal>,
|
||||||
|
which ensures that Nix will use the binary caches corresponding to
|
||||||
|
the channels installed by root. Do not set this option to read
|
||||||
|
files created by untrusted users!</para></listitem>
|
||||||
|
|
||||||
|
</varlistentry>
|
||||||
|
|
||||||
|
|
||||||
|
<varlistentry><term><literal>trusted-binary-caches</literal></term>
|
||||||
|
|
||||||
|
<listitem><para>A list of URLs of binary caches, separated by
|
||||||
|
whitespace. These are not used by default, but can be enabled by
|
||||||
|
users of the Nix daemon by specifying <literal>--option
|
||||||
|
binary-caches <replaceable>urls</replaceable></literal> on the
|
||||||
|
command line. Daemon users are only allowed to pass a subset of
|
||||||
|
the URLs listed in <literal>binary-caches</literal> and
|
||||||
|
<literal>trusted-binary-caches</literal>.</para></listitem>
|
||||||
|
|
||||||
|
</varlistentry>
|
||||||
|
|
||||||
|
|
||||||
|
<varlistentry><term><literal>binary-caches-parallel-connections</literal></term>
|
||||||
|
|
||||||
|
<listitem><para>The maximum number of parallel HTTP connections
|
||||||
|
used by the binary cache substituter to get NAR info files. This
|
||||||
|
number should be high to minimise latency. It defaults to
|
||||||
|
150.</para></listitem>
|
||||||
|
|
||||||
|
</varlistentry>
|
||||||
|
|
||||||
|
|
||||||
<varlistentry><term><literal>system</literal></term>
|
<varlistentry><term><literal>system</literal></term>
|
||||||
|
|
||||||
<listitem><para>This option specifies the canonical Nix system
|
<listitem><para>This option specifies the canonical Nix system
|
||||||
|
|
|
@ -94,6 +94,11 @@
|
||||||
<citerefentry><refentrytitle>nix.conf</refentrytitle><manvolnum>5</manvolnum></citerefentry>.</para>
|
<citerefentry><refentrytitle>nix.conf</refentrytitle><manvolnum>5</manvolnum></citerefentry>.</para>
|
||||||
</listitem>
|
</listitem>
|
||||||
|
|
||||||
|
<listitem>
|
||||||
|
<para>When using the Nix daemon, the <option>-s</option> flag in
|
||||||
|
<command>nix-env -qa</command> is now much faster.</para>
|
||||||
|
</listitem>
|
||||||
|
|
||||||
</itemizedlist>
|
</itemizedlist>
|
||||||
|
|
||||||
</section>
|
</section>
|
||||||
|
|
|
@ -16,6 +16,7 @@ BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX)
|
||||||
%endif
|
%endif
|
||||||
BuildRequires: perl(DBD::SQLite)
|
BuildRequires: perl(DBD::SQLite)
|
||||||
BuildRequires: perl(DBI)
|
BuildRequires: perl(DBI)
|
||||||
|
BuildRequires: perl(WWW::Curl)
|
||||||
BuildRequires: perl(ExtUtils::ParseXS)
|
BuildRequires: perl(ExtUtils::ParseXS)
|
||||||
Requires: /usr/bin/perl
|
Requires: /usr/bin/perl
|
||||||
Requires: curl
|
Requires: curl
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
PERL_MODULES = lib/Nix/Store.pm lib/Nix/Manifest.pm lib/Nix/GeneratePatches.pm lib/Nix/SSH.pm lib/Nix/CopyClosure.pm lib/Nix/Config.pm.in
|
PERL_MODULES = lib/Nix/Store.pm lib/Nix/Manifest.pm lib/Nix/GeneratePatches.pm lib/Nix/SSH.pm lib/Nix/CopyClosure.pm lib/Nix/Config.pm.in lib/Nix/Utils.pm
|
||||||
|
|
||||||
all: $(PERL_MODULES:.in=)
|
all: $(PERL_MODULES:.in=)
|
||||||
|
|
||||||
|
|
|
@ -1,27 +1,39 @@
|
||||||
package Nix::Config;
|
package Nix::Config;
|
||||||
|
|
||||||
|
$version = "@version@";
|
||||||
|
|
||||||
$binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
|
$binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
|
||||||
$libexecDir = $ENV{"NIX_LIBEXEC_DIR"} || "@libexecdir@";
|
$libexecDir = $ENV{"NIX_LIBEXEC_DIR"} || "@libexecdir@";
|
||||||
$stateDir = $ENV{"NIX_STATE_DIR"} || "@localstatedir@/nix";
|
$stateDir = $ENV{"NIX_STATE_DIR"} || "@localstatedir@/nix";
|
||||||
$manifestDir = $ENV{"NIX_MANIFESTS_DIR"} || "@localstatedir@/nix/manifests";
|
$manifestDir = $ENV{"NIX_MANIFESTS_DIR"} || "@localstatedir@/nix/manifests";
|
||||||
$logDir = $ENV{"NIX_LOG_DIR"} || "@localstatedir@/log/nix";
|
$logDir = $ENV{"NIX_LOG_DIR"} || "@localstatedir@/log/nix";
|
||||||
$confDir = $ENV{"NIX_CONF_DIR"} || "@sysconfdir@/nix";
|
$confDir = $ENV{"NIX_CONF_DIR"} || "@sysconfdir@/nix";
|
||||||
|
$storeDir = $ENV{"NIX_STORE_DIR"} || "@storedir@";
|
||||||
|
|
||||||
$bzip2 = $ENV{"NIX_BZIP2"} || "@bzip2@";
|
$bzip2 = "@bzip2@";
|
||||||
|
$xz = "@xz@";
|
||||||
$curl = "@curl@";
|
$curl = "@curl@";
|
||||||
|
|
||||||
$useBindings = "@perlbindings@" eq "yes";
|
$useBindings = "@perlbindings@" eq "yes";
|
||||||
|
|
||||||
|
%config = ();
|
||||||
|
|
||||||
sub readConfig {
|
sub readConfig {
|
||||||
my %config;
|
if (defined $ENV{'_NIX_OPTIONS'}) {
|
||||||
my $config = "@sysconfdir@/nix/nix.conf";
|
foreach my $s (split '\n', $ENV{'_NIX_OPTIONS'}) {
|
||||||
|
my ($n, $v) = split '=', $s, 2;
|
||||||
|
$config{$n} = $v;
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
my $config = "$confDir/nix.conf";
|
||||||
return unless -f $config;
|
return unless -f $config;
|
||||||
|
|
||||||
open CONFIG, "<$config" or die "cannot open `$config'";
|
open CONFIG, "<$config" or die "cannot open `$config'";
|
||||||
while (<CONFIG>) {
|
while (<CONFIG>) {
|
||||||
/^\s*([\w|-]+)\s*=\s*(.*)$/ or next;
|
/^\s*([\w|-]+)\s*=\s*(.*)$/ or next;
|
||||||
$config{$1} = $2;
|
$config{$1} = $2;
|
||||||
print "|$1| -> |$2|\n";
|
|
||||||
}
|
}
|
||||||
close CONFIG;
|
close CONFIG;
|
||||||
}
|
}
|
||||||
|
|
|
@ -19,7 +19,7 @@ void doInit()
|
||||||
{
|
{
|
||||||
if (!store) {
|
if (!store) {
|
||||||
try {
|
try {
|
||||||
setDefaultsFromEnvironment();
|
settings.processEnvironment();
|
||||||
store = openStore();
|
store = openStore();
|
||||||
} catch (Error & e) {
|
} catch (Error & e) {
|
||||||
croak(e.what());
|
croak(e.what());
|
||||||
|
|
19
perl/lib/Nix/Utils.pm
Normal file
19
perl/lib/Nix/Utils.pm
Normal file
|
@ -0,0 +1,19 @@
|
||||||
|
package Nix::Utils;
|
||||||
|
|
||||||
|
$urlRE = "(?: [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*]+ )";
|
||||||
|
|
||||||
|
sub checkURL {
|
||||||
|
my ($url) = @_;
|
||||||
|
die "invalid URL ‘$url’\n" unless $url =~ /^ $urlRE $ /x;
|
||||||
|
}
|
||||||
|
|
||||||
|
sub uniq {
|
||||||
|
my %seen;
|
||||||
|
my @res;
|
||||||
|
foreach my $name (@_) {
|
||||||
|
next if $seen{$name};
|
||||||
|
$seen{$name} = 1;
|
||||||
|
push @res, $name;
|
||||||
|
}
|
||||||
|
return @res;
|
||||||
|
}
|
|
@ -29,6 +29,7 @@ let
|
||||||
--with-xml-flags=--nonet
|
--with-xml-flags=--nonet
|
||||||
--with-dbi=${perlPackages.DBI}/${perl.libPrefix}
|
--with-dbi=${perlPackages.DBI}/${perl.libPrefix}
|
||||||
--with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}
|
--with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}
|
||||||
|
--with-www-curl=${perlPackages.WWWCurl}/${perl.libPrefix}
|
||||||
'';
|
'';
|
||||||
|
|
||||||
postUnpack = ''
|
postUnpack = ''
|
||||||
|
@ -77,6 +78,7 @@ let
|
||||||
--disable-init-state
|
--disable-init-state
|
||||||
--with-dbi=${perlPackages.DBI}/${perl.libPrefix}
|
--with-dbi=${perlPackages.DBI}/${perl.libPrefix}
|
||||||
--with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}
|
--with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}
|
||||||
|
--with-www-curl=${perlPackages.WWWCurl}/${perl.libPrefix}
|
||||||
--enable-gc
|
--enable-gc
|
||||||
'';
|
'';
|
||||||
|
|
||||||
|
@ -134,6 +136,7 @@ let
|
||||||
--disable-init-state
|
--disable-init-state
|
||||||
--with-dbi=${perlPackages.DBI}/${perl.libPrefix}
|
--with-dbi=${perlPackages.DBI}/${perl.libPrefix}
|
||||||
--with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}
|
--with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}
|
||||||
|
--with-www-curl=${perlPackages.WWWCurl}/${perl.libPrefix}
|
||||||
'';
|
'';
|
||||||
|
|
||||||
dontInstall = false;
|
dontInstall = false;
|
||||||
|
@ -192,7 +195,7 @@ let
|
||||||
name = "nix-rpm-${diskImage.name}";
|
name = "nix-rpm-${diskImage.name}";
|
||||||
src = jobs.tarball;
|
src = jobs.tarball;
|
||||||
diskImage = (diskImageFun vmTools.diskImageFuns)
|
diskImage = (diskImageFun vmTools.diskImageFuns)
|
||||||
{ extraPackages = [ "perl-DBD-SQLite" "perl-devel" "sqlite" "sqlite-devel" "bzip2-devel" "emacs" ]; };
|
{ extraPackages = [ "perl-DBD-SQLite" "perl-devel" "sqlite" "sqlite-devel" "bzip2-devel" "emacs" "perl-WWW-Curl" ]; };
|
||||||
memSize = 1024;
|
memSize = 1024;
|
||||||
meta.schedulingPriority = prio;
|
meta.schedulingPriority = prio;
|
||||||
postRPMInstall = "cd /tmp/rpmout/BUILD/nix-* && make installcheck";
|
postRPMInstall = "cd /tmp/rpmout/BUILD/nix-* && make installcheck";
|
||||||
|
@ -211,7 +214,7 @@ let
|
||||||
name = "nix-deb";
|
name = "nix-deb";
|
||||||
src = jobs.tarball;
|
src = jobs.tarball;
|
||||||
diskImage = (diskImageFun vmTools.diskImageFuns)
|
diskImage = (diskImageFun vmTools.diskImageFuns)
|
||||||
{ extraPackages = [ "libdbd-sqlite3-perl" "libsqlite3-dev" "libbz2-dev" ]; };
|
{ extraPackages = [ "libdbd-sqlite3-perl" "libsqlite3-dev" "libbz2-dev" "libwww-curl-perl" ]; };
|
||||||
memSize = 1024;
|
memSize = 1024;
|
||||||
meta.schedulingPriority = prio;
|
meta.schedulingPriority = prio;
|
||||||
configureFlags = "--sysconfdir=/etc";
|
configureFlags = "--sysconfdir=/etc";
|
||||||
|
|
|
@ -7,17 +7,14 @@ noinst_SCRIPTS = nix-profile.sh \
|
||||||
find-runtime-roots.pl build-remote.pl nix-reduce-build \
|
find-runtime-roots.pl build-remote.pl nix-reduce-build \
|
||||||
copy-from-other-stores.pl nix-http-export.cgi
|
copy-from-other-stores.pl nix-http-export.cgi
|
||||||
|
|
||||||
nix-pull nix-push: download-using-manifests.pl
|
install-exec-local: download-using-manifests.pl copy-from-other-stores.pl download-from-binary-cache.pl find-runtime-roots.pl
|
||||||
|
|
||||||
install-exec-local: download-using-manifests.pl copy-from-other-stores.pl find-runtime-roots.pl
|
|
||||||
$(INSTALL) -d $(DESTDIR)$(sysconfdir)/profile.d
|
$(INSTALL) -d $(DESTDIR)$(sysconfdir)/profile.d
|
||||||
$(INSTALL_DATA) nix-profile.sh $(DESTDIR)$(sysconfdir)/profile.d/nix.sh
|
$(INSTALL_DATA) nix-profile.sh $(DESTDIR)$(sysconfdir)/profile.d/nix.sh
|
||||||
$(INSTALL) -d $(DESTDIR)$(libexecdir)/nix
|
$(INSTALL) -d $(DESTDIR)$(libexecdir)/nix
|
||||||
$(INSTALL_PROGRAM) find-runtime-roots.pl $(DESTDIR)$(libexecdir)/nix
|
$(INSTALL_PROGRAM) find-runtime-roots.pl $(DESTDIR)$(libexecdir)/nix
|
||||||
$(INSTALL_PROGRAM) build-remote.pl $(DESTDIR)$(libexecdir)/nix
|
$(INSTALL_PROGRAM) build-remote.pl $(DESTDIR)$(libexecdir)/nix
|
||||||
$(INSTALL) -d $(DESTDIR)$(libexecdir)/nix/substituters
|
$(INSTALL) -d $(DESTDIR)$(libexecdir)/nix/substituters
|
||||||
$(INSTALL_PROGRAM) download-using-manifests.pl $(DESTDIR)$(libexecdir)/nix/substituters
|
$(INSTALL_PROGRAM) download-using-manifests.pl copy-from-other-stores.pl download-from-binary-cache.pl $(DESTDIR)$(libexecdir)/nix/substituters
|
||||||
$(INSTALL_PROGRAM) copy-from-other-stores.pl $(DESTDIR)$(libexecdir)/nix/substituters
|
|
||||||
$(INSTALL) -d $(DESTDIR)$(sysconfdir)/nix
|
$(INSTALL) -d $(DESTDIR)$(sysconfdir)/nix
|
||||||
|
|
||||||
include ../substitute.mk
|
include ../substitute.mk
|
||||||
|
@ -29,6 +26,7 @@ EXTRA_DIST = nix-collect-garbage.in \
|
||||||
nix-build.in \
|
nix-build.in \
|
||||||
download-using-manifests.pl.in \
|
download-using-manifests.pl.in \
|
||||||
copy-from-other-stores.pl.in \
|
copy-from-other-stores.pl.in \
|
||||||
|
download-from-binary-cache.pl.in \
|
||||||
nix-copy-closure.in \
|
nix-copy-closure.in \
|
||||||
find-runtime-roots.pl.in \
|
find-runtime-roots.pl.in \
|
||||||
build-remote.pl.in \
|
build-remote.pl.in \
|
||||||
|
|
|
@ -36,42 +36,45 @@ sub findStorePath {
|
||||||
if ($ARGV[0] eq "--query") {
|
if ($ARGV[0] eq "--query") {
|
||||||
|
|
||||||
while (<STDIN>) {
|
while (<STDIN>) {
|
||||||
my $cmd = $_; chomp $cmd;
|
chomp;
|
||||||
|
my ($cmd, @args) = split " ", $_;
|
||||||
|
|
||||||
if ($cmd eq "have") {
|
if ($cmd eq "have") {
|
||||||
my $storePath = <STDIN>; chomp $storePath;
|
foreach my $storePath (@args) {
|
||||||
print STDOUT (defined findStorePath($storePath) ? "1\n" : "0\n");
|
print "$storePath\n" if defined findStorePath($storePath);
|
||||||
|
}
|
||||||
|
print "\n";
|
||||||
}
|
}
|
||||||
|
|
||||||
elsif ($cmd eq "info") {
|
elsif ($cmd eq "info") {
|
||||||
my $storePath = <STDIN>; chomp $storePath;
|
foreach my $storePath (@args) {
|
||||||
my ($store, $sourcePath) = findStorePath($storePath);
|
my ($store, $sourcePath) = findStorePath($storePath);
|
||||||
if (!defined $store) {
|
next unless defined $store;
|
||||||
print "0\n";
|
|
||||||
next; # not an error
|
$ENV{"NIX_DB_DIR"} = "$store/var/nix/db";
|
||||||
|
|
||||||
|
my $deriver = `@bindir@/nix-store --query --deriver $storePath`;
|
||||||
|
die "cannot query deriver of `$storePath'" if $? != 0;
|
||||||
|
chomp $deriver;
|
||||||
|
$deriver = "" if $deriver eq "unknown-deriver";
|
||||||
|
|
||||||
|
my @references = split "\n",
|
||||||
|
`@bindir@/nix-store --query --references $storePath`;
|
||||||
|
die "cannot query references of `$storePath'" if $? != 0;
|
||||||
|
|
||||||
|
my $narSize = `@bindir@/nix-store --query --size $storePath`;
|
||||||
|
die "cannot query size of `$storePath'" if $? != 0;
|
||||||
|
chomp $narSize;
|
||||||
|
|
||||||
|
print "$storePath\n";
|
||||||
|
print "$deriver\n";
|
||||||
|
print scalar @references, "\n";
|
||||||
|
print "$_\n" foreach @references;
|
||||||
|
print "$narSize\n";
|
||||||
|
print "$narSize\n";
|
||||||
}
|
}
|
||||||
print "1\n";
|
|
||||||
|
|
||||||
$ENV{"NIX_DB_DIR"} = "$store/var/nix/db";
|
print "\n";
|
||||||
|
|
||||||
my $deriver = `@bindir@/nix-store --query --deriver $storePath`;
|
|
||||||
die "cannot query deriver of `$storePath'" if $? != 0;
|
|
||||||
chomp $deriver;
|
|
||||||
$deriver = "" if $deriver eq "unknown-deriver";
|
|
||||||
|
|
||||||
my @references = split "\n",
|
|
||||||
`@bindir@/nix-store --query --references $storePath`;
|
|
||||||
die "cannot query references of `$storePath'" if $? != 0;
|
|
||||||
|
|
||||||
my $narSize = `@bindir@/nix-store --query --size $storePath`;
|
|
||||||
die "cannot query size of `$storePath'" if $? != 0;
|
|
||||||
chomp $narSize;
|
|
||||||
|
|
||||||
print "$deriver\n";
|
|
||||||
print scalar @references, "\n";
|
|
||||||
print "$_\n" foreach @references;
|
|
||||||
print "$narSize\n";
|
|
||||||
print "$narSize\n";
|
|
||||||
}
|
}
|
||||||
|
|
||||||
else { die "unknown command `$cmd'"; }
|
else { die "unknown command `$cmd'"; }
|
||||||
|
@ -84,9 +87,10 @@ elsif ($ARGV[0] eq "--substitute") {
|
||||||
my $storePath = $ARGV[1];
|
my $storePath = $ARGV[1];
|
||||||
my ($store, $sourcePath) = findStorePath $storePath;
|
my ($store, $sourcePath) = findStorePath $storePath;
|
||||||
die unless $store;
|
die unless $store;
|
||||||
print "\n*** Copying `$storePath' from `$sourcePath'\n\n";
|
print STDERR "\n*** Copying `$storePath' from `$sourcePath'\n\n";
|
||||||
system("$binDir/nix-store --dump $sourcePath | $binDir/nix-store --restore $storePath") == 0
|
system("$binDir/nix-store --dump $sourcePath | $binDir/nix-store --restore $storePath") == 0
|
||||||
or die "cannot copy `$sourcePath' to `$storePath'";
|
or die "cannot copy `$sourcePath' to `$storePath'";
|
||||||
|
print "\n"; # no hash to verify
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
537
scripts/download-from-binary-cache.pl.in
Normal file
537
scripts/download-from-binary-cache.pl.in
Normal file
|
@ -0,0 +1,537 @@
|
||||||
|
#! @perl@ -w @perlFlags@
|
||||||
|
|
||||||
|
use DBI;
|
||||||
|
use File::Basename;
|
||||||
|
use IO::Select;
|
||||||
|
use Nix::Config;
|
||||||
|
use Nix::Store;
|
||||||
|
use Nix::Utils;
|
||||||
|
use WWW::Curl::Easy;
|
||||||
|
use WWW::Curl::Multi;
|
||||||
|
use strict;
|
||||||
|
|
||||||
|
|
||||||
|
Nix::Config::readConfig;
|
||||||
|
|
||||||
|
my @caches;
|
||||||
|
my $gotCaches = 0;
|
||||||
|
|
||||||
|
my $maxParallelRequests = int($Nix::Config::config{"binary-caches-parallel-connections"} // 150);
|
||||||
|
$maxParallelRequests = 1 if $maxParallelRequests < 1;
|
||||||
|
|
||||||
|
my $debug = ($ENV{"NIX_DEBUG_SUBST"} // "") eq 1;
|
||||||
|
|
||||||
|
my ($dbh, $queryCache, $insertNAR, $queryNAR, $insertNARExistence, $queryNARExistence);
|
||||||
|
|
||||||
|
my $curlm = WWW::Curl::Multi->new;
|
||||||
|
my $activeRequests = 0;
|
||||||
|
my $curlIdCount = 1;
|
||||||
|
my %requests;
|
||||||
|
my %scheduled;
|
||||||
|
my $caBundle = $ENV{"CURL_CA_BUNDLE"} // $ENV{"OPENSSL_X509_CERT_FILE"};
|
||||||
|
|
||||||
|
|
||||||
|
sub addRequest {
|
||||||
|
my ($storePath, $url, $head) = @_;
|
||||||
|
|
||||||
|
my $curl = WWW::Curl::Easy->new;
|
||||||
|
my $curlId = $curlIdCount++;
|
||||||
|
$requests{$curlId} = { storePath => $storePath, url => $url, handle => $curl, content => "", type => $head ? "HEAD" : "GET" };
|
||||||
|
|
||||||
|
$curl->setopt(CURLOPT_PRIVATE, $curlId);
|
||||||
|
$curl->setopt(CURLOPT_URL, $url);
|
||||||
|
$curl->setopt(CURLOPT_WRITEDATA, \$requests{$curlId}->{content});
|
||||||
|
$curl->setopt(CURLOPT_FOLLOWLOCATION, 1);
|
||||||
|
$curl->setopt(CURLOPT_CAINFO, $caBundle) if defined $caBundle;
|
||||||
|
$curl->setopt(CURLOPT_USERAGENT, "Nix/$Nix::Config::version");
|
||||||
|
$curl->setopt(CURLOPT_NOBODY, 1) if $head;
|
||||||
|
$curl->setopt(CURLOPT_FAILONERROR, 1);
|
||||||
|
|
||||||
|
if ($activeRequests >= $maxParallelRequests) {
|
||||||
|
$scheduled{$curlId} = 1;
|
||||||
|
} else {
|
||||||
|
$curlm->add_handle($curl);
|
||||||
|
$activeRequests++;
|
||||||
|
}
|
||||||
|
|
||||||
|
return $requests{$curlId};
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
sub processRequests {
|
||||||
|
while ($activeRequests) {
|
||||||
|
my ($rfds, $wfds, $efds) = $curlm->fdset();
|
||||||
|
#print STDERR "R = @{$rfds}, W = @{$wfds}, E = @{$efds}\n";
|
||||||
|
|
||||||
|
# Sleep until we can read or write some data.
|
||||||
|
if (scalar @{$rfds} + scalar @{$wfds} + scalar @{$efds} > 0) {
|
||||||
|
IO::Select->select(IO::Select->new(@{$rfds}), IO::Select->new(@{$wfds}), IO::Select->new(@{$efds}), 0.1);
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($curlm->perform() != $activeRequests) {
|
||||||
|
while (my ($id, $result) = $curlm->info_read) {
|
||||||
|
if ($id) {
|
||||||
|
my $request = $requests{$id} or die;
|
||||||
|
my $handle = $request->{handle};
|
||||||
|
$request->{result} = $result;
|
||||||
|
$request->{httpStatus} = $handle->getinfo(CURLINFO_RESPONSE_CODE);
|
||||||
|
|
||||||
|
print STDERR "$request->{type} on $request->{url} [$request->{result}, $request->{httpStatus}]\n" if $debug;
|
||||||
|
|
||||||
|
$activeRequests--;
|
||||||
|
delete $request->{handle};
|
||||||
|
|
||||||
|
if (scalar(keys %scheduled) > 0) {
|
||||||
|
my $id2 = (keys %scheduled)[0];
|
||||||
|
$curlm->add_handle($requests{$id2}->{handle});
|
||||||
|
$activeRequests++;
|
||||||
|
delete $scheduled{$id2};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
sub initCache {
|
||||||
|
my $dbPath = "$Nix::Config::stateDir/binary-cache-v1.sqlite";
|
||||||
|
|
||||||
|
# Open/create the database.
|
||||||
|
$dbh = DBI->connect("dbi:SQLite:dbname=$dbPath", "", "")
|
||||||
|
or die "cannot open database `$dbPath'";
|
||||||
|
$dbh->{RaiseError} = 1;
|
||||||
|
$dbh->{PrintError} = 0;
|
||||||
|
|
||||||
|
$dbh->do("pragma synchronous = off"); # we can always reproduce the cache
|
||||||
|
$dbh->do("pragma journal_mode = truncate");
|
||||||
|
|
||||||
|
# Initialise the database schema, if necessary.
|
||||||
|
$dbh->do(<<EOF);
|
||||||
|
create table if not exists BinaryCaches (
|
||||||
|
id integer primary key autoincrement not null,
|
||||||
|
url text unique not null,
|
||||||
|
timestamp integer not null,
|
||||||
|
storeDir text not null,
|
||||||
|
wantMassQuery integer not null
|
||||||
|
);
|
||||||
|
EOF
|
||||||
|
|
||||||
|
$dbh->do(<<EOF);
|
||||||
|
create table if not exists NARs (
|
||||||
|
cache integer not null,
|
||||||
|
storePath text not null,
|
||||||
|
url text not null,
|
||||||
|
compression text not null,
|
||||||
|
fileHash text,
|
||||||
|
fileSize integer,
|
||||||
|
narHash text,
|
||||||
|
narSize integer,
|
||||||
|
refs text,
|
||||||
|
deriver text,
|
||||||
|
system text,
|
||||||
|
timestamp integer not null,
|
||||||
|
primary key (cache, storePath),
|
||||||
|
foreign key (cache) references BinaryCaches(id) on delete cascade
|
||||||
|
);
|
||||||
|
EOF
|
||||||
|
|
||||||
|
$dbh->do(<<EOF);
|
||||||
|
create table if not exists NARExistence (
|
||||||
|
cache integer not null,
|
||||||
|
storePath text not null,
|
||||||
|
exist integer not null,
|
||||||
|
timestamp integer not null,
|
||||||
|
primary key (cache, storePath),
|
||||||
|
foreign key (cache) references BinaryCaches(id) on delete cascade
|
||||||
|
);
|
||||||
|
EOF
|
||||||
|
|
||||||
|
$queryCache = $dbh->prepare("select id, storeDir, wantMassQuery from BinaryCaches where url = ?") or die;
|
||||||
|
|
||||||
|
$insertNAR = $dbh->prepare(
|
||||||
|
"insert or replace into NARs(cache, storePath, url, compression, fileHash, fileSize, narHash, " .
|
||||||
|
"narSize, refs, deriver, system, timestamp) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") or die;
|
||||||
|
|
||||||
|
$queryNAR = $dbh->prepare("select * from NARs where cache = ? and storePath = ?") or die;
|
||||||
|
|
||||||
|
$insertNARExistence = $dbh->prepare(
|
||||||
|
"insert or replace into NARExistence(cache, storePath, exist, timestamp) values (?, ?, ?, ?)") or die;
|
||||||
|
|
||||||
|
$queryNARExistence = $dbh->prepare("select exist from NARExistence where cache = ? and storePath = ?") or die;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
sub getAvailableCaches {
|
||||||
|
return if $gotCaches;
|
||||||
|
$gotCaches = 1;
|
||||||
|
|
||||||
|
sub strToList {
|
||||||
|
my ($s) = @_;
|
||||||
|
return map { s/\/+$//; $_ } split(/ /, $s);
|
||||||
|
}
|
||||||
|
|
||||||
|
my @urls = strToList ($Nix::Config::config{"binary-caches"} // "");
|
||||||
|
# // ($Nix::Config::storeDir eq "/nix/store" ? "http://nixos.org/binary-cache" : ""));
|
||||||
|
|
||||||
|
my $urlsFiles = $Nix::Config::config{"binary-cache-files"}
|
||||||
|
// "/nix/var/nix/profiles/per-user/root/channels/binary-caches/*";
|
||||||
|
foreach my $urlFile (glob $urlsFiles) {
|
||||||
|
next unless -f $urlFile;
|
||||||
|
open FILE, "<$urlFile" or die "cannot open ‘$urlFile’\n";
|
||||||
|
my $url = <FILE>; chomp $url;
|
||||||
|
close FILE;
|
||||||
|
push @urls, strToList($url);
|
||||||
|
}
|
||||||
|
|
||||||
|
# Allow Nix daemon users to override the binary caches to a subset
|
||||||
|
# of those listed in the config file. Note that ‘untrusted-*’
|
||||||
|
# denotes options passed by the client.
|
||||||
|
if (defined $Nix::Config::config{"untrusted-binary-caches"}) {
|
||||||
|
my @untrustedUrls = strToList $Nix::Config::config{"untrusted-binary-caches"};
|
||||||
|
my @trustedUrls = (@urls, strToList($Nix::Config::config{"trusted-binary-caches"} // ""));
|
||||||
|
@urls = ();
|
||||||
|
foreach my $url (@untrustedUrls) {
|
||||||
|
die "binary cache ‘$url’ is not trusted (please add it to ‘trusted-binary-caches’ in $Nix::Config::confDir/nix.conf)\n"
|
||||||
|
unless grep { $url eq $_ } @trustedUrls > 0;
|
||||||
|
push @urls, $url;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach my $url (Nix::Utils::uniq @urls) {
|
||||||
|
|
||||||
|
# FIXME: not atomic.
|
||||||
|
$queryCache->execute($url);
|
||||||
|
my $res = $queryCache->fetchrow_hashref();
|
||||||
|
if (defined $res) {
|
||||||
|
next if $res->{storeDir} ne $Nix::Config::storeDir;
|
||||||
|
push @caches, { id => $res->{id}, url => $url, wantMassQuery => $res->{wantMassQuery} };
|
||||||
|
next;
|
||||||
|
}
|
||||||
|
|
||||||
|
# Get the cache info file.
|
||||||
|
my $request = addRequest(undef, $url . "/nix-cache-info");
|
||||||
|
processRequests;
|
||||||
|
|
||||||
|
if ($request->{result} != 0) {
|
||||||
|
print STDERR "could not download ‘$request->{url}’ (" .
|
||||||
|
($request->{result} != 0 ? "Curl error $request->{result}" : "HTTP status $request->{httpStatus}") . ")\n";
|
||||||
|
next;
|
||||||
|
}
|
||||||
|
|
||||||
|
my $storeDir = "/nix/store";
|
||||||
|
my $wantMassQuery = 0;
|
||||||
|
foreach my $line (split "\n", $request->{content}) {
|
||||||
|
unless ($line =~ /^(.*): (.*)$/) {
|
||||||
|
print STDERR "bad cache info file ‘$request->{url}’\n";
|
||||||
|
return undef;
|
||||||
|
}
|
||||||
|
if ($1 eq "StoreDir") { $storeDir = $2; }
|
||||||
|
elsif ($1 eq "WantMassQuery") { $wantMassQuery = int($2); }
|
||||||
|
}
|
||||||
|
|
||||||
|
$dbh->do("insert into BinaryCaches(url, timestamp, storeDir, wantMassQuery) values (?, ?, ?, ?)",
|
||||||
|
{}, $url, time(), $storeDir, $wantMassQuery);
|
||||||
|
my $id = $dbh->last_insert_id("", "", "", "");
|
||||||
|
next if $storeDir ne $Nix::Config::storeDir;
|
||||||
|
push @caches, { id => $id, url => $url, wantMassQuery => $wantMassQuery };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
sub processNARInfo {
|
||||||
|
my ($storePath, $cache, $request) = @_;
|
||||||
|
|
||||||
|
if ($request->{result} != 0) {
|
||||||
|
if ($request->{result} != 37 && $request->{httpStatus} != 404) {
|
||||||
|
print STDERR "could not download ‘$request->{url}’ (" .
|
||||||
|
($request->{result} != 0 ? "Curl error $request->{result}" : "HTTP status $request->{httpStatus}") . ")\n";
|
||||||
|
} else {
|
||||||
|
$insertNARExistence->execute($cache->{id}, basename($storePath), 0, time())
|
||||||
|
unless $request->{url} =~ /^file:/;
|
||||||
|
}
|
||||||
|
return undef;
|
||||||
|
}
|
||||||
|
|
||||||
|
my ($storePath2, $url, $fileHash, $fileSize, $narHash, $narSize, $deriver, $system);
|
||||||
|
my $compression = "bzip2";
|
||||||
|
my @refs;
|
||||||
|
foreach my $line (split "\n", $request->{content}) {
|
||||||
|
unless ($line =~ /^(.*): (.*)$/) {
|
||||||
|
print STDERR "bad NAR info file ‘$request->{url}’\n";
|
||||||
|
return undef;
|
||||||
|
}
|
||||||
|
if ($1 eq "StorePath") { $storePath2 = $2; }
|
||||||
|
elsif ($1 eq "URL") { $url = $2; }
|
||||||
|
elsif ($1 eq "Compression") { $compression = $2; }
|
||||||
|
elsif ($1 eq "FileHash") { $fileHash = $2; }
|
||||||
|
elsif ($1 eq "FileSize") { $fileSize = int($2); }
|
||||||
|
elsif ($1 eq "NarHash") { $narHash = $2; }
|
||||||
|
elsif ($1 eq "NarSize") { $narSize = int($2); }
|
||||||
|
elsif ($1 eq "References") { @refs = split / /, $2; }
|
||||||
|
elsif ($1 eq "Deriver") { $deriver = $2; }
|
||||||
|
elsif ($1 eq "System") { $system = $2; }
|
||||||
|
}
|
||||||
|
return undef if $storePath ne $storePath2;
|
||||||
|
if ($storePath ne $storePath2 || !defined $url || !defined $narHash) {
|
||||||
|
print STDERR "bad NAR info file ‘$request->{url}’\n";
|
||||||
|
return undef;
|
||||||
|
}
|
||||||
|
|
||||||
|
# Cache the result.
|
||||||
|
$insertNAR->execute(
|
||||||
|
$cache->{id}, basename($storePath), $url, $compression, $fileHash, $fileSize,
|
||||||
|
$narHash, $narSize, join(" ", @refs), $deriver, $system, time())
|
||||||
|
unless $request->{url} =~ /^file:/;
|
||||||
|
|
||||||
|
return
|
||||||
|
{ url => $url
|
||||||
|
, compression => $compression
|
||||||
|
, fileHash => $fileHash
|
||||||
|
, fileSize => $fileSize
|
||||||
|
, narHash => $narHash
|
||||||
|
, narSize => $narSize
|
||||||
|
, refs => [ @refs ]
|
||||||
|
, deriver => $deriver
|
||||||
|
, system => $system
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
sub getCachedInfoFrom {
|
||||||
|
my ($storePath, $cache) = @_;
|
||||||
|
|
||||||
|
$queryNAR->execute($cache->{id}, basename($storePath));
|
||||||
|
my $res = $queryNAR->fetchrow_hashref();
|
||||||
|
return undef unless defined $res;
|
||||||
|
|
||||||
|
return
|
||||||
|
{ url => $res->{url}
|
||||||
|
, compression => $res->{compression}
|
||||||
|
, fileHash => $res->{fileHash}
|
||||||
|
, fileSize => $res->{fileSize}
|
||||||
|
, narHash => $res->{narHash}
|
||||||
|
, narSize => $res->{narSize}
|
||||||
|
, refs => [ split " ", $res->{refs} ]
|
||||||
|
, deriver => $res->{deriver}
|
||||||
|
} if defined $res;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
sub negativeHit {
|
||||||
|
my ($storePath, $cache) = @_;
|
||||||
|
$queryNARExistence->execute($cache->{id}, basename($storePath));
|
||||||
|
my $res = $queryNARExistence->fetchrow_hashref();
|
||||||
|
return defined $res && $res->{exist} == 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
sub positiveHit {
|
||||||
|
my ($storePath, $cache) = @_;
|
||||||
|
return 1 if defined getCachedInfoFrom($storePath, $cache);
|
||||||
|
$queryNARExistence->execute($cache->{id}, basename($storePath));
|
||||||
|
my $res = $queryNARExistence->fetchrow_hashref();
|
||||||
|
return defined $res && $res->{exist} == 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
sub printInfo {
|
||||||
|
my ($storePath, $info) = @_;
|
||||||
|
print "$storePath\n";
|
||||||
|
print $info->{deriver} ? "$Nix::Config::storeDir/$info->{deriver}" : "", "\n";
|
||||||
|
print scalar @{$info->{refs}}, "\n";
|
||||||
|
print "$Nix::Config::storeDir/$_\n" foreach @{$info->{refs}};
|
||||||
|
print $info->{fileSize} || 0, "\n";
|
||||||
|
print $info->{narSize} || 0, "\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
sub infoUrl {
|
||||||
|
my ($binaryCacheUrl, $storePath) = @_;
|
||||||
|
my $pathHash = substr(basename($storePath), 0, 32);
|
||||||
|
my $infoUrl = "$binaryCacheUrl/$pathHash.narinfo";
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
sub printInfoParallel {
|
||||||
|
my @paths = @_;
|
||||||
|
|
||||||
|
# First print all paths for which we have cached info.
|
||||||
|
my @left;
|
||||||
|
foreach my $storePath (@paths) {
|
||||||
|
my $found = 0;
|
||||||
|
foreach my $cache (@caches) {
|
||||||
|
my $info = getCachedInfoFrom($storePath, $cache);
|
||||||
|
if (defined $info) {
|
||||||
|
printInfo($storePath, $info);
|
||||||
|
$found = 1;
|
||||||
|
last;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
push @left, $storePath if !$found;
|
||||||
|
}
|
||||||
|
|
||||||
|
return if scalar @left == 0;
|
||||||
|
|
||||||
|
foreach my $cache (@caches) {
|
||||||
|
|
||||||
|
my @left2;
|
||||||
|
%requests = ();
|
||||||
|
foreach my $storePath (@left) {
|
||||||
|
if (negativeHit($storePath, $cache)) {
|
||||||
|
push @left2, $storePath;
|
||||||
|
next;
|
||||||
|
}
|
||||||
|
addRequest($storePath, infoUrl($cache->{url}, $storePath));
|
||||||
|
}
|
||||||
|
|
||||||
|
processRequests;
|
||||||
|
|
||||||
|
foreach my $request (values %requests) {
|
||||||
|
my $info = processNARInfo($request->{storePath}, $cache, $request);
|
||||||
|
if (defined $info) {
|
||||||
|
printInfo($request->{storePath}, $info);
|
||||||
|
} else {
|
||||||
|
push @left2, $request->{storePath};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@left = @left2;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
sub printSubstitutablePaths {
|
||||||
|
my @paths = @_;
|
||||||
|
|
||||||
|
# First look for paths that have cached info.
|
||||||
|
my @left;
|
||||||
|
foreach my $storePath (@paths) {
|
||||||
|
my $found = 0;
|
||||||
|
foreach my $cache (@caches) {
|
||||||
|
next unless $cache->{wantMassQuery};
|
||||||
|
if (positiveHit($storePath, $cache)) {
|
||||||
|
print "$storePath\n";
|
||||||
|
$found = 1;
|
||||||
|
last;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
push @left, $storePath if !$found;
|
||||||
|
}
|
||||||
|
|
||||||
|
return if scalar @left == 0;
|
||||||
|
|
||||||
|
# For remaining paths, do HEAD requests.
|
||||||
|
foreach my $cache (@caches) {
|
||||||
|
next unless $cache->{wantMassQuery};
|
||||||
|
my @left2;
|
||||||
|
%requests = ();
|
||||||
|
foreach my $storePath (@left) {
|
||||||
|
if (negativeHit($storePath, $cache)) {
|
||||||
|
push @left2, $storePath;
|
||||||
|
next;
|
||||||
|
}
|
||||||
|
addRequest($storePath, infoUrl($cache->{url}, $storePath), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
processRequests;
|
||||||
|
|
||||||
|
foreach my $request (values %requests) {
|
||||||
|
if ($request->{result} != 0) {
|
||||||
|
if ($request->{result} != 37 && $request->{httpStatus} != 404) {
|
||||||
|
print STDERR "could not check ‘$request->{url}’ (" .
|
||||||
|
($request->{result} != 0 ? "Curl error $request->{result}" : "HTTP status $request->{httpStatus}") . ")\n";
|
||||||
|
} else {
|
||||||
|
$insertNARExistence->execute($cache->{id}, basename($request->{storePath}), 0, time())
|
||||||
|
unless $request->{url} =~ /^file:/;
|
||||||
|
}
|
||||||
|
push @left2, $request->{storePath};
|
||||||
|
} else {
|
||||||
|
$insertNARExistence->execute($cache->{id}, basename($request->{storePath}), 1, time())
|
||||||
|
unless $request->{url} =~ /^file:/;
|
||||||
|
print "$request->{storePath}\n";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@left = @left2;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
sub downloadBinary {
|
||||||
|
my ($storePath) = @_;
|
||||||
|
|
||||||
|
foreach my $cache (@caches) {
|
||||||
|
my $info = getCachedInfoFrom($storePath, $cache);
|
||||||
|
|
||||||
|
unless (defined $info) {
|
||||||
|
next if negativeHit($storePath, $cache);
|
||||||
|
my $request = addRequest($storePath, infoUrl($cache->{url}, $storePath));
|
||||||
|
processRequests;
|
||||||
|
$info = processNARInfo($storePath, $cache, $request);
|
||||||
|
}
|
||||||
|
|
||||||
|
next unless defined $info;
|
||||||
|
|
||||||
|
my $decompressor;
|
||||||
|
if ($info->{compression} eq "bzip2") { $decompressor = "$Nix::Config::bzip2 -d"; }
|
||||||
|
elsif ($info->{compression} eq "xz") { $decompressor = "$Nix::Config::xz -d"; }
|
||||||
|
else {
|
||||||
|
print STDERR "unknown compression method ‘$info->{compression}’\n";
|
||||||
|
next;
|
||||||
|
}
|
||||||
|
my $url = "$cache->{url}/$info->{url}"; # FIXME: handle non-relative URLs
|
||||||
|
print STDERR "\n*** Downloading ‘$url’ into ‘$storePath’...\n";
|
||||||
|
Nix::Utils::checkURL $url;
|
||||||
|
if (system("$Nix::Config::curl --fail --location --insecure '$url' | $decompressor | $Nix::Config::binDir/nix-store --restore $storePath") != 0) {
|
||||||
|
die "download of `$info->{url}' failed" . ($! ? ": $!" : "") . "\n" unless $? == 0;
|
||||||
|
next;
|
||||||
|
}
|
||||||
|
|
||||||
|
# Tell Nix about the expected hash so it can verify it.
|
||||||
|
print "$info->{narHash}\n";
|
||||||
|
|
||||||
|
print STDERR "\n";
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
print STDERR "could not download ‘$storePath’ from any binary cache\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
initCache();
|
||||||
|
|
||||||
|
|
||||||
|
if ($ARGV[0] eq "--query") {
|
||||||
|
|
||||||
|
while (<STDIN>) {
|
||||||
|
getAvailableCaches;
|
||||||
|
chomp;
|
||||||
|
my ($cmd, @args) = split " ", $_;
|
||||||
|
|
||||||
|
if ($cmd eq "have") {
|
||||||
|
printSubstitutablePaths(@args);
|
||||||
|
print "\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
elsif ($cmd eq "info") {
|
||||||
|
printInfoParallel(@args);
|
||||||
|
print "\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
else { die "unknown command `$cmd'"; }
|
||||||
|
|
||||||
|
flush STDOUT;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
elsif ($ARGV[0] eq "--substitute") {
|
||||||
|
my $storePath = $ARGV[1] or die;
|
||||||
|
getAvailableCaches;
|
||||||
|
downloadBinary($storePath);
|
||||||
|
}
|
||||||
|
|
||||||
|
else {
|
||||||
|
die;
|
||||||
|
}
|
|
@ -4,6 +4,7 @@ use strict;
|
||||||
use Nix::Config;
|
use Nix::Config;
|
||||||
use Nix::Manifest;
|
use Nix::Manifest;
|
||||||
use Nix::Store;
|
use Nix::Store;
|
||||||
|
use Nix::Utils;
|
||||||
use POSIX qw(strftime);
|
use POSIX qw(strftime);
|
||||||
use File::Temp qw(tempdir);
|
use File::Temp qw(tempdir);
|
||||||
|
|
||||||
|
@ -15,6 +16,9 @@ my $logFile = "$Nix::Config::logDir/downloads";
|
||||||
# estimating the expected download size.
|
# estimating the expected download size.
|
||||||
my $fast = 1;
|
my $fast = 1;
|
||||||
|
|
||||||
|
# ‘--insecure’ is fine because Nix verifies the hash of the result.
|
||||||
|
my $curl = "$Nix::Config::curl --fail --location --insecure";
|
||||||
|
|
||||||
|
|
||||||
# Open the manifest cache and update it if necessary.
|
# Open the manifest cache and update it if necessary.
|
||||||
my $dbh = updateManifestDB();
|
my $dbh = updateManifestDB();
|
||||||
|
@ -173,56 +177,54 @@ sub computeSmallestDownload {
|
||||||
if ($ARGV[0] eq "--query") {
|
if ($ARGV[0] eq "--query") {
|
||||||
|
|
||||||
while (<STDIN>) {
|
while (<STDIN>) {
|
||||||
my $cmd = $_; chomp $cmd;
|
chomp;
|
||||||
|
my ($cmd, @args) = split " ", $_;
|
||||||
|
|
||||||
if ($cmd eq "have") {
|
if ($cmd eq "have") {
|
||||||
my $storePath = <STDIN>; chomp $storePath;
|
foreach my $storePath (@args) {
|
||||||
print STDOUT (
|
print "$storePath\n" if scalar @{$dbh->selectcol_arrayref("select 1 from NARs where storePath = ?", {}, $storePath)} > 0;
|
||||||
scalar @{$dbh->selectcol_arrayref("select 1 from NARs where storePath = ?", {}, $storePath)} > 0
|
}
|
||||||
? "1\n" : "0\n");
|
print "\n";
|
||||||
}
|
}
|
||||||
|
|
||||||
elsif ($cmd eq "info") {
|
elsif ($cmd eq "info") {
|
||||||
my $storePath = <STDIN>; chomp $storePath;
|
foreach my $storePath (@args) {
|
||||||
|
|
||||||
my $infos = $dbh->selectall_arrayref(
|
my $infos = $dbh->selectall_arrayref(
|
||||||
"select * from NARs where storePath = ?",
|
"select * from NARs where storePath = ?",
|
||||||
{ Slice => {} }, $storePath);
|
{ Slice => {} }, $storePath);
|
||||||
|
|
||||||
my $info;
|
next unless scalar @{$infos} > 0;
|
||||||
if (scalar @{$infos} > 0) {
|
my $info = @{$infos}[0];
|
||||||
$info = @{$infos}[0];
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
print "0\n";
|
|
||||||
next; # not an error
|
|
||||||
}
|
|
||||||
|
|
||||||
print "1\n";
|
print "$storePath\n";
|
||||||
print "$info->{deriver}\n";
|
print "$info->{deriver}\n";
|
||||||
my @references = split " ", $info->{refs};
|
my @references = split " ", $info->{refs};
|
||||||
print scalar @references, "\n";
|
print scalar @references, "\n";
|
||||||
print "$_\n" foreach @references;
|
print "$_\n" foreach @references;
|
||||||
|
|
||||||
my @path = computeSmallestDownload $storePath;
|
my @path = computeSmallestDownload $storePath;
|
||||||
|
|
||||||
my $downloadSize = 0;
|
my $downloadSize = 0;
|
||||||
while (scalar @path > 0) {
|
while (scalar @path > 0) {
|
||||||
my $edge = pop @path;
|
my $edge = pop @path;
|
||||||
my $u = $edge->{start};
|
my $u = $edge->{start};
|
||||||
my $v = $edge->{end};
|
my $v = $edge->{end};
|
||||||
if ($edge->{type} eq "patch") {
|
if ($edge->{type} eq "patch") {
|
||||||
$downloadSize += $edge->{info}->{size} || 0;
|
$downloadSize += $edge->{info}->{size} || 0;
|
||||||
}
|
}
|
||||||
elsif ($edge->{type} eq "narfile") {
|
elsif ($edge->{type} eq "narfile") {
|
||||||
$downloadSize += $edge->{info}->{size} || 0;
|
$downloadSize += $edge->{info}->{size} || 0;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
print "$downloadSize\n";
|
||||||
|
|
||||||
|
my $narSize = $info->{narSize} || 0;
|
||||||
|
print "$narSize\n";
|
||||||
}
|
}
|
||||||
|
|
||||||
print "$downloadSize\n";
|
print "\n";
|
||||||
|
|
||||||
my $narSize = $info->{narSize} || 0;
|
|
||||||
print "$narSize\n";
|
|
||||||
}
|
}
|
||||||
|
|
||||||
else { die "unknown command `$cmd'"; }
|
else { die "unknown command `$cmd'"; }
|
||||||
|
@ -273,16 +275,6 @@ $dbh->disconnect;
|
||||||
my $curStep = 1;
|
my $curStep = 1;
|
||||||
my $maxStep = scalar @path;
|
my $maxStep = scalar @path;
|
||||||
|
|
||||||
sub downloadFile {
|
|
||||||
my $url = shift;
|
|
||||||
$ENV{"PRINT_PATH"} = 1;
|
|
||||||
$ENV{"QUIET"} = 1;
|
|
||||||
my ($hash, $path) = `$Nix::Config::binDir/nix-prefetch-url '$url'`;
|
|
||||||
die "download of `$url' failed" . ($! ? ": $!" : "") . "\n" unless $? == 0;
|
|
||||||
chomp $path;
|
|
||||||
return $path;
|
|
||||||
}
|
|
||||||
|
|
||||||
my $finalNarHash;
|
my $finalNarHash;
|
||||||
|
|
||||||
while (scalar @path > 0) {
|
while (scalar @path > 0) {
|
||||||
|
@ -314,13 +306,16 @@ while (scalar @path > 0) {
|
||||||
|
|
||||||
# Download the patch.
|
# Download the patch.
|
||||||
print STDERR " downloading patch...\n";
|
print STDERR " downloading patch...\n";
|
||||||
my $patchPath = downloadFile "$patch->{url}";
|
my $patchPath = "$tmpDir/patch";
|
||||||
|
Nix::Utils::checkURL $patch->{url};
|
||||||
|
system("$curl '$patch->{url}' -o $patchPath") == 0
|
||||||
|
or die "cannot download patch `$patch->{url}'\n";
|
||||||
|
|
||||||
# Apply the patch to the NAR archive produced in step 1 (for
|
# Apply the patch to the NAR archive produced in step 1 (for
|
||||||
# the already present path) or a later step (for patch sequences).
|
# the already present path) or a later step (for patch sequences).
|
||||||
print STDERR " applying patch...\n";
|
print STDERR " applying patch...\n";
|
||||||
system("$Nix::Config::libexecDir/bspatch $tmpNar $tmpNar2 $patchPath") == 0
|
system("$Nix::Config::libexecDir/bspatch $tmpNar $tmpNar2 $patchPath") == 0
|
||||||
or die "cannot apply patch `$patchPath' to $tmpNar";
|
or die "cannot apply patch `$patchPath' to $tmpNar\n";
|
||||||
|
|
||||||
if ($curStep < $maxStep) {
|
if ($curStep < $maxStep) {
|
||||||
# The archive will be used as the base of the next patch.
|
# The archive will be used as the base of the next patch.
|
||||||
|
@ -330,7 +325,7 @@ while (scalar @path > 0) {
|
||||||
# into the target path.
|
# into the target path.
|
||||||
print STDERR " unpacking patched archive...\n";
|
print STDERR " unpacking patched archive...\n";
|
||||||
system("$Nix::Config::binDir/nix-store --restore $v < $tmpNar2") == 0
|
system("$Nix::Config::binDir/nix-store --restore $v < $tmpNar2") == 0
|
||||||
or die "cannot unpack $tmpNar2 into `$v'";
|
or die "cannot unpack $tmpNar2 into `$v'\n";
|
||||||
}
|
}
|
||||||
|
|
||||||
$finalNarHash = $patch->{narHash};
|
$finalNarHash = $patch->{narHash};
|
||||||
|
@ -343,19 +338,15 @@ while (scalar @path > 0) {
|
||||||
my $size = $narFile->{size} || -1;
|
my $size = $narFile->{size} || -1;
|
||||||
print LOGFILE "$$ narfile $narFile->{url} $size $v\n";
|
print LOGFILE "$$ narfile $narFile->{url} $size $v\n";
|
||||||
|
|
||||||
# Download the archive.
|
Nix::Utils::checkURL $narFile->{url};
|
||||||
print STDERR " downloading archive...\n";
|
|
||||||
my $narFilePath = downloadFile "$narFile->{url}";
|
|
||||||
|
|
||||||
if ($curStep < $maxStep) {
|
if ($curStep < $maxStep) {
|
||||||
# The archive will be used a base to a patch.
|
# The archive will be used a base to a patch.
|
||||||
system("$Nix::Config::bzip2 -d < '$narFilePath' > $tmpNar") == 0
|
system("$curl '$narFile->{url}' | $Nix::Config::bzip2 -d > $tmpNar") == 0
|
||||||
or die "cannot unpack `$narFilePath' into `$v'";
|
or die "cannot download and unpack `$narFile->{url}' into `$v'\n";
|
||||||
} else {
|
} else {
|
||||||
# Unpack the archive into the target path.
|
# Unpack the archive into the target path.
|
||||||
print STDERR " unpacking archive...\n";
|
system("$curl '$narFile->{url}' | $Nix::Config::bzip2 -d | $Nix::Config::binDir/nix-store --restore '$v'") == 0
|
||||||
system("$Nix::Config::bzip2 -d < '$narFilePath' | $Nix::Config::binDir/nix-store --restore '$v'") == 0
|
or die "cannot download and unpack `$narFile->{url}' into `$v'\n";
|
||||||
or die "cannot unpack `$narFilePath' into `$v'";
|
|
||||||
}
|
}
|
||||||
|
|
||||||
$finalNarHash = $narFile->{narHash};
|
$finalNarHash = $narFile->{narHash};
|
||||||
|
@ -365,21 +356,10 @@ while (scalar @path > 0) {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
# Make sure that the hash declared in the manifest matches what we
|
# Tell Nix about the expected hash so it can verify it.
|
||||||
# downloaded and unpacked.
|
die "cannot check integrity of the downloaded path since its hash is not known\n"
|
||||||
|
unless defined $finalNarHash;
|
||||||
if (defined $finalNarHash) {
|
print "$finalNarHash\n";
|
||||||
my ($hashAlgo, $hash) = parseHash $finalNarHash;
|
|
||||||
|
|
||||||
# The hash in the manifest can be either in base-16 or base-32.
|
|
||||||
# Handle both.
|
|
||||||
my $hash2 = hashPath($hashAlgo, $hashAlgo eq "sha256" && length($hash) != 64, $targetPath);
|
|
||||||
|
|
||||||
die "hash mismatch in downloaded path $targetPath; expected $hash, got $hash2\n"
|
|
||||||
if $hash ne $hash2;
|
|
||||||
} else {
|
|
||||||
die "cannot check integrity of the downloaded path since its hash is not known\n";
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
print STDERR "\n";
|
print STDERR "\n";
|
||||||
|
|
|
@ -58,6 +58,11 @@ EOF
|
||||||
# '` hack
|
# '` hack
|
||||||
}
|
}
|
||||||
|
|
||||||
|
elsif ($arg eq "--version") {
|
||||||
|
print "nix-build (Nix) $Nix::Config::version\n";
|
||||||
|
exit 0;
|
||||||
|
}
|
||||||
|
|
||||||
elsif ($arg eq "--add-drv-link") {
|
elsif ($arg eq "--add-drv-link") {
|
||||||
$drvLink = "./derivation";
|
$drvLink = "./derivation";
|
||||||
}
|
}
|
||||||
|
|
|
@ -80,12 +80,6 @@ sub update {
|
||||||
|
|
||||||
readChannels;
|
readChannels;
|
||||||
|
|
||||||
# Create the manifests directory if it doesn't exist.
|
|
||||||
mkdir $manifestDir, 0755 unless -e $manifestDir;
|
|
||||||
|
|
||||||
# Do we have write permission to the manifests directory?
|
|
||||||
die "$0: you do not have write permission to `$manifestDir'!\n" unless -W $manifestDir;
|
|
||||||
|
|
||||||
# Download each channel.
|
# Download each channel.
|
||||||
my $exprs = "";
|
my $exprs = "";
|
||||||
foreach my $name (keys %channels) {
|
foreach my $name (keys %channels) {
|
||||||
|
@ -102,10 +96,19 @@ sub update {
|
||||||
$headers =~ s/\r//g;
|
$headers =~ s/\r//g;
|
||||||
$url = $1 if $headers =~ /^Location:\s*(.*)\s*$/m;
|
$url = $1 if $headers =~ /^Location:\s*(.*)\s*$/m;
|
||||||
|
|
||||||
# Pull the channel manifest.
|
# Check if the channel advertises a binary cache.
|
||||||
$ENV{'NIX_ORIG_URL'} = $origUrl;
|
my $binaryCacheURL = `$Nix::Config::curl --silent '$url'/binary-cache-url`;
|
||||||
system("$Nix::Config::binDir/nix-pull", "--skip-wrong-store", "$url/MANIFEST") == 0
|
my $extraAttrs = "";
|
||||||
or die "cannot pull manifest from `$url'\n";
|
if ($? == 0 && $binaryCacheURL ne "") {
|
||||||
|
$extraAttrs .= "binaryCacheURL = \"$binaryCacheURL\"; ";
|
||||||
|
} else {
|
||||||
|
# No binary cache, so pull the channel manifest.
|
||||||
|
mkdir $manifestDir, 0755 unless -e $manifestDir;
|
||||||
|
die "$0: you do not have write permission to `$manifestDir'!\n" unless -W $manifestDir;
|
||||||
|
$ENV{'NIX_ORIG_URL'} = $origUrl;
|
||||||
|
system("$Nix::Config::binDir/nix-pull", "--skip-wrong-store", "$url/MANIFEST") == 0
|
||||||
|
or die "cannot pull manifest from `$url'\n";
|
||||||
|
}
|
||||||
|
|
||||||
# Download the channel tarball.
|
# Download the channel tarball.
|
||||||
my $fullURL = "$url/nixexprs.tar.bz2";
|
my $fullURL = "$url/nixexprs.tar.bz2";
|
||||||
|
@ -120,7 +123,7 @@ sub update {
|
||||||
my $cname = $name;
|
my $cname = $name;
|
||||||
$cname .= $1 if basename($url) =~ /(-\d.*)$/;
|
$cname .= $1 if basename($url) =~ /(-\d.*)$/;
|
||||||
|
|
||||||
$exprs .= "'f: f { name = \"$cname\"; channelName = \"$name\"; src = builtins.storePath \"$path\"; }' ";
|
$exprs .= "'f: f { name = \"$cname\"; channelName = \"$name\"; src = builtins.storePath \"$path\"; $extraAttrs }' ";
|
||||||
}
|
}
|
||||||
|
|
||||||
# Unpack the channel tarballs into the Nix store and install them
|
# Unpack the channel tarballs into the Nix store and install them
|
||||||
|
@ -194,6 +197,11 @@ while (scalar @ARGV) {
|
||||||
usageError;
|
usageError;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
elsif ($arg eq "--version") {
|
||||||
|
print "nix-channel (Nix) $Nix::Config::version\n";
|
||||||
|
exit 0;
|
||||||
|
}
|
||||||
|
|
||||||
else {
|
else {
|
||||||
die "unknown argument `$arg'; try `--help'";
|
die "unknown argument `$arg'; try `--help'";
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,6 +3,7 @@
|
||||||
use strict;
|
use strict;
|
||||||
use File::Temp qw(tempdir);
|
use File::Temp qw(tempdir);
|
||||||
use Nix::Config;
|
use Nix::Config;
|
||||||
|
use Nix::Utils;
|
||||||
|
|
||||||
|
|
||||||
sub usageError {
|
sub usageError {
|
||||||
|
@ -72,7 +73,7 @@ my $tmpDir = tempdir("nix-install-package.XXXXXX", CLEANUP => 1, TMPDIR => 1)
|
||||||
|
|
||||||
sub barf {
|
sub barf {
|
||||||
my $msg = shift;
|
my $msg = shift;
|
||||||
print "$msg\n";
|
print "\nInstallation failed: $msg\n";
|
||||||
<STDIN> if $interactive;
|
<STDIN> if $interactive;
|
||||||
exit 1;
|
exit 1;
|
||||||
}
|
}
|
||||||
|
@ -92,7 +93,6 @@ open PKGFILE, "<$pkgFile" or barf "cannot open `$pkgFile': $!";
|
||||||
my $contents = <PKGFILE>;
|
my $contents = <PKGFILE>;
|
||||||
close PKGFILE;
|
close PKGFILE;
|
||||||
|
|
||||||
my $urlRE = "(?: [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\']+ )";
|
|
||||||
my $nameRE = "(?: [A-Za-z0-9\+\-\.\_\?\=]+ )"; # see checkStoreName()
|
my $nameRE = "(?: [A-Za-z0-9\+\-\.\_\?\=]+ )"; # see checkStoreName()
|
||||||
my $systemRE = "(?: [A-Za-z0-9\+\-\_]+ )";
|
my $systemRE = "(?: [A-Za-z0-9\+\-\_]+ )";
|
||||||
my $pathRE = "(?: \/ [\/A-Za-z0-9\+\-\.\_\?\=]* )";
|
my $pathRE = "(?: \/ [\/A-Za-z0-9\+\-\.\_\?\=]* )";
|
||||||
|
@ -101,7 +101,7 @@ my $pathRE = "(?: \/ [\/A-Za-z0-9\+\-\.\_\?\=]* )";
|
||||||
# store path. We'll let nix-env do that.
|
# store path. We'll let nix-env do that.
|
||||||
|
|
||||||
$contents =~
|
$contents =~
|
||||||
/ ^ \s* (\S+) \s+ ($urlRE) \s+ ($nameRE) \s+ ($systemRE) \s+ ($pathRE) \s+ ($pathRE) /x
|
/ ^ \s* (\S+) \s+ ($Nix::Utils::urlRE) \s+ ($nameRE) \s+ ($systemRE) \s+ ($pathRE) \s+ ($pathRE) ( \s+ ($Nix::Utils::urlRE) )? /x
|
||||||
or barf "invalid package contents";
|
or barf "invalid package contents";
|
||||||
my $version = $1;
|
my $version = $1;
|
||||||
my $manifestURL = $2;
|
my $manifestURL = $2;
|
||||||
|
@ -109,6 +109,7 @@ my $drvName = $3;
|
||||||
my $system = $4;
|
my $system = $4;
|
||||||
my $drvPath = $5;
|
my $drvPath = $5;
|
||||||
my $outPath = $6;
|
my $outPath = $6;
|
||||||
|
my $binaryCacheURL = $8;
|
||||||
|
|
||||||
barf "invalid package version `$version'" unless $version eq "NIXPKG1";
|
barf "invalid package version `$version'" unless $version eq "NIXPKG1";
|
||||||
|
|
||||||
|
@ -122,17 +123,25 @@ if ($interactive) {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
# Store the manifest in the temporary directory so that we don't
|
if (defined $binaryCacheURL) {
|
||||||
# pollute /nix/var/nix/manifests. This also requires that we don't
|
|
||||||
# use the Nix daemon (because otherwise download-using-manifests won't
|
|
||||||
# see our NIX_MANIFESTS_DIRS environment variable).
|
|
||||||
$ENV{NIX_MANIFESTS_DIR} = $tmpDir;
|
|
||||||
$ENV{NIX_REMOTE} = "";
|
|
||||||
|
|
||||||
|
push @extraNixEnvArgs, "--option", "binary-caches", $binaryCacheURL;
|
||||||
|
|
||||||
print "\nPulling manifests...\n";
|
} else {
|
||||||
system("$Nix::Config::binDir/nix-pull", $manifestURL) == 0
|
|
||||||
or barf "nix-pull failed: $?";
|
# Store the manifest in the temporary directory so that we don't
|
||||||
|
# pollute /nix/var/nix/manifests. This also requires that we
|
||||||
|
# don't use the Nix daemon (because otherwise
|
||||||
|
# download-using-manifests won't see our NIX_MANIFESTS_DIRS
|
||||||
|
# environment variable).
|
||||||
|
$ENV{NIX_MANIFESTS_DIR} = $tmpDir;
|
||||||
|
$ENV{NIX_REMOTE} = "";
|
||||||
|
|
||||||
|
print "\nPulling manifests...\n";
|
||||||
|
system("$Nix::Config::binDir/nix-pull", $manifestURL) == 0
|
||||||
|
or barf "nix-pull failed: $?";
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
print "\nInstalling package...\n";
|
print "\nInstalling package...\n";
|
||||||
|
|
|
@ -1,77 +1,77 @@
|
||||||
#! @perl@ -w @perlFlags@
|
#! @perl@ -w @perlFlags@
|
||||||
|
|
||||||
use strict;
|
use strict;
|
||||||
|
use File::Basename;
|
||||||
use File::Temp qw(tempdir);
|
use File::Temp qw(tempdir);
|
||||||
|
use File::Path qw(mkpath);
|
||||||
use File::stat;
|
use File::stat;
|
||||||
|
use File::Copy;
|
||||||
use Nix::Config;
|
use Nix::Config;
|
||||||
|
use Nix::Store;
|
||||||
use Nix::Manifest;
|
use Nix::Manifest;
|
||||||
|
|
||||||
my $hashAlgo = "sha256";
|
|
||||||
|
|
||||||
my $tmpDir = tempdir("nix-push.XXXXXX", CLEANUP => 1, TMPDIR => 1)
|
my $tmpDir = tempdir("nix-push.XXXXXX", CLEANUP => 1, TMPDIR => 1)
|
||||||
or die "cannot create a temporary directory";
|
or die "cannot create a temporary directory";
|
||||||
|
|
||||||
my $nixExpr = "$tmpDir/create-nars.nix";
|
my $nixExpr = "$tmpDir/create-nars.nix";
|
||||||
my $manifest = "$tmpDir/MANIFEST";
|
|
||||||
|
|
||||||
my $curl = "$Nix::Config::curl --fail --silent";
|
|
||||||
my $extraCurlFlags = ${ENV{'CURL_FLAGS'}};
|
|
||||||
$curl = "$curl $extraCurlFlags" if defined $extraCurlFlags;
|
|
||||||
|
|
||||||
|
|
||||||
# Parse the command line.
|
# Parse the command line.
|
||||||
my $localCopy;
|
my $compressionType = "xz";
|
||||||
my $localArchivesDir;
|
my $force = 0;
|
||||||
my $localManifestFile;
|
my $destDir;
|
||||||
|
my $writeManifest = 0;
|
||||||
my $targetArchivesUrl;
|
my $archivesURL;
|
||||||
|
my @roots;
|
||||||
my $archivesPutURL;
|
|
||||||
my $archivesGetURL;
|
|
||||||
my $manifestPutURL;
|
|
||||||
|
|
||||||
sub showSyntax {
|
sub showSyntax {
|
||||||
print STDERR <<EOF
|
print STDERR <<EOF
|
||||||
Usage: nix-push --copy ARCHIVES_DIR MANIFEST_FILE PATHS...
|
Usage: nix-push --dest DIR [--manifest] [--url-prefix URL] PATHS...
|
||||||
or: nix-push ARCHIVES_PUT_URL ARCHIVES_GET_URL MANIFEST_PUT_URL PATHS...
|
|
||||||
|
|
||||||
`nix-push' copies or uploads the closure of PATHS to the given
|
`nix-push' packs the closure of PATHS into a set of NAR files stored
|
||||||
destination.
|
in DIR. Optionally generate a manifest.
|
||||||
EOF
|
EOF
|
||||||
; # `
|
; # `
|
||||||
exit 1;
|
exit 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
showSyntax if scalar @ARGV < 1;
|
for (my $n = 0; $n < scalar @ARGV; $n++) {
|
||||||
|
my $arg = $ARGV[$n];
|
||||||
|
|
||||||
if ($ARGV[0] eq "--copy") {
|
if ($arg eq "--help") {
|
||||||
showSyntax if scalar @ARGV < 3;
|
showSyntax;
|
||||||
$localCopy = 1;
|
} elsif ($arg eq "--bzip2") {
|
||||||
shift @ARGV;
|
$compressionType = "bzip2";
|
||||||
$localArchivesDir = shift @ARGV;
|
} elsif ($arg eq "--force") {
|
||||||
$localManifestFile = shift @ARGV;
|
$force = 1;
|
||||||
if ($ARGV[0] eq "--target") {
|
} elsif ($arg eq "--dest") {
|
||||||
shift @ARGV;
|
$n++;
|
||||||
$targetArchivesUrl = shift @ARGV;
|
die "$0: `$arg' requires an argument\n" unless $n < scalar @ARGV;
|
||||||
}
|
$destDir = $ARGV[$n];
|
||||||
else {
|
mkpath($destDir, 0, 0755);
|
||||||
$targetArchivesUrl = "file://$localArchivesDir";
|
} elsif ($arg eq "--manifest") {
|
||||||
|
$writeManifest = 1;
|
||||||
|
} elsif ($arg eq "--url-prefix") {
|
||||||
|
$n++;
|
||||||
|
die "$0: `$arg' requires an argument\n" unless $n < scalar @ARGV;
|
||||||
|
$archivesURL = $ARGV[$n];
|
||||||
|
} elsif (substr($arg, 0, 1) eq "-") {
|
||||||
|
showSyntax;
|
||||||
|
} else {
|
||||||
|
push @roots, $arg;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else {
|
|
||||||
showSyntax if scalar @ARGV < 3;
|
showSyntax if !defined $destDir;
|
||||||
$localCopy = 0;
|
|
||||||
$archivesPutURL = shift @ARGV;
|
$archivesURL = "file://$destDir" unless defined $archivesURL;
|
||||||
$archivesGetURL = shift @ARGV;
|
|
||||||
$manifestPutURL = shift @ARGV;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# From the given store paths, determine the set of requisite store
|
# From the given store paths, determine the set of requisite store
|
||||||
# paths, i.e, the paths required to realise them.
|
# paths, i.e, the paths required to realise them.
|
||||||
my %storePaths;
|
my %storePaths;
|
||||||
|
|
||||||
foreach my $path (@ARGV) {
|
foreach my $path (@roots) {
|
||||||
die unless $path =~ /^\//;
|
die unless $path =~ /^\//;
|
||||||
|
|
||||||
# Get all paths referenced by the normalisation of the given
|
# Get all paths referenced by the normalisation of the given
|
||||||
|
@ -92,8 +92,8 @@ foreach my $path (@ARGV) {
|
||||||
my @storePaths = keys %storePaths;
|
my @storePaths = keys %storePaths;
|
||||||
|
|
||||||
|
|
||||||
# For each path, create a Nix expression that turns the path into
|
# Create a list of Nix derivations that turn each path into a Nix
|
||||||
# a Nix archive.
|
# archive.
|
||||||
open NIX, ">$nixExpr";
|
open NIX, ">$nixExpr";
|
||||||
print NIX "[";
|
print NIX "[";
|
||||||
|
|
||||||
|
@ -103,7 +103,7 @@ foreach my $storePath (@storePaths) {
|
||||||
# Construct a Nix expression that creates a Nix archive.
|
# Construct a Nix expression that creates a Nix archive.
|
||||||
my $nixexpr =
|
my $nixexpr =
|
||||||
"(import <nix/nar.nix> " .
|
"(import <nix/nar.nix> " .
|
||||||
"{ storePath = builtins.storePath \"$storePath\"; hashAlgo = \"$hashAlgo\"; }) ";
|
"{ storePath = builtins.storePath \"$storePath\"; hashAlgo = \"sha256\"; compressionType = \"$compressionType\"; }) ";
|
||||||
|
|
||||||
print NIX $nixexpr;
|
print NIX $nixexpr;
|
||||||
}
|
}
|
||||||
|
@ -112,172 +112,132 @@ print NIX "]";
|
||||||
close NIX;
|
close NIX;
|
||||||
|
|
||||||
|
|
||||||
# Instantiate store derivations from the Nix expression.
|
# Build the Nix expression.
|
||||||
my @storeExprs;
|
print STDERR "building compressed archives...\n";
|
||||||
print STDERR "instantiating store derivations...\n";
|
my @narPaths;
|
||||||
my $pid = open(READ, "$Nix::Config::binDir/nix-instantiate $nixExpr|")
|
my $pid = open(READ, "$Nix::Config::binDir/nix-build $nixExpr -o $tmpDir/result |")
|
||||||
or die "cannot run nix-instantiate";
|
or die "cannot run nix-build";
|
||||||
while (<READ>) {
|
while (<READ>) {
|
||||||
chomp;
|
chomp;
|
||||||
die unless /^\//;
|
die unless /^\//;
|
||||||
push @storeExprs, $_;
|
push @narPaths, $_;
|
||||||
}
|
}
|
||||||
close READ or die "nix-instantiate failed: $?";
|
close READ or die "nix-build failed: $?";
|
||||||
|
|
||||||
|
|
||||||
# Build the derivations.
|
# Write the cache info file.
|
||||||
print STDERR "creating archives...\n";
|
my $cacheInfoFile = "$destDir/nix-cache-info";
|
||||||
|
if (! -e $cacheInfoFile) {
|
||||||
my @narPaths;
|
open FILE, ">$cacheInfoFile" or die "cannot create $cacheInfoFile: $!";
|
||||||
|
print FILE "StoreDir: $Nix::Config::storeDir\n";
|
||||||
my @tmp = @storeExprs;
|
print FILE "WantMassQuery: 0\n"; # by default, don't hit this cache for "nix-env -qas"
|
||||||
while (scalar @tmp > 0) {
|
close FILE;
|
||||||
my $n = scalar @tmp;
|
|
||||||
if ($n > 256) { $n = 256 };
|
|
||||||
my @tmp2 = @tmp[0..$n - 1];
|
|
||||||
@tmp = @tmp[$n..scalar @tmp - 1];
|
|
||||||
|
|
||||||
my $pid = open(READ, "$Nix::Config::binDir/nix-store --realise @tmp2|")
|
|
||||||
or die "cannot run nix-store";
|
|
||||||
while (<READ>) {
|
|
||||||
chomp;
|
|
||||||
die unless (/^\//);
|
|
||||||
push @narPaths, "$_";
|
|
||||||
}
|
|
||||||
close READ or die "nix-store failed: $?";
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
# Create the manifest.
|
# Copy the archives and the corresponding NAR info files.
|
||||||
print STDERR "creating manifest...\n";
|
print STDERR "copying archives...\n";
|
||||||
|
|
||||||
|
my $totalNarSize = 0;
|
||||||
|
my $totalCompressedSize = 0;
|
||||||
|
|
||||||
my %narFiles;
|
my %narFiles;
|
||||||
my %patches;
|
|
||||||
|
|
||||||
my @narArchives;
|
|
||||||
for (my $n = 0; $n < scalar @storePaths; $n++) {
|
for (my $n = 0; $n < scalar @storePaths; $n++) {
|
||||||
my $storePath = $storePaths[$n];
|
my $storePath = $storePaths[$n];
|
||||||
my $narDir = $narPaths[$n];
|
my $narDir = $narPaths[$n];
|
||||||
|
my $baseName = basename $storePath;
|
||||||
|
|
||||||
$storePath =~ /\/([^\/]*)$/;
|
# Get info about the store path.
|
||||||
my $basename = $1;
|
my ($deriver, $narHash, $time, $narSize, $refs) = queryPathInfo($storePath, 1);
|
||||||
defined $basename or die;
|
|
||||||
|
|
||||||
open HASH, "$narDir/narbz2-hash" or die "cannot open narbz2-hash";
|
|
||||||
my $narbz2Hash = <HASH>;
|
|
||||||
chomp $narbz2Hash;
|
|
||||||
$narbz2Hash =~ /^[0-9a-z]+$/ or die "invalid hash";
|
|
||||||
close HASH;
|
|
||||||
|
|
||||||
my $narName = "$narbz2Hash.nar.bz2";
|
|
||||||
|
|
||||||
my $narFile = "$narDir/$narName";
|
|
||||||
(-f $narFile) or die "narfile for $storePath not found";
|
|
||||||
push @narArchives, $narFile;
|
|
||||||
|
|
||||||
my $narbz2Size = stat($narFile)->size;
|
|
||||||
|
|
||||||
my $references = `$Nix::Config::binDir/nix-store --query --references '$storePath'`;
|
|
||||||
die "cannot query references for `$storePath'" if $? != 0;
|
|
||||||
$references = join(" ", split(" ", $references));
|
|
||||||
|
|
||||||
my $deriver = `$Nix::Config::binDir/nix-store --query --deriver '$storePath'`;
|
|
||||||
die "cannot query deriver for `$storePath'" if $? != 0;
|
|
||||||
chomp $deriver;
|
|
||||||
$deriver = "" if $deriver eq "unknown-deriver";
|
|
||||||
|
|
||||||
my $narHash = `$Nix::Config::binDir/nix-store --query --hash '$storePath'`;
|
|
||||||
die "cannot query hash for `$storePath'" if $? != 0;
|
|
||||||
chomp $narHash;
|
|
||||||
|
|
||||||
# In some exceptional cases (such as VM tests that use the Nix
|
# In some exceptional cases (such as VM tests that use the Nix
|
||||||
# store of the host), the database doesn't contain the hash. So
|
# store of the host), the database doesn't contain the hash. So
|
||||||
# compute it.
|
# compute it.
|
||||||
if ($narHash =~ /^sha256:0*$/) {
|
if ($narHash =~ /^sha256:0*$/) {
|
||||||
$narHash = `$Nix::Config::binDir/nix-hash --type sha256 --base32 '$storePath'`;
|
my $nar = "$tmpDir/nar";
|
||||||
die "cannot hash `$storePath'" if $? != 0;
|
system("$Nix::Config::binDir/nix-store --dump $storePath > $nar") == 0
|
||||||
|
or die "cannot dump $storePath\n";
|
||||||
|
$narHash = `$Nix::Config::binDir/nix-hash --type sha256 --base32 --flat $nar`;
|
||||||
|
die "cannot hash `$nar'" if $? != 0;
|
||||||
chomp $narHash;
|
chomp $narHash;
|
||||||
$narHash = "sha256:$narHash";
|
$narHash = "sha256:$narHash";
|
||||||
|
$narSize = stat("$nar")->size;
|
||||||
|
unlink $nar or die;
|
||||||
}
|
}
|
||||||
|
|
||||||
my $narSize = `$Nix::Config::binDir/nix-store --query --size '$storePath'`;
|
$totalNarSize += $narSize;
|
||||||
die "cannot query size for `$storePath'" if $? != 0;
|
|
||||||
chomp $narSize;
|
|
||||||
|
|
||||||
my $url;
|
# Get info about the compressed NAR.
|
||||||
if ($localCopy) {
|
open HASH, "$narDir/nar-compressed-hash" or die "cannot open nar-compressed-hash";
|
||||||
$url = "$targetArchivesUrl/$narName";
|
my $compressedHash = <HASH>;
|
||||||
} else {
|
chomp $compressedHash;
|
||||||
$url = "$archivesGetURL/$narName";
|
$compressedHash =~ /^[0-9a-z]+$/ or die "invalid hash";
|
||||||
|
close HASH;
|
||||||
|
|
||||||
|
my $narName = "$compressedHash.nar." . ($compressionType eq "xz" ? "xz" : "bz2");
|
||||||
|
|
||||||
|
my $narFile = "$narDir/$narName";
|
||||||
|
(-f $narFile) or die "NAR file for $storePath not found";
|
||||||
|
|
||||||
|
my $compressedSize = stat($narFile)->size;
|
||||||
|
$totalCompressedSize += $compressedSize;
|
||||||
|
|
||||||
|
printf STDERR "%s [%.2f MiB, %.1f%%]\n", $storePath,
|
||||||
|
$compressedSize / (1024 * 1024), $compressedSize / $narSize * 100;
|
||||||
|
|
||||||
|
# Copy the compressed NAR.
|
||||||
|
my $dst = "$destDir/$narName";
|
||||||
|
if (! -f $dst) {
|
||||||
|
my $tmp = "$destDir/.tmp.$$.$narName";
|
||||||
|
copy($narFile, $tmp) or die "cannot copy $narFile to $tmp: $!\n";
|
||||||
|
rename($tmp, $dst) or die "cannot rename $tmp to $dst: $!\n";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Write the info file.
|
||||||
|
my $info;
|
||||||
|
$info .= "StorePath: $storePath\n";
|
||||||
|
$info .= "URL: $narName\n";
|
||||||
|
$info .= "Compression: $compressionType\n";
|
||||||
|
$info .= "FileHash: sha256:$compressedHash\n";
|
||||||
|
$info .= "FileSize: $compressedSize\n";
|
||||||
|
$info .= "NarHash: $narHash\n";
|
||||||
|
$info .= "NarSize: $narSize\n";
|
||||||
|
$info .= "References: " . join(" ", map { basename $_ } @{$refs}) . "\n";
|
||||||
|
if (defined $deriver) {
|
||||||
|
$info .= "Deriver: " . basename $deriver . "\n";
|
||||||
|
if (isValidPath($deriver)) {
|
||||||
|
my $drv = derivationFromPath($deriver);
|
||||||
|
$info .= "System: $drv->{platform}\n";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
my $pathHash = substr(basename($storePath), 0, 32);
|
||||||
|
|
||||||
|
$dst = "$destDir/$pathHash.narinfo";
|
||||||
|
if ($force || ! -f $dst) {
|
||||||
|
my $tmp = "$destDir/.tmp.$$.$pathHash.narinfo";
|
||||||
|
open INFO, ">$tmp" or die;
|
||||||
|
print INFO "$info" or die;
|
||||||
|
close INFO or die;
|
||||||
|
rename($tmp, $dst) or die "cannot rename $tmp to $dst: $!\n";
|
||||||
|
}
|
||||||
|
|
||||||
$narFiles{$storePath} = [
|
$narFiles{$storePath} = [
|
||||||
{ url => $url
|
{ url => "$archivesURL/$narName"
|
||||||
, hash => "$hashAlgo:$narbz2Hash"
|
, hash => "sha256:$compressedHash"
|
||||||
, size => $narbz2Size
|
, size => $compressedSize
|
||||||
, narHash => "$narHash"
|
, narHash => "$narHash"
|
||||||
, narSize => $narSize
|
, narSize => $narSize
|
||||||
, references => $references
|
, references => join(" ", @{$refs})
|
||||||
, deriver => $deriver
|
, deriver => $deriver
|
||||||
}
|
}
|
||||||
];
|
] if $writeManifest;
|
||||||
}
|
}
|
||||||
|
|
||||||
writeManifest $manifest, \%narFiles, \%patches;
|
printf STDERR "total compressed size %.2f MiB, %.1f%%\n",
|
||||||
|
$totalCompressedSize / (1024 * 1024), $totalCompressedSize / $totalNarSize * 100;
|
||||||
|
|
||||||
|
|
||||||
sub copyFile {
|
# Optionally write a manifest.
|
||||||
my $src = shift;
|
writeManifest "$destDir/MANIFEST", \%narFiles, \() if $writeManifest;
|
||||||
my $dst = shift;
|
|
||||||
my $tmp = "$dst.tmp.$$";
|
|
||||||
system("@coreutils@/cp", $src, $tmp) == 0 or die "cannot copy file";
|
|
||||||
rename($tmp, $dst) or die "cannot rename file: $!";
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# Upload/copy the archives.
|
|
||||||
print STDERR "uploading/copying archives...\n";
|
|
||||||
|
|
||||||
sub archiveExists {
|
|
||||||
my $name = shift;
|
|
||||||
print STDERR " HEAD on $archivesGetURL/$name\n";
|
|
||||||
return system("$curl --head $archivesGetURL/$name > /dev/null") == 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach my $narArchive (@narArchives) {
|
|
||||||
|
|
||||||
$narArchive =~ /\/([^\/]*)$/;
|
|
||||||
my $basename = $1;
|
|
||||||
|
|
||||||
if ($localCopy) {
|
|
||||||
# Since nix-push creates $dst atomically, if it exists we
|
|
||||||
# don't have to copy again.
|
|
||||||
my $dst = "$localArchivesDir/$basename";
|
|
||||||
if (! -f "$localArchivesDir/$basename") {
|
|
||||||
print STDERR " $narArchive\n";
|
|
||||||
copyFile $narArchive, $dst;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
if (!archiveExists("$basename")) {
|
|
||||||
print STDERR " $narArchive\n";
|
|
||||||
system("$curl --show-error --upload-file " .
|
|
||||||
"'$narArchive' '$archivesPutURL/$basename' > /dev/null") == 0 or
|
|
||||||
die "curl failed on $narArchive: $?";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# Upload the manifest.
|
|
||||||
print STDERR "uploading manifest...\n";
|
|
||||||
if ($localCopy) {
|
|
||||||
copyFile $manifest, $localManifestFile;
|
|
||||||
copyFile "$manifest.bz2", "$localManifestFile.bz2";
|
|
||||||
} else {
|
|
||||||
system("$curl --show-error --upload-file " .
|
|
||||||
"'$manifest' '$manifestPutURL' > /dev/null") == 0 or
|
|
||||||
die "curl failed on $manifest: $?";
|
|
||||||
system("$curl --show-error --upload-file " .
|
|
||||||
"'$manifest'.bz2 '$manifestPutURL'.bz2 > /dev/null") == 0 or
|
|
||||||
die "curl failed on $manifest: $?";
|
|
||||||
}
|
|
||||||
|
|
|
@ -181,7 +181,7 @@ EvalState::EvalState()
|
||||||
searchPathInsertionPoint = searchPath.end();
|
searchPathInsertionPoint = searchPath.end();
|
||||||
Strings paths = tokenizeString(getEnv("NIX_PATH", ""), ":");
|
Strings paths = tokenizeString(getEnv("NIX_PATH", ""), ":");
|
||||||
foreach (Strings::iterator, i, paths) addToSearchPath(*i);
|
foreach (Strings::iterator, i, paths) addToSearchPath(*i);
|
||||||
addToSearchPath("nix=" + nixDataDir + "/nix/corepkgs");
|
addToSearchPath("nix=" + settings.nixDataDir + "/nix/corepkgs");
|
||||||
searchPathInsertionPoint = searchPath.begin();
|
searchPathInsertionPoint = searchPath.begin();
|
||||||
|
|
||||||
createBaseEnv();
|
createBaseEnv();
|
||||||
|
@ -1091,7 +1091,7 @@ string EvalState::coerceToString(Value & v, PathSet & context,
|
||||||
if (srcToStore[path] != "")
|
if (srcToStore[path] != "")
|
||||||
dstPath = srcToStore[path];
|
dstPath = srcToStore[path];
|
||||||
else {
|
else {
|
||||||
dstPath = readOnlyMode
|
dstPath = settings.readOnlyMode
|
||||||
? computeStorePathForPath(path).first
|
? computeStorePathForPath(path).first
|
||||||
: store->addToStore(path);
|
: store->addToStore(path);
|
||||||
srcToStore[path] = dstPath;
|
srcToStore[path] = dstPath;
|
||||||
|
|
|
@ -32,6 +32,8 @@ private:
|
||||||
bool metaInfoRead;
|
bool metaInfoRead;
|
||||||
MetaInfo meta;
|
MetaInfo meta;
|
||||||
|
|
||||||
|
bool failed; // set if we get an AssertionError
|
||||||
|
|
||||||
public:
|
public:
|
||||||
string name;
|
string name;
|
||||||
string attrPath; /* path towards the derivation */
|
string attrPath; /* path towards the derivation */
|
||||||
|
@ -40,7 +42,7 @@ public:
|
||||||
/* !!! make this private */
|
/* !!! make this private */
|
||||||
Bindings * attrs;
|
Bindings * attrs;
|
||||||
|
|
||||||
DrvInfo() : metaInfoRead(false), attrs(0) { };
|
DrvInfo() : metaInfoRead(false), failed(false), attrs(0) { };
|
||||||
|
|
||||||
string queryDrvPath(EvalState & state) const;
|
string queryDrvPath(EvalState & state) const;
|
||||||
string queryOutPath(EvalState & state) const;
|
string queryOutPath(EvalState & state) const;
|
||||||
|
@ -58,6 +60,9 @@ public:
|
||||||
}
|
}
|
||||||
|
|
||||||
void setMetaInfo(const MetaInfo & meta);
|
void setMetaInfo(const MetaInfo & meta);
|
||||||
|
|
||||||
|
void setFailed() { failed = true; };
|
||||||
|
bool hasFailed() { return failed; };
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -51,6 +51,12 @@ static void prim_import(EvalState & state, Value * * args, Value & v)
|
||||||
% path % ctx);
|
% path % ctx);
|
||||||
if (isDerivation(ctx))
|
if (isDerivation(ctx))
|
||||||
try {
|
try {
|
||||||
|
/* For performance, prefetch all substitute info. */
|
||||||
|
PathSet willBuild, willSubstitute, unknown;
|
||||||
|
unsigned long long downloadSize, narSize;
|
||||||
|
queryMissing(*store, singleton<PathSet>(ctx),
|
||||||
|
willBuild, willSubstitute, unknown, downloadSize, narSize);
|
||||||
|
|
||||||
/* !!! If using a substitute, we only need to fetch
|
/* !!! If using a substitute, we only need to fetch
|
||||||
the selected output of this derivation. */
|
the selected output of this derivation. */
|
||||||
store->buildPaths(singleton<PathSet>(ctx));
|
store->buildPaths(singleton<PathSet>(ctx));
|
||||||
|
@ -617,7 +623,7 @@ static void prim_toFile(EvalState & state, Value * * args, Value & v)
|
||||||
refs.insert(path);
|
refs.insert(path);
|
||||||
}
|
}
|
||||||
|
|
||||||
Path storePath = readOnlyMode
|
Path storePath = settings.readOnlyMode
|
||||||
? computeStorePathForText(name, contents, refs)
|
? computeStorePathForText(name, contents, refs)
|
||||||
: store->addTextToStore(name, contents, refs);
|
: store->addTextToStore(name, contents, refs);
|
||||||
|
|
||||||
|
@ -681,7 +687,7 @@ static void prim_filterSource(EvalState & state, Value * * args, Value & v)
|
||||||
|
|
||||||
FilterFromExpr filter(state, *args[0]);
|
FilterFromExpr filter(state, *args[0]);
|
||||||
|
|
||||||
Path dstPath = readOnlyMode
|
Path dstPath = settings.readOnlyMode
|
||||||
? computeStorePathForPath(path, true, htSHA256, filter).first
|
? computeStorePathForPath(path, true, htSHA256, filter).first
|
||||||
: store->addToStore(path, true, htSHA256, filter);
|
: store->addToStore(path, true, htSHA256, filter);
|
||||||
|
|
||||||
|
@ -1134,7 +1140,7 @@ void EvalState::createBaseEnv()
|
||||||
mkInt(v, time(0));
|
mkInt(v, time(0));
|
||||||
addConstant("__currentTime", v);
|
addConstant("__currentTime", v);
|
||||||
|
|
||||||
mkString(v, thisSystem.c_str());
|
mkString(v, settings.thisSystem.c_str());
|
||||||
addConstant("__currentSystem", v);
|
addConstant("__currentSystem", v);
|
||||||
|
|
||||||
// Miscellaneous
|
// Miscellaneous
|
||||||
|
|
|
@ -64,7 +64,7 @@ void printMissing(StoreAPI & store, const PathSet & paths)
|
||||||
|
|
||||||
if (!unknown.empty()) {
|
if (!unknown.empty()) {
|
||||||
printMsg(lvlInfo, format("don't know how to build these paths%1%:")
|
printMsg(lvlInfo, format("don't know how to build these paths%1%:")
|
||||||
% (readOnlyMode ? " (may be caused by read-only store access)" : ""));
|
% (settings.readOnlyMode ? " (may be caused by read-only store access)" : ""));
|
||||||
foreach (PathSet::iterator, i, unknown)
|
foreach (PathSet::iterator, i, unknown)
|
||||||
printMsg(lvlInfo, format(" %1%") % *i);
|
printMsg(lvlInfo, format(" %1%") % *i);
|
||||||
}
|
}
|
||||||
|
@ -83,11 +83,20 @@ static void setLogType(string lt)
|
||||||
static bool showTrace = false;
|
static bool showTrace = false;
|
||||||
|
|
||||||
|
|
||||||
|
string getArg(const string & opt,
|
||||||
|
Strings::iterator & i, const Strings::iterator & end)
|
||||||
|
{
|
||||||
|
++i;
|
||||||
|
if (i == end) throw UsageError(format("`%1%' requires an argument") % opt);
|
||||||
|
return *i;
|
||||||
|
}
|
||||||
|
|
||||||
/* Initialize and reorder arguments, then call the actual argument
|
/* Initialize and reorder arguments, then call the actual argument
|
||||||
processor. */
|
processor. */
|
||||||
static void initAndRun(int argc, char * * argv)
|
static void initAndRun(int argc, char * * argv)
|
||||||
{
|
{
|
||||||
setDefaultsFromEnvironment();
|
settings.processEnvironment();
|
||||||
|
settings.loadConfFile();
|
||||||
|
|
||||||
/* Catch SIGINT. */
|
/* Catch SIGINT. */
|
||||||
struct sigaction act;
|
struct sigaction act;
|
||||||
|
@ -146,20 +155,19 @@ static void initAndRun(int argc, char * * argv)
|
||||||
remaining.clear();
|
remaining.clear();
|
||||||
|
|
||||||
/* Process default options. */
|
/* Process default options. */
|
||||||
int verbosityDelta = 0;
|
int verbosityDelta = lvlInfo;
|
||||||
for (Strings::iterator i = args.begin(); i != args.end(); ++i) {
|
for (Strings::iterator i = args.begin(); i != args.end(); ++i) {
|
||||||
string arg = *i;
|
string arg = *i;
|
||||||
if (arg == "--verbose" || arg == "-v") verbosityDelta++;
|
if (arg == "--verbose" || arg == "-v") verbosityDelta++;
|
||||||
else if (arg == "--quiet") verbosityDelta--;
|
else if (arg == "--quiet") verbosityDelta--;
|
||||||
else if (arg == "--log-type") {
|
else if (arg == "--log-type") {
|
||||||
++i;
|
string s = getArg(arg, i, args.end());
|
||||||
if (i == args.end()) throw UsageError("`--log-type' requires an argument");
|
setLogType(s);
|
||||||
setLogType(*i);
|
|
||||||
}
|
}
|
||||||
else if (arg == "--no-build-output" || arg == "-Q")
|
else if (arg == "--no-build-output" || arg == "-Q")
|
||||||
buildVerbosity = lvlVomit;
|
settings.buildVerbosity = lvlVomit;
|
||||||
else if (arg == "--print-build-trace")
|
else if (arg == "--print-build-trace")
|
||||||
printBuildTrace = true;
|
settings.printBuildTrace = true;
|
||||||
else if (arg == "--help") {
|
else if (arg == "--help") {
|
||||||
printHelp();
|
printHelp();
|
||||||
return;
|
return;
|
||||||
|
@ -169,23 +177,23 @@ static void initAndRun(int argc, char * * argv)
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
else if (arg == "--keep-failed" || arg == "-K")
|
else if (arg == "--keep-failed" || arg == "-K")
|
||||||
keepFailed = true;
|
settings.keepFailed = true;
|
||||||
else if (arg == "--keep-going" || arg == "-k")
|
else if (arg == "--keep-going" || arg == "-k")
|
||||||
keepGoing = true;
|
settings.keepGoing = true;
|
||||||
else if (arg == "--fallback")
|
else if (arg == "--fallback")
|
||||||
tryFallback = true;
|
settings.set("build-fallback", "true");
|
||||||
else if (arg == "--max-jobs" || arg == "-j")
|
else if (arg == "--max-jobs" || arg == "-j")
|
||||||
maxBuildJobs = getIntArg<unsigned int>(arg, i, args.end());
|
settings.set("build-max-jobs", getArg(arg, i, args.end()));
|
||||||
else if (arg == "--cores")
|
else if (arg == "--cores")
|
||||||
buildCores = getIntArg<unsigned int>(arg, i, args.end());
|
settings.set("build-cores", getArg(arg, i, args.end()));
|
||||||
else if (arg == "--readonly-mode")
|
else if (arg == "--readonly-mode")
|
||||||
readOnlyMode = true;
|
settings.readOnlyMode = true;
|
||||||
else if (arg == "--max-silent-time")
|
else if (arg == "--max-silent-time")
|
||||||
maxSilentTime = getIntArg<unsigned int>(arg, i, args.end());
|
settings.set("build-max-silent-time", getArg(arg, i, args.end()));
|
||||||
else if (arg == "--timeout")
|
else if (arg == "--timeout")
|
||||||
buildTimeout = getIntArg<unsigned int>(arg, i, args.end());
|
settings.set("build-timeout", getArg(arg, i, args.end()));
|
||||||
else if (arg == "--no-build-hook")
|
else if (arg == "--no-build-hook")
|
||||||
useBuildHook = false;
|
settings.useBuildHook = false;
|
||||||
else if (arg == "--show-trace")
|
else if (arg == "--show-trace")
|
||||||
showTrace = true;
|
showTrace = true;
|
||||||
else if (arg == "--option") {
|
else if (arg == "--option") {
|
||||||
|
@ -193,14 +201,15 @@ static void initAndRun(int argc, char * * argv)
|
||||||
string name = *i;
|
string name = *i;
|
||||||
++i; if (i == args.end()) throw UsageError("`--option' requires two arguments");
|
++i; if (i == args.end()) throw UsageError("`--option' requires two arguments");
|
||||||
string value = *i;
|
string value = *i;
|
||||||
overrideSetting(name, tokenizeString(value));
|
settings.set(name, value);
|
||||||
}
|
}
|
||||||
else remaining.push_back(arg);
|
else remaining.push_back(arg);
|
||||||
}
|
}
|
||||||
|
|
||||||
verbosityDelta += queryIntSetting("verbosity", lvlInfo);
|
|
||||||
verbosity = (Verbosity) (verbosityDelta < 0 ? 0 : verbosityDelta);
|
verbosity = (Verbosity) (verbosityDelta < 0 ? 0 : verbosityDelta);
|
||||||
|
|
||||||
|
settings.update();
|
||||||
|
|
||||||
run(remaining);
|
run(remaining);
|
||||||
|
|
||||||
/* Close the Nix database. */
|
/* Close the Nix database. */
|
||||||
|
|
|
@ -94,7 +94,7 @@ typedef map<Path, WeakGoalPtr> WeakGoalMap;
|
||||||
class Goal : public boost::enable_shared_from_this<Goal>
|
class Goal : public boost::enable_shared_from_this<Goal>
|
||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
typedef enum {ecBusy, ecSuccess, ecFailed} ExitCode;
|
typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters} ExitCode;
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
|
|
||||||
|
@ -111,6 +111,10 @@ protected:
|
||||||
/* Number of goals we are/were waiting for that have failed. */
|
/* Number of goals we are/were waiting for that have failed. */
|
||||||
unsigned int nrFailed;
|
unsigned int nrFailed;
|
||||||
|
|
||||||
|
/* Number of substitution goals we are/were waiting for that
|
||||||
|
failed because there are no substituters. */
|
||||||
|
unsigned int nrNoSubstituters;
|
||||||
|
|
||||||
/* Name of this goal for debugging purposes. */
|
/* Name of this goal for debugging purposes. */
|
||||||
string name;
|
string name;
|
||||||
|
|
||||||
|
@ -119,7 +123,7 @@ protected:
|
||||||
|
|
||||||
Goal(Worker & worker) : worker(worker)
|
Goal(Worker & worker) : worker(worker)
|
||||||
{
|
{
|
||||||
nrFailed = 0;
|
nrFailed = nrNoSubstituters = 0;
|
||||||
exitCode = ecBusy;
|
exitCode = ecBusy;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -225,8 +229,6 @@ private:
|
||||||
|
|
||||||
public:
|
public:
|
||||||
|
|
||||||
bool cacheFailure;
|
|
||||||
|
|
||||||
/* Set if at least one derivation had a BuildError (i.e. permanent
|
/* Set if at least one derivation had a BuildError (i.e. permanent
|
||||||
failure). */
|
failure). */
|
||||||
bool permanentFailure;
|
bool permanentFailure;
|
||||||
|
@ -306,9 +308,11 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
|
||||||
trace(format("waitee `%1%' done; %2% left") %
|
trace(format("waitee `%1%' done; %2% left") %
|
||||||
waitee->name % waitees.size());
|
waitee->name % waitees.size());
|
||||||
|
|
||||||
if (result == ecFailed) ++nrFailed;
|
if (result == ecFailed || result == ecNoSubstituters) ++nrFailed;
|
||||||
|
|
||||||
if (waitees.empty() || (result == ecFailed && !keepGoing)) {
|
if (result == ecNoSubstituters) ++nrNoSubstituters;
|
||||||
|
|
||||||
|
if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) {
|
||||||
|
|
||||||
/* If we failed and keepGoing is not set, we remove all
|
/* If we failed and keepGoing is not set, we remove all
|
||||||
remaining waitees. */
|
remaining waitees. */
|
||||||
|
@ -330,7 +334,7 @@ void Goal::amDone(ExitCode result)
|
||||||
{
|
{
|
||||||
trace("done");
|
trace("done");
|
||||||
assert(exitCode == ecBusy);
|
assert(exitCode == ecBusy);
|
||||||
assert(result == ecSuccess || result == ecFailed);
|
assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters);
|
||||||
exitCode = result;
|
exitCode = result;
|
||||||
foreach (WeakGoals::iterator, i, waiters) {
|
foreach (WeakGoals::iterator, i, waiters) {
|
||||||
GoalPtr goal = i->lock();
|
GoalPtr goal = i->lock();
|
||||||
|
@ -365,6 +369,7 @@ void commonChildInit(Pipe & logPipe)
|
||||||
if (dup2(logPipe.writeSide, STDERR_FILENO) == -1)
|
if (dup2(logPipe.writeSide, STDERR_FILENO) == -1)
|
||||||
throw SysError("cannot pipe standard error into log file");
|
throw SysError("cannot pipe standard error into log file");
|
||||||
logPipe.readSide.close();
|
logPipe.readSide.close();
|
||||||
|
logPipe.writeSide.close();
|
||||||
|
|
||||||
/* Dup stderr to stdout. */
|
/* Dup stderr to stdout. */
|
||||||
if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1)
|
if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1)
|
||||||
|
@ -459,14 +464,13 @@ void UserLock::acquire()
|
||||||
{
|
{
|
||||||
assert(uid == 0);
|
assert(uid == 0);
|
||||||
|
|
||||||
string buildUsersGroup = querySetting("build-users-group", "");
|
assert(settings.buildUsersGroup != "");
|
||||||
assert(buildUsersGroup != "");
|
|
||||||
|
|
||||||
/* Get the members of the build-users-group. */
|
/* Get the members of the build-users-group. */
|
||||||
struct group * gr = getgrnam(buildUsersGroup.c_str());
|
struct group * gr = getgrnam(settings.buildUsersGroup.c_str());
|
||||||
if (!gr)
|
if (!gr)
|
||||||
throw Error(format("the group `%1%' specified in `build-users-group' does not exist")
|
throw Error(format("the group `%1%' specified in `build-users-group' does not exist")
|
||||||
% buildUsersGroup);
|
% settings.buildUsersGroup);
|
||||||
gid = gr->gr_gid;
|
gid = gr->gr_gid;
|
||||||
|
|
||||||
/* Copy the result of getgrnam. */
|
/* Copy the result of getgrnam. */
|
||||||
|
@ -478,7 +482,7 @@ void UserLock::acquire()
|
||||||
|
|
||||||
if (users.empty())
|
if (users.empty())
|
||||||
throw Error(format("the build users group `%1%' has no members")
|
throw Error(format("the build users group `%1%' has no members")
|
||||||
% buildUsersGroup);
|
% settings.buildUsersGroup);
|
||||||
|
|
||||||
/* Find a user account that isn't currently in use for another
|
/* Find a user account that isn't currently in use for another
|
||||||
build. */
|
build. */
|
||||||
|
@ -488,11 +492,11 @@ void UserLock::acquire()
|
||||||
struct passwd * pw = getpwnam(i->c_str());
|
struct passwd * pw = getpwnam(i->c_str());
|
||||||
if (!pw)
|
if (!pw)
|
||||||
throw Error(format("the user `%1%' in the group `%2%' does not exist")
|
throw Error(format("the user `%1%' in the group `%2%' does not exist")
|
||||||
% *i % buildUsersGroup);
|
% *i % settings.buildUsersGroup);
|
||||||
|
|
||||||
createDirs(nixStateDir + "/userpool");
|
createDirs(settings.nixStateDir + "/userpool");
|
||||||
|
|
||||||
fnUserLock = (format("%1%/userpool/%2%") % nixStateDir % pw->pw_uid).str();
|
fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str();
|
||||||
|
|
||||||
if (lockedPaths.find(fnUserLock) != lockedPaths.end())
|
if (lockedPaths.find(fnUserLock) != lockedPaths.end())
|
||||||
/* We already have a lock on this one. */
|
/* We already have a lock on this one. */
|
||||||
|
@ -512,7 +516,7 @@ void UserLock::acquire()
|
||||||
/* Sanity check... */
|
/* Sanity check... */
|
||||||
if (uid == getuid() || uid == geteuid())
|
if (uid == getuid() || uid == geteuid())
|
||||||
throw Error(format("the Nix user should not be a member of `%1%'")
|
throw Error(format("the Nix user should not be a member of `%1%'")
|
||||||
% buildUsersGroup);
|
% settings.buildUsersGroup);
|
||||||
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -520,7 +524,7 @@ void UserLock::acquire()
|
||||||
|
|
||||||
throw Error(format("all build users are currently in use; "
|
throw Error(format("all build users are currently in use; "
|
||||||
"consider creating additional users and adding them to the `%1%' group")
|
"consider creating additional users and adding them to the `%1%' group")
|
||||||
% buildUsersGroup);
|
% settings.buildUsersGroup);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -539,7 +543,7 @@ static void runSetuidHelper(const string & command,
|
||||||
const string & arg)
|
const string & arg)
|
||||||
{
|
{
|
||||||
Path program = getEnv("NIX_SETUID_HELPER",
|
Path program = getEnv("NIX_SETUID_HELPER",
|
||||||
nixLibexecDir + "/nix-setuid-helper");
|
settings.nixLibexecDir + "/nix-setuid-helper");
|
||||||
|
|
||||||
/* Fork. */
|
/* Fork. */
|
||||||
Pid pid;
|
Pid pid;
|
||||||
|
@ -594,12 +598,6 @@ bool amPrivileged()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
bool haveBuildUsers()
|
|
||||||
{
|
|
||||||
return querySetting("build-users-group", "") != "";
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
void getOwnership(const Path & path)
|
void getOwnership(const Path & path)
|
||||||
{
|
{
|
||||||
runSetuidHelper("get-ownership", path);
|
runSetuidHelper("get-ownership", path);
|
||||||
|
@ -614,7 +612,7 @@ void deletePathWrapped(const Path & path, unsigned long long & bytesFreed)
|
||||||
} catch (SysError & e) {
|
} catch (SysError & e) {
|
||||||
/* If this failed due to a permission error, then try it with
|
/* If this failed due to a permission error, then try it with
|
||||||
the setuid helper. */
|
the setuid helper. */
|
||||||
if (haveBuildUsers() && !amPrivileged()) {
|
if (settings.buildUsersGroup != "" && !amPrivileged()) {
|
||||||
getOwnership(path);
|
getOwnership(path);
|
||||||
deletePath(path, bytesFreed);
|
deletePath(path, bytesFreed);
|
||||||
} else
|
} else
|
||||||
|
@ -692,10 +690,10 @@ HookInstance::HookInstance()
|
||||||
if (dup2(builderOut.writeSide, 4) == -1)
|
if (dup2(builderOut.writeSide, 4) == -1)
|
||||||
throw SysError("dupping builder's stdout/stderr");
|
throw SysError("dupping builder's stdout/stderr");
|
||||||
|
|
||||||
/* XXX: Pass `buildTimeout' to the hook? */
|
/* XXX: Pass `buildTimeout' to the hook? */
|
||||||
execl(buildHook.c_str(), buildHook.c_str(), thisSystem.c_str(),
|
execl(buildHook.c_str(), buildHook.c_str(), settings.thisSystem.c_str(),
|
||||||
(format("%1%") % maxSilentTime).str().c_str(),
|
(format("%1%") % settings.maxSilentTime).str().c_str(),
|
||||||
(format("%1%") % printBuildTrace).str().c_str(),
|
(format("%1%") % settings.printBuildTrace).str().c_str(),
|
||||||
NULL);
|
NULL);
|
||||||
|
|
||||||
throw SysError(format("executing `%1%'") % buildHook);
|
throw SysError(format("executing `%1%'") % buildHook);
|
||||||
|
@ -735,6 +733,8 @@ HookInstance::~HookInstance()
|
||||||
|
|
||||||
typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
|
typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
|
||||||
|
|
||||||
|
class SubstitutionGoal;
|
||||||
|
|
||||||
class DerivationGoal : public Goal
|
class DerivationGoal : public Goal
|
||||||
{
|
{
|
||||||
private:
|
private:
|
||||||
|
@ -933,7 +933,7 @@ void DerivationGoal::init()
|
||||||
{
|
{
|
||||||
trace("init");
|
trace("init");
|
||||||
|
|
||||||
if (readOnlyMode)
|
if (settings.readOnlyMode)
|
||||||
throw Error(format("cannot build derivation `%1%' - no write access to the Nix store") % drvPath);
|
throw Error(format("cannot build derivation `%1%' - no write access to the Nix store") % drvPath);
|
||||||
|
|
||||||
/* The first thing to do is to make sure that the derivation
|
/* The first thing to do is to make sure that the derivation
|
||||||
|
@ -985,10 +985,8 @@ void DerivationGoal::haveDerivation()
|
||||||
/* We are first going to try to create the invalid output paths
|
/* We are first going to try to create the invalid output paths
|
||||||
through substitutes. If that doesn't work, we'll build
|
through substitutes. If that doesn't work, we'll build
|
||||||
them. */
|
them. */
|
||||||
foreach (PathSet::iterator, i, invalidOutputs)
|
if (settings.useSubstitutes)
|
||||||
/* Don't bother creating a substitution goal if there are no
|
foreach (PathSet::iterator, i, invalidOutputs)
|
||||||
substitutes. */
|
|
||||||
if (queryBoolSetting("build-use-substitutes", true) && worker.store.hasSubstitutes(*i))
|
|
||||||
addWaitee(worker.makeSubstitutionGoal(*i));
|
addWaitee(worker.makeSubstitutionGoal(*i));
|
||||||
|
|
||||||
if (waitees.empty()) /* to prevent hang (no wake-up event) */
|
if (waitees.empty()) /* to prevent hang (no wake-up event) */
|
||||||
|
@ -1002,10 +1000,10 @@ void DerivationGoal::outputsSubstituted()
|
||||||
{
|
{
|
||||||
trace("all outputs substituted (maybe)");
|
trace("all outputs substituted (maybe)");
|
||||||
|
|
||||||
if (nrFailed > 0 && !tryFallback)
|
if (nrFailed > 0 && nrFailed > nrNoSubstituters && !settings.tryFallback)
|
||||||
throw Error(format("some substitutes for the outputs of derivation `%1%' failed; try `--fallback'") % drvPath);
|
throw Error(format("some substitutes for the outputs of derivation `%1%' failed; try `--fallback'") % drvPath);
|
||||||
|
|
||||||
nrFailed = 0;
|
nrFailed = nrNoSubstituters = 0;
|
||||||
|
|
||||||
if (checkPathValidity(false).size() == 0) {
|
if (checkPathValidity(false).size() == 0) {
|
||||||
amDone(ecSuccess);
|
amDone(ecSuccess);
|
||||||
|
@ -1101,9 +1099,9 @@ PathSet outputPaths(const DerivationOutputs & outputs)
|
||||||
|
|
||||||
static bool canBuildLocally(const string & platform)
|
static bool canBuildLocally(const string & platform)
|
||||||
{
|
{
|
||||||
return platform == thisSystem
|
return platform == settings.thisSystem
|
||||||
#ifdef CAN_DO_LINUX32_BUILDS
|
#ifdef CAN_DO_LINUX32_BUILDS
|
||||||
|| (platform == "i686-linux" && thisSystem == "x86_64-linux")
|
|| (platform == "i686-linux" && settings.thisSystem == "x86_64-linux")
|
||||||
#endif
|
#endif
|
||||||
;
|
;
|
||||||
}
|
}
|
||||||
|
@ -1205,7 +1203,7 @@ void DerivationGoal::tryToBuild()
|
||||||
derivation prefers to be done locally, do it even if
|
derivation prefers to be done locally, do it even if
|
||||||
maxBuildJobs is 0. */
|
maxBuildJobs is 0. */
|
||||||
unsigned int curBuilds = worker.getNrLocalBuilds();
|
unsigned int curBuilds = worker.getNrLocalBuilds();
|
||||||
if (curBuilds >= maxBuildJobs && !(preferLocalBuild && curBuilds == 0)) {
|
if (curBuilds >= settings.maxBuildJobs && !(preferLocalBuild && curBuilds == 0)) {
|
||||||
worker.waitForBuildSlot(shared_from_this());
|
worker.waitForBuildSlot(shared_from_this());
|
||||||
outputLocks.unlock();
|
outputLocks.unlock();
|
||||||
return;
|
return;
|
||||||
|
@ -1220,7 +1218,7 @@ void DerivationGoal::tryToBuild()
|
||||||
printMsg(lvlError, e.msg());
|
printMsg(lvlError, e.msg());
|
||||||
outputLocks.unlock();
|
outputLocks.unlock();
|
||||||
buildUser.release();
|
buildUser.release();
|
||||||
if (printBuildTrace)
|
if (settings.printBuildTrace)
|
||||||
printMsg(lvlError, format("@ build-failed %1% %2% %3% %4%")
|
printMsg(lvlError, format("@ build-failed %1% %2% %3% %4%")
|
||||||
% drvPath % drv.outputs["out"].path % 0 % e.msg());
|
% drvPath % drv.outputs["out"].path % 0 % e.msg());
|
||||||
worker.permanentFailure = true;
|
worker.permanentFailure = true;
|
||||||
|
@ -1352,7 +1350,7 @@ void DerivationGoal::buildDone()
|
||||||
bool hookError = hook &&
|
bool hookError = hook &&
|
||||||
(!WIFEXITED(status) || WEXITSTATUS(status) != 100);
|
(!WIFEXITED(status) || WEXITSTATUS(status) != 100);
|
||||||
|
|
||||||
if (printBuildTrace) {
|
if (settings.printBuildTrace) {
|
||||||
if (hook && hookError)
|
if (hook && hookError)
|
||||||
printMsg(lvlError, format("@ hook-failed %1% %2% %3% %4%")
|
printMsg(lvlError, format("@ hook-failed %1% %2% %3% %4%")
|
||||||
% drvPath % drv.outputs["out"].path % status % e.msg());
|
% drvPath % drv.outputs["out"].path % status % e.msg());
|
||||||
|
@ -1368,7 +1366,7 @@ void DerivationGoal::buildDone()
|
||||||
able to access the network). Hook errors (like
|
able to access the network). Hook errors (like
|
||||||
communication problems with the remote machine) shouldn't
|
communication problems with the remote machine) shouldn't
|
||||||
be cached either. */
|
be cached either. */
|
||||||
if (worker.cacheFailure && !hookError && !fixedOutput)
|
if (settings.cacheFailure && !hookError && !fixedOutput)
|
||||||
foreach (DerivationOutputs::iterator, i, drv.outputs)
|
foreach (DerivationOutputs::iterator, i, drv.outputs)
|
||||||
worker.store.registerFailedPath(i->second.path);
|
worker.store.registerFailedPath(i->second.path);
|
||||||
|
|
||||||
|
@ -1380,7 +1378,7 @@ void DerivationGoal::buildDone()
|
||||||
/* Release the build user, if applicable. */
|
/* Release the build user, if applicable. */
|
||||||
buildUser.release();
|
buildUser.release();
|
||||||
|
|
||||||
if (printBuildTrace) {
|
if (settings.printBuildTrace) {
|
||||||
printMsg(lvlError, format("@ build-succeeded %1% %2%")
|
printMsg(lvlError, format("@ build-succeeded %1% %2%")
|
||||||
% drvPath % drv.outputs["out"].path);
|
% drvPath % drv.outputs["out"].path);
|
||||||
}
|
}
|
||||||
|
@ -1391,7 +1389,7 @@ void DerivationGoal::buildDone()
|
||||||
|
|
||||||
HookReply DerivationGoal::tryBuildHook()
|
HookReply DerivationGoal::tryBuildHook()
|
||||||
{
|
{
|
||||||
if (!useBuildHook || getEnv("NIX_BUILD_HOOK") == "") return rpDecline;
|
if (!settings.useBuildHook || getEnv("NIX_BUILD_HOOK") == "") return rpDecline;
|
||||||
|
|
||||||
if (!worker.hook)
|
if (!worker.hook)
|
||||||
worker.hook = boost::shared_ptr<HookInstance>(new HookInstance);
|
worker.hook = boost::shared_ptr<HookInstance>(new HookInstance);
|
||||||
|
@ -1404,7 +1402,7 @@ HookReply DerivationGoal::tryBuildHook()
|
||||||
|
|
||||||
/* Send the request to the hook. */
|
/* Send the request to the hook. */
|
||||||
writeLine(worker.hook->toHook.writeSide, (format("%1% %2% %3% %4%")
|
writeLine(worker.hook->toHook.writeSide, (format("%1% %2% %3% %4%")
|
||||||
% (worker.getNrLocalBuilds() < maxBuildJobs ? "1" : "0")
|
% (worker.getNrLocalBuilds() < settings.maxBuildJobs ? "1" : "0")
|
||||||
% drv.platform % drvPath % concatStringsSep(",", features)).str());
|
% drv.platform % drvPath % concatStringsSep(",", features)).str());
|
||||||
|
|
||||||
/* Read the first line of input, which should be a word indicating
|
/* Read the first line of input, which should be a word indicating
|
||||||
|
@ -1463,7 +1461,7 @@ HookReply DerivationGoal::tryBuildHook()
|
||||||
fds.insert(hook->builderOut.readSide);
|
fds.insert(hook->builderOut.readSide);
|
||||||
worker.childStarted(shared_from_this(), hook->pid, fds, false, false);
|
worker.childStarted(shared_from_this(), hook->pid, fds, false, false);
|
||||||
|
|
||||||
if (printBuildTrace)
|
if (settings.printBuildTrace)
|
||||||
printMsg(lvlError, format("@ build-started %1% %2% %3% %4%")
|
printMsg(lvlError, format("@ build-started %1% %2% %3% %4%")
|
||||||
% drvPath % drv.outputs["out"].path % drv.platform % logFile);
|
% drvPath % drv.outputs["out"].path % drv.platform % logFile);
|
||||||
|
|
||||||
|
@ -1494,7 +1492,7 @@ void DerivationGoal::startBuilder()
|
||||||
if (!canBuildLocally(drv.platform))
|
if (!canBuildLocally(drv.platform))
|
||||||
throw Error(
|
throw Error(
|
||||||
format("a `%1%' is required to build `%3%', but I am a `%2%'")
|
format("a `%1%' is required to build `%3%', but I am a `%2%'")
|
||||||
% drv.platform % thisSystem % drvPath);
|
% drv.platform % settings.thisSystem % drvPath);
|
||||||
|
|
||||||
/* Construct the environment passed to the builder. */
|
/* Construct the environment passed to the builder. */
|
||||||
|
|
||||||
|
@ -1515,10 +1513,10 @@ void DerivationGoal::startBuilder()
|
||||||
shouldn't care, but this is useful for purity checking (e.g.,
|
shouldn't care, but this is useful for purity checking (e.g.,
|
||||||
the compiler or linker might only want to accept paths to files
|
the compiler or linker might only want to accept paths to files
|
||||||
in the store or in the build directory). */
|
in the store or in the build directory). */
|
||||||
env["NIX_STORE"] = nixStore;
|
env["NIX_STORE"] = settings.nixStore;
|
||||||
|
|
||||||
/* The maximum number of cores to utilize for parallel building. */
|
/* The maximum number of cores to utilize for parallel building. */
|
||||||
env["NIX_BUILD_CORES"] = (format("%d") % buildCores).str();
|
env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str();
|
||||||
|
|
||||||
/* Add all bindings specified in the derivation. */
|
/* Add all bindings specified in the derivation. */
|
||||||
foreach (StringPairs::iterator, i, drv.env)
|
foreach (StringPairs::iterator, i, drv.env)
|
||||||
|
@ -1610,7 +1608,7 @@ void DerivationGoal::startBuilder()
|
||||||
|
|
||||||
/* If `build-users-group' is not empty, then we have to build as
|
/* If `build-users-group' is not empty, then we have to build as
|
||||||
one of the members of that group. */
|
one of the members of that group. */
|
||||||
if (haveBuildUsers()) {
|
if (settings.buildUsersGroup != "") {
|
||||||
buildUser.acquire();
|
buildUser.acquire();
|
||||||
assert(buildUser.getUID() != 0);
|
assert(buildUser.getUID() != 0);
|
||||||
assert(buildUser.getGID() != 0);
|
assert(buildUser.getGID() != 0);
|
||||||
|
@ -1632,15 +1630,15 @@ void DerivationGoal::startBuilder()
|
||||||
the builder can create its output but not mess with the
|
the builder can create its output but not mess with the
|
||||||
outputs of other processes). */
|
outputs of other processes). */
|
||||||
struct stat st;
|
struct stat st;
|
||||||
if (stat(nixStore.c_str(), &st) == -1)
|
if (stat(settings.nixStore.c_str(), &st) == -1)
|
||||||
throw SysError(format("cannot stat `%1%'") % nixStore);
|
throw SysError(format("cannot stat `%1%'") % settings.nixStore);
|
||||||
if (!(st.st_mode & S_ISVTX) ||
|
if (!(st.st_mode & S_ISVTX) ||
|
||||||
((st.st_mode & S_IRWXG) != S_IRWXG) ||
|
((st.st_mode & S_IRWXG) != S_IRWXG) ||
|
||||||
(st.st_gid != buildUser.getGID()))
|
(st.st_gid != buildUser.getGID()))
|
||||||
throw Error(format(
|
throw Error(format(
|
||||||
"builder does not have write permission to `%2%'; "
|
"builder does not have write permission to `%2%'; "
|
||||||
"try `chgrp %1% %2%; chmod 1775 %2%'")
|
"try `chgrp %1% %2%; chmod 1775 %2%'")
|
||||||
% buildUser.getGID() % nixStore);
|
% buildUser.getGID() % settings.nixStore);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -1649,7 +1647,7 @@ void DerivationGoal::startBuilder()
|
||||||
functions like fetchurl (which needs a proper /etc/resolv.conf)
|
functions like fetchurl (which needs a proper /etc/resolv.conf)
|
||||||
work properly. Purity checking for fixed-output derivations
|
work properly. Purity checking for fixed-output derivations
|
||||||
is somewhat pointless anyway. */
|
is somewhat pointless anyway. */
|
||||||
useChroot = queryBoolSetting("build-use-chroot", false);
|
useChroot = settings.useChroot;
|
||||||
|
|
||||||
if (fixedOutput) useChroot = false;
|
if (fixedOutput) useChroot = false;
|
||||||
|
|
||||||
|
@ -1689,8 +1687,8 @@ void DerivationGoal::startBuilder()
|
||||||
% (buildUser.enabled() ? buildUser.getUID() : getuid())
|
% (buildUser.enabled() ? buildUser.getUID() : getuid())
|
||||||
% (buildUser.enabled() ? buildUser.getGID() : getgid())).str());
|
% (buildUser.enabled() ? buildUser.getGID() : getgid())).str());
|
||||||
|
|
||||||
/* Declare the build user's group so that programs get a consistent
|
/* Declare the build user's group so that programs get a consistent
|
||||||
view of the system (e.g., "id -gn"). */
|
view of the system (e.g., "id -gn"). */
|
||||||
writeFile(chrootRootDir + "/etc/group",
|
writeFile(chrootRootDir + "/etc/group",
|
||||||
(format("nixbld:!:%1%:\n")
|
(format("nixbld:!:%1%:\n")
|
||||||
% (buildUser.enabled() ? buildUser.getGID() : getgid())).str());
|
% (buildUser.enabled() ? buildUser.getGID() : getgid())).str());
|
||||||
|
@ -1699,16 +1697,8 @@ void DerivationGoal::startBuilder()
|
||||||
writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n");
|
writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n");
|
||||||
|
|
||||||
/* Bind-mount a user-configurable set of directories from the
|
/* Bind-mount a user-configurable set of directories from the
|
||||||
host file system. The `/dev/pts' directory must be mounted
|
host file system. */
|
||||||
separately so that newly-created pseudo-terminals show
|
dirsInChroot = settings.dirsInChroot;
|
||||||
up. */
|
|
||||||
Paths defaultDirs;
|
|
||||||
defaultDirs.push_back("/dev");
|
|
||||||
defaultDirs.push_back("/dev/pts");
|
|
||||||
|
|
||||||
Paths dirsInChroot_ = querySetting("build-chroot-dirs", defaultDirs);
|
|
||||||
dirsInChroot.insert(dirsInChroot_.begin(), dirsInChroot_.end());
|
|
||||||
|
|
||||||
dirsInChroot.insert(tmpDir);
|
dirsInChroot.insert(tmpDir);
|
||||||
|
|
||||||
/* Make the closure of the inputs available in the chroot,
|
/* Make the closure of the inputs available in the chroot,
|
||||||
|
@ -1718,8 +1708,8 @@ void DerivationGoal::startBuilder()
|
||||||
can be bind-mounted). !!! As an extra security
|
can be bind-mounted). !!! As an extra security
|
||||||
precaution, make the fake Nix store only writable by the
|
precaution, make the fake Nix store only writable by the
|
||||||
build user. */
|
build user. */
|
||||||
createDirs(chrootRootDir + nixStore);
|
createDirs(chrootRootDir + settings.nixStore);
|
||||||
chmod_(chrootRootDir + nixStore, 01777);
|
chmod_(chrootRootDir + settings.nixStore, 01777);
|
||||||
|
|
||||||
foreach (PathSet::iterator, i, inputPaths) {
|
foreach (PathSet::iterator, i, inputPaths) {
|
||||||
struct stat st;
|
struct stat st;
|
||||||
|
@ -1819,7 +1809,7 @@ void DerivationGoal::startBuilder()
|
||||||
worker.childStarted(shared_from_this(), pid,
|
worker.childStarted(shared_from_this(), pid,
|
||||||
singleton<set<int> >(builderOut.readSide), true, true);
|
singleton<set<int> >(builderOut.readSide), true, true);
|
||||||
|
|
||||||
if (printBuildTrace) {
|
if (settings.printBuildTrace) {
|
||||||
printMsg(lvlError, format("@ build-started %1% %2% %3% %4%")
|
printMsg(lvlError, format("@ build-started %1% %2% %3% %4%")
|
||||||
% drvPath % drv.outputs["out"].path % drv.platform % logFile);
|
% drvPath % drv.outputs["out"].path % drv.platform % logFile);
|
||||||
}
|
}
|
||||||
|
@ -1917,16 +1907,14 @@ void DerivationGoal::initChild()
|
||||||
#ifdef CAN_DO_LINUX32_BUILDS
|
#ifdef CAN_DO_LINUX32_BUILDS
|
||||||
/* Change the personality to 32-bit if we're doing an
|
/* Change the personality to 32-bit if we're doing an
|
||||||
i686-linux build on an x86_64-linux machine. */
|
i686-linux build on an x86_64-linux machine. */
|
||||||
if (drv.platform == "i686-linux" && thisSystem == "x86_64-linux") {
|
if (drv.platform == "i686-linux" && settings.thisSystem == "x86_64-linux") {
|
||||||
if (personality(0x0008 | 0x8000000 /* == PER_LINUX32_3GB */) == -1)
|
if (personality(0x0008 | 0x8000000 /* == PER_LINUX32_3GB */) == -1)
|
||||||
throw SysError("cannot set i686-linux personality");
|
throw SysError("cannot set i686-linux personality");
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Impersonate a Linux 2.6 machine to get some determinism in
|
/* Impersonate a Linux 2.6 machine to get some determinism in
|
||||||
builds that depend on the kernel version. */
|
builds that depend on the kernel version. */
|
||||||
if ((drv.platform == "i686-linux" || drv.platform == "x86_64-linux") &&
|
if ((drv.platform == "i686-linux" || drv.platform == "x86_64-linux") && settings.impersonateLinux26) {
|
||||||
queryBoolSetting("build-impersonate-linux-26", true))
|
|
||||||
{
|
|
||||||
int cur = personality(0xffffffff);
|
int cur = personality(0xffffffff);
|
||||||
if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */);
|
if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */);
|
||||||
}
|
}
|
||||||
|
@ -1968,7 +1956,7 @@ void DerivationGoal::initChild()
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
/* Let the setuid helper take care of it. */
|
/* Let the setuid helper take care of it. */
|
||||||
program = nixLibexecDir + "/nix-setuid-helper";
|
program = settings.nixLibexecDir + "/nix-setuid-helper";
|
||||||
args.push_back(program.c_str());
|
args.push_back(program.c_str());
|
||||||
args.push_back("run-builder");
|
args.push_back("run-builder");
|
||||||
user = buildUser.getUser().c_str();
|
user = buildUser.getUser().c_str();
|
||||||
|
@ -2078,12 +2066,12 @@ void DerivationGoal::computeClosure()
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Get rid of all weird permissions. */
|
/* Get rid of all weird permissions. */
|
||||||
canonicalisePathMetaData(path);
|
canonicalisePathMetaData(path);
|
||||||
|
|
||||||
/* For this output path, find the references to other paths
|
/* For this output path, find the references to other paths
|
||||||
contained in it. Compute the SHA-256 NAR hash at the same
|
contained in it. Compute the SHA-256 NAR hash at the same
|
||||||
time. The hash is stored in the database so that we can
|
time. The hash is stored in the database so that we can
|
||||||
verify later on whether nobody has messed with the store. */
|
verify later on whether nobody has messed with the store. */
|
||||||
HashResult hash;
|
HashResult hash;
|
||||||
PathSet references = scanForReferences(path, allPaths, hash);
|
PathSet references = scanForReferences(path, allPaths, hash);
|
||||||
contentHashes[path] = hash;
|
contentHashes[path] = hash;
|
||||||
|
@ -2136,13 +2124,13 @@ string drvsLogDir = "drvs";
|
||||||
|
|
||||||
Path DerivationGoal::openLogFile()
|
Path DerivationGoal::openLogFile()
|
||||||
{
|
{
|
||||||
if (!queryBoolSetting("build-keep-log", true)) return "";
|
if (!settings.keepLog) return "";
|
||||||
|
|
||||||
/* Create a log file. */
|
/* Create a log file. */
|
||||||
Path dir = (format("%1%/%2%") % nixLogDir % drvsLogDir).str();
|
Path dir = (format("%1%/%2%") % settings.nixLogDir % drvsLogDir).str();
|
||||||
createDirs(dir);
|
createDirs(dir);
|
||||||
|
|
||||||
if (queryBoolSetting("build-compress-log", true)) {
|
if (settings.compressLog) {
|
||||||
|
|
||||||
Path logFileName = (format("%1%/%2%.bz2") % dir % baseNameOf(drvPath)).str();
|
Path logFileName = (format("%1%/%2%.bz2") % dir % baseNameOf(drvPath)).str();
|
||||||
AutoCloseFD fd = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666);
|
AutoCloseFD fd = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666);
|
||||||
|
@ -2189,9 +2177,9 @@ void DerivationGoal::closeLogFile()
|
||||||
void DerivationGoal::deleteTmpDir(bool force)
|
void DerivationGoal::deleteTmpDir(bool force)
|
||||||
{
|
{
|
||||||
if (tmpDir != "") {
|
if (tmpDir != "") {
|
||||||
if (keepFailed && !force) {
|
if (settings.keepFailed && !force) {
|
||||||
printMsg(lvlError,
|
printMsg(lvlError,
|
||||||
format("builder for `%1%' failed; keeping build directory `%2%'")
|
format("builder for `%1%' failed; keeping build directory `%2%'")
|
||||||
% drvPath % tmpDir);
|
% drvPath % tmpDir);
|
||||||
if (buildUser.enabled() && !amPrivileged())
|
if (buildUser.enabled() && !amPrivileged())
|
||||||
getOwnership(tmpDir);
|
getOwnership(tmpDir);
|
||||||
|
@ -2209,7 +2197,7 @@ void DerivationGoal::handleChildOutput(int fd, const string & data)
|
||||||
if ((hook && fd == hook->builderOut.readSide) ||
|
if ((hook && fd == hook->builderOut.readSide) ||
|
||||||
(!hook && fd == builderOut.readSide))
|
(!hook && fd == builderOut.readSide))
|
||||||
{
|
{
|
||||||
if (verbosity >= buildVerbosity)
|
if (verbosity >= settings.buildVerbosity)
|
||||||
writeToStderr((unsigned char *) data.data(), data.size());
|
writeToStderr((unsigned char *) data.data(), data.size());
|
||||||
if (bzLogFile) {
|
if (bzLogFile) {
|
||||||
int err;
|
int err;
|
||||||
|
@ -2245,13 +2233,13 @@ PathSet DerivationGoal::checkPathValidity(bool returnValid)
|
||||||
|
|
||||||
bool DerivationGoal::pathFailed(const Path & path)
|
bool DerivationGoal::pathFailed(const Path & path)
|
||||||
{
|
{
|
||||||
if (!worker.cacheFailure) return false;
|
if (!settings.cacheFailure) return false;
|
||||||
|
|
||||||
if (!worker.store.hasPathFailed(path)) return false;
|
if (!worker.store.hasPathFailed(path)) return false;
|
||||||
|
|
||||||
printMsg(lvlError, format("builder for `%1%' failed previously (cached)") % path);
|
printMsg(lvlError, format("builder for `%1%' failed previously (cached)") % path);
|
||||||
|
|
||||||
if (printBuildTrace)
|
if (settings.printBuildTrace)
|
||||||
printMsg(lvlError, format("@ build-failed %1% %2% cached") % drvPath % path);
|
printMsg(lvlError, format("@ build-failed %1% %2% cached") % drvPath % path);
|
||||||
|
|
||||||
worker.permanentFailure = true;
|
worker.permanentFailure = true;
|
||||||
|
@ -2278,10 +2266,16 @@ private:
|
||||||
/* The current substituter. */
|
/* The current substituter. */
|
||||||
Path sub;
|
Path sub;
|
||||||
|
|
||||||
|
/* Whether any substituter can realise this path */
|
||||||
|
bool hasSubstitute;
|
||||||
|
|
||||||
/* Path info returned by the substituter's query info operation. */
|
/* Path info returned by the substituter's query info operation. */
|
||||||
SubstitutablePathInfo info;
|
SubstitutablePathInfo info;
|
||||||
|
|
||||||
/* Pipe for the substitute's standard output/error. */
|
/* Pipe for the substituter's standard output. */
|
||||||
|
Pipe outPipe;
|
||||||
|
|
||||||
|
/* Pipe for the substituter's standard error. */
|
||||||
Pipe logPipe;
|
Pipe logPipe;
|
||||||
|
|
||||||
/* The process ID of the builder. */
|
/* The process ID of the builder. */
|
||||||
|
@ -2319,6 +2313,7 @@ public:
|
||||||
|
|
||||||
SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker)
|
SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker)
|
||||||
: Goal(worker)
|
: Goal(worker)
|
||||||
|
, hasSubstitute(false)
|
||||||
{
|
{
|
||||||
this->storePath = storePath;
|
this->storePath = storePath;
|
||||||
state = &SubstitutionGoal::init;
|
state = &SubstitutionGoal::init;
|
||||||
|
@ -2365,10 +2360,10 @@ void SubstitutionGoal::init()
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (readOnlyMode)
|
if (settings.readOnlyMode)
|
||||||
throw Error(format("cannot substitute path `%1%' - no write access to the Nix store") % storePath);
|
throw Error(format("cannot substitute path `%1%' - no write access to the Nix store") % storePath);
|
||||||
|
|
||||||
subs = substituters;
|
subs = settings.substituters;
|
||||||
|
|
||||||
tryNext();
|
tryNext();
|
||||||
}
|
}
|
||||||
|
@ -2382,17 +2377,23 @@ void SubstitutionGoal::tryNext()
|
||||||
/* None left. Terminate this goal and let someone else deal
|
/* None left. Terminate this goal and let someone else deal
|
||||||
with it. */
|
with it. */
|
||||||
debug(format("path `%1%' is required, but there is no substituter that can build it") % storePath);
|
debug(format("path `%1%' is required, but there is no substituter that can build it") % storePath);
|
||||||
amDone(ecFailed);
|
/* Hack: don't indicate failure if there were no substituters.
|
||||||
|
In that case the calling derivation should just do a
|
||||||
|
build. */
|
||||||
|
amDone(hasSubstitute ? ecFailed : ecNoSubstituters);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
sub = subs.front();
|
sub = subs.front();
|
||||||
subs.pop_front();
|
subs.pop_front();
|
||||||
|
|
||||||
if (!worker.store.querySubstitutablePathInfo(sub, storePath, info)) {
|
SubstitutablePathInfos infos;
|
||||||
tryNext();
|
PathSet dummy(singleton<PathSet>(storePath));
|
||||||
return;
|
worker.store.querySubstitutablePathInfos(sub, dummy, infos);
|
||||||
}
|
SubstitutablePathInfos::iterator k = infos.find(storePath);
|
||||||
|
if (k == infos.end()) { tryNext(); return; }
|
||||||
|
info = k->second;
|
||||||
|
hasSubstitute = true;
|
||||||
|
|
||||||
/* To maintain the closure invariant, we first have to realise the
|
/* To maintain the closure invariant, we first have to realise the
|
||||||
paths referenced by this one. */
|
paths referenced by this one. */
|
||||||
|
@ -2434,7 +2435,7 @@ void SubstitutionGoal::tryToRun()
|
||||||
is maxBuildJobs == 0 (no local builds allowed), we still allow
|
is maxBuildJobs == 0 (no local builds allowed), we still allow
|
||||||
a substituter to run. This is because substitutions cannot be
|
a substituter to run. This is because substitutions cannot be
|
||||||
distributed to another machine via the build hook. */
|
distributed to another machine via the build hook. */
|
||||||
if (worker.getNrLocalBuilds() >= (maxBuildJobs == 0 ? 1 : maxBuildJobs)) {
|
if (worker.getNrLocalBuilds() >= (settings.maxBuildJobs == 0 ? 1 : settings.maxBuildJobs)) {
|
||||||
worker.waitForBuildSlot(shared_from_this());
|
worker.waitForBuildSlot(shared_from_this());
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -2467,6 +2468,7 @@ void SubstitutionGoal::tryToRun()
|
||||||
|
|
||||||
printMsg(lvlInfo, format("fetching path `%1%'...") % storePath);
|
printMsg(lvlInfo, format("fetching path `%1%'...") % storePath);
|
||||||
|
|
||||||
|
outPipe.create();
|
||||||
logPipe.create();
|
logPipe.create();
|
||||||
|
|
||||||
/* Remove the (stale) output path if it exists. */
|
/* Remove the (stale) output path if it exists. */
|
||||||
|
@ -2483,10 +2485,17 @@ void SubstitutionGoal::tryToRun()
|
||||||
case 0:
|
case 0:
|
||||||
try { /* child */
|
try { /* child */
|
||||||
|
|
||||||
logPipe.readSide.close();
|
|
||||||
|
|
||||||
commonChildInit(logPipe);
|
commonChildInit(logPipe);
|
||||||
|
|
||||||
|
if (dup2(outPipe.writeSide, STDOUT_FILENO) == -1)
|
||||||
|
throw SysError("cannot dup output pipe into stdout");
|
||||||
|
outPipe.readSide.close();
|
||||||
|
outPipe.writeSide.close();
|
||||||
|
|
||||||
|
/* Pass configuration options (including those overriden
|
||||||
|
with --option) to the substituter. */
|
||||||
|
setenv("_NIX_OPTIONS", settings.pack().c_str(), 1);
|
||||||
|
|
||||||
/* Fill in the arguments. */
|
/* Fill in the arguments. */
|
||||||
Strings args;
|
Strings args;
|
||||||
args.push_back(baseNameOf(sub));
|
args.push_back(baseNameOf(sub));
|
||||||
|
@ -2507,13 +2516,14 @@ void SubstitutionGoal::tryToRun()
|
||||||
/* parent */
|
/* parent */
|
||||||
pid.setSeparatePG(true);
|
pid.setSeparatePG(true);
|
||||||
pid.setKillSignal(SIGTERM);
|
pid.setKillSignal(SIGTERM);
|
||||||
|
outPipe.writeSide.close();
|
||||||
logPipe.writeSide.close();
|
logPipe.writeSide.close();
|
||||||
worker.childStarted(shared_from_this(),
|
worker.childStarted(shared_from_this(),
|
||||||
pid, singleton<set<int> >(logPipe.readSide), true, true);
|
pid, singleton<set<int> >(logPipe.readSide), true, true);
|
||||||
|
|
||||||
state = &SubstitutionGoal::finished;
|
state = &SubstitutionGoal::finished;
|
||||||
|
|
||||||
if (printBuildTrace) {
|
if (settings.printBuildTrace) {
|
||||||
printMsg(lvlError, format("@ substituter-started %1% %2%")
|
printMsg(lvlError, format("@ substituter-started %1% %2%")
|
||||||
% storePath % sub);
|
% storePath % sub);
|
||||||
}
|
}
|
||||||
|
@ -2535,9 +2545,12 @@ void SubstitutionGoal::finished()
|
||||||
/* Close the read side of the logger pipe. */
|
/* Close the read side of the logger pipe. */
|
||||||
logPipe.readSide.close();
|
logPipe.readSide.close();
|
||||||
|
|
||||||
debug(format("substitute for `%1%' finished") % storePath);
|
/* Get the hash info from stdout. */
|
||||||
|
string expectedHashStr = statusOk(status) ? readLine(outPipe.readSide) : "";
|
||||||
|
outPipe.readSide.close();
|
||||||
|
|
||||||
/* Check the exit status and the build result. */
|
/* Check the exit status and the build result. */
|
||||||
|
HashResult hash;
|
||||||
try {
|
try {
|
||||||
|
|
||||||
if (!statusOk(status))
|
if (!statusOk(status))
|
||||||
|
@ -2547,11 +2560,28 @@ void SubstitutionGoal::finished()
|
||||||
if (!pathExists(storePath))
|
if (!pathExists(storePath))
|
||||||
throw SubstError(format("substitute did not produce path `%1%'") % storePath);
|
throw SubstError(format("substitute did not produce path `%1%'") % storePath);
|
||||||
|
|
||||||
|
hash = hashPath(htSHA256, storePath);
|
||||||
|
|
||||||
|
/* Verify the expected hash we got from the substituer. */
|
||||||
|
if (expectedHashStr != "") {
|
||||||
|
size_t n = expectedHashStr.find(':');
|
||||||
|
if (n == string::npos)
|
||||||
|
throw Error(format("bad hash from substituter: %1%") % expectedHashStr);
|
||||||
|
HashType hashType = parseHashType(string(expectedHashStr, 0, n));
|
||||||
|
if (hashType == htUnknown)
|
||||||
|
throw Error(format("unknown hash algorithm in `%1%'") % expectedHashStr);
|
||||||
|
Hash expectedHash = parseHash16or32(hashType, string(expectedHashStr, n + 1));
|
||||||
|
Hash actualHash = hashType == htSHA256 ? hash.first : hashPath(hashType, storePath).first;
|
||||||
|
if (expectedHash != actualHash)
|
||||||
|
throw SubstError(format("hash mismatch in downloaded path `%1%': expected %2%, got %3%")
|
||||||
|
% storePath % printHash(expectedHash) % printHash(actualHash));
|
||||||
|
}
|
||||||
|
|
||||||
} catch (SubstError & e) {
|
} catch (SubstError & e) {
|
||||||
|
|
||||||
printMsg(lvlInfo, e.msg());
|
printMsg(lvlInfo, e.msg());
|
||||||
|
|
||||||
if (printBuildTrace) {
|
if (settings.printBuildTrace) {
|
||||||
printMsg(lvlError, format("@ substituter-failed %1% %2% %3%")
|
printMsg(lvlError, format("@ substituter-failed %1% %2% %3%")
|
||||||
% storePath % status % e.msg());
|
% storePath % status % e.msg());
|
||||||
}
|
}
|
||||||
|
@ -2564,8 +2594,6 @@ void SubstitutionGoal::finished()
|
||||||
|
|
||||||
canonicalisePathMetaData(storePath);
|
canonicalisePathMetaData(storePath);
|
||||||
|
|
||||||
HashResult hash = hashPath(htSHA256, storePath);
|
|
||||||
|
|
||||||
worker.store.optimisePath(storePath); // FIXME: combine with hashPath()
|
worker.store.optimisePath(storePath); // FIXME: combine with hashPath()
|
||||||
|
|
||||||
ValidPathInfo info2;
|
ValidPathInfo info2;
|
||||||
|
@ -2581,7 +2609,7 @@ void SubstitutionGoal::finished()
|
||||||
printMsg(lvlChatty,
|
printMsg(lvlChatty,
|
||||||
format("substitution of path `%1%' succeeded") % storePath);
|
format("substitution of path `%1%' succeeded") % storePath);
|
||||||
|
|
||||||
if (printBuildTrace) {
|
if (settings.printBuildTrace) {
|
||||||
printMsg(lvlError, format("@ substituter-succeeded %1%") % storePath);
|
printMsg(lvlError, format("@ substituter-succeeded %1%") % storePath);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -2592,7 +2620,7 @@ void SubstitutionGoal::finished()
|
||||||
void SubstitutionGoal::handleChildOutput(int fd, const string & data)
|
void SubstitutionGoal::handleChildOutput(int fd, const string & data)
|
||||||
{
|
{
|
||||||
assert(fd == logPipe.readSide);
|
assert(fd == logPipe.readSide);
|
||||||
if (verbosity >= buildVerbosity)
|
if (verbosity >= settings.buildVerbosity)
|
||||||
writeToStderr((unsigned char *) data.data(), data.size());
|
writeToStderr((unsigned char *) data.data(), data.size());
|
||||||
/* Don't write substitution output to a log file for now. We
|
/* Don't write substitution output to a log file for now. We
|
||||||
probably should, though. */
|
probably should, though. */
|
||||||
|
@ -2620,7 +2648,6 @@ Worker::Worker(LocalStore & store)
|
||||||
working = true;
|
working = true;
|
||||||
nrLocalBuilds = 0;
|
nrLocalBuilds = 0;
|
||||||
lastWokenUp = 0;
|
lastWokenUp = 0;
|
||||||
cacheFailure = queryBoolSetting("build-cache-failure", false);
|
|
||||||
permanentFailure = false;
|
permanentFailure = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -2685,7 +2712,7 @@ void Worker::removeGoal(GoalPtr goal)
|
||||||
topGoals.erase(goal);
|
topGoals.erase(goal);
|
||||||
/* If a top-level goal failed, then kill all other goals
|
/* If a top-level goal failed, then kill all other goals
|
||||||
(unless keepGoing was set). */
|
(unless keepGoing was set). */
|
||||||
if (goal->getExitCode() == Goal::ecFailed && !keepGoing)
|
if (goal->getExitCode() == Goal::ecFailed && !settings.keepGoing)
|
||||||
topGoals.clear();
|
topGoals.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -2757,7 +2784,7 @@ void Worker::childTerminated(pid_t pid, bool wakeSleepers)
|
||||||
void Worker::waitForBuildSlot(GoalPtr goal)
|
void Worker::waitForBuildSlot(GoalPtr goal)
|
||||||
{
|
{
|
||||||
debug("wait for build slot");
|
debug("wait for build slot");
|
||||||
if (getNrLocalBuilds() < maxBuildJobs)
|
if (getNrLocalBuilds() < settings.maxBuildJobs)
|
||||||
wakeUp(goal); /* we can do it right away */
|
wakeUp(goal); /* we can do it right away */
|
||||||
else
|
else
|
||||||
wantingToBuild.insert(goal);
|
wantingToBuild.insert(goal);
|
||||||
|
@ -2806,7 +2833,7 @@ void Worker::run(const Goals & _topGoals)
|
||||||
if (!children.empty() || !waitingForAWhile.empty())
|
if (!children.empty() || !waitingForAWhile.empty())
|
||||||
waitForInput();
|
waitForInput();
|
||||||
else {
|
else {
|
||||||
if (awake.empty() && maxBuildJobs == 0) throw Error(
|
if (awake.empty() && settings.maxBuildJobs == 0) throw Error(
|
||||||
"unable to start any build; either increase `--max-jobs' "
|
"unable to start any build; either increase `--max-jobs' "
|
||||||
"or enable distributed builds");
|
"or enable distributed builds");
|
||||||
assert(!awake.empty());
|
assert(!awake.empty());
|
||||||
|
@ -2816,9 +2843,9 @@ void Worker::run(const Goals & _topGoals)
|
||||||
/* If --keep-going is not set, it's possible that the main goal
|
/* If --keep-going is not set, it's possible that the main goal
|
||||||
exited while some of its subgoals were still active. But if
|
exited while some of its subgoals were still active. But if
|
||||||
--keep-going *is* set, then they must all be finished now. */
|
--keep-going *is* set, then they must all be finished now. */
|
||||||
assert(!keepGoing || awake.empty());
|
assert(!settings.keepGoing || awake.empty());
|
||||||
assert(!keepGoing || wantingToBuild.empty());
|
assert(!settings.keepGoing || wantingToBuild.empty());
|
||||||
assert(!keepGoing || children.empty());
|
assert(!settings.keepGoing || children.empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -2838,15 +2865,15 @@ void Worker::waitForInput()
|
||||||
time_t before = time(0);
|
time_t before = time(0);
|
||||||
|
|
||||||
/* If a global timeout has been set, sleep until it's done. */
|
/* If a global timeout has been set, sleep until it's done. */
|
||||||
if (buildTimeout != 0) {
|
if (settings.buildTimeout != 0) {
|
||||||
useTimeout = true;
|
useTimeout = true;
|
||||||
if (lastWait == 0 || lastWait > before) lastWait = before;
|
if (lastWait == 0 || lastWait > before) lastWait = before;
|
||||||
timeout.tv_sec = std::max((time_t) 0, lastWait + buildTimeout - before);
|
timeout.tv_sec = std::max((time_t) 0, lastWait + settings.buildTimeout - before);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* If we're monitoring for silence on stdout/stderr, sleep until
|
/* If we're monitoring for silence on stdout/stderr, sleep until
|
||||||
the first deadline for any child. */
|
the first deadline for any child. */
|
||||||
if (maxSilentTime != 0) {
|
if (settings.maxSilentTime != 0) {
|
||||||
time_t oldest = 0;
|
time_t oldest = 0;
|
||||||
foreach (Children::iterator, i, children) {
|
foreach (Children::iterator, i, children) {
|
||||||
if (i->second.monitorForSilence) {
|
if (i->second.monitorForSilence) {
|
||||||
|
@ -2855,10 +2882,10 @@ void Worker::waitForInput()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (oldest) {
|
if (oldest) {
|
||||||
time_t silenceTimeout = std::max((time_t) 0, oldest + maxSilentTime - before);
|
time_t silenceTimeout = std::max((time_t) 0, oldest + settings.maxSilentTime - before);
|
||||||
timeout.tv_sec = useTimeout
|
timeout.tv_sec = useTimeout
|
||||||
? std::min(silenceTimeout, timeout.tv_sec)
|
? std::min(silenceTimeout, timeout.tv_sec)
|
||||||
: silenceTimeout;
|
: silenceTimeout;
|
||||||
useTimeout = true;
|
useTimeout = true;
|
||||||
printMsg(lvlVomit, format("sleeping %1% seconds") % timeout.tv_sec);
|
printMsg(lvlVomit, format("sleeping %1% seconds") % timeout.tv_sec);
|
||||||
}
|
}
|
||||||
|
@ -2866,14 +2893,12 @@ void Worker::waitForInput()
|
||||||
|
|
||||||
/* If we are polling goals that are waiting for a lock, then wake
|
/* If we are polling goals that are waiting for a lock, then wake
|
||||||
up after a few seconds at most. */
|
up after a few seconds at most. */
|
||||||
int wakeUpInterval = queryIntSetting("build-poll-interval", 5);
|
|
||||||
|
|
||||||
if (!waitingForAWhile.empty()) {
|
if (!waitingForAWhile.empty()) {
|
||||||
useTimeout = true;
|
useTimeout = true;
|
||||||
if (lastWokenUp == 0)
|
if (lastWokenUp == 0)
|
||||||
printMsg(lvlError, "waiting for locks or build slots...");
|
printMsg(lvlError, "waiting for locks or build slots...");
|
||||||
if (lastWokenUp == 0 || lastWokenUp > before) lastWokenUp = before;
|
if (lastWokenUp == 0 || lastWokenUp > before) lastWokenUp = before;
|
||||||
timeout.tv_sec = std::max((time_t) 0, lastWokenUp + wakeUpInterval - before);
|
timeout.tv_sec = std::max((time_t) 0, (time_t) (lastWokenUp + settings.pollInterval - before));
|
||||||
} else lastWokenUp = 0;
|
} else lastWokenUp = 0;
|
||||||
|
|
||||||
using namespace std;
|
using namespace std;
|
||||||
|
@ -2939,27 +2964,27 @@ void Worker::waitForInput()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (maxSilentTime != 0 &&
|
if (settings.maxSilentTime != 0 &&
|
||||||
j->second.monitorForSilence &&
|
j->second.monitorForSilence &&
|
||||||
after - j->second.lastOutput >= (time_t) maxSilentTime)
|
after - j->second.lastOutput >= (time_t) settings.maxSilentTime)
|
||||||
{
|
{
|
||||||
printMsg(lvlError,
|
printMsg(lvlError,
|
||||||
format("%1% timed out after %2% seconds of silence")
|
format("%1% timed out after %2% seconds of silence")
|
||||||
% goal->getName() % maxSilentTime);
|
% goal->getName() % settings.maxSilentTime);
|
||||||
goal->cancel();
|
goal->cancel();
|
||||||
}
|
}
|
||||||
|
|
||||||
if (buildTimeout != 0 &&
|
if (settings.buildTimeout != 0 &&
|
||||||
after - before >= (time_t) buildTimeout)
|
after - before >= (time_t) settings.buildTimeout)
|
||||||
{
|
{
|
||||||
printMsg(lvlError,
|
printMsg(lvlError,
|
||||||
format("%1% timed out after %2% seconds of activity")
|
format("%1% timed out after %2% seconds of activity")
|
||||||
% goal->getName() % buildTimeout);
|
% goal->getName() % settings.buildTimeout);
|
||||||
goal->cancel();
|
goal->cancel();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!waitingForAWhile.empty() && lastWokenUp + wakeUpInterval <= after) {
|
if (!waitingForAWhile.empty() && lastWokenUp + settings.pollInterval <= after) {
|
||||||
lastWokenUp = after;
|
lastWokenUp = after;
|
||||||
foreach (WeakGoals::iterator, i, waitingForAWhile) {
|
foreach (WeakGoals::iterator, i, waitingForAWhile) {
|
||||||
GoalPtr goal = i->lock();
|
GoalPtr goal = i->lock();
|
||||||
|
|
|
@ -38,7 +38,7 @@ Path writeDerivation(StoreAPI & store,
|
||||||
held during a garbage collection). */
|
held during a garbage collection). */
|
||||||
string suffix = name + drvExtension;
|
string suffix = name + drvExtension;
|
||||||
string contents = unparseDerivation(drv);
|
string contents = unparseDerivation(drv);
|
||||||
return readOnlyMode
|
return settings.readOnlyMode
|
||||||
? computeStorePathForText(suffix, contents, references)
|
? computeStorePathForText(suffix, contents, references)
|
||||||
: store.addTextToStore(suffix, contents, references);
|
: store.addTextToStore(suffix, contents, references);
|
||||||
}
|
}
|
||||||
|
|
|
@ -34,7 +34,7 @@ static const int defaultGcLevel = 1000;
|
||||||
int LocalStore::openGCLock(LockType lockType)
|
int LocalStore::openGCLock(LockType lockType)
|
||||||
{
|
{
|
||||||
Path fnGCLock = (format("%1%/%2%")
|
Path fnGCLock = (format("%1%/%2%")
|
||||||
% nixStateDir % gcLockName).str();
|
% settings.nixStateDir % gcLockName).str();
|
||||||
|
|
||||||
debug(format("acquiring global GC lock `%1%'") % fnGCLock);
|
debug(format("acquiring global GC lock `%1%'") % fnGCLock);
|
||||||
|
|
||||||
|
@ -85,7 +85,7 @@ void LocalStore::addIndirectRoot(const Path & path)
|
||||||
{
|
{
|
||||||
string hash = printHash32(hashString(htSHA1, path));
|
string hash = printHash32(hashString(htSHA1, path));
|
||||||
Path realRoot = canonPath((format("%1%/%2%/auto/%3%")
|
Path realRoot = canonPath((format("%1%/%2%/auto/%3%")
|
||||||
% nixStateDir % gcRootsDir % hash).str());
|
% settings.nixStateDir % gcRootsDir % hash).str());
|
||||||
createSymlink(realRoot, path);
|
createSymlink(realRoot, path);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -113,7 +113,7 @@ Path addPermRoot(StoreAPI & store, const Path & _storePath,
|
||||||
|
|
||||||
else {
|
else {
|
||||||
if (!allowOutsideRootsDir) {
|
if (!allowOutsideRootsDir) {
|
||||||
Path rootsDir = canonPath((format("%1%/%2%") % nixStateDir % gcRootsDir).str());
|
Path rootsDir = canonPath((format("%1%/%2%") % settings.nixStateDir % gcRootsDir).str());
|
||||||
|
|
||||||
if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/")
|
if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/")
|
||||||
throw Error(format(
|
throw Error(format(
|
||||||
|
@ -130,7 +130,7 @@ Path addPermRoot(StoreAPI & store, const Path & _storePath,
|
||||||
Instead of reading all the roots, it would be more efficient to
|
Instead of reading all the roots, it would be more efficient to
|
||||||
check if the root is in a directory in or linked from the
|
check if the root is in a directory in or linked from the
|
||||||
gcroots directory. */
|
gcroots directory. */
|
||||||
if (queryBoolSetting("gc-check-reachability", false)) {
|
if (settings.checkRootReachability) {
|
||||||
Roots roots = store.findRoots();
|
Roots roots = store.findRoots();
|
||||||
if (roots.find(gcRoot) == roots.end())
|
if (roots.find(gcRoot) == roots.end())
|
||||||
printMsg(lvlError,
|
printMsg(lvlError,
|
||||||
|
@ -160,7 +160,7 @@ void LocalStore::addTempRoot(const Path & path)
|
||||||
if (fdTempRoots == -1) {
|
if (fdTempRoots == -1) {
|
||||||
|
|
||||||
while (1) {
|
while (1) {
|
||||||
Path dir = (format("%1%/%2%") % nixStateDir % tempRootsDir).str();
|
Path dir = (format("%1%/%2%") % settings.nixStateDir % tempRootsDir).str();
|
||||||
createDirs(dir);
|
createDirs(dir);
|
||||||
|
|
||||||
fnTempRoots = (format("%1%/%2%")
|
fnTempRoots = (format("%1%/%2%")
|
||||||
|
@ -173,7 +173,7 @@ void LocalStore::addTempRoot(const Path & path)
|
||||||
processes with the same pid. */
|
processes with the same pid. */
|
||||||
unlink(fnTempRoots.c_str());
|
unlink(fnTempRoots.c_str());
|
||||||
|
|
||||||
fdTempRoots = openLockFile(fnTempRoots, true);
|
fdTempRoots = openLockFile(fnTempRoots, true);
|
||||||
|
|
||||||
fdGCLock.close();
|
fdGCLock.close();
|
||||||
|
|
||||||
|
@ -238,10 +238,10 @@ static void readTempRoots(PathSet & tempRoots, FDs & fds)
|
||||||
/* Read the `temproots' directory for per-process temporary root
|
/* Read the `temproots' directory for per-process temporary root
|
||||||
files. */
|
files. */
|
||||||
Strings tempRootFiles = readDirectory(
|
Strings tempRootFiles = readDirectory(
|
||||||
(format("%1%/%2%") % nixStateDir % tempRootsDir).str());
|
(format("%1%/%2%") % settings.nixStateDir % tempRootsDir).str());
|
||||||
|
|
||||||
foreach (Strings::iterator, i, tempRootFiles) {
|
foreach (Strings::iterator, i, tempRootFiles) {
|
||||||
Path path = (format("%1%/%2%/%3%") % nixStateDir % tempRootsDir % *i).str();
|
Path path = (format("%1%/%2%/%3%") % settings.nixStateDir % tempRootsDir % *i).str();
|
||||||
|
|
||||||
debug(format("reading temporary root file `%1%'") % path);
|
debug(format("reading temporary root file `%1%'") % path);
|
||||||
FDPtr fd(new AutoCloseFD(open(path.c_str(), O_RDWR, 0666)));
|
FDPtr fd(new AutoCloseFD(open(path.c_str(), O_RDWR, 0666)));
|
||||||
|
@ -350,7 +350,7 @@ static void findRoots(StoreAPI & store, const Path & path,
|
||||||
static Roots findRoots(StoreAPI & store, bool deleteStale)
|
static Roots findRoots(StoreAPI & store, bool deleteStale)
|
||||||
{
|
{
|
||||||
Roots roots;
|
Roots roots;
|
||||||
Path rootsDir = canonPath((format("%1%/%2%") % nixStateDir % gcRootsDir).str());
|
Path rootsDir = canonPath((format("%1%/%2%") % settings.nixStateDir % gcRootsDir).str());
|
||||||
findRoots(store, rootsDir, true, deleteStale, roots);
|
findRoots(store, rootsDir, true, deleteStale, roots);
|
||||||
return roots;
|
return roots;
|
||||||
}
|
}
|
||||||
|
@ -365,7 +365,7 @@ Roots LocalStore::findRoots()
|
||||||
static void addAdditionalRoots(StoreAPI & store, PathSet & roots)
|
static void addAdditionalRoots(StoreAPI & store, PathSet & roots)
|
||||||
{
|
{
|
||||||
Path rootFinder = getEnv("NIX_ROOT_FINDER",
|
Path rootFinder = getEnv("NIX_ROOT_FINDER",
|
||||||
nixLibexecDir + "/nix/find-runtime-roots.pl");
|
settings.nixLibexecDir + "/nix/find-runtime-roots.pl");
|
||||||
|
|
||||||
if (rootFinder.empty()) return;
|
if (rootFinder.empty()) return;
|
||||||
|
|
||||||
|
@ -623,8 +623,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
|
||||||
GCState state(results);
|
GCState state(results);
|
||||||
state.options = options;
|
state.options = options;
|
||||||
|
|
||||||
state.gcKeepOutputs = queryBoolSetting("gc-keep-outputs", false);
|
state.gcKeepOutputs = settings.gcKeepOutputs;
|
||||||
state.gcKeepDerivations = queryBoolSetting("gc-keep-derivations", true);
|
state.gcKeepDerivations = settings.gcKeepDerivations;
|
||||||
|
|
||||||
/* Using `--ignore-liveness' with `--delete' can have unintended
|
/* Using `--ignore-liveness' with `--delete' can have unintended
|
||||||
consequences if `gc-keep-outputs' or `gc-keep-derivations' are
|
consequences if `gc-keep-outputs' or `gc-keep-derivations' are
|
||||||
|
@ -685,8 +685,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|
||||||
AutoCloseDir dir = opendir(nixStore.c_str());
|
AutoCloseDir dir = opendir(settings.nixStore.c_str());
|
||||||
if (!dir) throw SysError(format("opening directory `%1%'") % nixStore);
|
if (!dir) throw SysError(format("opening directory `%1%'") % settings.nixStore);
|
||||||
|
|
||||||
/* Read the store and immediately delete all paths that
|
/* Read the store and immediately delete all paths that
|
||||||
aren't valid. When using --max-freed etc., deleting
|
aren't valid. When using --max-freed etc., deleting
|
||||||
|
@ -700,14 +700,14 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
|
||||||
checkInterrupt();
|
checkInterrupt();
|
||||||
string name = dirent->d_name;
|
string name = dirent->d_name;
|
||||||
if (name == "." || name == "..") continue;
|
if (name == "." || name == "..") continue;
|
||||||
Path path = nixStore + "/" + name;
|
Path path = settings.nixStore + "/" + name;
|
||||||
if (isValidPath(path))
|
if (isValidPath(path))
|
||||||
entries.push_back(path);
|
entries.push_back(path);
|
||||||
else
|
else
|
||||||
tryToDelete(state, path);
|
tryToDelete(state, path);
|
||||||
}
|
}
|
||||||
|
|
||||||
dir.close();
|
dir.close();
|
||||||
|
|
||||||
/* Now delete the unreachable valid paths. Randomise the
|
/* Now delete the unreachable valid paths. Randomise the
|
||||||
order in which we delete entries to make the collector
|
order in which we delete entries to make the collector
|
||||||
|
|
|
@ -10,36 +10,63 @@
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
|
||||||
string nixStore = "/UNINIT";
|
Settings settings;
|
||||||
string nixDataDir = "/UNINIT";
|
|
||||||
string nixLogDir = "/UNINIT";
|
|
||||||
string nixStateDir = "/UNINIT";
|
|
||||||
string nixDBPath = "/UNINIT";
|
|
||||||
string nixConfDir = "/UNINIT";
|
|
||||||
string nixLibexecDir = "/UNINIT";
|
|
||||||
string nixBinDir = "/UNINIT";
|
|
||||||
|
|
||||||
bool keepFailed = false;
|
|
||||||
bool keepGoing = false;
|
|
||||||
bool tryFallback = false;
|
|
||||||
Verbosity buildVerbosity = lvlError;
|
|
||||||
unsigned int maxBuildJobs = 1;
|
|
||||||
unsigned int buildCores = 1;
|
|
||||||
bool readOnlyMode = false;
|
|
||||||
string thisSystem = "unset";
|
|
||||||
time_t maxSilentTime = 0;
|
|
||||||
time_t buildTimeout = 0;
|
|
||||||
Paths substituters;
|
|
||||||
bool useBuildHook = true;
|
|
||||||
bool printBuildTrace = false;
|
|
||||||
|
|
||||||
|
|
||||||
static bool settingsRead = false;
|
Settings::Settings()
|
||||||
|
{
|
||||||
|
keepFailed = false;
|
||||||
|
keepGoing = false;
|
||||||
|
tryFallback = false;
|
||||||
|
buildVerbosity = lvlError;
|
||||||
|
maxBuildJobs = 1;
|
||||||
|
buildCores = 1;
|
||||||
|
readOnlyMode = false;
|
||||||
|
thisSystem = SYSTEM;
|
||||||
|
maxSilentTime = 0;
|
||||||
|
buildTimeout = 0;
|
||||||
|
useBuildHook = true;
|
||||||
|
printBuildTrace = false;
|
||||||
|
reservedSize = 1024 * 1024;
|
||||||
|
fsyncMetadata = true;
|
||||||
|
useSQLiteWAL = true;
|
||||||
|
syncBeforeRegistering = false;
|
||||||
|
useSubstitutes = true;
|
||||||
|
useChroot = false;
|
||||||
|
dirsInChroot.insert("/dev");
|
||||||
|
dirsInChroot.insert("/dev/pts");
|
||||||
|
impersonateLinux26 = false;
|
||||||
|
keepLog = true;
|
||||||
|
compressLog = true;
|
||||||
|
cacheFailure = false;
|
||||||
|
pollInterval = 5;
|
||||||
|
checkRootReachability = false;
|
||||||
|
gcKeepOutputs = false;
|
||||||
|
gcKeepDerivations = true;
|
||||||
|
autoOptimiseStore = true;
|
||||||
|
envKeepDerivations = false;
|
||||||
|
}
|
||||||
|
|
||||||
static std::map<string, Strings> settings;
|
|
||||||
|
|
||||||
/* Overriden settings. */
|
void Settings::processEnvironment()
|
||||||
std::map<string, Strings> settingsCmdline;
|
{
|
||||||
|
nixStore = canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR)));
|
||||||
|
nixDataDir = canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR));
|
||||||
|
nixLogDir = canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR));
|
||||||
|
nixStateDir = canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR));
|
||||||
|
nixDBPath = getEnv("NIX_DB_DIR", nixStateDir + "/db");
|
||||||
|
nixConfDir = canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR));
|
||||||
|
nixLibexecDir = canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR));
|
||||||
|
nixBinDir = canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR));
|
||||||
|
|
||||||
|
string subs = getEnv("NIX_SUBSTITUTERS", "default");
|
||||||
|
if (subs == "default") {
|
||||||
|
substituters.push_back(nixLibexecDir + "/nix/substituters/copy-from-other-stores.pl");
|
||||||
|
substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl");
|
||||||
|
substituters.push_back(nixLibexecDir + "/nix/substituters/download-from-binary-cache.pl");
|
||||||
|
} else
|
||||||
|
substituters = tokenizeString(subs, ":");
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
string & at(Strings & ss, unsigned int n)
|
string & at(Strings & ss, unsigned int n)
|
||||||
|
@ -50,7 +77,7 @@ string & at(Strings & ss, unsigned int n)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
static void readSettings()
|
void Settings::loadConfFile()
|
||||||
{
|
{
|
||||||
Path settingsFile = (format("%1%/%2%") % nixConfDir % "nix.conf").str();
|
Path settingsFile = (format("%1%/%2%") % nixConfDir % "nix.conf").str();
|
||||||
if (!pathExists(settingsFile)) return;
|
if (!pathExists(settingsFile)) return;
|
||||||
|
@ -78,94 +105,102 @@ static void readSettings()
|
||||||
|
|
||||||
Strings::iterator i = tokens.begin();
|
Strings::iterator i = tokens.begin();
|
||||||
advance(i, 2);
|
advance(i, 2);
|
||||||
settings[name] = Strings(i, tokens.end());
|
settings[name] = concatStringsSep(" ", Strings(i, tokens.end())); // FIXME: slow
|
||||||
};
|
};
|
||||||
|
|
||||||
settings.insert(settingsCmdline.begin(), settingsCmdline.end());
|
|
||||||
|
|
||||||
settingsRead = true;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
Strings querySetting(const string & name, const Strings & def)
|
void Settings::set(const string & name, const string & value)
|
||||||
{
|
{
|
||||||
if (!settingsRead) readSettings();
|
settings[name] = value;
|
||||||
std::map<string, Strings>::iterator i = settings.find(name);
|
overrides[name] = value;
|
||||||
return i == settings.end() ? def : i->second;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
string querySetting(const string & name, const string & def)
|
void Settings::update()
|
||||||
{
|
{
|
||||||
Strings defs;
|
get(tryFallback, "build-fallback");
|
||||||
defs.push_back(def);
|
get(maxBuildJobs, "build-max-jobs");
|
||||||
|
get(buildCores, "build-cores");
|
||||||
Strings value = querySetting(name, defs);
|
get(thisSystem, "system");
|
||||||
if (value.size() != 1)
|
get(maxSilentTime, "build-max-silent-time");
|
||||||
throw Error(format("configuration option `%1%' should not be a list") % name);
|
get(buildTimeout, "build-timeout");
|
||||||
|
get(reservedSize, "gc-reserved-space");
|
||||||
return value.front();
|
get(fsyncMetadata, "fsync-metadata");
|
||||||
|
get(useSQLiteWAL, "use-sqlite-wal");
|
||||||
|
get(syncBeforeRegistering, "sync-before-registering");
|
||||||
|
get(useSubstitutes, "build-use-substitutes");
|
||||||
|
get(buildUsersGroup, "build-users-group");
|
||||||
|
get(useChroot, "build-use-chroot");
|
||||||
|
get(dirsInChroot, "build-chroot-dirs");
|
||||||
|
get(impersonateLinux26, "build-impersonate-linux-26");
|
||||||
|
get(keepLog, "build-keep-log");
|
||||||
|
get(compressLog, "build-compress-log");
|
||||||
|
get(cacheFailure, "build-cache-failure");
|
||||||
|
get(pollInterval, "build-poll-interval");
|
||||||
|
get(checkRootReachability, "gc-check-reachability");
|
||||||
|
get(gcKeepOutputs, "gc-keep-outputs");
|
||||||
|
get(gcKeepDerivations, "gc-keep-derivations");
|
||||||
|
get(autoOptimiseStore, "auto-optimise-store");
|
||||||
|
get(envKeepDerivations, "env-keep-derivations");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
bool queryBoolSetting(const string & name, bool def)
|
void Settings::get(string & res, const string & name)
|
||||||
{
|
{
|
||||||
string v = querySetting(name, def ? "true" : "false");
|
SettingsMap::iterator i = settings.find(name);
|
||||||
if (v == "true") return true;
|
if (i == settings.end()) return;
|
||||||
else if (v == "false") return false;
|
res = i->second;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
void Settings::get(bool & res, const string & name)
|
||||||
|
{
|
||||||
|
SettingsMap::iterator i = settings.find(name);
|
||||||
|
if (i == settings.end()) return;
|
||||||
|
if (i->second == "true") res = true;
|
||||||
|
else if (i->second == "false") res = false;
|
||||||
else throw Error(format("configuration option `%1%' should be either `true' or `false', not `%2%'")
|
else throw Error(format("configuration option `%1%' should be either `true' or `false', not `%2%'")
|
||||||
% name % v);
|
% name % i->second);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
unsigned int queryIntSetting(const string & name, unsigned int def)
|
void Settings::get(PathSet & res, const string & name)
|
||||||
{
|
{
|
||||||
int n;
|
SettingsMap::iterator i = settings.find(name);
|
||||||
if (!string2Int(querySetting(name, int2String(def)), n) || n < 0)
|
if (i == settings.end()) return;
|
||||||
|
res.clear();
|
||||||
|
Strings ss = tokenizeString(i->second);
|
||||||
|
res.insert(ss.begin(), ss.end());
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
template<class N> void Settings::get(N & res, const string & name)
|
||||||
|
{
|
||||||
|
SettingsMap::iterator i = settings.find(name);
|
||||||
|
if (i == settings.end()) return;
|
||||||
|
if (!string2Int(i->second, res))
|
||||||
throw Error(format("configuration setting `%1%' should have an integer value") % name);
|
throw Error(format("configuration setting `%1%' should have an integer value") % name);
|
||||||
return n;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void overrideSetting(const string & name, const Strings & value)
|
string Settings::pack()
|
||||||
{
|
{
|
||||||
if (settingsRead) settings[name] = value;
|
string s;
|
||||||
settingsCmdline[name] = value;
|
foreach (SettingsMap::iterator, i, settings) {
|
||||||
|
if (i->first.find('\n') != string::npos ||
|
||||||
|
i->first.find('=') != string::npos ||
|
||||||
|
i->second.find('\n') != string::npos)
|
||||||
|
throw Error("illegal option name/value");
|
||||||
|
s += i->first; s += '='; s += i->second; s += '\n';
|
||||||
|
}
|
||||||
|
return s;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void reloadSettings()
|
Settings::SettingsMap Settings::getOverrides()
|
||||||
{
|
{
|
||||||
settingsRead = false;
|
return overrides;
|
||||||
settings.clear();
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
void setDefaultsFromEnvironment()
|
|
||||||
{
|
|
||||||
/* Setup Nix paths. */
|
|
||||||
nixStore = canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR)));
|
|
||||||
nixDataDir = canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR));
|
|
||||||
nixLogDir = canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR));
|
|
||||||
nixStateDir = canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR));
|
|
||||||
nixDBPath = getEnv("NIX_DB_DIR", nixStateDir + "/db");
|
|
||||||
nixConfDir = canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR));
|
|
||||||
nixLibexecDir = canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR));
|
|
||||||
nixBinDir = canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR));
|
|
||||||
|
|
||||||
string subs = getEnv("NIX_SUBSTITUTERS", "default");
|
|
||||||
if (subs == "default") {
|
|
||||||
substituters.push_back(nixLibexecDir + "/nix/substituters/copy-from-other-stores.pl");
|
|
||||||
substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl");
|
|
||||||
} else
|
|
||||||
substituters = tokenizeString(subs, ":");
|
|
||||||
|
|
||||||
/* Get some settings from the configuration file. */
|
|
||||||
thisSystem = querySetting("system", SYSTEM);
|
|
||||||
maxBuildJobs = queryIntSetting("build-max-jobs", 1);
|
|
||||||
buildCores = queryIntSetting("build-cores", 1);
|
|
||||||
maxSilentTime = queryIntSetting("build-max-silent-time", 0);
|
|
||||||
buildTimeout = queryIntSetting("build-timeout", 0);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -2,118 +2,191 @@
|
||||||
|
|
||||||
#include "types.hh"
|
#include "types.hh"
|
||||||
|
|
||||||
|
#include <map>
|
||||||
|
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
|
||||||
/* Path names. */
|
struct Settings {
|
||||||
|
|
||||||
/* nixStore is the directory where we generally store atomic and
|
typedef std::map<string, string> SettingsMap;
|
||||||
derived files. */
|
|
||||||
extern string nixStore;
|
|
||||||
|
|
||||||
extern string nixDataDir; /* !!! fix */
|
Settings();
|
||||||
|
|
||||||
/* nixLogDir is the directory where we log various operations. */
|
void processEnvironment();
|
||||||
extern string nixLogDir;
|
|
||||||
|
|
||||||
/* nixStateDir is the directory where state is stored. */
|
void loadConfFile();
|
||||||
extern string nixStateDir;
|
|
||||||
|
|
||||||
/* nixDBPath is the path name of our Berkeley DB environment. */
|
void set(const string & name, const string & value);
|
||||||
extern string nixDBPath;
|
|
||||||
|
|
||||||
/* nixConfDir is the directory where configuration files are
|
void update();
|
||||||
stored. */
|
|
||||||
extern string nixConfDir;
|
|
||||||
|
|
||||||
/* nixLibexecDir is the directory where internal helper programs are
|
string pack();
|
||||||
stored. */
|
|
||||||
extern string nixLibexecDir;
|
|
||||||
|
|
||||||
/* nixBinDir is the directory where the main programs are stored. */
|
SettingsMap getOverrides();
|
||||||
extern string nixBinDir;
|
|
||||||
|
/* The directory where we store sources and derived files. */
|
||||||
|
Path nixStore;
|
||||||
|
|
||||||
|
Path nixDataDir; /* !!! fix */
|
||||||
|
|
||||||
|
/* The directory where we log various operations. */
|
||||||
|
Path nixLogDir;
|
||||||
|
|
||||||
|
/* The directory where state is stored. */
|
||||||
|
Path nixStateDir;
|
||||||
|
|
||||||
|
/* The directory where we keep the SQLite database. */
|
||||||
|
Path nixDBPath;
|
||||||
|
|
||||||
|
/* The directory where configuration files are stored. */
|
||||||
|
Path nixConfDir;
|
||||||
|
|
||||||
|
/* The directory where internal helper programs are stored. */
|
||||||
|
Path nixLibexecDir;
|
||||||
|
|
||||||
|
/* The directory where the main programs are stored. */
|
||||||
|
Path nixBinDir;
|
||||||
|
|
||||||
|
/* Whether to keep temporary directories of failed builds. */
|
||||||
|
bool keepFailed;
|
||||||
|
|
||||||
|
/* Whether to keep building subgoals when a sibling (another
|
||||||
|
subgoal of the same goal) fails. */
|
||||||
|
bool keepGoing;
|
||||||
|
|
||||||
|
/* Whether, if we cannot realise the known closure corresponding
|
||||||
|
to a derivation, we should try to normalise the derivation
|
||||||
|
instead. */
|
||||||
|
bool tryFallback;
|
||||||
|
|
||||||
|
/* Verbosity level for build output. */
|
||||||
|
Verbosity buildVerbosity;
|
||||||
|
|
||||||
|
/* Maximum number of parallel build jobs. 0 means unlimited. */
|
||||||
|
unsigned int maxBuildJobs;
|
||||||
|
|
||||||
|
/* Number of CPU cores to utilize in parallel within a build,
|
||||||
|
i.e. by passing this number to Make via '-j'. 0 means that the
|
||||||
|
number of actual CPU cores on the local host ought to be
|
||||||
|
auto-detected. */
|
||||||
|
unsigned int buildCores;
|
||||||
|
|
||||||
|
/* Read-only mode. Don't copy stuff to the store, don't change
|
||||||
|
the database. */
|
||||||
|
bool readOnlyMode;
|
||||||
|
|
||||||
|
/* The canonical system name, as returned by config.guess. */
|
||||||
|
string thisSystem;
|
||||||
|
|
||||||
|
/* The maximum time in seconds that a builer can go without
|
||||||
|
producing any output on stdout/stderr before it is killed. 0
|
||||||
|
means infinity. */
|
||||||
|
time_t maxSilentTime;
|
||||||
|
|
||||||
|
/* The maximum duration in seconds that a builder can run. 0
|
||||||
|
means infinity. */
|
||||||
|
time_t buildTimeout;
|
||||||
|
|
||||||
|
/* The substituters. There are programs that can somehow realise
|
||||||
|
a store path without building, e.g., by downloading it or
|
||||||
|
copying it from a CD. */
|
||||||
|
Paths substituters;
|
||||||
|
|
||||||
|
/* Whether to use build hooks (for distributed builds). Sometimes
|
||||||
|
users want to disable this from the command-line. */
|
||||||
|
bool useBuildHook;
|
||||||
|
|
||||||
|
/* Whether buildDerivations() should print out lines on stderr in
|
||||||
|
a fixed format to allow its progress to be monitored. Each
|
||||||
|
line starts with a "@". The following are defined:
|
||||||
|
|
||||||
|
@ build-started <drvpath> <outpath> <system> <logfile>
|
||||||
|
@ build-failed <drvpath> <outpath> <exitcode> <error text>
|
||||||
|
@ build-succeeded <drvpath> <outpath>
|
||||||
|
@ substituter-started <outpath> <substituter>
|
||||||
|
@ substituter-failed <outpath> <exitcode> <error text>
|
||||||
|
@ substituter-succeeded <outpath>
|
||||||
|
|
||||||
|
Best combined with --no-build-output, otherwise stderr might
|
||||||
|
conceivably contain lines in this format printed by the
|
||||||
|
builders. */
|
||||||
|
bool printBuildTrace;
|
||||||
|
|
||||||
|
/* Amount of reserved space for the garbage collector
|
||||||
|
(/nix/var/nix/db/reserved). */
|
||||||
|
off_t reservedSize;
|
||||||
|
|
||||||
|
/* Whether SQLite should use fsync. */
|
||||||
|
bool fsyncMetadata;
|
||||||
|
|
||||||
|
/* Whether SQLite should use WAL mode. */
|
||||||
|
bool useSQLiteWAL;
|
||||||
|
|
||||||
|
/* Whether to call sync() before registering a path as valid. */
|
||||||
|
bool syncBeforeRegistering;
|
||||||
|
|
||||||
|
/* Whether to use substitutes. */
|
||||||
|
bool useSubstitutes;
|
||||||
|
|
||||||
|
/* The Unix group that contains the build users. */
|
||||||
|
string buildUsersGroup;
|
||||||
|
|
||||||
|
/* Whether to build in chroot. */
|
||||||
|
bool useChroot;
|
||||||
|
|
||||||
|
/* The directories from the host filesystem to be included in the
|
||||||
|
chroot. */
|
||||||
|
PathSet dirsInChroot;
|
||||||
|
|
||||||
|
/* Whether to impersonate a Linux 2.6 machine on newer kernels. */
|
||||||
|
bool impersonateLinux26;
|
||||||
|
|
||||||
|
/* Whether to store build logs. */
|
||||||
|
bool keepLog;
|
||||||
|
|
||||||
|
/* Whether to compress logs. */
|
||||||
|
bool compressLog;
|
||||||
|
|
||||||
|
/* Whether to cache build failures. */
|
||||||
|
bool cacheFailure;
|
||||||
|
|
||||||
|
/* How often (in seconds) to poll for locks. */
|
||||||
|
unsigned int pollInterval;
|
||||||
|
|
||||||
|
/* Whether to check if new GC roots can in fact be found by the
|
||||||
|
garbage collector. */
|
||||||
|
bool checkRootReachability;
|
||||||
|
|
||||||
|
/* Whether the garbage collector should keep outputs of live
|
||||||
|
derivations. */
|
||||||
|
bool gcKeepOutputs;
|
||||||
|
|
||||||
|
/* Whether the garbage collector should keep derivers of live
|
||||||
|
paths. */
|
||||||
|
bool gcKeepDerivations;
|
||||||
|
|
||||||
|
/* Whether to automatically replace files with identical contents
|
||||||
|
with hard links. */
|
||||||
|
bool autoOptimiseStore;
|
||||||
|
|
||||||
|
/* Whether to add derivations as a dependency of user environments
|
||||||
|
(to prevent them from being GCed). */
|
||||||
|
bool envKeepDerivations;
|
||||||
|
|
||||||
|
private:
|
||||||
|
SettingsMap settings, overrides;
|
||||||
|
|
||||||
|
void get(string & res, const string & name);
|
||||||
|
void get(bool & res, const string & name);
|
||||||
|
void get(PathSet & res, const string & name);
|
||||||
|
template<class N> void get(N & res, const string & name);
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
/* Misc. global flags. */
|
// FIXME: don't use a global variable.
|
||||||
|
extern Settings settings;
|
||||||
/* Whether to keep temporary directories of failed builds. */
|
|
||||||
extern bool keepFailed;
|
|
||||||
|
|
||||||
/* Whether to keep building subgoals when a sibling (another subgoal
|
|
||||||
of the same goal) fails. */
|
|
||||||
extern bool keepGoing;
|
|
||||||
|
|
||||||
/* Whether, if we cannot realise the known closure corresponding to a
|
|
||||||
derivation, we should try to normalise the derivation instead. */
|
|
||||||
extern bool tryFallback;
|
|
||||||
|
|
||||||
/* Verbosity level for build output. */
|
|
||||||
extern Verbosity buildVerbosity;
|
|
||||||
|
|
||||||
/* Maximum number of parallel build jobs. 0 means unlimited. */
|
|
||||||
extern unsigned int maxBuildJobs;
|
|
||||||
|
|
||||||
/* Number of CPU cores to utilize in parallel within a build, i.e. by passing
|
|
||||||
this number to Make via '-j'. 0 means that the number of actual CPU cores on
|
|
||||||
the local host ought to be auto-detected. */
|
|
||||||
extern unsigned int buildCores;
|
|
||||||
|
|
||||||
/* Read-only mode. Don't copy stuff to the store, don't change the
|
|
||||||
database. */
|
|
||||||
extern bool readOnlyMode;
|
|
||||||
|
|
||||||
/* The canonical system name, as returned by config.guess. */
|
|
||||||
extern string thisSystem;
|
|
||||||
|
|
||||||
/* The maximum time in seconds that a builer can go without producing
|
|
||||||
any output on stdout/stderr before it is killed. 0 means
|
|
||||||
infinity. */
|
|
||||||
extern time_t maxSilentTime;
|
|
||||||
|
|
||||||
/* The maximum duration in seconds that a builder can run. 0 means
|
|
||||||
infinity. */
|
|
||||||
extern time_t buildTimeout;
|
|
||||||
|
|
||||||
/* The substituters. There are programs that can somehow realise a
|
|
||||||
store path without building, e.g., by downloading it or copying it
|
|
||||||
from a CD. */
|
|
||||||
extern Paths substituters;
|
|
||||||
|
|
||||||
/* Whether to use build hooks (for distributed builds). Sometimes
|
|
||||||
users want to disable this from the command-line. */
|
|
||||||
extern bool useBuildHook;
|
|
||||||
|
|
||||||
/* Whether buildDerivations() should print out lines on stderr in a
|
|
||||||
fixed format to allow its progress to be monitored. Each line
|
|
||||||
starts with a "@". The following are defined:
|
|
||||||
|
|
||||||
@ build-started <drvpath> <outpath> <system> <logfile>
|
|
||||||
@ build-failed <drvpath> <outpath> <exitcode> <error text>
|
|
||||||
@ build-succeeded <drvpath> <outpath>
|
|
||||||
@ substituter-started <outpath> <substituter>
|
|
||||||
@ substituter-failed <outpath> <exitcode> <error text>
|
|
||||||
@ substituter-succeeded <outpath>
|
|
||||||
|
|
||||||
Best combined with --no-build-output, otherwise stderr might
|
|
||||||
conceivably contain lines in this format printed by the builders.
|
|
||||||
*/
|
|
||||||
extern bool printBuildTrace;
|
|
||||||
|
|
||||||
|
|
||||||
Strings querySetting(const string & name, const Strings & def);
|
|
||||||
|
|
||||||
string querySetting(const string & name, const string & def);
|
|
||||||
|
|
||||||
bool queryBoolSetting(const string & name, bool def);
|
|
||||||
|
|
||||||
unsigned int queryIntSetting(const string & name, unsigned int def);
|
|
||||||
|
|
||||||
void overrideSetting(const string & name, const Strings & value);
|
|
||||||
|
|
||||||
void reloadSettings();
|
|
||||||
|
|
||||||
void setDefaultsFromEnvironment();
|
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -181,7 +181,7 @@ struct SQLiteTxn
|
||||||
void checkStoreNotSymlink()
|
void checkStoreNotSymlink()
|
||||||
{
|
{
|
||||||
if (getEnv("NIX_IGNORE_SYMLINK_STORE") == "1") return;
|
if (getEnv("NIX_IGNORE_SYMLINK_STORE") == "1") return;
|
||||||
Path path = nixStore;
|
Path path = settings.nixStore;
|
||||||
struct stat st;
|
struct stat st;
|
||||||
while (path != "/") {
|
while (path != "/") {
|
||||||
if (lstat(path.c_str(), &st))
|
if (lstat(path.c_str(), &st))
|
||||||
|
@ -198,23 +198,21 @@ void checkStoreNotSymlink()
|
||||||
|
|
||||||
LocalStore::LocalStore(bool reserveSpace)
|
LocalStore::LocalStore(bool reserveSpace)
|
||||||
{
|
{
|
||||||
substitutablePathsLoaded = false;
|
schemaPath = settings.nixDBPath + "/schema";
|
||||||
|
|
||||||
schemaPath = nixDBPath + "/schema";
|
if (settings.readOnlyMode) {
|
||||||
|
|
||||||
if (readOnlyMode) {
|
|
||||||
openDB(false);
|
openDB(false);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Create missing state directories if they don't already exist. */
|
/* Create missing state directories if they don't already exist. */
|
||||||
createDirs(nixStore);
|
createDirs(settings.nixStore);
|
||||||
createDirs(linksDir = nixStore + "/.links");
|
createDirs(linksDir = settings.nixStore + "/.links");
|
||||||
Path profilesDir = nixStateDir + "/profiles";
|
Path profilesDir = settings.nixStateDir + "/profiles";
|
||||||
createDirs(nixStateDir + "/profiles");
|
createDirs(settings.nixStateDir + "/profiles");
|
||||||
createDirs(nixStateDir + "/temproots");
|
createDirs(settings.nixStateDir + "/temproots");
|
||||||
createDirs(nixDBPath);
|
createDirs(settings.nixDBPath);
|
||||||
Path gcRootsDir = nixStateDir + "/gcroots";
|
Path gcRootsDir = settings.nixStateDir + "/gcroots";
|
||||||
if (!pathExists(gcRootsDir)) {
|
if (!pathExists(gcRootsDir)) {
|
||||||
createDirs(gcRootsDir);
|
createDirs(gcRootsDir);
|
||||||
if (symlink(profilesDir.c_str(), (gcRootsDir + "/profiles").c_str()) == -1)
|
if (symlink(profilesDir.c_str(), (gcRootsDir + "/profiles").c_str()) == -1)
|
||||||
|
@ -228,13 +226,12 @@ LocalStore::LocalStore(bool reserveSpace)
|
||||||
needed, we reserve some dummy space that we can free just
|
needed, we reserve some dummy space that we can free just
|
||||||
before doing a garbage collection. */
|
before doing a garbage collection. */
|
||||||
try {
|
try {
|
||||||
Path reservedPath = nixDBPath + "/reserved";
|
Path reservedPath = settings.nixDBPath + "/reserved";
|
||||||
if (reserveSpace) {
|
if (reserveSpace) {
|
||||||
int reservedSize = queryIntSetting("gc-reserved-space", 1024 * 1024);
|
|
||||||
struct stat st;
|
struct stat st;
|
||||||
if (stat(reservedPath.c_str(), &st) == -1 ||
|
if (stat(reservedPath.c_str(), &st) == -1 ||
|
||||||
st.st_size != reservedSize)
|
st.st_size != settings.reservedSize)
|
||||||
writeFile(reservedPath, string(reservedSize, 'X'));
|
writeFile(reservedPath, string(settings.reservedSize, 'X'));
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
deletePath(reservedPath);
|
deletePath(reservedPath);
|
||||||
|
@ -244,11 +241,11 @@ LocalStore::LocalStore(bool reserveSpace)
|
||||||
/* Acquire the big fat lock in shared mode to make sure that no
|
/* Acquire the big fat lock in shared mode to make sure that no
|
||||||
schema upgrade is in progress. */
|
schema upgrade is in progress. */
|
||||||
try {
|
try {
|
||||||
Path globalLockPath = nixDBPath + "/big-lock";
|
Path globalLockPath = settings.nixDBPath + "/big-lock";
|
||||||
globalLock = openLockFile(globalLockPath.c_str(), true);
|
globalLock = openLockFile(globalLockPath.c_str(), true);
|
||||||
} catch (SysError & e) {
|
} catch (SysError & e) {
|
||||||
if (e.errNo != EACCES) throw;
|
if (e.errNo != EACCES) throw;
|
||||||
readOnlyMode = true;
|
settings.readOnlyMode = true;
|
||||||
openDB(false);
|
openDB(false);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -327,7 +324,7 @@ int LocalStore::getSchema()
|
||||||
void LocalStore::openDB(bool create)
|
void LocalStore::openDB(bool create)
|
||||||
{
|
{
|
||||||
/* Open the Nix database. */
|
/* Open the Nix database. */
|
||||||
if (sqlite3_open_v2((nixDBPath + "/db.sqlite").c_str(), &db.db,
|
if (sqlite3_open_v2((settings.nixDBPath + "/db.sqlite").c_str(), &db.db,
|
||||||
SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK)
|
SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK)
|
||||||
throw Error("cannot open SQLite database");
|
throw Error("cannot open SQLite database");
|
||||||
|
|
||||||
|
@ -344,13 +341,13 @@ void LocalStore::openDB(bool create)
|
||||||
should be safe enough. If the user asks for it, don't sync at
|
should be safe enough. If the user asks for it, don't sync at
|
||||||
all. This can cause database corruption if the system
|
all. This can cause database corruption if the system
|
||||||
crashes. */
|
crashes. */
|
||||||
string syncMode = queryBoolSetting("fsync-metadata", true) ? "normal" : "off";
|
string syncMode = settings.fsyncMetadata ? "normal" : "off";
|
||||||
if (sqlite3_exec(db, ("pragma synchronous = " + syncMode + ";").c_str(), 0, 0, 0) != SQLITE_OK)
|
if (sqlite3_exec(db, ("pragma synchronous = " + syncMode + ";").c_str(), 0, 0, 0) != SQLITE_OK)
|
||||||
throwSQLiteError(db, "setting synchronous mode");
|
throwSQLiteError(db, "setting synchronous mode");
|
||||||
|
|
||||||
/* Set the SQLite journal mode. WAL mode is fastest, so it's the
|
/* Set the SQLite journal mode. WAL mode is fastest, so it's the
|
||||||
default. */
|
default. */
|
||||||
string mode = queryBoolSetting("use-sqlite-wal", true) ? "wal" : "truncate";
|
string mode = settings.useSQLiteWAL ? "wal" : "truncate";
|
||||||
string prevMode;
|
string prevMode;
|
||||||
{
|
{
|
||||||
SQLiteStmt stmt;
|
SQLiteStmt stmt;
|
||||||
|
@ -423,7 +420,7 @@ void canonicalisePathMetaData(const Path & path, bool recurse)
|
||||||
|
|
||||||
struct stat st;
|
struct stat st;
|
||||||
if (lstat(path.c_str(), &st))
|
if (lstat(path.c_str(), &st))
|
||||||
throw SysError(format("getting attributes of path `%1%'") % path);
|
throw SysError(format("getting attributes of path `%1%'") % path);
|
||||||
|
|
||||||
/* Really make sure that the path is of a supported type. This
|
/* Really make sure that the path is of a supported type. This
|
||||||
has already been checked in dumpPath(). */
|
has already been checked in dumpPath(). */
|
||||||
|
@ -478,8 +475,8 @@ void canonicalisePathMetaData(const Path & path, bool recurse)
|
||||||
|
|
||||||
if (recurse && S_ISDIR(st.st_mode)) {
|
if (recurse && S_ISDIR(st.st_mode)) {
|
||||||
Strings names = readDirectory(path);
|
Strings names = readDirectory(path);
|
||||||
foreach (Strings::iterator, i, names)
|
foreach (Strings::iterator, i, names)
|
||||||
canonicalisePathMetaData(path + "/" + *i, true);
|
canonicalisePathMetaData(path + "/" + *i, true);
|
||||||
}
|
}
|
||||||
|
|
||||||
makeImmutable(path);
|
makeImmutable(path);
|
||||||
|
@ -494,7 +491,7 @@ void canonicalisePathMetaData(const Path & path)
|
||||||
be a symlink, since we can't change its ownership. */
|
be a symlink, since we can't change its ownership. */
|
||||||
struct stat st;
|
struct stat st;
|
||||||
if (lstat(path.c_str(), &st))
|
if (lstat(path.c_str(), &st))
|
||||||
throw SysError(format("getting attributes of path `%1%'") % path);
|
throw SysError(format("getting attributes of path `%1%'") % path);
|
||||||
|
|
||||||
if (st.st_uid != geteuid()) {
|
if (st.st_uid != geteuid()) {
|
||||||
assert(S_ISLNK(st.st_mode));
|
assert(S_ISLNK(st.st_mode));
|
||||||
|
@ -756,7 +753,16 @@ bool LocalStore::isValidPath(const Path & path)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
PathSet LocalStore::queryValidPaths()
|
PathSet LocalStore::queryValidPaths(const PathSet & paths)
|
||||||
|
{
|
||||||
|
PathSet res;
|
||||||
|
foreach (PathSet::const_iterator, i, paths)
|
||||||
|
if (isValidPath(*i)) res.insert(*i);
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
PathSet LocalStore::queryAllValidPaths()
|
||||||
{
|
{
|
||||||
SQLiteStmt stmt;
|
SQLiteStmt stmt;
|
||||||
stmt.create(db, "select path from ValidPaths");
|
stmt.create(db, "select path from ValidPaths");
|
||||||
|
@ -883,7 +889,7 @@ Path LocalStore::queryPathFromHashPart(const string & hashPart)
|
||||||
|
|
||||||
SQLiteTxn txn(db);
|
SQLiteTxn txn(db);
|
||||||
|
|
||||||
Path prefix = nixStore + "/" + hashPart;
|
Path prefix = settings.nixStore + "/" + hashPart;
|
||||||
|
|
||||||
SQLiteStmtUse use(stmtQueryPathFromHashPart);
|
SQLiteStmtUse use(stmtQueryPathFromHashPart);
|
||||||
stmtQueryPathFromHashPart.bind(prefix);
|
stmtQueryPathFromHashPart.bind(prefix);
|
||||||
|
@ -903,10 +909,11 @@ void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter &
|
||||||
|
|
||||||
debug(format("starting substituter program `%1%'") % substituter);
|
debug(format("starting substituter program `%1%'") % substituter);
|
||||||
|
|
||||||
Pipe toPipe, fromPipe;
|
Pipe toPipe, fromPipe, errorPipe;
|
||||||
|
|
||||||
toPipe.create();
|
toPipe.create();
|
||||||
fromPipe.create();
|
fromPipe.create();
|
||||||
|
errorPipe.create();
|
||||||
|
|
||||||
run.pid = fork();
|
run.pid = fork();
|
||||||
|
|
||||||
|
@ -924,12 +931,18 @@ void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter &
|
||||||
written in Perl (i.e. all of them) fail. */
|
written in Perl (i.e. all of them) fail. */
|
||||||
unsetenv("DYLD_LIBRARY_PATH");
|
unsetenv("DYLD_LIBRARY_PATH");
|
||||||
|
|
||||||
|
/* Pass configuration options (including those overriden
|
||||||
|
with --option) to the substituter. */
|
||||||
|
setenv("_NIX_OPTIONS", settings.pack().c_str(), 1);
|
||||||
|
|
||||||
fromPipe.readSide.close();
|
fromPipe.readSide.close();
|
||||||
toPipe.writeSide.close();
|
toPipe.writeSide.close();
|
||||||
if (dup2(toPipe.readSide, STDIN_FILENO) == -1)
|
if (dup2(toPipe.readSide, STDIN_FILENO) == -1)
|
||||||
throw SysError("dupping stdin");
|
throw SysError("dupping stdin");
|
||||||
if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1)
|
if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1)
|
||||||
throw SysError("dupping stdout");
|
throw SysError("dupping stdout");
|
||||||
|
if (dup2(errorPipe.writeSide, STDERR_FILENO) == -1)
|
||||||
|
throw SysError("dupping stderr");
|
||||||
closeMostFDs(set<int>());
|
closeMostFDs(set<int>());
|
||||||
execl(substituter.c_str(), substituter.c_str(), "--query", NULL);
|
execl(substituter.c_str(), substituter.c_str(), "--query", NULL);
|
||||||
throw SysError(format("executing `%1%'") % substituter);
|
throw SysError(format("executing `%1%'") % substituter);
|
||||||
|
@ -943,6 +956,7 @@ void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter &
|
||||||
|
|
||||||
run.to = toPipe.writeSide.borrow();
|
run.to = toPipe.writeSide.borrow();
|
||||||
run.from = fromPipe.readSide.borrow();
|
run.from = fromPipe.readSide.borrow();
|
||||||
|
run.error = errorPipe.readSide.borrow();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -955,50 +969,79 @@ template<class T> T getIntLine(int fd)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
bool LocalStore::hasSubstitutes(const Path & path)
|
PathSet LocalStore::querySubstitutablePaths(const PathSet & paths)
|
||||||
{
|
{
|
||||||
foreach (Paths::iterator, i, substituters) {
|
PathSet res;
|
||||||
|
foreach (Paths::iterator, i, settings.substituters) {
|
||||||
|
if (res.size() == paths.size()) break;
|
||||||
RunningSubstituter & run(runningSubstituters[*i]);
|
RunningSubstituter & run(runningSubstituters[*i]);
|
||||||
startSubstituter(*i, run);
|
startSubstituter(*i, run);
|
||||||
writeLine(run.to, "have\n" + path);
|
string s = "have ";
|
||||||
if (getIntLine<int>(run.from)) return true;
|
foreach (PathSet::const_iterator, j, paths)
|
||||||
|
if (res.find(*j) == res.end()) { s += *j; s += " "; }
|
||||||
|
writeLine(run.to, s);
|
||||||
|
while (true) {
|
||||||
|
/* FIXME: we only read stderr when an error occurs, so
|
||||||
|
substituters should only write (short) messages to
|
||||||
|
stderr when they fail. I.e. they shouldn't write debug
|
||||||
|
output. */
|
||||||
|
try {
|
||||||
|
Path path = readLine(run.from);
|
||||||
|
if (path == "") break;
|
||||||
|
res.insert(path);
|
||||||
|
} catch (EndOfFile e) {
|
||||||
|
throw Error(format("substituter `%1%' failed: %2%") % *i % chomp(drainFD(run.error)));
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
return res;
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
bool LocalStore::querySubstitutablePathInfo(const Path & substituter,
|
void LocalStore::querySubstitutablePathInfos(const Path & substituter,
|
||||||
const Path & path, SubstitutablePathInfo & info)
|
PathSet & paths, SubstitutablePathInfos & infos)
|
||||||
{
|
{
|
||||||
RunningSubstituter & run(runningSubstituters[substituter]);
|
RunningSubstituter & run(runningSubstituters[substituter]);
|
||||||
startSubstituter(substituter, run);
|
startSubstituter(substituter, run);
|
||||||
|
|
||||||
writeLine(run.to, "info\n" + path);
|
string s = "info ";
|
||||||
|
foreach (PathSet::const_iterator, i, paths)
|
||||||
|
if (infos.find(*i) == infos.end()) { s += *i; s += " "; }
|
||||||
|
writeLine(run.to, s);
|
||||||
|
|
||||||
if (!getIntLine<int>(run.from)) return false;
|
while (true) {
|
||||||
|
try {
|
||||||
info.deriver = readLine(run.from);
|
Path path = readLine(run.from);
|
||||||
if (info.deriver != "") assertStorePath(info.deriver);
|
if (path == "") break;
|
||||||
int nrRefs = getIntLine<int>(run.from);
|
if (paths.find(path) == paths.end())
|
||||||
while (nrRefs--) {
|
throw Error(format("got unexpected path `%1%' from substituter") % path);
|
||||||
Path p = readLine(run.from);
|
paths.erase(path);
|
||||||
assertStorePath(p);
|
SubstitutablePathInfo & info(infos[path]);
|
||||||
info.references.insert(p);
|
info.deriver = readLine(run.from);
|
||||||
|
if (info.deriver != "") assertStorePath(info.deriver);
|
||||||
|
int nrRefs = getIntLine<int>(run.from);
|
||||||
|
while (nrRefs--) {
|
||||||
|
Path p = readLine(run.from);
|
||||||
|
assertStorePath(p);
|
||||||
|
info.references.insert(p);
|
||||||
|
}
|
||||||
|
info.downloadSize = getIntLine<long long>(run.from);
|
||||||
|
info.narSize = getIntLine<long long>(run.from);
|
||||||
|
} catch (EndOfFile e) {
|
||||||
|
throw Error(format("substituter `%1%' failed: %2%") % substituter % chomp(drainFD(run.error)));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
info.downloadSize = getIntLine<long long>(run.from);
|
|
||||||
info.narSize = getIntLine<long long>(run.from);
|
|
||||||
|
|
||||||
return true;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
bool LocalStore::querySubstitutablePathInfo(const Path & path,
|
void LocalStore::querySubstitutablePathInfos(const PathSet & paths,
|
||||||
SubstitutablePathInfo & info)
|
SubstitutablePathInfos & infos)
|
||||||
{
|
{
|
||||||
foreach (Paths::iterator, i, substituters)
|
PathSet todo = paths;
|
||||||
if (querySubstitutablePathInfo(*i, path, info)) return true;
|
foreach (Paths::iterator, i, settings.substituters) {
|
||||||
return false;
|
if (todo.empty()) break;
|
||||||
|
querySubstitutablePathInfos(*i, todo, infos);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -1018,11 +1061,10 @@ void LocalStore::registerValidPath(const ValidPathInfo & info)
|
||||||
|
|
||||||
void LocalStore::registerValidPaths(const ValidPathInfos & infos)
|
void LocalStore::registerValidPaths(const ValidPathInfos & infos)
|
||||||
{
|
{
|
||||||
/* sqlite will fsync by default, but the new valid paths may not be fsync-ed.
|
/* SQLite will fsync by default, but the new valid paths may not be fsync-ed.
|
||||||
* So some may want to fsync them before registering the validity, at the
|
* So some may want to fsync them before registering the validity, at the
|
||||||
* expense of some speed of the path registering operation. */
|
* expense of some speed of the path registering operation. */
|
||||||
if (queryBoolSetting("sync-before-registering", false))
|
if (settings.syncBeforeRegistering) sync();
|
||||||
sync();
|
|
||||||
|
|
||||||
while (1) {
|
while (1) {
|
||||||
try {
|
try {
|
||||||
|
@ -1266,7 +1308,7 @@ void LocalStore::exportPath(const Path & path, bool sign,
|
||||||
Path hashFile = tmpDir + "/hash";
|
Path hashFile = tmpDir + "/hash";
|
||||||
writeFile(hashFile, printHash(hash));
|
writeFile(hashFile, printHash(hash));
|
||||||
|
|
||||||
Path secretKey = nixConfDir + "/signing-key.sec";
|
Path secretKey = settings.nixConfDir + "/signing-key.sec";
|
||||||
checkSecrecy(secretKey);
|
checkSecrecy(secretKey);
|
||||||
|
|
||||||
Strings args;
|
Strings args;
|
||||||
|
@ -1312,7 +1354,7 @@ Path LocalStore::createTempDirInStore()
|
||||||
/* There is a slight possibility that `tmpDir' gets deleted by
|
/* There is a slight possibility that `tmpDir' gets deleted by
|
||||||
the GC between createTempDir() and addTempRoot(), so repeat
|
the GC between createTempDir() and addTempRoot(), so repeat
|
||||||
until `tmpDir' exists. */
|
until `tmpDir' exists. */
|
||||||
tmpDir = createTempDir(nixStore);
|
tmpDir = createTempDir(settings.nixStore);
|
||||||
addTempRoot(tmpDir);
|
addTempRoot(tmpDir);
|
||||||
} while (!pathExists(tmpDir));
|
} while (!pathExists(tmpDir));
|
||||||
return tmpDir;
|
return tmpDir;
|
||||||
|
@ -1364,7 +1406,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source)
|
||||||
args.push_back("rsautl");
|
args.push_back("rsautl");
|
||||||
args.push_back("-verify");
|
args.push_back("-verify");
|
||||||
args.push_back("-inkey");
|
args.push_back("-inkey");
|
||||||
args.push_back(nixConfDir + "/signing-key.pub");
|
args.push_back(settings.nixConfDir + "/signing-key.pub");
|
||||||
args.push_back("-pubin");
|
args.push_back("-pubin");
|
||||||
args.push_back("-in");
|
args.push_back("-in");
|
||||||
args.push_back(sigFile);
|
args.push_back(sigFile);
|
||||||
|
@ -1473,13 +1515,13 @@ void LocalStore::verifyStore(bool checkContents)
|
||||||
/* Acquire the global GC lock to prevent a garbage collection. */
|
/* Acquire the global GC lock to prevent a garbage collection. */
|
||||||
AutoCloseFD fdGCLock = openGCLock(ltWrite);
|
AutoCloseFD fdGCLock = openGCLock(ltWrite);
|
||||||
|
|
||||||
Paths entries = readDirectory(nixStore);
|
Paths entries = readDirectory(settings.nixStore);
|
||||||
PathSet store(entries.begin(), entries.end());
|
PathSet store(entries.begin(), entries.end());
|
||||||
|
|
||||||
/* Check whether all valid paths actually exist. */
|
/* Check whether all valid paths actually exist. */
|
||||||
printMsg(lvlInfo, "checking path existence...");
|
printMsg(lvlInfo, "checking path existence...");
|
||||||
|
|
||||||
PathSet validPaths2 = queryValidPaths(), validPaths, done;
|
PathSet validPaths2 = queryAllValidPaths(), validPaths, done;
|
||||||
|
|
||||||
foreach (PathSet::iterator, i, validPaths2)
|
foreach (PathSet::iterator, i, validPaths2)
|
||||||
verifyPath(*i, store, done, validPaths);
|
verifyPath(*i, store, done, validPaths);
|
||||||
|
@ -1583,9 +1625,9 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store,
|
||||||
PathSet LocalStore::queryValidPathsOld()
|
PathSet LocalStore::queryValidPathsOld()
|
||||||
{
|
{
|
||||||
PathSet paths;
|
PathSet paths;
|
||||||
Strings entries = readDirectory(nixDBPath + "/info");
|
Strings entries = readDirectory(settings.nixDBPath + "/info");
|
||||||
foreach (Strings::iterator, i, entries)
|
foreach (Strings::iterator, i, entries)
|
||||||
if (i->at(0) != '.') paths.insert(nixStore + "/" + *i);
|
if (i->at(0) != '.') paths.insert(settings.nixStore + "/" + *i);
|
||||||
return paths;
|
return paths;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1597,7 +1639,7 @@ ValidPathInfo LocalStore::queryPathInfoOld(const Path & path)
|
||||||
|
|
||||||
/* Read the info file. */
|
/* Read the info file. */
|
||||||
string baseName = baseNameOf(path);
|
string baseName = baseNameOf(path);
|
||||||
Path infoFile = (format("%1%/info/%2%") % nixDBPath % baseName).str();
|
Path infoFile = (format("%1%/info/%2%") % settings.nixDBPath % baseName).str();
|
||||||
if (!pathExists(infoFile))
|
if (!pathExists(infoFile))
|
||||||
throw Error(format("path `%1%' is not valid") % path);
|
throw Error(format("path `%1%' is not valid") % path);
|
||||||
string info = readFile(infoFile);
|
string info = readFile(infoFile);
|
||||||
|
|
|
@ -45,7 +45,7 @@ struct OptimiseStats
|
||||||
struct RunningSubstituter
|
struct RunningSubstituter
|
||||||
{
|
{
|
||||||
Pid pid;
|
Pid pid;
|
||||||
AutoCloseFD to, from;
|
AutoCloseFD to, from, error;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
|
@ -80,9 +80,6 @@ struct SQLiteStmt
|
||||||
class LocalStore : public StoreAPI
|
class LocalStore : public StoreAPI
|
||||||
{
|
{
|
||||||
private:
|
private:
|
||||||
bool substitutablePathsLoaded;
|
|
||||||
PathSet substitutablePaths;
|
|
||||||
|
|
||||||
typedef std::map<Path, RunningSubstituter> RunningSubstituters;
|
typedef std::map<Path, RunningSubstituter> RunningSubstituters;
|
||||||
RunningSubstituters runningSubstituters;
|
RunningSubstituters runningSubstituters;
|
||||||
|
|
||||||
|
@ -100,7 +97,9 @@ public:
|
||||||
|
|
||||||
bool isValidPath(const Path & path);
|
bool isValidPath(const Path & path);
|
||||||
|
|
||||||
PathSet queryValidPaths();
|
PathSet queryValidPaths(const PathSet & paths);
|
||||||
|
|
||||||
|
PathSet queryAllValidPaths();
|
||||||
|
|
||||||
ValidPathInfo queryPathInfo(const Path & path);
|
ValidPathInfo queryPathInfo(const Path & path);
|
||||||
|
|
||||||
|
@ -124,15 +123,13 @@ public:
|
||||||
|
|
||||||
Path queryPathFromHashPart(const string & hashPart);
|
Path queryPathFromHashPart(const string & hashPart);
|
||||||
|
|
||||||
PathSet querySubstitutablePaths();
|
PathSet querySubstitutablePaths(const PathSet & paths);
|
||||||
|
|
||||||
bool hasSubstitutes(const Path & path);
|
void querySubstitutablePathInfos(const Path & substituter,
|
||||||
|
PathSet & paths, SubstitutablePathInfos & infos);
|
||||||
|
|
||||||
bool querySubstitutablePathInfo(const Path & path,
|
void querySubstitutablePathInfos(const PathSet & paths,
|
||||||
SubstitutablePathInfo & info);
|
SubstitutablePathInfos & infos);
|
||||||
|
|
||||||
bool querySubstitutablePathInfo(const Path & substituter,
|
|
||||||
const Path & path, SubstitutablePathInfo & info);
|
|
||||||
|
|
||||||
Path addToStore(const Path & srcPath,
|
Path addToStore(const Path & srcPath,
|
||||||
bool recursive = true, HashType hashAlgo = htSHA256,
|
bool recursive = true, HashType hashAlgo = htSHA256,
|
||||||
|
@ -293,9 +290,6 @@ void canonicalisePathMetaData(const Path & path, bool recurse);
|
||||||
|
|
||||||
MakeError(PathInUse, Error);
|
MakeError(PathInUse, Error);
|
||||||
|
|
||||||
/* Whether we are in build users mode. */
|
|
||||||
bool haveBuildUsers();
|
|
||||||
|
|
||||||
/* Whether we are root. */
|
/* Whether we are root. */
|
||||||
bool amPrivileged();
|
bool amPrivileged();
|
||||||
|
|
||||||
|
|
|
@ -55,45 +55,95 @@ void queryMissing(StoreAPI & store, const PathSet & targets,
|
||||||
|
|
||||||
PathSet todo(targets.begin(), targets.end()), done;
|
PathSet todo(targets.begin(), targets.end()), done;
|
||||||
|
|
||||||
while (!todo.empty()) {
|
/* Getting substitute info has high latency when using the binary
|
||||||
Path p = *(todo.begin());
|
cache substituter. Thus it's essential to do substitute
|
||||||
todo.erase(p);
|
queries in parallel as much as possible. To accomplish this
|
||||||
if (done.find(p) != done.end()) continue;
|
we do the following:
|
||||||
done.insert(p);
|
|
||||||
|
|
||||||
if (isDerivation(p)) {
|
- For all paths still to be processed (‘todo’), we add all
|
||||||
if (!store.isValidPath(p)) {
|
paths for which we need info to the set ‘query’. For an
|
||||||
unknown.insert(p);
|
unbuilt derivation this is the output paths; otherwise, it's
|
||||||
continue;
|
the path itself.
|
||||||
|
|
||||||
|
- We get info about all paths in ‘query’ in parallel.
|
||||||
|
|
||||||
|
- We process the results and add new items to ‘todo’ if
|
||||||
|
necessary. E.g. if a path is substitutable, then we need to
|
||||||
|
get info on its references.
|
||||||
|
|
||||||
|
- Repeat until ‘todo’ is empty.
|
||||||
|
*/
|
||||||
|
|
||||||
|
while (!todo.empty()) {
|
||||||
|
|
||||||
|
PathSet query, todoDrv, todoNonDrv;
|
||||||
|
|
||||||
|
foreach (PathSet::iterator, i, todo) {
|
||||||
|
if (done.find(*i) != done.end()) continue;
|
||||||
|
done.insert(*i);
|
||||||
|
|
||||||
|
if (isDerivation(*i)) {
|
||||||
|
if (!store.isValidPath(*i)) {
|
||||||
|
// FIXME: we could try to substitute p.
|
||||||
|
unknown.insert(*i);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
Derivation drv = derivationFromPath(store, *i);
|
||||||
|
|
||||||
|
PathSet invalid;
|
||||||
|
foreach (DerivationOutputs::iterator, j, drv.outputs)
|
||||||
|
if (!store.isValidPath(j->second.path)) invalid.insert(j->second.path);
|
||||||
|
if (invalid.empty()) continue;
|
||||||
|
|
||||||
|
todoDrv.insert(*i);
|
||||||
|
if (settings.useSubstitutes) query.insert(invalid.begin(), invalid.end());
|
||||||
}
|
}
|
||||||
Derivation drv = derivationFromPath(store, p);
|
|
||||||
|
else {
|
||||||
|
if (store.isValidPath(*i)) continue;
|
||||||
|
query.insert(*i);
|
||||||
|
todoNonDrv.insert(*i);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
todo.clear();
|
||||||
|
|
||||||
|
SubstitutablePathInfos infos;
|
||||||
|
store.querySubstitutablePathInfos(query, infos);
|
||||||
|
|
||||||
|
foreach (PathSet::iterator, i, todoDrv) {
|
||||||
|
// FIXME: cache this
|
||||||
|
Derivation drv = derivationFromPath(store, *i);
|
||||||
|
|
||||||
bool mustBuild = false;
|
bool mustBuild = false;
|
||||||
foreach (DerivationOutputs::iterator, i, drv.outputs)
|
if (settings.useSubstitutes) {
|
||||||
if (!store.isValidPath(i->second.path) &&
|
foreach (DerivationOutputs::iterator, j, drv.outputs)
|
||||||
!(queryBoolSetting("build-use-substitutes", true) && store.hasSubstitutes(i->second.path)))
|
if (!store.isValidPath(j->second.path) &&
|
||||||
mustBuild = true;
|
infos.find(j->second.path) == infos.end())
|
||||||
|
mustBuild = true;
|
||||||
|
} else
|
||||||
|
mustBuild = true;
|
||||||
|
|
||||||
if (mustBuild) {
|
if (mustBuild) {
|
||||||
willBuild.insert(p);
|
willBuild.insert(*i);
|
||||||
todo.insert(drv.inputSrcs.begin(), drv.inputSrcs.end());
|
todo.insert(drv.inputSrcs.begin(), drv.inputSrcs.end());
|
||||||
foreach (DerivationInputs::iterator, i, drv.inputDrvs)
|
foreach (DerivationInputs::iterator, i, drv.inputDrvs)
|
||||||
todo.insert(i->first);
|
todo.insert(i->first);
|
||||||
} else
|
} else
|
||||||
foreach (DerivationOutputs::iterator, i, drv.outputs)
|
foreach (DerivationOutputs::iterator, i, drv.outputs)
|
||||||
todo.insert(i->second.path);
|
todoNonDrv.insert(i->second.path);
|
||||||
}
|
}
|
||||||
|
|
||||||
else {
|
foreach (PathSet::iterator, i, todoNonDrv) {
|
||||||
if (store.isValidPath(p)) continue;
|
done.insert(*i);
|
||||||
SubstitutablePathInfo info;
|
SubstitutablePathInfos::iterator info = infos.find(*i);
|
||||||
if (store.querySubstitutablePathInfo(p, info)) {
|
if (info != infos.end()) {
|
||||||
willSubstitute.insert(p);
|
willSubstitute.insert(*i);
|
||||||
downloadSize += info.downloadSize;
|
downloadSize += info->second.downloadSize;
|
||||||
narSize += info.narSize;
|
narSize += info->second.narSize;
|
||||||
todo.insert(info.references.begin(), info.references.end());
|
todo.insert(info->second.references.begin(), info->second.references.end());
|
||||||
} else
|
} else
|
||||||
unknown.insert(p);
|
unknown.insert(*i);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -19,7 +19,7 @@ static void makeWritable(const Path & path)
|
||||||
{
|
{
|
||||||
struct stat st;
|
struct stat st;
|
||||||
if (lstat(path.c_str(), &st))
|
if (lstat(path.c_str(), &st))
|
||||||
throw SysError(format("getting attributes of path `%1%'") % path);
|
throw SysError(format("getting attributes of path `%1%'") % path);
|
||||||
if (S_ISDIR(st.st_mode) || S_ISREG(st.st_mode)) makeMutable(path);
|
if (S_ISDIR(st.st_mode) || S_ISREG(st.st_mode)) makeMutable(path);
|
||||||
if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
|
if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
|
||||||
throw SysError(format("changing writability of `%1%'") % path);
|
throw SysError(format("changing writability of `%1%'") % path);
|
||||||
|
@ -57,12 +57,12 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
|
||||||
|
|
||||||
struct stat st;
|
struct stat st;
|
||||||
if (lstat(path.c_str(), &st))
|
if (lstat(path.c_str(), &st))
|
||||||
throw SysError(format("getting attributes of path `%1%'") % path);
|
throw SysError(format("getting attributes of path `%1%'") % path);
|
||||||
|
|
||||||
if (S_ISDIR(st.st_mode)) {
|
if (S_ISDIR(st.st_mode)) {
|
||||||
Strings names = readDirectory(path);
|
Strings names = readDirectory(path);
|
||||||
foreach (Strings::iterator, i, names)
|
foreach (Strings::iterator, i, names)
|
||||||
optimisePath_(stats, path + "/" + *i);
|
optimisePath_(stats, path + "/" + *i);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -113,7 +113,7 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
|
||||||
current file with a hard link to that file. */
|
current file with a hard link to that file. */
|
||||||
struct stat stLink;
|
struct stat stLink;
|
||||||
if (lstat(linkPath.c_str(), &stLink))
|
if (lstat(linkPath.c_str(), &stLink))
|
||||||
throw SysError(format("getting attributes of path `%1%'") % linkPath);
|
throw SysError(format("getting attributes of path `%1%'") % linkPath);
|
||||||
|
|
||||||
stats.sameContents++;
|
stats.sameContents++;
|
||||||
if (st.st_ino == stLink.st_ino) {
|
if (st.st_ino == stLink.st_ino) {
|
||||||
|
@ -149,7 +149,7 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
|
||||||
MakeImmutable mk1(linkPath);
|
MakeImmutable mk1(linkPath);
|
||||||
|
|
||||||
Path tempLink = (format("%1%/.tmp-link-%2%-%3%")
|
Path tempLink = (format("%1%/.tmp-link-%2%-%3%")
|
||||||
% nixStore % getpid() % rand()).str();
|
% settings.nixStore % getpid() % rand()).str();
|
||||||
|
|
||||||
if (link(linkPath.c_str(), tempLink.c_str()) == -1) {
|
if (link(linkPath.c_str(), tempLink.c_str()) == -1) {
|
||||||
if (errno == EMLINK) {
|
if (errno == EMLINK) {
|
||||||
|
@ -194,7 +194,7 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
|
||||||
|
|
||||||
void LocalStore::optimiseStore(OptimiseStats & stats)
|
void LocalStore::optimiseStore(OptimiseStats & stats)
|
||||||
{
|
{
|
||||||
PathSet paths = queryValidPaths();
|
PathSet paths = queryAllValidPaths();
|
||||||
|
|
||||||
foreach (PathSet::iterator, i, paths) {
|
foreach (PathSet::iterator, i, paths) {
|
||||||
addTempRoot(*i);
|
addTempRoot(*i);
|
||||||
|
@ -207,10 +207,8 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
|
||||||
|
|
||||||
void LocalStore::optimisePath(const Path & path)
|
void LocalStore::optimisePath(const Path & path)
|
||||||
{
|
{
|
||||||
if (queryBoolSetting("auto-optimise-store", true)) {
|
OptimiseStats stats;
|
||||||
OptimiseStats stats;
|
if (settings.autoOptimiseStore) optimisePath_(stats, path);
|
||||||
optimisePath_(stats, path);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -100,7 +100,7 @@ void RemoteStore::forkSlave()
|
||||||
/* Start the worker. */
|
/* Start the worker. */
|
||||||
Path worker = getEnv("NIX_WORKER");
|
Path worker = getEnv("NIX_WORKER");
|
||||||
if (worker == "")
|
if (worker == "")
|
||||||
worker = nixBinDir + "/nix-worker";
|
worker = settings.nixBinDir + "/nix-worker";
|
||||||
|
|
||||||
child = fork();
|
child = fork();
|
||||||
|
|
||||||
|
@ -142,7 +142,7 @@ void RemoteStore::connectToDaemon()
|
||||||
if (fdSocket == -1)
|
if (fdSocket == -1)
|
||||||
throw SysError("cannot create Unix domain socket");
|
throw SysError("cannot create Unix domain socket");
|
||||||
|
|
||||||
string socketPath = nixStateDir + DEFAULT_SOCKET_PATH;
|
string socketPath = settings.nixStateDir + DEFAULT_SOCKET_PATH;
|
||||||
|
|
||||||
/* Urgh, sockaddr_un allows path names of only 108 characters. So
|
/* Urgh, sockaddr_un allows path names of only 108 characters. So
|
||||||
chdir to the socket directory so that we can pass a relative
|
chdir to the socket directory so that we can pass a relative
|
||||||
|
@ -184,23 +184,33 @@ RemoteStore::~RemoteStore()
|
||||||
void RemoteStore::setOptions()
|
void RemoteStore::setOptions()
|
||||||
{
|
{
|
||||||
writeInt(wopSetOptions, to);
|
writeInt(wopSetOptions, to);
|
||||||
writeInt(keepFailed, to);
|
|
||||||
writeInt(keepGoing, to);
|
writeInt(settings.keepFailed, to);
|
||||||
writeInt(tryFallback, to);
|
writeInt(settings.keepGoing, to);
|
||||||
|
writeInt(settings.tryFallback, to);
|
||||||
writeInt(verbosity, to);
|
writeInt(verbosity, to);
|
||||||
writeInt(maxBuildJobs, to);
|
writeInt(settings.maxBuildJobs, to);
|
||||||
writeInt(maxSilentTime, to);
|
writeInt(settings.maxSilentTime, to);
|
||||||
if (GET_PROTOCOL_MINOR(daemonVersion) >= 2)
|
if (GET_PROTOCOL_MINOR(daemonVersion) >= 2)
|
||||||
writeInt(useBuildHook, to);
|
writeInt(settings.useBuildHook, to);
|
||||||
if (GET_PROTOCOL_MINOR(daemonVersion) >= 4) {
|
if (GET_PROTOCOL_MINOR(daemonVersion) >= 4) {
|
||||||
writeInt(buildVerbosity, to);
|
writeInt(settings.buildVerbosity, to);
|
||||||
writeInt(logType, to);
|
writeInt(logType, to);
|
||||||
writeInt(printBuildTrace, to);
|
writeInt(settings.printBuildTrace, to);
|
||||||
}
|
}
|
||||||
if (GET_PROTOCOL_MINOR(daemonVersion) >= 6)
|
if (GET_PROTOCOL_MINOR(daemonVersion) >= 6)
|
||||||
writeInt(buildCores, to);
|
writeInt(settings.buildCores, to);
|
||||||
if (GET_PROTOCOL_MINOR(daemonVersion) >= 10)
|
if (GET_PROTOCOL_MINOR(daemonVersion) >= 10)
|
||||||
writeInt(queryBoolSetting("build-use-substitutes", true), to);
|
writeInt(settings.useSubstitutes, to);
|
||||||
|
|
||||||
|
if (GET_PROTOCOL_MINOR(daemonVersion) >= 12) {
|
||||||
|
Settings::SettingsMap overrides = settings.getOverrides();
|
||||||
|
writeInt(overrides.size(), to);
|
||||||
|
foreach (Settings::SettingsMap::iterator, i, overrides) {
|
||||||
|
writeString(i->first, to);
|
||||||
|
writeString(i->second, to);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
processStderr();
|
processStderr();
|
||||||
}
|
}
|
||||||
|
@ -217,42 +227,96 @@ bool RemoteStore::isValidPath(const Path & path)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
PathSet RemoteStore::queryValidPaths()
|
PathSet RemoteStore::queryValidPaths(const PathSet & paths)
|
||||||
{
|
{
|
||||||
openConnection();
|
openConnection();
|
||||||
writeInt(wopQueryValidPaths, to);
|
if (GET_PROTOCOL_MINOR(daemonVersion) < 12) {
|
||||||
|
PathSet res;
|
||||||
|
foreach (PathSet::const_iterator, i, paths)
|
||||||
|
if (isValidPath(*i)) res.insert(*i);
|
||||||
|
return res;
|
||||||
|
} else {
|
||||||
|
writeInt(wopQueryValidPaths, to);
|
||||||
|
writeStrings(paths, to);
|
||||||
|
processStderr();
|
||||||
|
return readStorePaths<PathSet>(from);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
PathSet RemoteStore::queryAllValidPaths()
|
||||||
|
{
|
||||||
|
openConnection();
|
||||||
|
writeInt(wopQueryAllValidPaths, to);
|
||||||
processStderr();
|
processStderr();
|
||||||
return readStorePaths<PathSet>(from);
|
return readStorePaths<PathSet>(from);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
bool RemoteStore::hasSubstitutes(const Path & path)
|
PathSet RemoteStore::querySubstitutablePaths(const PathSet & paths)
|
||||||
{
|
{
|
||||||
openConnection();
|
openConnection();
|
||||||
writeInt(wopHasSubstitutes, to);
|
if (GET_PROTOCOL_MINOR(daemonVersion) < 12) {
|
||||||
writeString(path, to);
|
PathSet res;
|
||||||
processStderr();
|
foreach (PathSet::const_iterator, i, paths) {
|
||||||
unsigned int reply = readInt(from);
|
writeInt(wopHasSubstitutes, to);
|
||||||
return reply != 0;
|
writeString(*i, to);
|
||||||
|
processStderr();
|
||||||
|
if (readInt(from)) res.insert(*i);
|
||||||
|
}
|
||||||
|
return res;
|
||||||
|
} else {
|
||||||
|
writeInt(wopQuerySubstitutablePaths, to);
|
||||||
|
writeStrings(paths, to);
|
||||||
|
processStderr();
|
||||||
|
return readStorePaths<PathSet>(from);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
bool RemoteStore::querySubstitutablePathInfo(const Path & path,
|
void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
|
||||||
SubstitutablePathInfo & info)
|
SubstitutablePathInfos & infos)
|
||||||
{
|
{
|
||||||
|
if (paths.empty()) return;
|
||||||
|
|
||||||
openConnection();
|
openConnection();
|
||||||
if (GET_PROTOCOL_MINOR(daemonVersion) < 3) return false;
|
|
||||||
writeInt(wopQuerySubstitutablePathInfo, to);
|
if (GET_PROTOCOL_MINOR(daemonVersion) < 3) return;
|
||||||
writeString(path, to);
|
|
||||||
processStderr();
|
if (GET_PROTOCOL_MINOR(daemonVersion) < 12) {
|
||||||
unsigned int reply = readInt(from);
|
|
||||||
if (reply == 0) return false;
|
foreach (PathSet::const_iterator, i, paths) {
|
||||||
info.deriver = readString(from);
|
SubstitutablePathInfo info;
|
||||||
if (info.deriver != "") assertStorePath(info.deriver);
|
writeInt(wopQuerySubstitutablePathInfo, to);
|
||||||
info.references = readStorePaths<PathSet>(from);
|
writeString(*i, to);
|
||||||
info.downloadSize = readLongLong(from);
|
processStderr();
|
||||||
info.narSize = GET_PROTOCOL_MINOR(daemonVersion) >= 7 ? readLongLong(from) : 0;
|
unsigned int reply = readInt(from);
|
||||||
return true;
|
if (reply == 0) continue;
|
||||||
|
info.deriver = readString(from);
|
||||||
|
if (info.deriver != "") assertStorePath(info.deriver);
|
||||||
|
info.references = readStorePaths<PathSet>(from);
|
||||||
|
info.downloadSize = readLongLong(from);
|
||||||
|
info.narSize = GET_PROTOCOL_MINOR(daemonVersion) >= 7 ? readLongLong(from) : 0;
|
||||||
|
infos[*i] = info;
|
||||||
|
}
|
||||||
|
|
||||||
|
} else {
|
||||||
|
|
||||||
|
writeInt(wopQuerySubstitutablePathInfos, to);
|
||||||
|
writeStrings(paths, to);
|
||||||
|
processStderr();
|
||||||
|
unsigned int count = readInt(from);
|
||||||
|
for (unsigned int n = 0; n < count; n++) {
|
||||||
|
Path path = readStorePath(from);
|
||||||
|
SubstitutablePathInfo & info(infos[path]);
|
||||||
|
info.deriver = readString(from);
|
||||||
|
if (info.deriver != "") assertStorePath(info.deriver);
|
||||||
|
info.references = readStorePaths<PathSet>(from);
|
||||||
|
info.downloadSize = readLongLong(from);
|
||||||
|
info.narSize = readLongLong(from);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -26,7 +26,9 @@ public:
|
||||||
|
|
||||||
bool isValidPath(const Path & path);
|
bool isValidPath(const Path & path);
|
||||||
|
|
||||||
PathSet queryValidPaths();
|
PathSet queryValidPaths(const PathSet & paths);
|
||||||
|
|
||||||
|
PathSet queryAllValidPaths();
|
||||||
|
|
||||||
ValidPathInfo queryPathInfo(const Path & path);
|
ValidPathInfo queryPathInfo(const Path & path);
|
||||||
|
|
||||||
|
@ -44,10 +46,10 @@ public:
|
||||||
|
|
||||||
Path queryPathFromHashPart(const string & hashPart);
|
Path queryPathFromHashPart(const string & hashPart);
|
||||||
|
|
||||||
bool hasSubstitutes(const Path & path);
|
PathSet querySubstitutablePaths(const PathSet & paths);
|
||||||
|
|
||||||
bool querySubstitutablePathInfo(const Path & path,
|
void querySubstitutablePathInfos(const PathSet & paths,
|
||||||
SubstitutablePathInfo & info);
|
SubstitutablePathInfos & infos);
|
||||||
|
|
||||||
Path addToStore(const Path & srcPath,
|
Path addToStore(const Path & srcPath,
|
||||||
bool recursive = true, HashType hashAlgo = htSHA256,
|
bool recursive = true, HashType hashAlgo = htSHA256,
|
||||||
|
|
|
@ -19,16 +19,16 @@ GCOptions::GCOptions()
|
||||||
bool isInStore(const Path & path)
|
bool isInStore(const Path & path)
|
||||||
{
|
{
|
||||||
return path[0] == '/'
|
return path[0] == '/'
|
||||||
&& string(path, 0, nixStore.size()) == nixStore
|
&& string(path, 0, settings.nixStore.size()) == settings.nixStore
|
||||||
&& path.size() >= nixStore.size() + 2
|
&& path.size() >= settings.nixStore.size() + 2
|
||||||
&& path[nixStore.size()] == '/';
|
&& path[settings.nixStore.size()] == '/';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
bool isStorePath(const Path & path)
|
bool isStorePath(const Path & path)
|
||||||
{
|
{
|
||||||
return isInStore(path)
|
return isInStore(path)
|
||||||
&& path.find('/', nixStore.size() + 1) == Path::npos;
|
&& path.find('/', settings.nixStore.size() + 1) == Path::npos;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -43,7 +43,7 @@ Path toStorePath(const Path & path)
|
||||||
{
|
{
|
||||||
if (!isInStore(path))
|
if (!isInStore(path))
|
||||||
throw Error(format("path `%1%' is not in the Nix store") % path);
|
throw Error(format("path `%1%' is not in the Nix store") % path);
|
||||||
Path::size_type slash = path.find('/', nixStore.size() + 1);
|
Path::size_type slash = path.find('/', settings.nixStore.size() + 1);
|
||||||
if (slash == Path::npos)
|
if (slash == Path::npos)
|
||||||
return path;
|
return path;
|
||||||
else
|
else
|
||||||
|
@ -74,7 +74,7 @@ Path followLinksToStorePath(const Path & path)
|
||||||
string storePathToName(const Path & path)
|
string storePathToName(const Path & path)
|
||||||
{
|
{
|
||||||
assertStorePath(path);
|
assertStorePath(path);
|
||||||
return string(path, nixStore.size() + 34);
|
return string(path, settings.nixStore.size() + 34);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -173,11 +173,11 @@ Path makeStorePath(const string & type,
|
||||||
{
|
{
|
||||||
/* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
|
/* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
|
||||||
string s = type + ":sha256:" + printHash(hash) + ":"
|
string s = type + ":sha256:" + printHash(hash) + ":"
|
||||||
+ nixStore + ":" + name;
|
+ settings.nixStore + ":" + name;
|
||||||
|
|
||||||
checkStoreName(name);
|
checkStoreName(name);
|
||||||
|
|
||||||
return nixStore + "/"
|
return settings.nixStore + "/"
|
||||||
+ printHash32(compressHash(hashString(htSHA256, s), 20))
|
+ printHash32(compressHash(hashString(htSHA256, s), 20))
|
||||||
+ "-" + name;
|
+ "-" + name;
|
||||||
}
|
}
|
||||||
|
|
|
@ -80,6 +80,8 @@ struct SubstitutablePathInfo
|
||||||
unsigned long long narSize; /* 0 = unknown */
|
unsigned long long narSize; /* 0 = unknown */
|
||||||
};
|
};
|
||||||
|
|
||||||
|
typedef std::map<Path, SubstitutablePathInfo> SubstitutablePathInfos;
|
||||||
|
|
||||||
|
|
||||||
struct ValidPathInfo
|
struct ValidPathInfo
|
||||||
{
|
{
|
||||||
|
@ -102,20 +104,23 @@ public:
|
||||||
|
|
||||||
virtual ~StoreAPI() { }
|
virtual ~StoreAPI() { }
|
||||||
|
|
||||||
/* Checks whether a path is valid. */
|
/* Check whether a path is valid. */
|
||||||
virtual bool isValidPath(const Path & path) = 0;
|
virtual bool isValidPath(const Path & path) = 0;
|
||||||
|
|
||||||
/* Query the set of valid paths. */
|
/* Query which of the given paths is valid. */
|
||||||
virtual PathSet queryValidPaths() = 0;
|
virtual PathSet queryValidPaths(const PathSet & paths) = 0;
|
||||||
|
|
||||||
|
/* Query the set of all valid paths. */
|
||||||
|
virtual PathSet queryAllValidPaths() = 0;
|
||||||
|
|
||||||
/* Query information about a valid path. */
|
/* Query information about a valid path. */
|
||||||
virtual ValidPathInfo queryPathInfo(const Path & path) = 0;
|
virtual ValidPathInfo queryPathInfo(const Path & path) = 0;
|
||||||
|
|
||||||
/* Queries the hash of a valid path. */
|
/* Query the hash of a valid path. */
|
||||||
virtual Hash queryPathHash(const Path & path) = 0;
|
virtual Hash queryPathHash(const Path & path) = 0;
|
||||||
|
|
||||||
/* Queries the set of outgoing FS references for a store path.
|
/* Query the set of outgoing FS references for a store path. The
|
||||||
The result is not cleared. */
|
result is not cleared. */
|
||||||
virtual void queryReferences(const Path & path,
|
virtual void queryReferences(const Path & path,
|
||||||
PathSet & references) = 0;
|
PathSet & references) = 0;
|
||||||
|
|
||||||
|
@ -138,13 +143,14 @@ public:
|
||||||
path, or "" if the path doesn't exist. */
|
path, or "" if the path doesn't exist. */
|
||||||
virtual Path queryPathFromHashPart(const string & hashPart) = 0;
|
virtual Path queryPathFromHashPart(const string & hashPart) = 0;
|
||||||
|
|
||||||
/* Query whether a path has substitutes. */
|
/* Query which of the given paths have substitutes. */
|
||||||
virtual bool hasSubstitutes(const Path & path) = 0;
|
virtual PathSet querySubstitutablePaths(const PathSet & paths) = 0;
|
||||||
|
|
||||||
/* Query the references, deriver and download size of a
|
/* Query substitute info (i.e. references, derivers and download
|
||||||
substitutable path. */
|
sizes) of a set of paths. If a path does not have substitute
|
||||||
virtual bool querySubstitutablePathInfo(const Path & path,
|
info, it's omitted from the resulting ‘infos’ map. */
|
||||||
SubstitutablePathInfo & info) = 0;
|
virtual void querySubstitutablePathInfos(const PathSet & paths,
|
||||||
|
SubstitutablePathInfos & infos) = 0;
|
||||||
|
|
||||||
/* Copy the contents of a path to the store and register the
|
/* Copy the contents of a path to the store and register the
|
||||||
validity the resulting path. The resulting path is returned.
|
validity the resulting path. The resulting path is returned.
|
||||||
|
|
|
@ -6,7 +6,7 @@ namespace nix {
|
||||||
#define WORKER_MAGIC_1 0x6e697863
|
#define WORKER_MAGIC_1 0x6e697863
|
||||||
#define WORKER_MAGIC_2 0x6478696f
|
#define WORKER_MAGIC_2 0x6478696f
|
||||||
|
|
||||||
#define PROTOCOL_VERSION 0x10b
|
#define PROTOCOL_VERSION 0x10c
|
||||||
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
|
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
|
||||||
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
|
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
|
||||||
|
|
||||||
|
@ -32,13 +32,16 @@ typedef enum {
|
||||||
wopCollectGarbage = 20,
|
wopCollectGarbage = 20,
|
||||||
wopQuerySubstitutablePathInfo = 21,
|
wopQuerySubstitutablePathInfo = 21,
|
||||||
wopQueryDerivationOutputs = 22,
|
wopQueryDerivationOutputs = 22,
|
||||||
wopQueryValidPaths = 23,
|
wopQueryAllValidPaths = 23,
|
||||||
wopQueryFailedPaths = 24,
|
wopQueryFailedPaths = 24,
|
||||||
wopClearFailedPaths = 25,
|
wopClearFailedPaths = 25,
|
||||||
wopQueryPathInfo = 26,
|
wopQueryPathInfo = 26,
|
||||||
wopImportPaths = 27,
|
wopImportPaths = 27,
|
||||||
wopQueryDerivationOutputNames = 28,
|
wopQueryDerivationOutputNames = 28,
|
||||||
wopQueryPathFromHashPart = 29,
|
wopQueryPathFromHashPart = 29,
|
||||||
|
wopQuerySubstitutablePathInfos = 30,
|
||||||
|
wopQueryValidPaths = 31,
|
||||||
|
wopQuerySubstitutablePaths = 32,
|
||||||
} WorkerOp;
|
} WorkerOp;
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -253,7 +253,7 @@ string readLine(int fd)
|
||||||
if (errno != EINTR)
|
if (errno != EINTR)
|
||||||
throw SysError("reading a line");
|
throw SysError("reading a line");
|
||||||
} else if (rd == 0)
|
} else if (rd == 0)
|
||||||
throw Error("unexpected EOF reading a line");
|
throw EndOfFile("unexpected EOF reading a line");
|
||||||
else {
|
else {
|
||||||
if (ch == '\n') return s;
|
if (ch == '\n') return s;
|
||||||
s += ch;
|
s += ch;
|
||||||
|
@ -1010,6 +1010,13 @@ string concatStringsSep(const string & sep, const Strings & ss)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
string chomp(const string & s)
|
||||||
|
{
|
||||||
|
size_t i = s.find_last_not_of(" \n\r\t");
|
||||||
|
return i == string::npos ? "" : string(s, 0, i + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
string statusToString(int status)
|
string statusToString(int status)
|
||||||
{
|
{
|
||||||
if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
|
if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
|
||||||
|
|
|
@ -294,6 +294,10 @@ Strings tokenizeString(const string & s, const string & separators = " \t\n\r");
|
||||||
string concatStringsSep(const string & sep, const Strings & ss);
|
string concatStringsSep(const string & sep, const Strings & ss);
|
||||||
|
|
||||||
|
|
||||||
|
/* Remove trailing whitespace from a string. */
|
||||||
|
string chomp(const string & s);
|
||||||
|
|
||||||
|
|
||||||
/* Convert the exit status of a child as returned by wait() into an
|
/* Convert the exit status of a child as returned by wait() into an
|
||||||
error string. */
|
error string. */
|
||||||
string statusToString(int status);
|
string statusToString(int status);
|
||||||
|
|
|
@ -55,7 +55,6 @@ struct Globals
|
||||||
EvalState state;
|
EvalState state;
|
||||||
bool dryRun;
|
bool dryRun;
|
||||||
bool preserveInstalled;
|
bool preserveInstalled;
|
||||||
bool keepDerivations;
|
|
||||||
string forceName;
|
string forceName;
|
||||||
bool prebuiltOnly;
|
bool prebuiltOnly;
|
||||||
};
|
};
|
||||||
|
@ -113,6 +112,11 @@ static void getAllExprs(EvalState & state,
|
||||||
StringSet namesSorted(names.begin(), names.end());
|
StringSet namesSorted(names.begin(), names.end());
|
||||||
|
|
||||||
foreach (StringSet::iterator, i, namesSorted) {
|
foreach (StringSet::iterator, i, namesSorted) {
|
||||||
|
/* Ignore the manifest.nix used by profiles. This is
|
||||||
|
necessary to prevent it from showing up in channels (which
|
||||||
|
are implemented using profiles). */
|
||||||
|
if (*i == "manifest.nix") continue;
|
||||||
|
|
||||||
Path path2 = path + "/" + *i;
|
Path path2 = path + "/" + *i;
|
||||||
|
|
||||||
struct stat st;
|
struct stat st;
|
||||||
|
@ -211,9 +215,12 @@ static int comparePriorities(EvalState & state,
|
||||||
|
|
||||||
static bool isPrebuilt(EvalState & state, const DrvInfo & elem)
|
static bool isPrebuilt(EvalState & state, const DrvInfo & elem)
|
||||||
{
|
{
|
||||||
|
assert(false);
|
||||||
|
#if 0
|
||||||
return
|
return
|
||||||
store->isValidPath(elem.queryOutPath(state)) ||
|
store->isValidPath(elem.queryOutPath(state)) ||
|
||||||
store->hasSubstitutes(elem.queryOutPath(state));
|
store->hasSubstitutes(elem.queryOutPath(state));
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -263,8 +270,8 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems,
|
||||||
|
|
||||||
if (k != newest.end()) {
|
if (k != newest.end()) {
|
||||||
d = j->first.system == k->second.first.system ? 0 :
|
d = j->first.system == k->second.first.system ? 0 :
|
||||||
j->first.system == thisSystem ? 1 :
|
j->first.system == settings.thisSystem ? 1 :
|
||||||
k->second.first.system == thisSystem ? -1 : 0;
|
k->second.first.system == settings.thisSystem ? -1 : 0;
|
||||||
if (d == 0)
|
if (d == 0)
|
||||||
d = comparePriorities(state, j->first, k->second.first);
|
d = comparePriorities(state, j->first, k->second.first);
|
||||||
if (d == 0)
|
if (d == 0)
|
||||||
|
@ -495,7 +502,7 @@ static void installDerivations(Globals & globals,
|
||||||
if (globals.dryRun) return;
|
if (globals.dryRun) return;
|
||||||
|
|
||||||
if (createUserEnv(globals.state, allElems,
|
if (createUserEnv(globals.state, allElems,
|
||||||
profile, globals.keepDerivations, lockToken)) break;
|
profile, settings.envKeepDerivations, lockToken)) break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -602,7 +609,7 @@ static void upgradeDerivations(Globals & globals,
|
||||||
if (globals.dryRun) return;
|
if (globals.dryRun) return;
|
||||||
|
|
||||||
if (createUserEnv(globals.state, newElems,
|
if (createUserEnv(globals.state, newElems,
|
||||||
globals.profile, globals.keepDerivations, lockToken)) break;
|
globals.profile, settings.envKeepDerivations, lockToken)) break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -669,7 +676,7 @@ static void opSetFlag(Globals & globals,
|
||||||
|
|
||||||
/* Write the new user environment. */
|
/* Write the new user environment. */
|
||||||
if (createUserEnv(globals.state, installedElems,
|
if (createUserEnv(globals.state, installedElems,
|
||||||
globals.profile, globals.keepDerivations, lockToken)) break;
|
globals.profile, settings.envKeepDerivations, lockToken)) break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -737,7 +744,7 @@ static void uninstallDerivations(Globals & globals, Strings & selectors,
|
||||||
if (globals.dryRun) return;
|
if (globals.dryRun) return;
|
||||||
|
|
||||||
if (createUserEnv(globals.state, newElems,
|
if (createUserEnv(globals.state, newElems,
|
||||||
profile, globals.keepDerivations, lockToken)) break;
|
profile, settings.envKeepDerivations, lockToken)) break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -866,7 +873,7 @@ static void opQuery(Globals & globals,
|
||||||
|
|
||||||
enum { sInstalled, sAvailable } source = sInstalled;
|
enum { sInstalled, sAvailable } source = sInstalled;
|
||||||
|
|
||||||
readOnlyMode = true; /* makes evaluation a bit faster */
|
settings.readOnlyMode = true; /* makes evaluation a bit faster */
|
||||||
|
|
||||||
for (Strings::iterator i = args.begin(); i != args.end(); ) {
|
for (Strings::iterator i = args.begin(); i != args.end(); ) {
|
||||||
string arg = *i++;
|
string arg = *i++;
|
||||||
|
@ -930,6 +937,22 @@ static void opQuery(Globals & globals,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/* Query which paths have substitutes. */
|
||||||
|
PathSet validPaths, substitutablePaths;
|
||||||
|
if (printStatus) {
|
||||||
|
PathSet paths;
|
||||||
|
foreach (vector<DrvInfo>::iterator, i, elems2)
|
||||||
|
try {
|
||||||
|
paths.insert(i->queryOutPath(globals.state));
|
||||||
|
} catch (AssertionError & e) {
|
||||||
|
printMsg(lvlTalkative, format("skipping derivation named `%1%' which gives an assertion failure") % i->name);
|
||||||
|
i->setFailed();
|
||||||
|
}
|
||||||
|
validPaths = store->queryValidPaths(paths);
|
||||||
|
substitutablePaths = store->querySubstitutablePaths(paths);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
/* Print the desired columns, or XML output. */
|
/* Print the desired columns, or XML output. */
|
||||||
Table table;
|
Table table;
|
||||||
std::ostringstream dummy;
|
std::ostringstream dummy;
|
||||||
|
@ -938,6 +961,8 @@ static void opQuery(Globals & globals,
|
||||||
|
|
||||||
foreach (vector<DrvInfo>::iterator, i, elems2) {
|
foreach (vector<DrvInfo>::iterator, i, elems2) {
|
||||||
try {
|
try {
|
||||||
|
if (i->hasFailed()) continue;
|
||||||
|
|
||||||
startNest(nest, lvlDebug, format("outputting query result `%1%'") % i->attrPath);
|
startNest(nest, lvlDebug, format("outputting query result `%1%'") % i->attrPath);
|
||||||
|
|
||||||
if (globals.prebuiltOnly && !isPrebuilt(globals.state, *i)) continue;
|
if (globals.prebuiltOnly && !isPrebuilt(globals.state, *i)) continue;
|
||||||
|
@ -949,9 +974,10 @@ static void opQuery(Globals & globals,
|
||||||
XMLAttrs attrs;
|
XMLAttrs attrs;
|
||||||
|
|
||||||
if (printStatus) {
|
if (printStatus) {
|
||||||
bool hasSubs = store->hasSubstitutes(i->queryOutPath(globals.state));
|
Path outPath = i->queryOutPath(globals.state);
|
||||||
bool isInstalled = installed.find(i->queryOutPath(globals.state)) != installed.end();
|
bool hasSubs = substitutablePaths.find(outPath) != substitutablePaths.end();
|
||||||
bool isValid = store->isValidPath(i->queryOutPath(globals.state));
|
bool isInstalled = installed.find(outPath) != installed.end();
|
||||||
|
bool isValid = validPaths.find(outPath) != validPaths.end();
|
||||||
if (xmlOutput) {
|
if (xmlOutput) {
|
||||||
attrs["installed"] = isInstalled ? "1" : "0";
|
attrs["installed"] = isInstalled ? "1" : "0";
|
||||||
attrs["valid"] = isValid ? "1" : "0";
|
attrs["valid"] = isValid ? "1" : "0";
|
||||||
|
@ -1240,9 +1266,6 @@ void run(Strings args)
|
||||||
globals.preserveInstalled = false;
|
globals.preserveInstalled = false;
|
||||||
globals.prebuiltOnly = false;
|
globals.prebuiltOnly = false;
|
||||||
|
|
||||||
globals.keepDerivations =
|
|
||||||
queryBoolSetting("env-keep-derivations", false);
|
|
||||||
|
|
||||||
for (Strings::iterator i = args.begin(); i != args.end(); ) {
|
for (Strings::iterator i = args.begin(); i != args.end(); ) {
|
||||||
string arg = *i++;
|
string arg = *i++;
|
||||||
|
|
||||||
|
@ -1309,7 +1332,7 @@ void run(Strings args)
|
||||||
Path profileLink = getHomeDir() + "/.nix-profile";
|
Path profileLink = getHomeDir() + "/.nix-profile";
|
||||||
globals.profile = pathExists(profileLink)
|
globals.profile = pathExists(profileLink)
|
||||||
? absPath(readLink(profileLink), dirOf(profileLink))
|
? absPath(readLink(profileLink), dirOf(profileLink))
|
||||||
: canonPath(nixStateDir + "/profiles/default");
|
: canonPath(settings.nixStateDir + "/profiles/default");
|
||||||
}
|
}
|
||||||
|
|
||||||
store = openStore();
|
store = openStore();
|
||||||
|
|
|
@ -96,11 +96,11 @@ void run(Strings args)
|
||||||
if (arg == "-")
|
if (arg == "-")
|
||||||
readStdin = true;
|
readStdin = true;
|
||||||
else if (arg == "--eval-only") {
|
else if (arg == "--eval-only") {
|
||||||
readOnlyMode = true;
|
settings.readOnlyMode = true;
|
||||||
evalOnly = true;
|
evalOnly = true;
|
||||||
}
|
}
|
||||||
else if (arg == "--parse-only") {
|
else if (arg == "--parse-only") {
|
||||||
readOnlyMode = true;
|
settings.readOnlyMode = true;
|
||||||
parseOnly = evalOnly = true;
|
parseOnly = evalOnly = true;
|
||||||
}
|
}
|
||||||
else if (arg == "--find-file")
|
else if (arg == "--find-file")
|
||||||
|
|
|
@ -432,7 +432,7 @@ static void opReadLog(Strings opFlags, Strings opArgs)
|
||||||
Path path = useDeriver(followLinksToStorePath(*i));
|
Path path = useDeriver(followLinksToStorePath(*i));
|
||||||
|
|
||||||
Path logPath = (format("%1%/%2%/%3%") %
|
Path logPath = (format("%1%/%2%/%3%") %
|
||||||
nixLogDir % drvsLogDir % baseNameOf(path)).str();
|
settings.nixLogDir % drvsLogDir % baseNameOf(path)).str();
|
||||||
Path logBz2Path = logPath + ".bz2";
|
Path logBz2Path = logPath + ".bz2";
|
||||||
|
|
||||||
if (pathExists(logPath)) {
|
if (pathExists(logPath)) {
|
||||||
|
@ -469,7 +469,7 @@ static void opDumpDB(Strings opFlags, Strings opArgs)
|
||||||
if (!opFlags.empty()) throw UsageError("unknown flag");
|
if (!opFlags.empty()) throw UsageError("unknown flag");
|
||||||
if (!opArgs.empty())
|
if (!opArgs.empty())
|
||||||
throw UsageError("no arguments expected");
|
throw UsageError("no arguments expected");
|
||||||
PathSet validPaths = store->queryValidPaths();
|
PathSet validPaths = store->queryAllValidPaths();
|
||||||
foreach (PathSet::iterator, i, validPaths)
|
foreach (PathSet::iterator, i, validPaths)
|
||||||
cout << store->makeValidityRegistration(singleton<PathSet>(*i), true, true);
|
cout << store->makeValidityRegistration(singleton<PathSet>(*i), true, true);
|
||||||
}
|
}
|
||||||
|
|
|
@ -297,12 +297,30 @@ static void performOp(unsigned int clientVersion,
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
case wopQueryValidPaths: {
|
||||||
|
PathSet paths = readStorePaths<PathSet>(from);
|
||||||
|
startWork();
|
||||||
|
PathSet res = store->queryValidPaths(paths);
|
||||||
|
stopWork();
|
||||||
|
writeStrings(res, to);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
case wopHasSubstitutes: {
|
case wopHasSubstitutes: {
|
||||||
Path path = readStorePath(from);
|
Path path = readStorePath(from);
|
||||||
startWork();
|
startWork();
|
||||||
bool result = store->hasSubstitutes(path);
|
PathSet res = store->querySubstitutablePaths(singleton<PathSet>(path));
|
||||||
stopWork();
|
stopWork();
|
||||||
writeInt(result, to);
|
writeInt(res.find(path) != res.end(), to);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
case wopQuerySubstitutablePaths: {
|
||||||
|
PathSet paths = readStorePaths<PathSet>(from);
|
||||||
|
startWork();
|
||||||
|
PathSet res = store->querySubstitutablePaths(paths);
|
||||||
|
stopWork();
|
||||||
|
writeStrings(res, to);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -509,26 +527,30 @@ static void performOp(unsigned int clientVersion,
|
||||||
}
|
}
|
||||||
|
|
||||||
case wopSetOptions: {
|
case wopSetOptions: {
|
||||||
keepFailed = readInt(from) != 0;
|
settings.keepFailed = readInt(from) != 0;
|
||||||
keepGoing = readInt(from) != 0;
|
settings.keepGoing = readInt(from) != 0;
|
||||||
tryFallback = readInt(from) != 0;
|
settings.tryFallback = readInt(from) != 0;
|
||||||
verbosity = (Verbosity) readInt(from);
|
verbosity = (Verbosity) readInt(from);
|
||||||
maxBuildJobs = readInt(from);
|
settings.maxBuildJobs = readInt(from);
|
||||||
maxSilentTime = readInt(from);
|
settings.maxSilentTime = readInt(from);
|
||||||
if (GET_PROTOCOL_MINOR(clientVersion) >= 2)
|
if (GET_PROTOCOL_MINOR(clientVersion) >= 2)
|
||||||
useBuildHook = readInt(from) != 0;
|
settings.useBuildHook = readInt(from) != 0;
|
||||||
if (GET_PROTOCOL_MINOR(clientVersion) >= 4) {
|
if (GET_PROTOCOL_MINOR(clientVersion) >= 4) {
|
||||||
buildVerbosity = (Verbosity) readInt(from);
|
settings.buildVerbosity = (Verbosity) readInt(from);
|
||||||
logType = (LogType) readInt(from);
|
logType = (LogType) readInt(from);
|
||||||
printBuildTrace = readInt(from) != 0;
|
settings.printBuildTrace = readInt(from) != 0;
|
||||||
}
|
}
|
||||||
if (GET_PROTOCOL_MINOR(clientVersion) >= 6)
|
if (GET_PROTOCOL_MINOR(clientVersion) >= 6)
|
||||||
buildCores = readInt(from);
|
settings.buildCores = readInt(from);
|
||||||
if (GET_PROTOCOL_MINOR(clientVersion) >= 10) {
|
if (GET_PROTOCOL_MINOR(clientVersion) >= 10)
|
||||||
int x = readInt(from);
|
settings.useSubstitutes = readInt(from) != 0;
|
||||||
Strings ss;
|
if (GET_PROTOCOL_MINOR(clientVersion) >= 12) {
|
||||||
ss.push_back(x == 0 ? "false" : "true");
|
unsigned int n = readInt(from);
|
||||||
overrideSetting("build-use-substitutes", ss);
|
for (unsigned int i = 0; i < n; i++) {
|
||||||
|
string name = readString(from);
|
||||||
|
string value = readString(from);
|
||||||
|
settings.set("untrusted-" + name, value);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
startWork();
|
startWork();
|
||||||
stopWork();
|
stopWork();
|
||||||
|
@ -538,23 +560,43 @@ static void performOp(unsigned int clientVersion,
|
||||||
case wopQuerySubstitutablePathInfo: {
|
case wopQuerySubstitutablePathInfo: {
|
||||||
Path path = absPath(readString(from));
|
Path path = absPath(readString(from));
|
||||||
startWork();
|
startWork();
|
||||||
SubstitutablePathInfo info;
|
SubstitutablePathInfos infos;
|
||||||
bool res = store->querySubstitutablePathInfo(path, info);
|
store->querySubstitutablePathInfos(singleton<PathSet>(path), infos);
|
||||||
stopWork();
|
stopWork();
|
||||||
writeInt(res ? 1 : 0, to);
|
SubstitutablePathInfos::iterator i = infos.find(path);
|
||||||
if (res) {
|
if (i == infos.end())
|
||||||
writeString(info.deriver, to);
|
writeInt(0, to);
|
||||||
writeStrings(info.references, to);
|
else {
|
||||||
writeLongLong(info.downloadSize, to);
|
writeInt(1, to);
|
||||||
|
writeString(i->second.deriver, to);
|
||||||
|
writeStrings(i->second.references, to);
|
||||||
|
writeLongLong(i->second.downloadSize, to);
|
||||||
if (GET_PROTOCOL_MINOR(clientVersion) >= 7)
|
if (GET_PROTOCOL_MINOR(clientVersion) >= 7)
|
||||||
writeLongLong(info.narSize, to);
|
writeLongLong(i->second.narSize, to);
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
case wopQueryValidPaths: {
|
case wopQuerySubstitutablePathInfos: {
|
||||||
|
PathSet paths = readStorePaths<PathSet>(from);
|
||||||
startWork();
|
startWork();
|
||||||
PathSet paths = store->queryValidPaths();
|
SubstitutablePathInfos infos;
|
||||||
|
store->querySubstitutablePathInfos(paths, infos);
|
||||||
|
stopWork();
|
||||||
|
writeInt(infos.size(), to);
|
||||||
|
foreach (SubstitutablePathInfos::iterator, i, infos) {
|
||||||
|
writeString(i->first, to);
|
||||||
|
writeString(i->second.deriver, to);
|
||||||
|
writeStrings(i->second.references, to);
|
||||||
|
writeLongLong(i->second.downloadSize, to);
|
||||||
|
writeLongLong(i->second.narSize, to);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
case wopQueryAllValidPaths: {
|
||||||
|
startWork();
|
||||||
|
PathSet paths = store->queryAllValidPaths();
|
||||||
stopWork();
|
stopWork();
|
||||||
writeStrings(paths, to);
|
writeStrings(paths, to);
|
||||||
break;
|
break;
|
||||||
|
@ -730,7 +772,7 @@ static void daemonLoop()
|
||||||
if (fdSocket == -1)
|
if (fdSocket == -1)
|
||||||
throw SysError("cannot create Unix domain socket");
|
throw SysError("cannot create Unix domain socket");
|
||||||
|
|
||||||
string socketPath = nixStateDir + DEFAULT_SOCKET_PATH;
|
string socketPath = settings.nixStateDir + DEFAULT_SOCKET_PATH;
|
||||||
|
|
||||||
createDirs(dirOf(socketPath));
|
createDirs(dirOf(socketPath));
|
||||||
|
|
||||||
|
@ -781,10 +823,10 @@ static void daemonLoop()
|
||||||
(struct sockaddr *) &remoteAddr, &remoteAddrLen);
|
(struct sockaddr *) &remoteAddr, &remoteAddrLen);
|
||||||
checkInterrupt();
|
checkInterrupt();
|
||||||
if (remote == -1) {
|
if (remote == -1) {
|
||||||
if (errno == EINTR)
|
if (errno == EINTR)
|
||||||
continue;
|
continue;
|
||||||
else
|
else
|
||||||
throw SysError("accepting connection");
|
throw SysError("accepting connection");
|
||||||
}
|
}
|
||||||
|
|
||||||
closeOnExec(remote);
|
closeOnExec(remote);
|
||||||
|
@ -829,10 +871,6 @@ static void daemonLoop()
|
||||||
strncpy(argvSaved[1], processName.c_str(), strlen(argvSaved[1]));
|
strncpy(argvSaved[1], processName.c_str(), strlen(argvSaved[1]));
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Since the daemon can be long-running, the
|
|
||||||
settings may have changed. So force a reload. */
|
|
||||||
reloadSettings();
|
|
||||||
|
|
||||||
/* Handle the connection. */
|
/* Handle the connection. */
|
||||||
from.fd = remote;
|
from.fd = remote;
|
||||||
to.fd = remote;
|
to.fd = remote;
|
||||||
|
|
|
@ -16,11 +16,13 @@
|
||||||
-e "s^@shell\@^$(bash)^g" \
|
-e "s^@shell\@^$(bash)^g" \
|
||||||
-e "s^@curl\@^$(curl)^g" \
|
-e "s^@curl\@^$(curl)^g" \
|
||||||
-e "s^@bzip2\@^$(bzip2)^g" \
|
-e "s^@bzip2\@^$(bzip2)^g" \
|
||||||
|
-e "s^@xz\@^$(xz)^g" \
|
||||||
-e "s^@perl\@^$(perl)^g" \
|
-e "s^@perl\@^$(perl)^g" \
|
||||||
-e "s^@perlFlags\@^$(perlFlags)^g" \
|
-e "s^@perlFlags\@^$(perlFlags)^g" \
|
||||||
-e "s^@coreutils\@^$(coreutils)^g" \
|
-e "s^@coreutils\@^$(coreutils)^g" \
|
||||||
-e "s^@sed\@^$(sed)^g" \
|
-e "s^@sed\@^$(sed)^g" \
|
||||||
-e "s^@tar\@^$(tar)^g" \
|
-e "s^@tar\@^$(tar)^g" \
|
||||||
|
-e "s^@tarFlags\@^$(tarFlags)^g" \
|
||||||
-e "s^@gzip\@^$(gzip)^g" \
|
-e "s^@gzip\@^$(gzip)^g" \
|
||||||
-e "s^@pv\@^$(pv)^g" \
|
-e "s^@pv\@^$(pv)^g" \
|
||||||
-e "s^@tr\@^$(tr)^g" \
|
-e "s^@tr\@^$(tr)^g" \
|
||||||
|
|
|
@ -9,7 +9,8 @@ TESTS = init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \
|
||||||
gc-runtime.sh install-package.sh check-refs.sh filter-source.sh \
|
gc-runtime.sh install-package.sh check-refs.sh filter-source.sh \
|
||||||
remote-store.sh export.sh export-graph.sh negative-caching.sh \
|
remote-store.sh export.sh export-graph.sh negative-caching.sh \
|
||||||
binary-patching.sh timeout.sh secure-drv-outputs.sh nix-channel.sh \
|
binary-patching.sh timeout.sh secure-drv-outputs.sh nix-channel.sh \
|
||||||
multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh
|
multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \
|
||||||
|
binary-cache.sh
|
||||||
|
|
||||||
XFAIL_TESTS =
|
XFAIL_TESTS =
|
||||||
|
|
||||||
|
|
35
tests/binary-cache.sh
Normal file
35
tests/binary-cache.sh
Normal file
|
@ -0,0 +1,35 @@
|
||||||
|
source common.sh
|
||||||
|
|
||||||
|
clearStore
|
||||||
|
|
||||||
|
# Create the binary cache.
|
||||||
|
cacheDir=$TEST_ROOT/binary-cache
|
||||||
|
rm -rf $cacheDir
|
||||||
|
|
||||||
|
outPath=$(nix-build dependencies.nix --no-out-link)
|
||||||
|
|
||||||
|
nix-push --dest $cacheDir $outPath
|
||||||
|
|
||||||
|
|
||||||
|
# By default, a binary cache doesn't support "nix-env -qas", but does
|
||||||
|
# support installation.
|
||||||
|
clearStore
|
||||||
|
rm -f $NIX_STATE_DIR/binary-cache*
|
||||||
|
|
||||||
|
nix-env --option binary-caches "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "---"
|
||||||
|
|
||||||
|
nix-store --option binary-caches "file://$cacheDir" -r $outPath
|
||||||
|
|
||||||
|
|
||||||
|
# But with the right configuration, "nix-env -qas" should also work.
|
||||||
|
clearStore
|
||||||
|
rm -f $NIX_STATE_DIR/binary-cache*
|
||||||
|
echo "WantMassQuery: 1" >> $cacheDir/nix-cache-info
|
||||||
|
|
||||||
|
nix-env --option binary-caches "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "--S"
|
||||||
|
|
||||||
|
nix-store --option binary-caches "file://$cacheDir" -r $outPath
|
||||||
|
|
||||||
|
nix-store --check-validity $outPath
|
||||||
|
nix-store -qR $outPath | grep input-2
|
||||||
|
|
|
@ -7,14 +7,17 @@ mkdir -p $TEST_ROOT/cache2 $TEST_ROOT/patches
|
||||||
RESULT=$TEST_ROOT/result
|
RESULT=$TEST_ROOT/result
|
||||||
|
|
||||||
# Build version 1 and 2 of the "foo" package.
|
# Build version 1 and 2 of the "foo" package.
|
||||||
nix-push --copy $TEST_ROOT/cache2 $TEST_ROOT/manifest1 \
|
nix-push --dest $TEST_ROOT/cache2 --manifest --bzip2 \
|
||||||
$(nix-build -o $RESULT binary-patching.nix --arg version 1)
|
$(nix-build -o $RESULT binary-patching.nix --arg version 1)
|
||||||
|
mv $TEST_ROOT/cache2/MANIFEST $TEST_ROOT/manifest1
|
||||||
|
|
||||||
out2=$(nix-build -o $RESULT binary-patching.nix --arg version 2)
|
out2=$(nix-build -o $RESULT binary-patching.nix --arg version 2)
|
||||||
nix-push --copy $TEST_ROOT/cache2 $TEST_ROOT/manifest2 $out2
|
nix-push --dest $TEST_ROOT/cache2 --manifest --bzip2 $out2
|
||||||
|
mv $TEST_ROOT/cache2/MANIFEST $TEST_ROOT/manifest2
|
||||||
|
|
||||||
out3=$(nix-build -o $RESULT binary-patching.nix --arg version 3)
|
out3=$(nix-build -o $RESULT binary-patching.nix --arg version 3)
|
||||||
nix-push --copy $TEST_ROOT/cache2 $TEST_ROOT/manifest3 $out3
|
nix-push --dest $TEST_ROOT/cache2 --manifest --bzip2 $out3
|
||||||
|
mv $TEST_ROOT/cache2/MANIFEST $TEST_ROOT/manifest3
|
||||||
|
|
||||||
rm $RESULT
|
rm $RESULT
|
||||||
|
|
||||||
|
|
|
@ -16,6 +16,7 @@ export NIX_DB_DIR=$TEST_ROOT/db
|
||||||
export NIX_CONF_DIR=$TEST_ROOT/etc
|
export NIX_CONF_DIR=$TEST_ROOT/etc
|
||||||
export NIX_MANIFESTS_DIR=$TEST_ROOT/var/nix/manifests
|
export NIX_MANIFESTS_DIR=$TEST_ROOT/var/nix/manifests
|
||||||
export SHARED=$TEST_ROOT/shared
|
export SHARED=$TEST_ROOT/shared
|
||||||
|
export NIX_REMOTE=$NIX_REMOTE_
|
||||||
|
|
||||||
export PATH=@bindir@:$PATH
|
export PATH=@bindir@:$PATH
|
||||||
|
|
||||||
|
@ -79,3 +80,5 @@ fail() {
|
||||||
echo "$1"
|
echo "$1"
|
||||||
exit 1
|
exit 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
set -x
|
||||||
|
|
|
@ -9,7 +9,7 @@ clearStore
|
||||||
clearProfiles
|
clearProfiles
|
||||||
|
|
||||||
cat > $TEST_ROOT/foo.nixpkg <<EOF
|
cat > $TEST_ROOT/foo.nixpkg <<EOF
|
||||||
NIXPKG1 file://$TEST_ROOT/manifest simple $system $drvPath $outPath
|
NIXPKG1 file://$TEST_ROOT/cache/MANIFEST simple $system $drvPath $outPath
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
nix-install-package --non-interactive -p $profiles/test $TEST_ROOT/foo.nixpkg
|
nix-install-package --non-interactive -p $profiles/test $TEST_ROOT/foo.nixpkg
|
||||||
|
|
|
@ -19,7 +19,7 @@ nix-channel --remove xyzzy
|
||||||
# Create a channel.
|
# Create a channel.
|
||||||
rm -rf $TEST_ROOT/foo
|
rm -rf $TEST_ROOT/foo
|
||||||
mkdir -p $TEST_ROOT/foo
|
mkdir -p $TEST_ROOT/foo
|
||||||
nix-push --copy $TEST_ROOT/foo $TEST_ROOT/foo/MANIFEST $(nix-store -r $(nix-instantiate dependencies.nix))
|
nix-push --dest $TEST_ROOT/foo --manifest --bzip2 $(nix-store -r $(nix-instantiate dependencies.nix))
|
||||||
rm -rf $TEST_ROOT/nixexprs
|
rm -rf $TEST_ROOT/nixexprs
|
||||||
mkdir -p $TEST_ROOT/nixexprs
|
mkdir -p $TEST_ROOT/nixexprs
|
||||||
cp config.nix dependencies.nix dependencies.builder*.sh $TEST_ROOT/nixexprs/
|
cp config.nix dependencies.nix dependencies.builder*.sh $TEST_ROOT/nixexprs/
|
||||||
|
|
|
@ -2,7 +2,7 @@ source common.sh
|
||||||
|
|
||||||
pullCache () {
|
pullCache () {
|
||||||
echo "pulling cache..."
|
echo "pulling cache..."
|
||||||
nix-pull file://$TEST_ROOT/manifest
|
nix-pull file://$TEST_ROOT/cache/MANIFEST
|
||||||
}
|
}
|
||||||
|
|
||||||
clearStore
|
clearStore
|
||||||
|
|
|
@ -1,5 +1,7 @@
|
||||||
source common.sh
|
source common.sh
|
||||||
|
|
||||||
|
clearStore
|
||||||
|
|
||||||
drvPath=$(nix-instantiate dependencies.nix)
|
drvPath=$(nix-instantiate dependencies.nix)
|
||||||
outPath=$(nix-store -r $drvPath)
|
outPath=$(nix-store -r $drvPath)
|
||||||
|
|
||||||
|
@ -7,4 +9,4 @@ echo "pushing $drvPath"
|
||||||
|
|
||||||
mkdir -p $TEST_ROOT/cache
|
mkdir -p $TEST_ROOT/cache
|
||||||
|
|
||||||
nix-push --copy $TEST_ROOT/cache $TEST_ROOT/manifest $drvPath
|
nix-push --dest $TEST_ROOT/cache --manifest $drvPath --bzip2
|
||||||
|
|
|
@ -10,6 +10,7 @@ touch $reference
|
||||||
|
|
||||||
echo "making registration..."
|
echo "making registration..."
|
||||||
|
|
||||||
|
set +x
|
||||||
for ((n = 0; n < $max; n++)); do
|
for ((n = 0; n < $max; n++)); do
|
||||||
storePath=$NIX_STORE_DIR/$n
|
storePath=$NIX_STORE_DIR/$n
|
||||||
echo -n > $storePath
|
echo -n > $storePath
|
||||||
|
@ -19,6 +20,7 @@ for ((n = 0; n < $max; n++)); do
|
||||||
fi
|
fi
|
||||||
echo $storePath; echo; echo 2; echo $reference; echo $ref2
|
echo $storePath; echo; echo 2; echo $reference; echo $ref2
|
||||||
done > $TEST_ROOT/reg_info
|
done > $TEST_ROOT/reg_info
|
||||||
|
set -x
|
||||||
|
|
||||||
echo "registering..."
|
echo "registering..."
|
||||||
|
|
||||||
|
|
|
@ -3,7 +3,7 @@ source common.sh
|
||||||
echo '*** testing slave mode ***'
|
echo '*** testing slave mode ***'
|
||||||
clearStore
|
clearStore
|
||||||
clearManifests
|
clearManifests
|
||||||
NIX_REMOTE=slave $SHELL ./user-envs.sh
|
NIX_REMOTE_=slave $SHELL ./user-envs.sh
|
||||||
|
|
||||||
echo '*** testing daemon mode ***'
|
echo '*** testing daemon mode ***'
|
||||||
clearStore
|
clearStore
|
||||||
|
|
|
@ -2,22 +2,25 @@
|
||||||
echo substituter args: $* >&2
|
echo substituter args: $* >&2
|
||||||
|
|
||||||
if test $1 = "--query"; then
|
if test $1 = "--query"; then
|
||||||
while read cmd; do
|
while read cmd args; do
|
||||||
echo FOO $cmd >&2
|
echo "CMD = $cmd, ARGS = $args" >&2
|
||||||
if test "$cmd" = "have"; then
|
if test "$cmd" = "have"; then
|
||||||
read path
|
for path in $args; do
|
||||||
if grep -q "$path" $TEST_ROOT/sub-paths; then
|
read path
|
||||||
echo 1
|
if grep -q "$path" $TEST_ROOT/sub-paths; then
|
||||||
else
|
echo $path
|
||||||
echo 0
|
fi
|
||||||
fi
|
done
|
||||||
|
echo
|
||||||
elif test "$cmd" = "info"; then
|
elif test "$cmd" = "info"; then
|
||||||
read path
|
for path in $args; do
|
||||||
echo 1
|
echo $path
|
||||||
echo "" # deriver
|
echo "" # deriver
|
||||||
echo 0 # nr of refs
|
echo 0 # nr of refs
|
||||||
echo $((1 * 1024 * 1024)) # download size
|
echo $((1 * 1024 * 1024)) # download size
|
||||||
echo $((2 * 1024 * 1024)) # nar size
|
echo $((2 * 1024 * 1024)) # nar size
|
||||||
|
done
|
||||||
|
echo
|
||||||
else
|
else
|
||||||
echo "bad command $cmd"
|
echo "bad command $cmd"
|
||||||
exit 1
|
exit 1
|
||||||
|
@ -26,6 +29,7 @@ if test $1 = "--query"; then
|
||||||
elif test $1 = "--substitute"; then
|
elif test $1 = "--substitute"; then
|
||||||
mkdir $2
|
mkdir $2
|
||||||
echo "Hallo Wereld" > $2/hello
|
echo "Hallo Wereld" > $2/hello
|
||||||
|
echo # no expected hash
|
||||||
else
|
else
|
||||||
echo "unknown substituter operation"
|
echo "unknown substituter operation"
|
||||||
exit 1
|
exit 1
|
||||||
|
|
|
@ -2,21 +2,23 @@
|
||||||
echo substituter2 args: $* >&2
|
echo substituter2 args: $* >&2
|
||||||
|
|
||||||
if test $1 = "--query"; then
|
if test $1 = "--query"; then
|
||||||
while read cmd; do
|
while read cmd args; do
|
||||||
if test "$cmd" = "have"; then
|
if test "$cmd" = have; then
|
||||||
read path
|
for path in $args; do
|
||||||
if grep -q "$path" $TEST_ROOT/sub-paths; then
|
if grep -q "$path" $TEST_ROOT/sub-paths; then
|
||||||
echo 1
|
echo $path
|
||||||
else
|
fi
|
||||||
echo 0
|
done
|
||||||
fi
|
echo
|
||||||
elif test "$cmd" = "info"; then
|
elif test "$cmd" = info; then
|
||||||
read path
|
for path in $args; do
|
||||||
echo 1
|
echo $path
|
||||||
echo "" # deriver
|
echo "" # deriver
|
||||||
echo 0 # nr of refs
|
echo 0 # nr of refs
|
||||||
echo 0 # download size
|
echo 0 # download size
|
||||||
echo 0 # nar size
|
echo 0 # nar size
|
||||||
|
done
|
||||||
|
echo
|
||||||
else
|
else
|
||||||
echo "bad command $cmd"
|
echo "bad command $cmd"
|
||||||
exit 1
|
exit 1
|
||||||
|
|
Loading…
Reference in a new issue