* Merged the SQLite branch.

This commit is contained in:
Eelco Dolstra 2011-02-09 12:41:54 +00:00
commit d0eda1f3e9
82 changed files with 2703 additions and 1917 deletions

View file

@ -115,3 +115,35 @@
fun:* fun:*
fun:AT_collect fun:AT_collect
} }
{
ATerm library conservatively scans for GC roots
Memcheck:Value4
fun:*
fun:*
fun:mark_phase
}
{
ATerm library conservatively scans for GC roots
Memcheck:Cond
fun:*
fun:*
fun:mark_phase
}
{
ATerm library conservatively scans for GC roots
Memcheck:Value4
fun:*
fun:*
fun:mark_phase_young
}
{
ATerm library conservatively scans for GC roots
Memcheck:Cond
fun:*
fun:*
fun:mark_phase_young
}

View file

@ -1,4 +1,5 @@
#! /bin/sh -e #! /bin/sh -e
rm -f aclocal.m4
mkdir -p config mkdir -p config
libtoolize --copy libtoolize --copy
aclocal aclocal

View file

@ -50,39 +50,24 @@ AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier (`cpu-os')])
test "$localstatedir" = '${prefix}/var' && localstatedir=/nix/var test "$localstatedir" = '${prefix}/var' && localstatedir=/nix/var
# Whether to produce a statically linked binary. On Cygwin, this is # Windows-specific stuff. On Cygwin, dynamically linking against the
# the default: dynamically linking against the ATerm DLL does work, # ATerm DLL works, except that it requires the ATerm "lib" directory
# except that it requires the ATerm "lib" directory to be in $PATH, as # to be in $PATH, as Windows doesn't have anything like an RPATH
# Windows doesn't have anything like an RPATH embedded in executable. # embedded in executable. Since this is kind of annoying, we use
# Since this is kind of annoying, we use static libraries for now. # static libraries for now.
if test "$sys_name" = "cygwin"; then
AC_ARG_ENABLE(static-nix, AC_HELP_STRING([--enable-static-nix],
[produce statically linked binaries]),
static_nix=$enableval, static_nix=no)
if test "$sys_name" = cygwin; then
static_nix=yes
fi
if test "$static_nix" = yes; then
AC_DISABLE_SHARED AC_DISABLE_SHARED
AC_ENABLE_STATIC AC_ENABLE_STATIC
fi fi
# Windows-specific stuff.
if test "$sys_name" = "cygwin"; then
# We cannot delete open files.
AC_DEFINE(CANNOT_DELETE_OPEN_FILES, 1, [Whether it is impossible to delete open files.])
fi
# Solaris-specific stuff. # Solaris-specific stuff.
if test "$sys_name" = "sunos"; then if test "$sys_name" = "sunos"; then
# Solaris requires -lsocket -lnsl for network functions # Solaris requires -lsocket -lnsl for network functions
ADDITIONAL_NETWORK_LIBS="-lsocket -lnsl" LIBS="-lsocket -lnsl $LIBS"
AC_SUBST(ADDITIONAL_NETWORK_LIBS)
fi fi
AC_PROG_CC AC_PROG_CC
AC_PROG_CXX AC_PROG_CXX
@ -101,6 +86,13 @@ AC_DISABLE_STATIC
AC_ENABLE_SHARED AC_ENABLE_SHARED
AC_PROG_LIBTOOL AC_PROG_LIBTOOL
if test "$enable_shared" = yes; then
SUB_CONFIGURE_FLAGS="--enable-shared --disable-static"
else
SUB_CONFIGURE_FLAGS="--enable-static --disable-shared"
fi
AC_SUBST(SUB_CONFIGURE_FLAGS)
# Use 64-bit file system calls so that we can support files > 2 GiB. # Use 64-bit file system calls so that we can support files > 2 GiB.
AC_SYS_LARGEFILE AC_SYS_LARGEFILE
@ -229,6 +221,8 @@ AC_ARG_WITH(bzip2, AC_HELP_STRING([--with-bzip2=PATH],
[prefix of bzip2]), [prefix of bzip2]),
bzip2=$withval, bzip2=) bzip2=$withval, bzip2=)
AM_CONDITIONAL(HAVE_BZIP2, test -n "$bzip2") AM_CONDITIONAL(HAVE_BZIP2, test -n "$bzip2")
ATERM_VERSION=2.5
AC_SUBST(ATERM_VERSION)
if test -z "$bzip2"; then if test -z "$bzip2"; then
# Headers and libraries will be used from the temporary installation # Headers and libraries will be used from the temporary installation
# in externals/inst-bzip2. # in externals/inst-bzip2.
@ -249,6 +243,24 @@ AC_SUBST(bzip2_include)
AC_SUBST(bzip2_bin) AC_SUBST(bzip2_bin)
AC_SUBST(bzip2_bin_test) AC_SUBST(bzip2_bin_test)
AC_ARG_WITH(sqlite, AC_HELP_STRING([--with-sqlite=PATH],
[prefix of SQLite]),
sqlite=$withval, sqlite=)
AM_CONDITIONAL(HAVE_SQLITE, test -n "$sqlite")
SQLITE_VERSION=3070500
AC_SUBST(SQLITE_VERSION)
if test -z "$sqlite"; then
sqlite_lib='${top_builddir}/externals/sqlite-autoconf-$(SQLITE_VERSION)/libsqlite3.la'
sqlite_include='-I${top_builddir}/externals/sqlite-autoconf-$(SQLITE_VERSION)'
sqlite_bin='${top_builddir}/externals/sqlite-autoconf-$(SQLITE_VERSION)'
else
sqlite_lib="-L$sqlite/lib -lsqlite3"
sqlite_include="-I$sqlite/include"
sqlite_bin="$sqlite/bin"
fi
AC_SUBST(sqlite_lib)
AC_SUBST(sqlite_include)
AC_SUBST(sqlite_bin)
# Whether to use the Boehm garbage collector. # Whether to use the Boehm garbage collector.
AC_ARG_ENABLE(gc, AC_HELP_STRING([--enable-gc], AC_ARG_ENABLE(gc, AC_HELP_STRING([--enable-gc],
@ -274,8 +286,7 @@ AC_CHECK_FUNCS([setresuid setreuid lchown])
# Nice to have, but not essential. # Nice to have, but not essential.
AC_CHECK_FUNCS([strsignal]) AC_CHECK_FUNCS([strsignal posix_fallocate nanosleep])
AC_CHECK_FUNCS([posix_fallocate])
# This is needed if ATerm or bzip2 are static libraries, # This is needed if ATerm or bzip2 are static libraries,
@ -285,14 +296,6 @@ if test "$(uname)" = "Darwin"; then
fi fi
if test "$static_nix" = yes; then
# `-all-static' has to be added at the end of configure, because
# the C compiler doesn't know about -all-static (it's filtered out
# by libtool, but configure doesn't use libtool).
LDFLAGS="-all-static $LDFLAGS"
fi
AM_CONFIG_HEADER([config.h]) AM_CONFIG_HEADER([config.h])
AC_CONFIG_FILES([Makefile AC_CONFIG_FILES([Makefile
externals/Makefile externals/Makefile

View file

@ -11,4 +11,8 @@ derivation {
paths = derivations; paths = derivations;
active = map (x: if x ? meta && x.meta ? active then x.meta.active else "true") derivations; active = map (x: if x ? meta && x.meta ? active then x.meta.active else "true") derivations;
priority = map (x: if x ? meta && x.meta ? priority then x.meta.priority else "5") derivations; priority = map (x: if x ? meta && x.meta ? priority then x.meta.priority else "5") derivations;
# Building user environments remotely just causes huge amounts of
# network traffic, so don't do that.
preferLocalBuild = true;
} }

View file

@ -7,8 +7,6 @@ dst=$out/tmp.nar.bz2
@bzip2@ < tmp > $dst @bzip2@ < tmp > $dst
@bindir@/nix-hash -vvvvv --flat --type $hashAlgo --base32 tmp > $out/nar-hash
@bindir@/nix-hash --flat --type $hashAlgo --base32 $dst > $out/narbz2-hash @bindir@/nix-hash --flat --type $hashAlgo --base32 $dst > $out/narbz2-hash
@coreutils@/mv $out/tmp.nar.bz2 $out/$(@coreutils@/cat $out/narbz2-hash).nar.bz2 @coreutils@/mv $out/tmp.nar.bz2 $out/$(@coreutils@/cat $out/narbz2-hash).nar.bz2

View file

@ -260,7 +260,7 @@ build-use-chroot = /dev /proc /bin</programlisting>
Nix store metadata (in <filename>/nix/var/nix/db</filename>) are Nix store metadata (in <filename>/nix/var/nix/db</filename>) are
synchronously flushed to disk. This improves robustness in case synchronously flushed to disk. This improves robustness in case
of system crashes, but reduces performance. The default is of system crashes, but reduces performance. The default is
<literal>false</literal>.</para></listitem> <literal>true</literal>.</para></listitem>
</varlistentry> </varlistentry>

View file

@ -404,6 +404,7 @@ error: cannot delete path `/nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4'
<arg choice='plain'><option>--tree</option></arg> <arg choice='plain'><option>--tree</option></arg>
<arg choice='plain'><option>--binding</option> <replaceable>name</replaceable></arg> <arg choice='plain'><option>--binding</option> <replaceable>name</replaceable></arg>
<arg choice='plain'><option>--hash</option></arg> <arg choice='plain'><option>--hash</option></arg>
<arg choice='plain'><option>--size</option></arg>
<arg choice='plain'><option>--roots</option></arg> <arg choice='plain'><option>--roots</option></arg>
</group> </group>
<arg><option>--use-output</option></arg> <arg><option>--use-output</option></arg>
@ -587,9 +588,21 @@ query is applied to the target of the symlink.</para>
<varlistentry><term><option>--hash</option></term> <varlistentry><term><option>--hash</option></term>
<listitem><para>Prints the SHA-256 hash of the contents of the <listitem><para>Prints the SHA-256 hash of the contents of the
store paths <replaceable>paths</replaceable>. Since the hash is store paths <replaceable>paths</replaceable> (that is, the hash of
stored in the Nix database, this is a fast the output of <command>nix-store --dump</command> on the given
operation.</para></listitem> paths). Since the hash is stored in the Nix database, this is a
fast operation.</para></listitem>
</varlistentry>
<varlistentry><term><option>--size</option></term>
<listitem><para>Prints the size in bytes of the contents of the
store paths <replaceable>paths</replaceable> — to be precise, the
size of the output of <command>nix-store --dump</command> on the
given paths. Note that the actual disk space required by the
store paths may be higher, especially on filesystems with large
cluster sizes.</para></listitem>
</varlistentry> </varlistentry>

View file

@ -60,7 +60,7 @@ available remotely.</para></listitem>
in the channel: in the channel:
<screen> <screen>
$ nix-env -qa * <lineannotation>(mind the quotes!)</lineannotation> $ nix-env -qa \*
docbook-xml-4.2 docbook-xml-4.2
firefox-1.0pre-PR-0.10.1 firefox-1.0pre-PR-0.10.1
hello-2.1.1 hello-2.1.1

56
externals/Makefile.am vendored
View file

@ -12,30 +12,56 @@ $(BZIP2).tar.gz:
$(BZIP2): $(BZIP2).tar.gz $(BZIP2): $(BZIP2).tar.gz
gunzip < $(srcdir)/$(BZIP2).tar.gz | tar xvf - gunzip < $(srcdir)/$(BZIP2).tar.gz | tar xvf -
have-bzip2:
$(MAKE) $(BZIP2)
touch have-bzip2
if HAVE_BZIP2 if HAVE_BZIP2
build-bzip2: build-bzip2:
else else
build-bzip2: have-bzip2 build-bzip2: $(BZIP2)
(pfx=`pwd` && \ (cd $(BZIP2) && \
cd $(BZIP2) && \ $(MAKE) CC="$(CC)" && \
$(MAKE) && \ $(MAKE) install PREFIX=$(abs_builddir)/inst-bzip2)
$(MAKE) install PREFIX=$$pfx/inst-bzip2)
touch build-bzip2 touch build-bzip2
install: install-exec-local:: build-bzip2
mkdir -p $(DESTDIR)${bzip2_bin} mkdir -p $(DESTDIR)${bzip2_bin}
$(INSTALL_PROGRAM) $(bzip2_bin_test)/bzip2 $(bzip2_bin_test)/bunzip2 $(DESTDIR)${bzip2_bin} $(INSTALL_PROGRAM) $(bzip2_bin_test)/bzip2 $(bzip2_bin_test)/bunzip2 $(DESTDIR)${bzip2_bin}
endif endif
all: build-bzip2 # SQLite
EXTRA_DIST = $(BZIP2).tar.gz SQLITE = sqlite-autoconf-$(SQLITE_VERSION)
SQLITE_TAR = sqlite-autoconf-$(SQLITE_VERSION).tar.gz
ext-clean: $(SQLITE_TAR):
$(RM) -f have-bzip2 build-bzip2 @echo "Nix requires the SQLite library to build."
$(RM) -rf $(BZIP2) @echo "Please download version $(SQLITE_VERSION) from"
@echo " http://www.sqlite.org/$(SQLITE_TAR)"
@echo "and place it in the externals/ directory."
false
$(SQLITE): $(SQLITE_TAR)
gzip -d < $(srcdir)/$(SQLITE_TAR) | tar xvf -
if HAVE_SQLITE
build-sqlite:
else
build-sqlite: $(SQLITE)
(cd $(SQLITE) && \
CC="$(CC)" CFLAGS="-DSQLITE_ENABLE_COLUMN_METADATA=1" ./configure --disable-static --prefix=$(pkglibdir)/dummy --libdir=${pkglibdir} $(SUB_CONFIGURE_FLAGS) && \
$(MAKE) )
touch build-sqlite
install-exec-local:: build-sqlite
cd $(SQLITE) && $(MAKE) install
rm -rf "$(DESTDIR)/$(pkglibdir)/dummy"
endif
all: build-bzip2 build-sqlite
EXTRA_DIST = $(BZIP2).tar.gz $(SQLITE_TAR)
clean:
$(RM) -f build-bzip2 build-sqlite
$(RM) -rf $(BZIP2) $(SQLITE)
$(RM) -rf inst-bzip2

View file

@ -33,6 +33,9 @@ let
stripHash ${bzip2.src} stripHash ${bzip2.src}
cp -pv ${bzip2.src} externals/$strippedName cp -pv ${bzip2.src} externals/$strippedName
stripHash ${sqlite.src}
cp -pv ${sqlite.src} externals/$strippedName
# TeX needs a writable font cache. # TeX needs a writable font cache.
export VARTEXFONTS=$TMPDIR/texfonts export VARTEXFONTS=$TMPDIR/texfonts
''; '';
@ -71,7 +74,7 @@ let
configureFlags = '' configureFlags = ''
--disable-init-state --disable-init-state
--with-bzip2=${bzip2} --with-bzip2=${bzip2} --with-sqlite=${sqlite}
--enable-gc --enable-gc
''; '';
}; };
@ -92,7 +95,7 @@ let
configureFlags = '' configureFlags = ''
--disable-init-state --disable-shared --disable-init-state --disable-shared
--with-bzip2=${bzip2} --with-bzip2=${bzip2} --with-sqlite=${sqlite}
''; '';
lcovFilter = [ "*/boost/*" "*-tab.*" ]; lcovFilter = [ "*/boost/*" "*-tab.*" ];
@ -144,11 +147,11 @@ let
with import nixpkgs { inherit system; }; with import nixpkgs { inherit system; };
releaseTools.rpmBuild rec { releaseTools.rpmBuild rec {
name = "nix-rpm"; name = "nix-rpm-${diskImage.name}";
src = jobs.tarball; src = jobs.tarball;
diskImage = diskImageFun vmTools.diskImages; diskImage = diskImageFun vmTools.diskImages;
memSize = 1024; memSize = 1024;
meta = { schedulingPriority = prio; }; meta.schedulingPriority = prio;
}; };
@ -165,7 +168,7 @@ let
src = jobs.tarball; src = jobs.tarball;
diskImage = diskImageFun vmTools.diskImages; diskImage = diskImageFun vmTools.diskImages;
memSize = 1024; memSize = 1024;
meta = { schedulingPriority = prio; }; meta.schedulingPriority = prio;
configureFlags = "--sysconfdir=/etc"; configureFlags = "--sysconfdir=/etc";
debRequires = [ "curl" ]; debRequires = [ "curl" ];
}; };

334
scripts/GeneratePatches.pm.in Executable file
View file

@ -0,0 +1,334 @@
#! @perl@ -w -I@libexecdir@/nix
use strict;
use File::Temp qw(tempdir);
# Some patch generations options.
# Max size of NAR archives to generate patches for.
my $maxNarSize = $ENV{"NIX_MAX_NAR_SIZE"};
$maxNarSize = 160 * 1024 * 1024 if !defined $maxNarSize;
# If patch is bigger than this fraction of full archive, reject.
my $maxPatchFraction = $ENV{"NIX_PATCH_FRACTION"};
$maxPatchFraction = 0.60 if !defined $maxPatchFraction;
my $timeLimit = $ENV{"NIX_BSDIFF_TIME_LIMIT"};
$timeLimit = 180 if !defined $timeLimit;
my $hashAlgo = "sha256";
sub findOutputPaths {
my $narFiles = shift;
my %outPaths;
foreach my $p (keys %{$narFiles}) {
# Ignore derivations.
next if ($p =~ /\.drv$/);
# Ignore builders (too much ambiguity -- they're all called
# `builder.sh').
next if ($p =~ /\.sh$/);
next if ($p =~ /\.patch$/);
# Don't bother including tar files etc.
next if ($p =~ /\.tar$/ || $p =~ /\.tar\.(gz|bz2|Z|lzma|xz)$/ || $p =~ /\.zip$/ || $p =~ /\.bin$/ || $p =~ /\.tgz$/ || $p =~ /\.rpm$/ || $p =~ /cvs-export$/ || $p =~ /fetchhg$/);
$outPaths{$p} = 1;
}
return %outPaths;
}
sub getNameVersion {
my $p = shift;
$p =~ /\/[0-9a-z]+((?:-[a-zA-Z][^\/-]*)+)([^\/]*)$/;
my $name = $1;
my $version = $2;
return undef unless defined $name && defined $version;
$name =~ s/^-//;
$version =~ s/^-//;
return ($name, $version);
}
# A quick hack to get a measure of the `distance' between two
# versions: it's just the position of the first character that differs
# (or 999 if they are the same).
sub versionDiff {
my $s = shift;
my $t = shift;
my $i;
return 999 if $s eq $t;
for ($i = 0; $i < length $s; $i++) {
return $i if $i >= length $t or
substr($s, $i, 1) ne substr($t, $i, 1);
}
return $i;
}
sub getNarBz2 {
my $narPath = shift;
my $narFiles = shift;
my $storePath = shift;
my $narFileList = $$narFiles{$storePath};
die "missing path $storePath" unless defined $narFileList;
my $narFile = @{$narFileList}[0];
die unless defined $narFile;
$narFile->{url} =~ /\/([^\/]+)$/;
die unless defined $1;
return "$narPath/$1";
}
sub containsPatch {
my $patches = shift;
my $storePath = shift;
my $basePath = shift;
my $patchList = $$patches{$storePath};
return 0 if !defined $patchList;
my $found = 0;
foreach my $patch (@{$patchList}) {
# !!! baseHash might differ
return 1 if $patch->{basePath} eq $basePath;
}
return 0;
}
sub generatePatches {
my ($srcNarFiles, $dstNarFiles, $srcPatches, $dstPatches, $narPath, $patchesPath, $patchesURL, $tmpDir) = @_;
my %srcOutPaths = findOutputPaths $srcNarFiles;
my %dstOutPaths = findOutputPaths $dstNarFiles;
# For each output path in the destination, see if we need to / can
# create a patch.
print STDERR "creating patches...\n";
foreach my $p (keys %dstOutPaths) {
# If exactly the same path already exists in the source, skip it.
next if defined $srcOutPaths{$p};
print " $p\n";
# If not, then we should find the paths in the source that are
# `most' likely to be present on a system that wants to
# install this path.
(my $name, my $version) = getNameVersion $p;
next unless defined $name && defined $version;
my @closest = ();
my $closestVersion;
my $minDist = -1; # actually, larger means closer
# Find all source paths with the same name.
foreach my $q (keys %srcOutPaths) {
(my $name2, my $version2) = getNameVersion $q;
next unless defined $name2 && defined $version2;
if ($name eq $name2) {
my $srcSystem = @{$$dstNarFiles{$p}}[0]->{system};
my $dstSystem = @{$$srcNarFiles{$q}}[0]->{system};
if (defined $srcSystem && defined $dstSystem && $srcSystem ne $dstSystem) {
print " SKIPPING $q due to different systems ($srcSystem vs. $dstSystem)\n";
next;
}
# If the sizes differ too much, then skip. This
# disambiguates between, e.g., a real component and a
# wrapper component (cf. Firefox in Nixpkgs).
my $srcSize = @{$$srcNarFiles{$q}}[0]->{size};
my $dstSize = @{$$dstNarFiles{$p}}[0]->{size};
my $ratio = $srcSize / $dstSize;
$ratio = 1 / $ratio if $ratio < 1;
# print " SIZE $srcSize $dstSize $ratio $q\n";
if ($ratio >= 3) {
print " SKIPPING $q due to size ratio $ratio ($srcSize vs. $dstSize)\n";
next;
}
# If there are multiple matching names, include the
# ones with the closest version numbers.
my $dist = versionDiff $version, $version2;
if ($dist > $minDist) {
$minDist = $dist;
@closest = ($q);
$closestVersion = $version2;
} elsif ($dist == $minDist) {
push @closest, $q;
}
}
}
if (scalar(@closest) == 0) {
print " NO BASE: $p\n";
next;
}
foreach my $closest (@closest) {
# Generate a patch between $closest and $p.
print STDERR " $p <- $closest\n";
# If the patch already exists, skip it.
if (containsPatch($srcPatches, $p, $closest) ||
containsPatch($dstPatches, $p, $closest))
{
print " skipping, already exists\n";
next;
}
my $srcNarBz2 = getNarBz2 $narPath, $srcNarFiles, $closest;
my $dstNarBz2 = getNarBz2 $narPath, $dstNarFiles, $p;
if (! -f $srcNarBz2) {
warn "patch source archive $srcNarBz2 is missing\n";
next;
}
system("@bunzip2@ < $srcNarBz2 > $tmpDir/A") == 0
or die "cannot unpack $srcNarBz2";
if ((stat "$tmpDir/A")[7] >= $maxNarSize) {
print " skipping, source is too large\n";
next;
}
system("@bunzip2@ < $dstNarBz2 > $tmpDir/B") == 0
or die "cannot unpack $dstNarBz2";
if ((stat "$tmpDir/B")[7] >= $maxNarSize) {
print " skipping, destination is too large\n";
next;
}
my $time1 = time();
my $res = system("ulimit -t $timeLimit; @libexecdir@/bsdiff $tmpDir/A $tmpDir/B $tmpDir/DIFF");
my $time2 = time();
if ($res) {
warn "binary diff computation aborted after ", $time2 - $time1, " seconds\n";
next;
}
my $baseHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpDir/A` or die;
chomp $baseHash;
my $narHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpDir/B` or die;
chomp $narHash;
my $narDiffHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpDir/DIFF` or die;
chomp $narDiffHash;
my $narDiffSize = (stat "$tmpDir/DIFF")[7];
my $dstNarBz2Size = (stat $dstNarBz2)[7];
print " size $narDiffSize; full size $dstNarBz2Size; ", $time2 - $time1, " seconds\n";
if ($narDiffSize >= $dstNarBz2Size) {
print " rejecting; patch bigger than full archive\n";
next;
}
if ($narDiffSize / $dstNarBz2Size >= $maxPatchFraction) {
print " rejecting; patch too large relative to full archive\n";
next;
}
my $finalName = "$narDiffHash.nar-bsdiff";
if (-e "$patchesPath/$finalName") {
print " not copying, already exists\n";
}
else {
system("cp '$tmpDir/DIFF' '$patchesPath/$finalName.tmp'") == 0
or die "cannot copy diff";
rename("$patchesPath/$finalName.tmp", "$patchesPath/$finalName")
or die "cannot rename $patchesPath/$finalName.tmp";
}
# Add the patch to the manifest.
addPatch $dstPatches, $p,
{ url => "$patchesURL/$finalName", hash => "$hashAlgo:$narDiffHash"
, size => $narDiffSize, basePath => $closest, baseHash => "$hashAlgo:$baseHash"
, narHash => "$hashAlgo:$narHash", patchType => "nar-bsdiff"
};
}
}
}
# Propagate useful patches from $srcPatches to $dstPatches. A patch
# is useful if it produces either paths in the $dstNarFiles or paths
# that can be used as the base for other useful patches.
sub propagatePatches {
my ($srcPatches, $dstNarFiles, $dstPatches) = @_;
print STDERR "propagating patches...\n";
my $changed;
do {
# !!! we repeat this to reach the transitive closure; inefficient
$changed = 0;
print STDERR "loop\n";
my %dstBasePaths;
foreach my $q (keys %{$dstPatches}) {
foreach my $patch (@{$$dstPatches{$q}}) {
$dstBasePaths{$patch->{basePath}} = 1;
}
}
foreach my $p (keys %{$srcPatches}) {
my $patchList = $$srcPatches{$p};
my $include = 0;
# Is path $p included in the destination? If so, include
# patches that produce it.
$include = 1 if defined $$dstNarFiles{$p};
# Is path $p a path that serves as a base for paths in the
# destination? If so, include patches that produce it.
# !!! check baseHash
$include = 1 if defined $dstBasePaths{$p};
if ($include) {
foreach my $patch (@{$patchList}) {
$changed = 1 if addPatch $dstPatches, $p, $patch;
}
}
}
} while $changed;
}
# Add all new patches in $srcPatches to $dstPatches.
sub copyPatches {
my ($srcPatches, $dstPatches) = @_;
foreach my $p (keys %{$srcPatches}) {
addPatch $dstPatches, $p, $_ foreach @{$$srcPatches{$p}};
}
}
return 1;

View file

@ -1,23 +1,23 @@
bin_SCRIPTS = nix-collect-garbage \ bin_SCRIPTS = nix-collect-garbage \
nix-pull nix-push nix-prefetch-url \ nix-pull nix-push nix-prefetch-url \
nix-install-package nix-channel nix-build \ nix-install-package nix-channel nix-build \
nix-copy-closure nix-copy-closure nix-generate-patches
noinst_SCRIPTS = nix-profile.sh generate-patches.pl \ noinst_SCRIPTS = nix-profile.sh GeneratePatches.pm \
find-runtime-roots.pl build-remote.pl nix-reduce-build \ find-runtime-roots.pl build-remote.pl nix-reduce-build \
copy-from-other-stores.pl nix-http-export.cgi copy-from-other-stores.pl nix-http-export.cgi
nix-pull nix-push: readmanifest.pm readconfig.pm download-using-manifests.pl nix-pull nix-push: NixManifest.pm NixConfig.pm download-using-manifests.pl
install-exec-local: readmanifest.pm download-using-manifests.pl copy-from-other-stores.pl find-runtime-roots.pl install-exec-local: NixManifest.pm GeneratePatches.pm download-using-manifests.pl copy-from-other-stores.pl find-runtime-roots.pl
$(INSTALL) -d $(DESTDIR)$(sysconfdir)/profile.d $(INSTALL) -d $(DESTDIR)$(sysconfdir)/profile.d
$(INSTALL_PROGRAM) nix-profile.sh $(DESTDIR)$(sysconfdir)/profile.d/nix.sh $(INSTALL_PROGRAM) nix-profile.sh $(DESTDIR)$(sysconfdir)/profile.d/nix.sh
$(INSTALL) -d $(DESTDIR)$(libexecdir)/nix $(INSTALL) -d $(DESTDIR)$(libexecdir)/nix
$(INSTALL_DATA) readmanifest.pm $(DESTDIR)$(libexecdir)/nix $(INSTALL_DATA) NixManifest.pm $(DESTDIR)$(libexecdir)/nix
$(INSTALL_DATA) readconfig.pm $(DESTDIR)$(libexecdir)/nix $(INSTALL_DATA) NixConfig.pm $(DESTDIR)$(libexecdir)/nix
$(INSTALL_DATA) ssh.pm $(DESTDIR)$(libexecdir)/nix $(INSTALL_DATA) SSH.pm $(DESTDIR)$(libexecdir)/nix
$(INSTALL_DATA) GeneratePatches.pm $(DESTDIR)$(libexecdir)/nix
$(INSTALL_PROGRAM) find-runtime-roots.pl $(DESTDIR)$(libexecdir)/nix $(INSTALL_PROGRAM) find-runtime-roots.pl $(DESTDIR)$(libexecdir)/nix
$(INSTALL_PROGRAM) generate-patches.pl $(DESTDIR)$(libexecdir)/nix
$(INSTALL_PROGRAM) build-remote.pl $(DESTDIR)$(libexecdir)/nix $(INSTALL_PROGRAM) build-remote.pl $(DESTDIR)$(libexecdir)/nix
$(INSTALL) -d $(DESTDIR)$(libexecdir)/nix/substituters $(INSTALL) -d $(DESTDIR)$(libexecdir)/nix/substituters
$(INSTALL_PROGRAM) download-using-manifests.pl $(DESTDIR)$(libexecdir)/nix/substituters $(INSTALL_PROGRAM) download-using-manifests.pl $(DESTDIR)$(libexecdir)/nix/substituters
@ -30,15 +30,16 @@ EXTRA_DIST = nix-collect-garbage.in \
nix-pull.in nix-push.in nix-profile.sh.in \ nix-pull.in nix-push.in nix-profile.sh.in \
nix-prefetch-url.in nix-install-package.in \ nix-prefetch-url.in nix-install-package.in \
nix-channel.in \ nix-channel.in \
readmanifest.pm.in \ NixManifest.pm.in \
readconfig.pm.in \ NixConfig.pm.in \
ssh.pm \ SSH.pm \
GeneratePatches.pm.in \
nix-build.in \ nix-build.in \
download-using-manifests.pl.in \ download-using-manifests.pl.in \
copy-from-other-stores.pl.in \ copy-from-other-stores.pl.in \
generate-patches.pl.in \
nix-copy-closure.in \ nix-copy-closure.in \
find-runtime-roots.pl.in \ find-runtime-roots.pl.in \
build-remote.pl.in \ build-remote.pl.in \
nix-reduce-build.in \ nix-reduce-build.in \
nix-http-export.cgi.in nix-http-export.cgi.in \
nix-generate-patches.in

View file

@ -33,18 +33,8 @@ sub readManifest {
my $manifestVersion = 2; my $manifestVersion = 2;
my $storePath; my ($storePath, $url, $hash, $size, $basePath, $baseHash, $patchType);
my $url; my ($narHash, $narSize, $references, $deriver, $hashAlgo, $copyFrom, $system);
my $hash;
my $size;
my $basePath;
my $baseHash;
my $patchType;
my $narHash;
my $references;
my $deriver;
my $hashAlgo;
my $copyFrom;
while (<MANIFEST>) { while (<MANIFEST>) {
chomp; chomp;
@ -62,9 +52,11 @@ sub readManifest {
undef $hash; undef $hash;
undef $size; undef $size;
undef $narHash; undef $narHash;
undef $narSize;
undef $basePath; undef $basePath;
undef $baseHash; undef $baseHash;
undef $patchType; undef $patchType;
undef $system;
$references = ""; $references = "";
$deriver = ""; $deriver = "";
$hashAlgo = "md5"; $hashAlgo = "md5";
@ -89,8 +81,10 @@ sub readManifest {
if (!$found) { if (!$found) {
push @{$narFileList}, push @{$narFileList},
{ url => $url, hash => $hash, size => $size { url => $url, hash => $hash, size => $size
, narHash => $narHash, references => $references , narHash => $narHash, narSize => $narSize
, references => $references
, deriver => $deriver, hashAlgo => $hashAlgo , deriver => $deriver, hashAlgo => $hashAlgo
, system => $system
}; };
} }
@ -100,8 +94,8 @@ sub readManifest {
addPatch $patches, $storePath, addPatch $patches, $storePath,
{ url => $url, hash => $hash, size => $size { url => $url, hash => $hash, size => $size
, basePath => $basePath, baseHash => $baseHash , basePath => $basePath, baseHash => $baseHash
, narHash => $narHash, patchType => $patchType , narHash => $narHash, narSize => $narSize
, hashAlgo => $hashAlgo , patchType => $patchType, hashAlgo => $hashAlgo
}; };
} }
@ -132,9 +126,11 @@ sub readManifest {
elsif (/^\s*BaseHash:\s*(\S+)\s*$/) { $baseHash = $1; } elsif (/^\s*BaseHash:\s*(\S+)\s*$/) { $baseHash = $1; }
elsif (/^\s*Type:\s*(\S+)\s*$/) { $patchType = $1; } elsif (/^\s*Type:\s*(\S+)\s*$/) { $patchType = $1; }
elsif (/^\s*NarHash:\s*(\S+)\s*$/) { $narHash = $1; } elsif (/^\s*NarHash:\s*(\S+)\s*$/) { $narHash = $1; }
elsif (/^\s*NarSize:\s*(\d+)\s*$/) { $narSize = $1; }
elsif (/^\s*References:\s*(.*)\s*$/) { $references = $1; } elsif (/^\s*References:\s*(.*)\s*$/) { $references = $1; }
elsif (/^\s*Deriver:\s*(\S+)\s*$/) { $deriver = $1; } elsif (/^\s*Deriver:\s*(\S+)\s*$/) { $deriver = $1; }
elsif (/^\s*ManifestVersion:\s*(\d+)\s*$/) { $manifestVersion = $1; } elsif (/^\s*ManifestVersion:\s*(\d+)\s*$/) { $manifestVersion = $1; }
elsif (/^\s*System:\s*(\S+)\s*$/) { $system = $1; }
# Compatibility; # Compatibility;
elsif (/^\s*NarURL:\s*(\S+)\s*$/) { $url = $1; } elsif (/^\s*NarURL:\s*(\S+)\s*$/) { $url = $1; }
@ -150,7 +146,7 @@ sub readManifest {
sub writeManifest { sub writeManifest {
my ($manifest, $narFiles, $patches) = @_; my ($manifest, $narFiles, $patches, $noCompress) = @_;
open MANIFEST, ">$manifest.tmp"; # !!! check exclusive open MANIFEST, ">$manifest.tmp"; # !!! check exclusive
@ -165,12 +161,14 @@ sub writeManifest {
print MANIFEST " StorePath: $storePath\n"; print MANIFEST " StorePath: $storePath\n";
print MANIFEST " NarURL: $narFile->{url}\n"; print MANIFEST " NarURL: $narFile->{url}\n";
print MANIFEST " Hash: $narFile->{hash}\n" if defined $narFile->{hash}; print MANIFEST " Hash: $narFile->{hash}\n" if defined $narFile->{hash};
print MANIFEST " NarHash: $narFile->{narHash}\n";
print MANIFEST " Size: $narFile->{size}\n" if defined $narFile->{size}; print MANIFEST " Size: $narFile->{size}\n" if defined $narFile->{size};
print MANIFEST " NarHash: $narFile->{narHash}\n";
print MANIFEST " NarSize: $narFile->{narSize}\n" if $narFile->{narSize};
print MANIFEST " References: $narFile->{references}\n" print MANIFEST " References: $narFile->{references}\n"
if defined $narFile->{references} && $narFile->{references} ne ""; if defined $narFile->{references} && $narFile->{references} ne "";
print MANIFEST " Deriver: $narFile->{deriver}\n" print MANIFEST " Deriver: $narFile->{deriver}\n"
if defined $narFile->{deriver} && $narFile->{deriver} ne ""; if defined $narFile->{deriver} && $narFile->{deriver} ne "";
print MANIFEST " System: $narFile->{system}\n" if defined $narFile->{system};
print MANIFEST "}\n"; print MANIFEST "}\n";
} }
} }
@ -182,8 +180,9 @@ sub writeManifest {
print MANIFEST " StorePath: $storePath\n"; print MANIFEST " StorePath: $storePath\n";
print MANIFEST " NarURL: $patch->{url}\n"; print MANIFEST " NarURL: $patch->{url}\n";
print MANIFEST " Hash: $patch->{hash}\n"; print MANIFEST " Hash: $patch->{hash}\n";
print MANIFEST " NarHash: $patch->{narHash}\n";
print MANIFEST " Size: $patch->{size}\n"; print MANIFEST " Size: $patch->{size}\n";
print MANIFEST " NarHash: $patch->{narHash}\n";
print MANIFEST " NarSize: $patch->{narSize}\n" if $patch->{narSize};
print MANIFEST " BasePath: $patch->{basePath}\n"; print MANIFEST " BasePath: $patch->{basePath}\n";
print MANIFEST " BaseHash: $patch->{baseHash}\n"; print MANIFEST " BaseHash: $patch->{baseHash}\n";
print MANIFEST " Type: $patch->{patchType}\n"; print MANIFEST " Type: $patch->{patchType}\n";
@ -199,12 +198,14 @@ sub writeManifest {
# Create a bzipped manifest. # Create a bzipped manifest.
unless (defined $noCompress) {
system("@bzip2@ < $manifest > $manifest.bz2.tmp") == 0 system("@bzip2@ < $manifest > $manifest.bz2.tmp") == 0
or die "cannot compress manifest"; or die "cannot compress manifest";
rename("$manifest.bz2.tmp", "$manifest.bz2") rename("$manifest.bz2.tmp", "$manifest.bz2")
or die "cannot rename $manifest.bz2.tmp: $!"; or die "cannot rename $manifest.bz2.tmp: $!";
} }
}
return 1; return 1;

View file

@ -3,6 +3,8 @@ use File::Temp qw(tempdir);
our @sshOpts = split ' ', ($ENV{"NIX_SSHOPTS"} or ""); our @sshOpts = split ' ', ($ENV{"NIX_SSHOPTS"} or "");
push @sshOpts, "-x";
my $sshStarted = 0; my $sshStarted = 0;
my $sshHost; my $sshHost;
@ -24,15 +26,18 @@ sub openSSHConnection {
# child continues to run if we are killed. So instead make SSH # child continues to run if we are killed. So instead make SSH
# print "started" when it has established the connection, and wait # print "started" when it has established the connection, and wait
# until we see that. # until we see that.
open SSH, "ssh $sshHost @sshOpts -M -N -o LocalCommand='echo started' -o PermitLocalCommand=yes |" or die; open SSHPIPE, "ssh $sshHost @sshOpts -M -N -o LocalCommand='echo started' -o PermitLocalCommand=yes |" or die;
while (<SSH>) {
chomp;
last if /started/;
}
while (<SSHPIPE>) {
chomp;
if ($_ eq "started") {
$sshStarted = 1; $sshStarted = 1;
return 1; return 1;
} }
}
return 0;
}
# Tell the master SSH client to exit. # Tell the master SSH client to exit.
sub closeSSHConnection { sub closeSSHConnection {

View file

@ -3,7 +3,8 @@
use Fcntl ':flock'; use Fcntl ':flock';
use English '-no_match_vars'; use English '-no_match_vars';
use IO::Handle; use IO::Handle;
use ssh qw/sshOpts openSSHConnection/; use SSH qw/sshOpts openSSHConnection/;
no warnings('once');
# General operation: # General operation:
@ -31,57 +32,22 @@ $ENV{"DISPLAY"} = "";
$ENV{"SSH_ASKPASS"} = ""; $ENV{"SSH_ASKPASS"} = "";
my $loadIncreased = 0;
my ($amWilling, $localSystem, $neededSystem, $drvPath, $maxSilentTime) = @ARGV;
$maxSilentTime = 0 unless defined $maxSilentTime;
sub sendReply { sub sendReply {
my $reply = shift; my $reply = shift;
print STDERR "# $reply\n"; print STDERR "# $reply\n";
} }
sub decline { sub all { $_ || return 0 for @_; 1 }
sendReply "decline";
exit 0;
} # Initialisation.
my $loadIncreased = 0;
my ($localSystem, $maxSilentTime, $printBuildTrace) = @ARGV;
$maxSilentTime = 0 unless defined $maxSilentTime;
my $currentLoad = $ENV{"NIX_CURRENT_LOAD"}; my $currentLoad = $ENV{"NIX_CURRENT_LOAD"};
decline unless defined $currentLoad;
mkdir $currentLoad, 0777 or die unless -d $currentLoad;
my $conf = $ENV{"NIX_REMOTE_SYSTEMS"}; my $conf = $ENV{"NIX_REMOTE_SYSTEMS"};
decline if !defined $conf || ! -e $conf;
my $canBuildLocally = $amWilling && ($localSystem eq $neededSystem);
# Read the list of machines.
my @machines;
open CONF, "< $conf" or die;
while (<CONF>) {
chomp;
s/\#.*$//g;
next if /^\s*$/;
/^\s*(\S+)\s+(\S+)\s+(\S+)\s+(\d+)(\s+([0-9\.]+))?\s*$/ or die;
push @machines,
{ hostName => $1
, systemTypes => [split(/,/, $2)]
, sshKeys => $3
, maxJobs => $4
, speedFactor => 1.0 * ($6 || 1)
, enabled => 1
};
}
close CONF;
# Acquire the exclusive lock on $currentLoad/main-lock.
my $mainLock = "$currentLoad/main-lock";
open MAINLOCK, ">>$mainLock" or die;
flock(MAINLOCK, LOCK_EX) or die;
sub openSlotLock { sub openSlotLock {
@ -93,17 +59,64 @@ sub openSlotLock {
} }
my $hostName; # Read the list of machines.
my $slotLock; my @machines;
if (defined $conf && -e $conf) {
open CONF, "< $conf" or die;
while (<CONF>) {
chomp;
s/\#.*$//g;
next if /^\s*$/;
my @tokens = split /\s/, $_;
push @machines,
{ hostName => $tokens[0]
, systemTypes => [ split(/,/, $tokens[1]) ]
, sshKeys => $tokens[2]
, maxJobs => int($tokens[3])
, speedFactor => 1.0 * (defined $tokens[4] ? int($tokens[4]) : 1)
, features => [ split(/,/, $tokens[5] || "") ]
, enabled => 1
};
}
close CONF;
}
# Wait for the calling process to ask us whether we can build some derivation.
my ($drvPath, $hostName, $slotLock);
REQ: while (1) {
$_ = <STDIN> || exit 0;
my ($amWilling, $neededSystem);
($amWilling, $neededSystem, $drvPath, $requiredFeatures) = split;
my @requiredFeatures = split /,/, $requiredFeatures;
my $canBuildLocally = $amWilling && ($localSystem eq $neededSystem);
if (!defined $currentLoad) {
sendReply "decline";
next;
}
# Acquire the exclusive lock on $currentLoad/main-lock.
mkdir $currentLoad, 0777 or die unless -d $currentLoad;
my $mainLock = "$currentLoad/main-lock";
open MAINLOCK, ">>$mainLock" or die;
flock(MAINLOCK, LOCK_EX) or die;
while (1) { while (1) {
# Find all machine that can execute this build, i.e., that
# Find all machine that can execute this build, i.e., that support # support builds for the given platform and features, and are
# builds for the given platform and are not at their job limit. # not at their job limit.
my $rightType = 0; my $rightType = 0;
my @available = (); my @available = ();
LOOP: foreach my $cur (@machines) { LOOP: foreach my $cur (@machines) {
if ($cur->{enabled} && grep { $neededSystem eq $_ } @{$cur->{systemTypes}}) { if ($cur->{enabled}
&& (grep { $neededSystem eq $_ } @{$cur->{systemTypes}})
&& all(map { my $f = $_; 0 != grep { $f eq $_ } @{$cur->{features}} } @requiredFeatures))
{
$rightType = 1; $rightType = 1;
# We have a machine of the right type. Determine the load on # We have a machine of the right type. Determine the load on
@ -136,14 +149,15 @@ while (1) {
# Didn't find any available machine? Then decline or postpone. # Didn't find any available machine? Then decline or postpone.
if (scalar @available == 0) { if (scalar @available == 0) {
# Postpone if we have a machine of the right type, except if the # Postpone if we have a machine of the right type, except
# local system can and wants to do the build. # if the local system can and wants to do the build.
if ($rightType && !$canBuildLocally) { if ($rightType && !$canBuildLocally) {
sendReply "postpone"; sendReply "postpone";
exit 0;
} else { } else {
decline; sendReply "decline";
} }
close MAINLOCK;
next REQ;
} }
@ -175,66 +189,81 @@ while (1) {
# Connect to the selected machine. # Connect to the selected machine.
@sshOpts = ("-i", $machine->{sshKeys}, "-x"); @sshOpts = ("-i", $machine->{sshKeys}, "-x");
$hostName = $machine->{hostName}; $hostName = $machine->{hostName};
last if openSSHConnection $hostName; last REQ if openSSHConnection $hostName;
warn "unable to open SSH connection to $hostName, trying other available machines...\n"; warn "unable to open SSH connection to $hostName, trying other available machines...\n";
$machine->{enabled} = 0; $machine->{enabled} = 0;
} }
}
# Tell Nix we've accepted the build. # Tell Nix we've accepted the build.
sendReply "accept"; sendReply "accept";
my $x = <STDIN>; my @inputs = split /\s/, readline(STDIN);
chomp $x; my @outputs = split /\s/, readline(STDIN);
if ($x ne "okay") {
exit 0;
}
# Do the actual build.
print STDERR "building `$drvPath' on `$hostName'\n"; print STDERR "building `$drvPath' on `$hostName'\n";
print STDERR "@ build-remote $drvPath $hostName\n" if $printBuildTrace;
my $inputs = `cat inputs`; die if ($? != 0);
$inputs =~ s/\n/ /g;
my $outputs = `cat outputs`; die if ($? != 0);
$outputs =~ s/\n/ /g;
print "copying inputs...\n";
my $maybeSign = ""; my $maybeSign = "";
$maybeSign = "--sign" if -e "/nix/etc/nix/signing-key.sec"; $maybeSign = "--sign" if -e "/nix/etc/nix/signing-key.sec";
system("NIX_SSHOPTS=\"@sshOpts\" @bindir@/nix-copy-closure $hostName $maybeSign $drvPath $inputs") == 0
# Register the derivation as a temporary GC root. Note that $PPID is
# the PID of the remote SSH process, which, due to the use of a
# persistant SSH connection, should be the same across all remote
# command invocations for this session.
my $rootsDir = "@localstatedir@/nix/gcroots/tmp";
system("ssh $hostName @sshOpts 'mkdir -m 1777 -p $rootsDir; ln -sfn $drvPath $rootsDir/\$PPID.drv'");
sub removeRoots {
system("ssh $hostName @sshOpts 'rm -f $rootsDir/\$PPID.drv $rootsDir/\$PPID.out'");
}
# Copy the derivation and its dependencies to the build machine.
system("NIX_SSHOPTS=\"@sshOpts\" @bindir@/nix-copy-closure $hostName $maybeSign $drvPath @inputs") == 0
or die "cannot copy inputs to $hostName: $?"; or die "cannot copy inputs to $hostName: $?";
print "building...\n";
my $buildFlags = "--max-silent-time $maxSilentTime --fallback"; # Perform the build.
my $buildFlags = "--max-silent-time $maxSilentTime --fallback --add-root $rootsDir/\$PPID.out --option verbosity 0";
# `-tt' forces allocation of a pseudo-terminal. This is required to # We let the remote side kill its process group when the connection is
# make the remote nix-store process receive a signal when the # closed unexpectedly. This is necessary to ensure that no processes
# connection dies. Without it, the remote process might continue to # are left running on the remote system if the local Nix process is
# run indefinitely (that is, until it next tries to write to # killed. (SSH itself doesn't kill child processes if the connection
# stdout/stderr). # is interrupted unless the `-tt' flag is used to force a pseudo-tty,
if (system("ssh $hostName @sshOpts -tt 'nix-store -r $drvPath $buildFlags > /dev/null'") != 0) { # in which case every child receives SIGHUP; however, `-tt' doesn't
# If we couldn't run ssh or there was an ssh problem (indicated by # work on some platforms when connection sharing is used.)
# exit code 255), then we return exit code 1; otherwise we assume pipe STDIN, DUMMY; # make sure we have a readable STDIN
# that the builder failed, which we indicate to Nix using exit if (system("ssh $hostName @sshOpts '(read; kill -INT -\$\$) <&0 & nix-store -r $drvPath $buildFlags > /dev/null' 2>&4") != 0) {
# code 100. It's important to distinguish between the two because # Note that if we get exit code 100 from `nix-store -r', it
# the first is a transient failure and the latter is permanent. # denotes a permanent build failure (as opposed to an SSH problem
my $res = $? == -1 || ($? >> 8) == 255 ? 1 : 100; # or a temporary Nix problem). We propagate this to the caller to
print STDERR "build of `$drvPath' on `$hostName' failed with exit code $?\n"; # allow it to distinguish between transient and permanent
# failures.
my $res = $? >> 8;
print STDERR "build of `$drvPath' on `$hostName' failed with exit code $res\n";
removeRoots;
exit $res; exit $res;
} }
print "build of `$drvPath' on `$hostName' succeeded\n"; #print "build of `$drvPath' on `$hostName' succeeded\n";
foreach my $output (split '\n', $outputs) {
# Copy the output from the build machine.
foreach my $output (@outputs) {
my $maybeSignRemote = ""; my $maybeSignRemote = "";
$maybeSignRemote = "--sign" if $UID != 0; $maybeSignRemote = "--sign" if $UID != 0;
system("ssh $hostName @sshOpts 'nix-store --export $maybeSignRemote $output' | @bindir@/nix-store --import > /dev/null") == 0 system("ssh $hostName @sshOpts 'nix-store --export $maybeSignRemote $output'" .
"| NIX_HELD_LOCKS=$output @bindir@/nix-store --import > /dev/null") == 0
or die "cannot copy $output from $hostName: $?"; or die "cannot copy $output from $hostName: $?";
} }
# Get rid of the temporary GC roots.
removeRoots;

View file

@ -17,25 +17,19 @@ foreach my $dir (@remoteStoresAll) {
} }
$ENV{"NIX_REMOTE"} = "";
sub findStorePath { sub findStorePath {
my $storePath = shift; my $storePath = shift;
my $storePathName = basename $storePath;
foreach my $store (@remoteStores) { foreach my $store (@remoteStores) {
# Determine whether $storePath exists by looking for the my $sourcePath = "$store/store/" . basename $storePath;
# existence of the info file, and if so, get store path info next unless -e $sourcePath || -l $sourcePath;
# from that file. This rather breaks abstraction: we should $ENV{"NIX_DB_DIR"} = "$store/var/nix/db";
# be using `nix-store' for that. But right now there is no return ($store, $sourcePath) if
# good way to tell nix-store to access a store mounted under a system("@bindir@/nix-store --check-validity $storePath") == 0;
# different location (there's $NIX_STORE, but that only works
# if the remote store is mounted under its "real" location).
my $infoFile = "$store/var/nix/db/info/$storePathName";
my $storePath2 = "$store/store/$storePathName";
if (-f $infoFile && -e $storePath2) {
return ($infoFile, $storePath2);
}
} }
return undef;
} }
@ -46,37 +40,38 @@ if ($ARGV[0] eq "--query") {
if ($cmd eq "have") { if ($cmd eq "have") {
my $storePath = <STDIN>; chomp $storePath; my $storePath = <STDIN>; chomp $storePath;
(my $infoFile) = findStorePath $storePath; print STDOUT (defined findStorePath($storePath) ? "1\n" : "0\n");
print STDOUT ($infoFile ? "1\n" : "0\n");
} }
elsif ($cmd eq "info") { elsif ($cmd eq "info") {
my $storePath = <STDIN>; chomp $storePath; my $storePath = <STDIN>; chomp $storePath;
(my $infoFile) = findStorePath $storePath; my ($store, $sourcePath) = findStorePath($storePath);
if (!$infoFile) { if (!defined $store) {
print "0\n"; print "0\n";
next; # not an error next; # not an error
} }
print "1\n"; print "1\n";
my $deriver = ""; $ENV{"NIX_DB_DIR"} = "$store/var/nix/db";
my @references = ();
open INFO, "<$infoFile" or die "cannot read info file $infoFile\n"; my $deriver = `@bindir@/nix-store --query --deriver $storePath`;
while (<INFO>) { die "cannot query deriver of `$storePath'" if $? != 0;
chomp; chomp $deriver;
/^([\w-]+): (.*)$/ or die "bad info file"; $deriver = "" if $deriver eq "unknown-deriver";
my $key = $1;
my $value = $2; my @references = split "\n",
if ($key eq "Deriver") { $deriver = $value; } `@bindir@/nix-store --query --references $storePath`;
elsif ($key eq "References") { @references = split ' ', $value; } die "cannot query references of `$storePath'" if $? != 0;
}
close INFO; my $narSize = `@bindir@/nix-store --query --size $storePath`;
die "cannot query size of `$storePath'" if $? != 0;
chomp $narSize;
print "$deriver\n"; print "$deriver\n";
print scalar @references, "\n"; print scalar @references, "\n";
print "$_\n" foreach @references; print "$_\n" foreach @references;
print "0\n"; # !!! showing size not supported (yet) print "$narSize\n";
print "$narSize\n";
} }
else { die "unknown command `$cmd'"; } else { die "unknown command `$cmd'"; }
@ -87,8 +82,8 @@ if ($ARGV[0] eq "--query") {
elsif ($ARGV[0] eq "--substitute") { elsif ($ARGV[0] eq "--substitute") {
die unless scalar @ARGV == 2; die unless scalar @ARGV == 2;
my $storePath = $ARGV[1]; my $storePath = $ARGV[1];
(my $infoFile, my $sourcePath) = findStorePath $storePath; my ($store, $sourcePath) = findStorePath $storePath;
die unless $infoFile; die unless $store;
print "\n*** Copying `$storePath' from `$sourcePath'\n\n"; print "\n*** Copying `$storePath' from `$sourcePath'\n\n";
system("$binDir/nix-store --dump $sourcePath | $binDir/nix-store --restore $storePath") == 0 system("$binDir/nix-store --dump $sourcePath | $binDir/nix-store --restore $storePath") == 0
or die "cannot copy `$sourcePath' to `$storePath'"; or die "cannot copy `$sourcePath' to `$storePath'";

View file

@ -1,7 +1,7 @@
#! @perl@ -w -I@libexecdir@/nix #! @perl@ -w -I@libexecdir@/nix
use strict; use strict;
use readmanifest; use NixManifest;
use POSIX qw(strftime); use POSIX qw(strftime);
use File::Temp qw(tempdir); use File::Temp qw(tempdir);
@ -12,6 +12,10 @@ STDOUT->autoflush(1);
my $manifestDir = ($ENV{"NIX_MANIFESTS_DIR"} or "@localstatedir@/nix/manifests"); my $manifestDir = ($ENV{"NIX_MANIFESTS_DIR"} or "@localstatedir@/nix/manifests");
my $logFile = "@localstatedir@/log/nix/downloads"; my $logFile = "@localstatedir@/log/nix/downloads";
# For queries, skip expensive calls to nix-hash etc. We're just
# estimating the expected download size.
my $fast = 1;
# Load all manifests. # Load all manifests.
my %narFiles; my %narFiles;
@ -31,6 +35,151 @@ for my $manifest (glob "$manifestDir/*.nixmanifest") {
} }
sub isValidPath {
my $p = shift;
if ($fast) {
return -e $p;
} else {
return system("$binDir/nix-store --check-validity '$p' 2> /dev/null") == 0;
}
}
sub parseHash {
my $hash = shift;
if ($hash =~ /^(.+):(.+)$/) {
return ($1, $2);
} else {
return ("md5", $hash);
}
}
# Compute the most efficient sequence of downloads to produce the
# given path.
sub computeSmallestDownload {
my $targetPath = shift;
# Build a graph of all store paths that might contribute to the
# construction of $targetPath, and the special node "start". The
# edges are either patch operations, or downloads of full NAR
# files. The latter edges only occur between "start" and a store
# path.
my %graph;
$graph{"start"} = {d => 0, pred => undef, edges => []};
my @queue = ();
my $queueFront = 0;
my %done;
sub addNode {
my $graph = shift;
my $u = shift;
$$graph{$u} = {d => 999999999999, pred => undef, edges => []}
unless defined $$graph{$u};
}
sub addEdge {
my $graph = shift;
my $u = shift;
my $v = shift;
my $w = shift;
my $type = shift;
my $info = shift;
addNode $graph, $u;
push @{$$graph{$u}->{edges}},
{weight => $w, start => $u, end => $v, type => $type, info => $info};
my $n = scalar @{$$graph{$u}->{edges}};
}
push @queue, $targetPath;
while ($queueFront < scalar @queue) {
my $u = $queue[$queueFront++];
next if defined $done{$u};
$done{$u} = 1;
addNode \%graph, $u;
# If the path already exists, it has distance 0 from the
# "start" node.
if (isValidPath($u)) {
addEdge \%graph, "start", $u, 0, "present", undef;
}
else {
# Add patch edges.
my $patchList = $patches{$u};
foreach my $patch (@{$patchList}) {
if (isValidPath($patch->{basePath})) {
# !!! this should be cached
my ($baseHashAlgo, $baseHash) = parseHash $patch->{baseHash};
my $format = "--base32";
$format = "" if $baseHashAlgo eq "md5";
my $hash = $fast && $baseHashAlgo eq "sha256"
? `$binDir/nix-store -q --hash "$patch->{basePath}"`
: `$binDir/nix-hash --type '$baseHashAlgo' $format "$patch->{basePath}"`;
chomp $hash;
$hash =~ s/.*://;
next if $hash ne $baseHash;
}
push @queue, $patch->{basePath};
addEdge \%graph, $patch->{basePath}, $u, $patch->{size}, "patch", $patch;
}
# Add NAR file edges to the start node.
my $narFileList = $narFiles{$u};
foreach my $narFile (@{$narFileList}) {
# !!! how to handle files whose size is not known in advance?
# For now, assume some arbitrary size (1 MB).
addEdge \%graph, "start", $u, ($narFile->{size} || 1000000), "narfile", $narFile;
}
}
}
# Run Dijkstra's shortest path algorithm to determine the shortest
# sequence of download and/or patch actions that will produce
# $targetPath.
my @todo = keys %graph;
while (scalar @todo > 0) {
# Remove the closest element from the todo list.
# !!! inefficient, use a priority queue
@todo = sort { -($graph{$a}->{d} <=> $graph{$b}->{d}) } @todo;
my $u = pop @todo;
my $u_ = $graph{$u};
foreach my $edge (@{$u_->{edges}}) {
my $v_ = $graph{$edge->{end}};
if ($v_->{d} > $u_->{d} + $edge->{weight}) {
$v_->{d} = $u_->{d} + $edge->{weight};
# Store the edge; to edge->start is actually the
# predecessor.
$v_->{pred} = $edge;
}
}
}
# Retrieve the shortest path from "start" to $targetPath.
my @path = ();
my $cur = $targetPath;
return () unless defined $graph{$targetPath}->{pred};
while ($cur ne "start") {
push @path, $graph{$cur}->{pred};
$cur = $graph{$cur}->{pred}->{start};
}
return @path;
}
# Parse the arguments. # Parse the arguments.
if ($ARGV[0] eq "--query") { if ($ARGV[0] eq "--query") {
@ -46,6 +195,7 @@ if ($ARGV[0] eq "--query") {
elsif ($cmd eq "info") { elsif ($cmd eq "info") {
my $storePath = <STDIN>; chomp $storePath; my $storePath = <STDIN>; chomp $storePath;
my $info; my $info;
if (defined $narFiles{$storePath}) { if (defined $narFiles{$storePath}) {
$info = @{$narFiles{$storePath}}[0]; $info = @{$narFiles{$storePath}}[0];
@ -57,13 +207,32 @@ if ($ARGV[0] eq "--query") {
print "0\n"; print "0\n";
next; # not an error next; # not an error
} }
print "1\n"; print "1\n";
print "$info->{deriver}\n"; print "$info->{deriver}\n";
my @references = split " ", $info->{references}; my @references = split " ", $info->{references};
print scalar @references, "\n"; print scalar @references, "\n";
print "$_\n" foreach @references; print "$_\n" foreach @references;
my $size = $info->{size} || 0;
print "$size\n"; my @path = computeSmallestDownload $storePath;
my $downloadSize = 0;
while (scalar @path > 0) {
my $edge = pop @path;
my $u = $edge->{start};
my $v = $edge->{end};
if ($edge->{type} eq "patch") {
$downloadSize += $edge->{info}->{size} || 0;
}
elsif ($edge->{type} eq "narfile") {
$downloadSize += $edge->{info}->{size} || 0;
}
}
print "$downloadSize\n";
my $narSize = $info->{narSize} || 0;
print "$narSize\n";
} }
else { die "unknown command `$cmd'"; } else { die "unknown command `$cmd'"; }
@ -79,6 +248,7 @@ elsif ($ARGV[0] ne "--substitute") {
die unless scalar @ARGV == 2; die unless scalar @ARGV == 2;
my $targetPath = $ARGV[1]; my $targetPath = $ARGV[1];
$fast = 0;
# Create a temporary directory. # Create a temporary directory.
@ -110,148 +280,9 @@ foreach my $localPath (@{$localPathList}) {
} }
# Build a graph of all store paths that might contribute to the # Compute the shortest path.
# construction of $targetPath, and the special node "start". The my @path = computeSmallestDownload $targetPath;
# edges are either patch operations, or downloads of full NAR files. die "don't know how to produce $targetPath\n" if scalar @path == 0;
# The latter edges only occur between "start" and a store path.
my %graph;
$graph{"start"} = {d => 0, pred => undef, edges => []};
my @queue = ();
my $queueFront = 0;
my %done;
sub addToQueue {
my $v = shift;
return if defined $done{$v};
$done{$v} = 1;
push @queue, $v;
}
sub addNode {
my $u = shift;
$graph{$u} = {d => 999999999999, pred => undef, edges => []}
unless defined $graph{$u};
}
sub addEdge {
my $u = shift;
my $v = shift;
my $w = shift;
my $type = shift;
my $info = shift;
addNode $u;
push @{$graph{$u}->{edges}},
{weight => $w, start => $u, end => $v, type => $type, info => $info};
my $n = scalar @{$graph{$u}->{edges}};
}
addToQueue $targetPath;
sub isValidPath {
my $p = shift;
return system("$binDir/nix-store --check-validity '$p' 2> /dev/null") == 0;
}
sub parseHash {
my $hash = shift;
if ($hash =~ /^(.+):(.+)$/) {
return ($1, $2);
} else {
return ("md5", $hash);
}
}
while ($queueFront < scalar @queue) {
my $u = $queue[$queueFront++];
# print "$u\n";
addNode $u;
# If the path already exists, it has distance 0 from the "start"
# node.
if (isValidPath($u)) {
addEdge "start", $u, 0, "present", undef;
}
else {
# Add patch edges.
my $patchList = $patches{$u};
foreach my $patch (@{$patchList}) {
if (isValidPath($patch->{basePath})) {
# !!! this should be cached
my ($baseHashAlgo, $baseHash) = parseHash $patch->{baseHash};
my $format = "--base32";
$format = "" if $baseHashAlgo eq "md5";
my $hash = `$binDir/nix-hash --type '$baseHashAlgo' $format "$patch->{basePath}"`;
chomp $hash;
if ($hash ne $baseHash) {
print LOGFILE "$$ rejecting $patch->{basePath}\n";
next;
}
}
addToQueue $patch->{basePath};
addEdge $patch->{basePath}, $u, $patch->{size}, "patch", $patch;
}
# Add NAR file edges to the start node.
my $narFileList = $narFiles{$u};
foreach my $narFile (@{$narFileList}) {
# !!! how to handle files whose size is not known in advance?
# For now, assume some arbitrary size (1 MB).
addEdge "start", $u, ($narFile->{size} || 1000000), "narfile", $narFile;
if ($u eq $targetPath) {
my $size = $narFile->{size} || -1;
print LOGFILE "$$ full-download-would-be $size\n";
}
}
}
}
# Run Dijkstra's shortest path algorithm to determine the shortest
# sequence of download and/or patch actions that will produce
# $targetPath.
sub byDistance { # sort by distance, reversed
return -($graph{$a}->{d} <=> $graph{$b}->{d});
}
my @todo = keys %graph;
while (scalar @todo > 0) {
# Remove the closest element from the todo list.
@todo = sort byDistance @todo;
my $u = pop @todo;
my $u_ = $graph{$u};
foreach my $edge (@{$u_->{edges}}) {
my $v_ = $graph{$edge->{end}};
if ($v_->{d} > $u_->{d} + $edge->{weight}) {
$v_->{d} = $u_->{d} + $edge->{weight};
# Store the edge; to edge->start is actually the
# predecessor.
$v_->{pred} = $edge;
}
}
}
# Retrieve the shortest path from "start" to $targetPath.
my @path = ();
my $cur = $targetPath;
die "don't know how to produce $targetPath\n"
unless defined $graph{$targetPath}->{pred};
while ($cur ne "start") {
push @path, $graph{$cur}->{pred};
$cur = $graph{$cur}->{pred}->{start};
}
# Traverse the shortest path, perform the actions described by the # Traverse the shortest path, perform the actions described by the

View file

@ -1,416 +0,0 @@
#! @perl@ -w -I@libexecdir@/nix
use strict;
use File::Temp qw(tempdir);
use readmanifest;
# Some patch generations options.
# Max size of NAR archives to generate patches for.
my $maxNarSize = $ENV{"NIX_MAX_NAR_SIZE"};
$maxNarSize = 100 * 1024 * 1024 if !defined $maxNarSize;
# If patch is bigger than this fraction of full archive, reject.
my $maxPatchFraction = $ENV{"NIX_PATCH_FRACTION"};
$maxPatchFraction = 0.60 if !defined $maxPatchFraction;
die unless scalar @ARGV == 5;
my $hashAlgo = "sha256";
my $narDir = $ARGV[0];
my $patchesDir = $ARGV[1];
my $patchesURL = $ARGV[2];
my $srcManifest = $ARGV[3];
my $dstManifest = $ARGV[4];
my $tmpDir = tempdir("nix-generate-patches.XXXXXX", CLEANUP => 1, TMPDIR => 1)
or die "cannot create a temporary directory";
print "TEMP = $tmpDir\n";
#END { rmdir $tmpDir; }
my %srcNarFiles;
my %srcLocalPaths;
my %srcPatches;
my %dstNarFiles;
my %dstLocalPaths;
my %dstPatches;
readManifest "$srcManifest",
\%srcNarFiles, \%srcLocalPaths, \%srcPatches;
readManifest "$dstManifest",
\%dstNarFiles, \%dstLocalPaths, \%dstPatches;
sub findOutputPaths {
my $narFiles = shift;
my %outPaths;
foreach my $p (keys %{$narFiles}) {
# Ignore derivations.
next if ($p =~ /\.drv$/);
# Ignore builders (too much ambiguity -- they're all called
# `builder.sh').
next if ($p =~ /\.sh$/);
next if ($p =~ /\.patch$/);
# Don't bother including tar files etc.
next if ($p =~ /\.tar$/ || $p =~ /\.tar\.(gz|bz2|Z|lzma|xz)$/ || $p =~ /\.zip$/ || $p =~ /\.bin$/ || $p =~ /\.tgz$/ || $p =~ /\.rpm$/ || $p =~ /cvs-export$/ || $p =~ /fetchhg$/);
$outPaths{$p} = 1;
}
return %outPaths;
}
print "finding src output paths...\n";
my %srcOutPaths = findOutputPaths \%srcNarFiles;
print "finding dst output paths...\n";
my %dstOutPaths = findOutputPaths \%dstNarFiles;
sub getNameVersion {
my $p = shift;
$p =~ /\/[0-9a-z]+((?:-[a-zA-Z][^\/-]*)+)([^\/]*)$/;
my $name = $1;
my $version = $2;
return undef unless defined $name && defined $version;
$name =~ s/^-//;
$version =~ s/^-//;
return ($name, $version);
}
# A quick hack to get a measure of the `distance' between two
# versions: it's just the position of the first character that differs
# (or 999 if they are the same).
sub versionDiff {
my $s = shift;
my $t = shift;
my $i;
return 999 if $s eq $t;
for ($i = 0; $i < length $s; $i++) {
return $i if $i >= length $t or
substr($s, $i, 1) ne substr($t, $i, 1);
}
return $i;
}
sub getNarBz2 {
my $narFiles = shift;
my $storePath = shift;
my $narFileList = $$narFiles{$storePath};
die "missing path $storePath" unless defined $narFileList;
my $narFile = @{$narFileList}[0];
die unless defined $narFile;
$narFile->{url} =~ /\/([^\/]+)$/;
die unless defined $1;
return "$narDir/$1";
}
sub containsPatch {
my $patches = shift;
my $storePath = shift;
my $basePath = shift;
my $patchList = $$patches{$storePath};
return 0 if !defined $patchList;
my $found = 0;
foreach my $patch (@{$patchList}) {
# !!! baseHash might differ
return 1 if $patch->{basePath} eq $basePath;
}
return 0;
}
# Compute the "weighted" number of uses of a path in the build graph.
sub computeUses {
my $narFiles = shift;
my $path = shift;
# Find the deriver of $path.
return 1 unless defined $$narFiles{$path};
my $deriver = @{$$narFiles{$path}}[0]->{deriver};
return 1 unless defined $deriver && $deriver ne "";
# print " DERIVER $deriver\n";
# Optimisation: build the referrers graph from the references
# graph.
my %referrers;
foreach my $q (keys %{$narFiles}) {
my @refs = split " ", @{$$narFiles{$q}}[0]->{references};
foreach my $r (@refs) {
$referrers{$r} = [] unless defined $referrers{$r};
push @{$referrers{$r}}, $q;
}
}
# Determine the shortest path from $deriver to all other reachable
# paths in the `referrers' graph.
my %dist;
$dist{$deriver} = 0;
my @queue = ($deriver);
my $pos = 0;
while ($pos < scalar @queue) {
my $p = $queue[$pos];
$pos++;
foreach my $q (@{$referrers{$p}}) {
if (!defined $dist{$q}) {
$dist{$q} = $dist{$p} + 1;
# print " $q $dist{$q}\n";
push @queue, $q;
}
}
}
my $wuse = 1.0;
foreach my $user (keys %dist) {
next if $user eq $deriver;
# print " $user $dist{$user}\n";
$wuse += 1.0 / 2.0**$dist{$user};
}
# print " XXX $path $wuse\n";
return $wuse;
}
# For each output path in the destination, see if we need to / can
# create a patch.
print "creating patches...\n";
foreach my $p (keys %dstOutPaths) {
# If exactly the same path already exists in the source, skip it.
next if defined $srcOutPaths{$p};
print " $p\n";
# If not, then we should find the paths in the source that are
# `most' likely to be present on a system that wants to install
# this path.
(my $name, my $version) = getNameVersion $p;
next unless defined $name && defined $version;
my @closest = ();
my $closestVersion;
my $minDist = -1; # actually, larger means closer
# Find all source paths with the same name.
foreach my $q (keys %srcOutPaths) {
(my $name2, my $version2) = getNameVersion $q;
next unless defined $name2 && defined $version2;
if ($name eq $name2) {
# If the sizes differ too much, then skip. This
# disambiguates between, e.g., a real component and a
# wrapper component (cf. Firefox in Nixpkgs).
my $srcSize = @{$srcNarFiles{$q}}[0]->{size};
my $dstSize = @{$dstNarFiles{$p}}[0]->{size};
my $ratio = $srcSize / $dstSize;
$ratio = 1 / $ratio if $ratio < 1;
# print " SIZE $srcSize $dstSize $ratio $q\n";
if ($ratio >= 3) {
print " SKIPPING $q due to size ratio $ratio ($srcSize $dstSize)\n";
next;
}
# If the numbers of weighted uses differ too much, then
# skip. This disambiguates between, e.g., the bootstrap
# GCC and the final GCC in Nixpkgs.
# my $srcUses = computeUses \%srcNarFiles, $q;
# my $dstUses = computeUses \%dstNarFiles, $p;
# $ratio = $srcUses / $dstUses;
# $ratio = 1 / $ratio if $ratio < 1;
# print " USE $srcUses $dstUses $ratio $q\n";
# if ($ratio >= 2) {
# print " SKIPPING $q due to use ratio $ratio ($srcUses $dstUses)\n";
# next;
# }
# If there are multiple matching names, include the ones
# with the closest version numbers.
my $dist = versionDiff $version, $version2;
if ($dist > $minDist) {
$minDist = $dist;
@closest = ($q);
$closestVersion = $version2;
} elsif ($dist == $minDist) {
push @closest, $q;
}
}
}
if (scalar(@closest) == 0) {
print " NO BASE: $p\n";
next;
}
foreach my $closest (@closest) {
# Generate a patch between $closest and $p.
print " $p <- $closest\n";
# If the patch already exists, skip it.
if (containsPatch(\%srcPatches, $p, $closest) ||
containsPatch(\%dstPatches, $p, $closest))
{
print " skipping, already exists\n";
next;
}
# next;
my $srcNarBz2 = getNarBz2 \%srcNarFiles, $closest;
my $dstNarBz2 = getNarBz2 \%dstNarFiles, $p;
if (! -f $srcNarBz2) {
warn "patch source archive $srcNarBz2 is missing\n";
next;
}
system("@bunzip2@ < $srcNarBz2 > $tmpDir/A") == 0
or die "cannot unpack $srcNarBz2";
if ((stat "$tmpDir/A")[7] >= $maxNarSize) {
print " skipping, source is too large\n";
next;
}
system("@bunzip2@ < $dstNarBz2 > $tmpDir/B") == 0
or die "cannot unpack $dstNarBz2";
if ((stat "$tmpDir/B")[7] >= $maxNarSize) {
print " skipping, destination is too large\n";
next;
}
system("@libexecdir@/bsdiff $tmpDir/A $tmpDir/B $tmpDir/DIFF") == 0
or die "cannot compute binary diff";
my $baseHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpDir/A` or die;
chomp $baseHash;
my $narHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpDir/B` or die;
chomp $narHash;
my $narDiffHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpDir/DIFF` or die;
chomp $narDiffHash;
my $narDiffSize = (stat "$tmpDir/DIFF")[7];
my $dstNarBz2Size = (stat $dstNarBz2)[7];
print " size $narDiffSize; full size $dstNarBz2Size\n";
if ($narDiffSize >= $dstNarBz2Size) {
print " rejecting; patch bigger than full archive\n";
next;
}
if ($narDiffSize / $dstNarBz2Size >= $maxPatchFraction) {
print " rejecting; patch too large relative to full archive\n";
next;
}
my $finalName =
"$narDiffHash.nar-bsdiff";
if (-e "$patchesDir/$finalName") {
print " not copying, already exists\n";
}
else {
system("cp '$tmpDir/DIFF' '$patchesDir/$finalName.tmp'") == 0
or die "cannot copy diff";
rename("$patchesDir/$finalName.tmp", "$patchesDir/$finalName")
or die "cannot rename $patchesDir/$finalName.tmp";
}
# Add the patch to the manifest.
addPatch \%dstPatches, $p,
{ url => "$patchesURL/$finalName", hash => "$hashAlgo:$narDiffHash"
, size => $narDiffSize, basePath => $closest, baseHash => "$hashAlgo:$baseHash"
, narHash => "$hashAlgo:$narHash", patchType => "nar-bsdiff"
}, 0;
}
}
# Add in any potentially useful patches in the source (namely, those
# patches that produce either paths in the destination or paths that
# can be used as the base for other useful patches).
print "propagating patches...\n";
my $changed;
do {
# !!! we repeat this to reach the transitive closure; inefficient
$changed = 0;
print "loop\n";
my %dstBasePaths;
foreach my $q (keys %dstPatches) {
foreach my $patch (@{$dstPatches{$q}}) {
$dstBasePaths{$patch->{basePath}} = 1;
}
}
foreach my $p (keys %srcPatches) {
my $patchList = $srcPatches{$p};
my $include = 0;
# Is path $p included in the destination? If so, include
# patches that produce it.
$include = 1 if defined $dstNarFiles{$p};
# Is path $p a path that serves as a base for paths in the
# destination? If so, include patches that produce it.
# !!! check baseHash
$include = 1 if defined $dstBasePaths{$p};
if ($include) {
foreach my $patch (@{$patchList}) {
$changed = 1 if addPatch \%dstPatches, $p, $patch;
}
}
}
} while $changed;
# Rewrite the manifest of the destination (with the new patches).
writeManifest "${dstManifest}",
\%dstNarFiles, \%dstPatches;

View file

@ -123,6 +123,11 @@ EOF
$verbose = 1; $verbose = 1;
} }
elsif ($arg eq "--quiet") {
push @buildArgs, $arg;
push @instArgs, $arg;
}
elsif (substr($arg, 0, 1) eq "-") { elsif (substr($arg, 0, 1) eq "-") {
push @buildArgs, $arg; push @buildArgs, $arg;
} }
@ -165,7 +170,7 @@ foreach my $expr (@exprs) {
# Build. # Build.
my @outPaths; my @outPaths;
$pid = open(OUTPATHS, "-|") || exec "$binDir/nix-store", "--add-root", $outLink, "--indirect", "-rv", $pid = open(OUTPATHS, "-|") || exec "$binDir/nix-store", "--add-root", $outLink, "--indirect", "-r",
@buildArgs, @drvPaths; @buildArgs, @drvPaths;
while (<OUTPATHS>) {chomp; push @outPaths, $_;} while (<OUTPATHS>) {chomp; push @outPaths, $_;}
if (!close OUTPATHS) { if (!close OUTPATHS) {

View file

@ -1,6 +1,6 @@
#! @perl@ -w -I@libexecdir@/nix #! @perl@ -w -I@libexecdir@/nix
use ssh; use SSH;
my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@"; my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
@ -61,7 +61,7 @@ if ($toMode) { # Copy TO the remote machine.
my @allStorePaths; my @allStorePaths;
# Get the closure of this path. # Get the closure of this path.
my $pid = open(READ, "$binDir/nix-store --query --requisites @storePaths|") or die; my $pid = open(READ, "set -f; $binDir/nix-store --query --requisites @storePaths|") or die;
while (<READ>) { while (<READ>) {
chomp; chomp;
@ -73,7 +73,7 @@ if ($toMode) { # Copy TO the remote machine.
# Ask the remote host which paths are invalid. # Ask the remote host which paths are invalid.
open(READ, "ssh $sshHost @sshOpts nix-store --check-validity --print-invalid @allStorePaths|"); open(READ, "set -f; ssh $sshHost @sshOpts nix-store --check-validity --print-invalid @allStorePaths|");
my @missing = (); my @missing = ();
while (<READ>) { while (<READ>) {
chomp; chomp;
@ -88,7 +88,7 @@ if ($toMode) { # Copy TO the remote machine.
print STDERR " $_\n" foreach @missing; print STDERR " $_\n" foreach @missing;
my $extraOpts = ""; my $extraOpts = "";
$extraOpts .= "--sign" if $sign == 1; $extraOpts .= "--sign" if $sign == 1;
system("nix-store --export $extraOpts @missing $compressor | ssh $sshHost @sshOpts '$decompressor nix-store --import'") == 0 system("set -f; nix-store --export $extraOpts @missing $compressor | ssh $sshHost @sshOpts '$decompressor nix-store --import'") == 0
or die "copying store paths to remote machine `$sshHost' failed: $?"; or die "copying store paths to remote machine `$sshHost' failed: $?";
} }
@ -101,7 +101,7 @@ else { # Copy FROM the remote machine.
# machine. Paths are assumed to be store paths; there is no # machine. Paths are assumed to be store paths; there is no
# resolution (following of symlinks). # resolution (following of symlinks).
my $pid = open(READ, my $pid = open(READ,
"ssh @sshOpts $sshHost nix-store --query --requisites @storePaths|") or die; "set -f; ssh @sshOpts $sshHost nix-store --query --requisites @storePaths|") or die;
my @allStorePaths; my @allStorePaths;
@ -115,7 +115,7 @@ else { # Copy FROM the remote machine.
# What paths are already valid locally? # What paths are already valid locally?
open(READ, "@bindir@/nix-store --check-validity --print-invalid @allStorePaths|"); open(READ, "set -f; @bindir@/nix-store --check-validity --print-invalid @allStorePaths|");
my @missing = (); my @missing = ();
while (<READ>) { while (<READ>) {
chomp; chomp;
@ -130,7 +130,7 @@ else { # Copy FROM the remote machine.
print STDERR " $_\n" foreach @missing; print STDERR " $_\n" foreach @missing;
my $extraOpts = ""; my $extraOpts = "";
$extraOpts .= "--sign" if $sign == 1; $extraOpts .= "--sign" if $sign == 1;
system("ssh $sshHost @sshOpts 'nix-store --export $extraOpts @missing $compressor' | $decompressor @bindir@/nix-store --import") == 0 system("set -f; ssh $sshHost @sshOpts 'nix-store --export $extraOpts @missing $compressor' | $decompressor @bindir@/nix-store --import") == 0
or die "copying store paths from remote machine `$sshHost' failed: $?"; or die "copying store paths from remote machine `$sshHost' failed: $?";
} }

View file

@ -0,0 +1,42 @@
#! @perl@ -w -I@libexecdir@/nix
use strict;
use File::Temp qw(tempdir);
use NixManifest;
use GeneratePatches;
if (scalar @ARGV != 5) {
print STDERR <<EOF;
Usage: nix-generate-patches NAR-DIR PATCH-DIR PATCH-URI OLD-MANIFEST NEW-MANIFEST
This command generates binary patches between NAR files listed in
OLD-MANIFEST and NEW-MANIFEST. The patches are written to the
directory PATCH-DIR, and the prefix PATCH-URI is used to generate URIs
for the patches. The patches are added to NEW-MANIFEST. All NARs are
required to exist in NAR-DIR. Patches are generated between
succeeding versions of packages with the same name.
EOF
exit 1;
}
my $narPath = $ARGV[0];
my $patchesPath = $ARGV[1];
my $patchesURL = $ARGV[2];
my $srcManifest = $ARGV[3];
my $dstManifest = $ARGV[4];
my (%srcNarFiles, %srcLocalPaths, %srcPatches);
readManifest $srcManifest, \%srcNarFiles, \%srcLocalPaths, \%srcPatches;
my (%dstNarFiles, %dstLocalPaths, %dstPatches);
readManifest $dstManifest, \%dstNarFiles, \%dstLocalPaths, \%dstPatches;
my $tmpDir = tempdir("nix-generate-patches.XXXXXX", CLEANUP => 1, TMPDIR => 1)
or die "cannot create a temporary directory";
generatePatches \%srcNarFiles, \%dstNarFiles, \%srcPatches, \%dstPatches,
$narPath, $patchesPath, $patchesURL, $tmpDir;
propagatePatches \%srcPatches, \%dstNarFiles, \%dstPatches;
writeManifest $dstManifest, \%dstNarFiles, \%dstPatches;

View file

@ -2,7 +2,7 @@
use strict; use strict;
use File::Temp qw(tempdir); use File::Temp qw(tempdir);
use readmanifest; use NixManifest;
my $tmpDir = tempdir("nix-pull.XXXXXX", CLEANUP => 1, TMPDIR => 1) my $tmpDir = tempdir("nix-pull.XXXXXX", CLEANUP => 1, TMPDIR => 1)
or die "cannot create a temporary directory"; or die "cannot create a temporary directory";

View file

@ -2,7 +2,7 @@
use strict; use strict;
use File::Temp qw(tempdir); use File::Temp qw(tempdir);
use readmanifest; use NixManifest;
my $hashAlgo = "sha256"; my $hashAlgo = "sha256";
@ -172,12 +172,6 @@ for (my $n = 0; $n < scalar @storePaths; $n++) {
$narbz2Hash =~ /^[0-9a-z]+$/ or die "invalid hash"; $narbz2Hash =~ /^[0-9a-z]+$/ or die "invalid hash";
close HASH; close HASH;
open HASH, "$narDir/nar-hash" or die "cannot open nar-hash";
my $narHash = <HASH>;
chomp $narHash;
$narHash =~ /^[0-9a-z]+$/ or die "invalid hash";
close HASH;
my $narName = "$narbz2Hash.nar.bz2"; my $narName = "$narbz2Hash.nar.bz2";
my $narFile = "$narDir/$narName"; my $narFile = "$narDir/$narName";
@ -195,6 +189,14 @@ for (my $n = 0; $n < scalar @storePaths; $n++) {
chomp $deriver; chomp $deriver;
$deriver = "" if $deriver eq "unknown-deriver"; $deriver = "" if $deriver eq "unknown-deriver";
my $narHash = `$binDir/nix-store --query --hash '$storePath'`;
die "cannot query hash for `$storePath'" if $? != 0;
chomp $narHash;
my $narSize = `$binDir/nix-store --query --size '$storePath'`;
die "cannot query size for `$storePath'" if $? != 0;
chomp $narSize;
my $url; my $url;
if ($localCopy) { if ($localCopy) {
$url = "$targetArchivesUrl/$narName"; $url = "$targetArchivesUrl/$narName";
@ -205,7 +207,8 @@ for (my $n = 0; $n < scalar @storePaths; $n++) {
{ url => $url { url => $url
, hash => "$hashAlgo:$narbz2Hash" , hash => "$hashAlgo:$narbz2Hash"
, size => $narbz2Size , size => $narbz2Size
, narHash => "$hashAlgo:$narHash" , narHash => "$narHash"
, narSize => $narSize
, references => $references , references => $references
, deriver => $deriver , deriver => $deriver
} }

View file

@ -18,6 +18,6 @@ int main(int argc, char * * argv)
while ((c = getchar()) != EOF) { while ((c = getchar()) != EOF) {
print("0x%02x, ", (unsigned char) c); print("0x%02x, ", (unsigned char) c);
} }
print("};\n"); print("0 };\n");
return 0; return 0;
} }

View file

@ -277,6 +277,7 @@ int main(int argc,char *argv[])
for(scsc=scan+=len;scan<newsize;scan++) { for(scsc=scan+=len;scan<newsize;scan++) {
len=search(I,old,oldsize,new+scan,newsize-scan, len=search(I,old,oldsize,new+scan,newsize-scan,
0,oldsize,&pos); 0,oldsize,&pos);
if (len > 64 * 1024) break;
for(;scsc<scan+len;scsc++) for(;scsc<scan+len;scsc++)
if((scsc+lastoffset<oldsize) && if((scsc+lastoffset<oldsize) &&

View file

@ -965,7 +965,7 @@ static void prim_substring(EvalState & state, Value * * args, Value & v)
if (start < 0) throw EvalError("negative start position in `substring'"); if (start < 0) throw EvalError("negative start position in `substring'");
mkString(v, string(s, start, len), context); mkString(v, start >= s.size() ? "" : string(s, start, len), context);
} }

View file

@ -54,25 +54,26 @@ void printGCWarning()
void printMissing(const PathSet & paths) void printMissing(const PathSet & paths)
{ {
unsigned long long downloadSize; unsigned long long downloadSize, narSize;
PathSet willBuild, willSubstitute, unknown; PathSet willBuild, willSubstitute, unknown;
queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize); queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
if (!willBuild.empty()) { if (!willBuild.empty()) {
printMsg(lvlInfo, format("the following derivations will be built:")); printMsg(lvlInfo, format("these derivations will be built:"));
foreach (PathSet::iterator, i, willBuild) foreach (PathSet::iterator, i, willBuild)
printMsg(lvlInfo, format(" %1%") % *i); printMsg(lvlInfo, format(" %1%") % *i);
} }
if (!willSubstitute.empty()) { if (!willSubstitute.empty()) {
printMsg(lvlInfo, format("the following paths will be downloaded/copied (%.2f MiB):") % printMsg(lvlInfo, format("these paths will be downloaded/copied (%.2f MiB download, %.2f MiB unpacked):")
(downloadSize / (1024.0 * 1024.0))); % (downloadSize / (1024.0 * 1024.0))
% (narSize / (1024.0 * 1024.0)));
foreach (PathSet::iterator, i, willSubstitute) foreach (PathSet::iterator, i, willSubstitute)
printMsg(lvlInfo, format(" %1%") % *i); printMsg(lvlInfo, format(" %1%") % *i);
} }
if (!unknown.empty()) { if (!unknown.empty()) {
printMsg(lvlInfo, format("don't know how to build the following paths%1%:") printMsg(lvlInfo, format("don't know how to build these paths%1%:")
% (readOnlyMode ? " (may be caused by read-only store access)" : "")); % (readOnlyMode ? " (may be caused by read-only store access)" : ""));
foreach (PathSet::iterator, i, unknown) foreach (PathSet::iterator, i, unknown)
printMsg(lvlInfo, format(" %1%") % *i); printMsg(lvlInfo, format(" %1%") % *i);
@ -200,17 +201,16 @@ static void initAndRun(int argc, char * * argv)
remaining.clear(); remaining.clear();
/* Process default options. */ /* Process default options. */
int verbosityDelta = 0;
for (Strings::iterator i = args.begin(); i != args.end(); ++i) { for (Strings::iterator i = args.begin(); i != args.end(); ++i) {
string arg = *i; string arg = *i;
if (arg == "--verbose" || arg == "-v") if (arg == "--verbose" || arg == "-v") verbosityDelta++;
verbosity = (Verbosity) ((int) verbosity + 1); else if (arg == "--quiet") verbosityDelta--;
else if (arg == "--log-type") { else if (arg == "--log-type") {
++i; ++i;
if (i == args.end()) throw UsageError("`--log-type' requires an argument"); if (i == args.end()) throw UsageError("`--log-type' requires an argument");
setLogType(*i); setLogType(*i);
} }
else if (arg == "--build-output" || arg == "-B")
; /* !!! obsolete - remove eventually */
else if (arg == "--no-build-output" || arg == "-Q") else if (arg == "--no-build-output" || arg == "-Q")
buildVerbosity = lvlVomit; buildVerbosity = lvlVomit;
else if (arg == "--print-build-trace") else if (arg == "--print-build-trace")
@ -251,6 +251,9 @@ static void initAndRun(int argc, char * * argv)
else remaining.push_back(arg); else remaining.push_back(arg);
} }
verbosityDelta += queryIntSetting("verbosity", lvlInfo);
verbosity = (Verbosity) (verbosityDelta < 0 ? 0 : verbosityDelta);
/* Automatically clean up the temporary roots file when we /* Automatically clean up the temporary roots file when we
exit. */ exit. */
RemoveTempRoots removeTempRoots __attribute__((unused)); RemoveTempRoots removeTempRoots __attribute__((unused));
@ -390,7 +393,7 @@ int main(int argc, char * * argv)
printMsg(lvlError, format("error: %1%%2%") % (showTrace ? e.prefix() : "") % e.msg()); printMsg(lvlError, format("error: %1%%2%") % (showTrace ? e.prefix() : "") % e.msg());
if (e.prefix() != "" && !showTrace) if (e.prefix() != "" && !showTrace)
printMsg(lvlError, "(use `--show-trace' to show detailed location information)"); printMsg(lvlError, "(use `--show-trace' to show detailed location information)");
return 1; return e.status;
} catch (std::exception & e) { } catch (std::exception & e) {
printMsg(lvlError, format("error: %1%") % e.what()); printMsg(lvlError, format("error: %1%") % e.what());
return 1; return 1;

View file

@ -1,7 +1,7 @@
#ifndef __SHARED_H #ifndef __SHARED_H
#define __SHARED_H #define __SHARED_H
#include "types.hh" #include "util.hh"
#include <signal.h> #include <signal.h>

View file

@ -10,7 +10,14 @@ pkginclude_HEADERS = \
globals.hh references.hh pathlocks.hh \ globals.hh references.hh pathlocks.hh \
worker-protocol.hh worker-protocol.hh
libstore_la_LIBADD = ../libutil/libutil.la ../boost/format/libformat.la @ADDITIONAL_NETWORK_LIBS@ libstore_la_LIBADD = ../libutil/libutil.la ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib}
EXTRA_DIST = schema.sql
AM_CXXFLAGS = -Wall \ AM_CXXFLAGS = -Wall \
-I$(srcdir)/.. -I$(srcdir)/../libutil ${sqlite_include} -I$(srcdir)/.. -I$(srcdir)/../libutil
local-store.lo: schema.sql.hh
%.sql.hh: %.sql
../bin2c/bin2c schema < $< > $@ || (rm $@ && exit 1)

View file

@ -64,6 +64,7 @@ static const uid_t rootUserId = 0;
/* Forward definition. */ /* Forward definition. */
class Worker; class Worker;
class HookInstance;
/* A pointer to a goal. */ /* A pointer to a goal. */
@ -213,8 +214,14 @@ public:
bool cacheFailure; bool cacheFailure;
/* Set if at least one derivation had a BuildError (i.e. permanent
failure). */
bool permanentFailure;
LocalStore & store; LocalStore & store;
boost::shared_ptr<HookInstance> hook;
Worker(LocalStore & store); Worker(LocalStore & store);
~Worker(); ~Worker();
@ -264,6 +271,7 @@ public:
/* Wait for input to become available. */ /* Wait for input to become available. */
void waitForInput(); void waitForInput();
unsigned int exitStatus();
}; };
@ -615,6 +623,107 @@ void deletePathWrapped(const Path & path)
////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////
struct HookInstance
{
/* Pipes for talking to the build hook. */
Pipe toHook;
/* Pipe for the hook's standard output/error. */
Pipe fromHook;
/* Pipe for the builder's standard output/error. */
Pipe builderOut;
/* The process ID of the hook. */
Pid pid;
HookInstance();
~HookInstance();
};
HookInstance::HookInstance()
{
debug("starting build hook");
Path buildHook = absPath(getEnv("NIX_BUILD_HOOK"));
/* Create a pipe to get the output of the child. */
fromHook.create();
/* Create the communication pipes. */
toHook.create();
/* Create a pipe to get the output of the builder. */
builderOut.create();
/* Fork the hook. */
pid = fork();
switch (pid) {
case -1:
throw SysError("unable to fork");
case 0:
try { /* child */
commonChildInit(fromHook);
if (chdir("/") == -1) throw SysError("changing into `/");
/* Dup the communication pipes. */
toHook.writeSide.close();
if (dup2(toHook.readSide, STDIN_FILENO) == -1)
throw SysError("dupping to-hook read side");
/* Use fd 4 for the builder's stdout/stderr. */
builderOut.readSide.close();
if (dup2(builderOut.writeSide, 4) == -1)
throw SysError("dupping builder's stdout/stderr");
execl(buildHook.c_str(), buildHook.c_str(), thisSystem.c_str(),
(format("%1%") % maxSilentTime).str().c_str(),
(format("%1%") % printBuildTrace).str().c_str(),
NULL);
throw SysError(format("executing `%1%'") % buildHook);
} catch (std::exception & e) {
std::cerr << format("build hook error: %1%") % e.what() << std::endl;
}
quickExit(1);
}
/* parent */
pid.setSeparatePG(true);
pid.setKillSignal(SIGTERM);
fromHook.writeSide.close();
toHook.readSide.close();
}
HookInstance::~HookInstance()
{
try {
/* Cleanly shut down the hook by closing its stdin if it's not
already building. Otherwise pid's destructor will kill
it. */
if (pid != -1 && toHook.writeSide != -1) {
toHook.writeSide.close();
pid.wait(true);
}
} catch (...) {
ignoreException();
}
}
//////////////////////////////////////////////////////////////////////
typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
class DerivationGoal : public Goal class DerivationGoal : public Goal
{ {
private: private:
@ -649,13 +758,10 @@ private:
AutoCloseFD fdLogFile; AutoCloseFD fdLogFile;
/* Pipe for the builder's standard output/error. */ /* Pipe for the builder's standard output/error. */
Pipe logPipe; Pipe builderOut;
/* Whether we're building using a build hook. */ /* The build hook. */
bool usingBuildHook; boost::shared_ptr<HookInstance> hook;
/* Pipes for talking to the build hook (if any). */
Pipe toHook;
/* Whether we're currently doing a chroot build. */ /* Whether we're currently doing a chroot build. */
bool useChroot; bool useChroot;
@ -694,12 +800,8 @@ private:
void buildDone(); void buildDone();
/* Is the build hook willing to perform the build? */ /* Is the build hook willing to perform the build? */
typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
HookReply tryBuildHook(); HookReply tryBuildHook();
/* Synchronously wait for a build hook to finish. */
void terminateBuildHook(bool kill = false);
/* Start building a derivation. */ /* Start building a derivation. */
void startBuilder(); void startBuilder();
@ -711,10 +813,6 @@ private:
/* Open a log file and a pipe to it. */ /* Open a log file and a pipe to it. */
Path openLogFile(); Path openLogFile();
/* Common initialisation to be performed in child processes (i.e.,
both in builders and in build hooks). */
void initChild();
/* Delete the temporary directory, if we have one. */ /* Delete the temporary directory, if we have one. */
void deleteTmpDir(bool force); void deleteTmpDir(bool force);
@ -742,6 +840,7 @@ DerivationGoal::DerivationGoal(const Path & drvPath, Worker & worker)
trace("created"); trace("created");
} }
DerivationGoal::~DerivationGoal() DerivationGoal::~DerivationGoal()
{ {
/* Careful: we should never ever throw an exception from a /* Careful: we should never ever throw an exception from a
@ -754,6 +853,7 @@ DerivationGoal::~DerivationGoal()
} }
} }
void DerivationGoal::killChild() void DerivationGoal::killChild()
{ {
if (pid != -1) { if (pid != -1) {
@ -778,6 +878,8 @@ void DerivationGoal::killChild()
assert(pid == -1); assert(pid == -1);
} }
hook.reset();
} }
@ -887,6 +989,9 @@ void DerivationGoal::outputsSubstituted()
foreach (PathSet::iterator, i, drv.inputSrcs) foreach (PathSet::iterator, i, drv.inputSrcs)
addWaitee(worker.makeSubstitutionGoal(*i)); addWaitee(worker.makeSubstitutionGoal(*i));
if (waitees.empty()) /* to prevent hang (no wake-up event) */
inputsRealised();
else
state = &DerivationGoal::inputsRealised; state = &DerivationGoal::inputsRealised;
} }
@ -961,6 +1066,16 @@ PathSet outputPaths(const DerivationOutputs & outputs)
} }
static bool canBuildLocally(const string & platform)
{
return platform == thisSystem
#ifdef CAN_DO_LINUX32_BUILDS
|| (platform == "i686-linux" && thisSystem == "x86_64-linux")
#endif
;
}
void DerivationGoal::tryToBuild() void DerivationGoal::tryToBuild()
{ {
trace("trying to build"); trace("trying to build");
@ -1028,16 +1143,22 @@ void DerivationGoal::tryToBuild()
foreach (DerivationOutputs::iterator, i, drv.outputs) foreach (DerivationOutputs::iterator, i, drv.outputs)
if (pathFailed(i->second.path)) return; if (pathFailed(i->second.path)) return;
/* Don't do a remote build if the derivation has the attribute
`preferLocalBuild' set. */
bool preferLocalBuild =
drv.env["preferLocalBuild"] == "1" && canBuildLocally(drv.platform);
/* Is the build hook willing to accept this job? */ /* Is the build hook willing to accept this job? */
usingBuildHook = true; if (!preferLocalBuild) {
switch (tryBuildHook()) { switch (tryBuildHook()) {
case rpAccept: case rpAccept:
/* Yes, it has started doing so. Wait until we get EOF /* Yes, it has started doing so. Wait until we get
from the hook. */ EOF from the hook. */
state = &DerivationGoal::buildDone; state = &DerivationGoal::buildDone;
return; return;
case rpPostpone: case rpPostpone:
/* Not now; wait until at least one child finishes. */ /* Not now; wait until at least one child finishes or
the wake-up timeout expires. */
worker.waitForAWhile(shared_from_this()); worker.waitForAWhile(shared_from_this());
outputLocks.unlock(); outputLocks.unlock();
return; return;
@ -1045,11 +1166,13 @@ void DerivationGoal::tryToBuild()
/* We should do it ourselves. */ /* We should do it ourselves. */
break; break;
} }
}
usingBuildHook = false; /* Make sure that we are allowed to start a build. If this
derivation prefers to be done locally, do it even if
/* Make sure that we are allowed to start a build. */ maxBuildJobs is 0. */
if (worker.getNrLocalBuilds() >= maxBuildJobs) { unsigned int curBuilds = worker.getNrLocalBuilds();
if (curBuilds >= maxBuildJobs && !(preferLocalBuild && curBuilds == 0)) {
worker.waitForBuildSlot(shared_from_this()); worker.waitForBuildSlot(shared_from_this());
outputLocks.unlock(); outputLocks.unlock();
return; return;
@ -1067,6 +1190,7 @@ void DerivationGoal::tryToBuild()
if (printBuildTrace) if (printBuildTrace)
printMsg(lvlError, format("@ build-failed %1% %2% %3% %4%") printMsg(lvlError, format("@ build-failed %1% %2% %3% %4%")
% drvPath % drv.outputs["out"].path % 0 % e.msg()); % drvPath % drv.outputs["out"].path % 0 % e.msg());
worker.permanentFailure = true;
amDone(ecFailed); amDone(ecFailed);
return; return;
} }
@ -1085,10 +1209,17 @@ void DerivationGoal::buildDone()
to have terminated. In fact, the builder could also have to have terminated. In fact, the builder could also have
simply have closed its end of the pipe --- just don't do that simply have closed its end of the pipe --- just don't do that
:-) */ :-) */
int status;
pid_t savedPid;
if (hook) {
savedPid = hook->pid;
status = hook->pid.wait(true);
} else {
/* !!! this could block! security problem! solution: kill the /* !!! this could block! security problem! solution: kill the
child */ child */
pid_t savedPid = pid; savedPid = pid;
int status = pid.wait(true); status = pid.wait(true);
}
debug(format("builder process for `%1%' finished") % drvPath); debug(format("builder process for `%1%' finished") % drvPath);
@ -1096,7 +1227,11 @@ void DerivationGoal::buildDone()
worker.childTerminated(savedPid); worker.childTerminated(savedPid);
/* Close the read side of the logger pipe. */ /* Close the read side of the logger pipe. */
logPipe.readSide.close(); if (hook) {
hook->builderOut.readSide.close();
hook->fromHook.readSide.close();
}
else builderOut.readSide.close();
/* Close the log file. */ /* Close the log file. */
fdLogFile.close(); fdLogFile.close();
@ -1169,11 +1304,11 @@ void DerivationGoal::buildDone()
/* When using a build hook, the hook will return a remote /* When using a build hook, the hook will return a remote
build failure using exit code 100. Anything else is a hook build failure using exit code 100. Anything else is a hook
problem. */ problem. */
bool hookError = usingBuildHook && bool hookError = hook &&
(!WIFEXITED(status) || WEXITSTATUS(status) != 100); (!WIFEXITED(status) || WEXITSTATUS(status) != 100);
if (printBuildTrace) { if (printBuildTrace) {
if (usingBuildHook && hookError) if (hook && hookError)
printMsg(lvlError, format("@ hook-failed %1% %2% %3% %4%") printMsg(lvlError, format("@ hook-failed %1% %2% %3% %4%")
% drvPath % drv.outputs["out"].path % status % e.msg()); % drvPath % drv.outputs["out"].path % status % e.msg());
else else
@ -1192,6 +1327,7 @@ void DerivationGoal::buildDone()
foreach (DerivationOutputs::iterator, i, drv.outputs) foreach (DerivationOutputs::iterator, i, drv.outputs)
worker.store.registerFailedPath(i->second.path); worker.store.registerFailedPath(i->second.path);
worker.permanentFailure = !hookError && !fixedOutput;
amDone(ecFailed); amDone(ecFailed);
return; return;
} }
@ -1208,135 +1344,79 @@ void DerivationGoal::buildDone()
} }
DerivationGoal::HookReply DerivationGoal::tryBuildHook() HookReply DerivationGoal::tryBuildHook()
{ {
if (!useBuildHook) return rpDecline; if (!useBuildHook || getEnv("NIX_BUILD_HOOK") == "") return rpDecline;
Path buildHook = getEnv("NIX_BUILD_HOOK");
if (buildHook == "") return rpDecline;
buildHook = absPath(buildHook);
/* Create a directory where we will store files used for if (!worker.hook)
communication between us and the build hook. */ worker.hook = boost::shared_ptr<HookInstance>(new HookInstance);
tmpDir = createTempDir();
/* Create the log file and pipe. */ /* Tell the hook about system features (beyond the system type)
Path logFile = openLogFile(); required from the build machine. (The hook could parse the
drv file itself, but this is easier.) */
Strings features = tokenizeString(drv.env["requiredSystemFeatures"]);
foreach (Strings::iterator, i, features) checkStoreName(*i); /* !!! abuse */
/* Create the communication pipes. */ /* Send the request to the hook. */
toHook.create(); writeLine(worker.hook->toHook.writeSide, (format("%1% %2% %3% %4%")
% (worker.getNrLocalBuilds() < maxBuildJobs ? "1" : "0")
/* Fork the hook. */ % drv.platform % drvPath % concatStringsSep(",", features)).str());
pid = fork();
switch (pid) {
case -1:
throw SysError("unable to fork");
case 0:
try { /* child */
initChild();
string s;
foreach (DerivationOutputs::const_iterator, i, drv.outputs)
s += i->second.path + " ";
if (setenv("NIX_HELD_LOCKS", s.c_str(), 1))
throw SysError("setting an environment variable");
execl(buildHook.c_str(), buildHook.c_str(),
(worker.getNrLocalBuilds() < maxBuildJobs ? (string) "1" : "0").c_str(),
thisSystem.c_str(),
drv.platform.c_str(),
drvPath.c_str(),
(format("%1%") % maxSilentTime).str().c_str(),
NULL);
throw SysError(format("executing `%1%'") % buildHook);
} catch (std::exception & e) {
std::cerr << format("build hook error: %1%") % e.what() << std::endl;
}
quickExit(1);
}
/* parent */
pid.setSeparatePG(true);
pid.setKillSignal(SIGTERM);
logPipe.writeSide.close();
worker.childStarted(shared_from_this(),
pid, singleton<set<int> >(logPipe.readSide), false, false);
toHook.readSide.close();
/* Read the first line of input, which should be a word indicating /* Read the first line of input, which should be a word indicating
whether the hook wishes to perform the build. */ whether the hook wishes to perform the build. */
string reply; string reply;
try {
while (true) { while (true) {
string s = readLine(logPipe.readSide); string s = readLine(worker.hook->fromHook.readSide);
if (string(s, 0, 2) == "# ") { if (string(s, 0, 2) == "# ") {
reply = string(s, 2); reply = string(s, 2);
break; break;
} }
handleChildOutput(logPipe.readSide, s + "\n"); s += "\n";
} writeToStderr((unsigned char *) s.c_str(), s.size());
} catch (Error & e) {
terminateBuildHook(true);
throw;
} }
debug(format("hook reply is `%1%'") % reply); debug(format("hook reply is `%1%'") % reply);
if (reply == "decline" || reply == "postpone") { if (reply == "decline" || reply == "postpone")
/* Clean up the child. !!! hacky / should verify */
terminateBuildHook();
return reply == "decline" ? rpDecline : rpPostpone; return reply == "decline" ? rpDecline : rpPostpone;
} else if (reply != "accept")
throw Error(format("bad hook reply `%1%'") % reply);
else if (reply == "accept") { printMsg(lvlTalkative, format("using hook to build path(s) %1%")
printMsg(lvlInfo, format("using hook to build path(s) %1%")
% showPaths(outputPaths(drv.outputs))); % showPaths(outputPaths(drv.outputs)));
/* Write the information that the hook needs to perform the hook = worker.hook;
build, i.e., the set of input paths, the set of output worker.hook.reset();
paths, and the references (pointer graph) in the input
paths. */
Path inputListFN = tmpDir + "/inputs"; /* Tell the hook all the inputs that have to be copied to the
Path outputListFN = tmpDir + "/outputs"; remote system. This unfortunately has to contain the entire
Path referencesFN = tmpDir + "/references"; derivation closure to ensure that the validity invariant holds
on the remote system. (I.e., it's unfortunate that we have to
/* The `inputs' file lists all inputs that have to be copied list it since the remote system *probably* already has it.) */
to the remote system. This unfortunately has to contain
the entire derivation closure to ensure that the validity
invariant holds on the remote system. (I.e., it's
unfortunate that we have to list it since the remote system
*probably* already has it.) */
PathSet allInputs; PathSet allInputs;
allInputs.insert(inputPaths.begin(), inputPaths.end()); allInputs.insert(inputPaths.begin(), inputPaths.end());
computeFSClosure(drvPath, allInputs); computeFSClosure(drvPath, allInputs);
string s; string s;
foreach (PathSet::iterator, i, allInputs) s += *i + "\n"; foreach (PathSet::iterator, i, allInputs) s += *i + " ";
writeLine(hook->toHook.writeSide, s);
writeFile(inputListFN, s); /* Tell the hooks the outputs that have to be copied back from the
remote system. */
/* The `outputs' file lists all outputs that have to be copied
from the remote system. */
s = ""; s = "";
foreach (DerivationOutputs::iterator, i, drv.outputs) foreach (DerivationOutputs::iterator, i, drv.outputs)
s += i->second.path + "\n"; s += i->second.path + " ";
writeFile(outputListFN, s); writeLine(hook->toHook.writeSide, s);
/* The `references' file has exactly the format accepted by hook->toHook.writeSide.close();
`nix-store --register-validity'. */
writeFile(referencesFN,
makeValidityRegistration(allInputs, true, false));
/* Tell the hook to proceed. */ /* Create the log file and pipe. */
writeLine(toHook.writeSide, "okay"); Path logFile = openLogFile();
toHook.writeSide.close();
set<int> fds;
fds.insert(hook->fromHook.readSide);
fds.insert(hook->builderOut.readSide);
worker.childStarted(shared_from_this(), hook->pid, fds, false, false);
if (printBuildTrace) if (printBuildTrace)
printMsg(lvlError, format("@ build-started %1% %2% %3% %4%") printMsg(lvlError, format("@ build-started %1% %2% %3% %4%")
@ -1345,27 +1425,6 @@ DerivationGoal::HookReply DerivationGoal::tryBuildHook()
return rpAccept; return rpAccept;
} }
else throw Error(format("bad hook reply `%1%'") % reply);
}
void DerivationGoal::terminateBuildHook(bool kill)
{
debug("terminating build hook");
pid_t savedPid = pid;
if (kill)
pid.kill();
else
pid.wait(true);
/* `false' means don't wake up waiting goals, since we want to
keep this build slot ourselves. */
worker.childTerminated(savedPid, false);
toHook.writeSide.close();
fdLogFile.close();
logPipe.readSide.close();
deleteTmpDir(true); /* get rid of the hook's temporary directory */
}
void chmod(const Path & path, mode_t mode) void chmod(const Path & path, mode_t mode)
{ {
@ -1380,11 +1439,7 @@ void DerivationGoal::startBuilder()
format("building path(s) %1%") % showPaths(outputPaths(drv.outputs))) format("building path(s) %1%") % showPaths(outputPaths(drv.outputs)))
/* Right platform? */ /* Right platform? */
if (drv.platform != thisSystem if (!canBuildLocally(drv.platform))
#ifdef CAN_DO_LINUX32_BUILDS
&& !(drv.platform == "i686-linux" && thisSystem == "x86_64-linux")
#endif
)
throw Error( throw Error(
format("a `%1%' is required to build `%3%', but I am a `%2%'") format("a `%1%' is required to build `%3%', but I am a `%2%'")
% drv.platform % thisSystem % drvPath); % drv.platform % thisSystem % drvPath);
@ -1499,7 +1554,7 @@ void DerivationGoal::startBuilder()
/* Write closure info to `fileName'. */ /* Write closure info to `fileName'. */
writeFile(tmpDir + "/" + fileName, writeFile(tmpDir + "/" + fileName,
makeValidityRegistration(paths, false, false)); worker.store.makeValidityRegistration(paths, false, false));
} }
@ -1549,6 +1604,9 @@ void DerivationGoal::startBuilder()
if (fixedOutput) useChroot = false; if (fixedOutput) useChroot = false;
/* Hack to allow derivations to disable chroot builds. */
if (drv.env["__noChroot"] == "1") useChroot = false;
if (useChroot) { if (useChroot) {
#if CHROOT_ENABLED #if CHROOT_ENABLED
/* Create a temporary directory in which we set up the chroot /* Create a temporary directory in which we set up the chroot
@ -1645,9 +1703,12 @@ void DerivationGoal::startBuilder()
printMsg(lvlChatty, format("executing builder `%1%'") % printMsg(lvlChatty, format("executing builder `%1%'") %
drv.builder); drv.builder);
/* Create the log file and pipe. */ /* Create the log file. */
Path logFile = openLogFile(); Path logFile = openLogFile();
/* Create a pipe to get the output of the builder. */
builderOut.create();
/* Fork a child to build the package. Note that while we /* Fork a child to build the package. Note that while we
currently use forks to run and wait for the children, it currently use forks to run and wait for the children, it
shouldn't be hard to use threads for this on systems where shouldn't be hard to use threads for this on systems where
@ -1661,7 +1722,7 @@ void DerivationGoal::startBuilder()
case 0: case 0:
/* Warning: in the child we should absolutely not make any /* Warning: in the child we should absolutely not make any
Berkeley DB calls! */ SQLite calls! */
try { /* child */ try { /* child */
@ -1688,18 +1749,23 @@ void DerivationGoal::startBuilder()
throw SysError(format("bind mount from `%1%' to `%2%' failed") % source % target); throw SysError(format("bind mount from `%1%' to `%2%' failed") % source % target);
} }
/* Do the chroot(). initChild() will do a chdir() to /* Do the chroot(). Below we do a chdir() to the
the temporary build directory to make sure the temporary build directory to make sure the current
current directory is in the chroot. (Actually the directory is in the chroot. (Actually the order
order doesn't matter, since due to the bind mount doesn't matter, since due to the bind mount tmpDir
tmpDir and tmpRootDit/tmpDir are the same and tmpRootDit/tmpDir are the same directories.) */
directories.) */
if (chroot(chrootRootDir.c_str()) == -1) if (chroot(chrootRootDir.c_str()) == -1)
throw SysError(format("cannot change root directory to `%1%'") % chrootRootDir); throw SysError(format("cannot change root directory to `%1%'") % chrootRootDir);
} }
#endif #endif
initChild(); commonChildInit(builderOut);
if (chdir(tmpDir.c_str()) == -1)
throw SysError(format("changing into `%1%'") % tmpDir);
/* Close all other file descriptors. */
closeMostFDs(set<int>());
#ifdef CAN_DO_LINUX32_BUILDS #ifdef CAN_DO_LINUX32_BUILDS
if (drv.platform == "i686-linux" && thisSystem == "x86_64-linux") { if (drv.platform == "i686-linux" && thisSystem == "x86_64-linux") {
@ -1720,10 +1786,10 @@ void DerivationGoal::startBuilder()
/* If we are running in `build-users' mode, then switch to /* If we are running in `build-users' mode, then switch to
the user we allocated above. Make sure that we drop the user we allocated above. Make sure that we drop
all root privileges. Note that initChild() above has all root privileges. Note that above we have closed
closed all file descriptors except std*, so that's all file descriptors except std*, so that's safe. Also
safe. Also note that setuid() when run as root sets note that setuid() when run as root sets the real,
the real, effective and saved UIDs. */ effective and saved UIDs. */
if (buildUser.enabled()) { if (buildUser.enabled()) {
printMsg(lvlChatty, format("switching to user `%1%'") % buildUser.getUser()); printMsg(lvlChatty, format("switching to user `%1%'") % buildUser.getUser());
@ -1777,9 +1843,9 @@ void DerivationGoal::startBuilder()
/* parent */ /* parent */
pid.setSeparatePG(true); pid.setSeparatePG(true);
logPipe.writeSide.close(); builderOut.writeSide.close();
worker.childStarted(shared_from_this(), pid, worker.childStarted(shared_from_this(), pid,
singleton<set<int> >(logPipe.readSide), true, true); singleton<set<int> >(builderOut.readSide), true, true);
if (printBuildTrace) { if (printBuildTrace) {
printMsg(lvlError, format("@ build-started %1% %2% %3% %4%") printMsg(lvlError, format("@ build-started %1% %2% %3% %4%")
@ -1811,12 +1877,12 @@ PathSet parseReferenceSpecifiers(const Derivation & drv, string attr)
void DerivationGoal::computeClosure() void DerivationGoal::computeClosure()
{ {
map<Path, PathSet> allReferences; map<Path, PathSet> allReferences;
map<Path, Hash> contentHashes; map<Path, HashResult> contentHashes;
/* When using a build hook, the build hook can register the output /* When using a build hook, the build hook can register the output
as valid (by doing `nix-store --import'). If so we don't have as valid (by doing `nix-store --import'). If so we don't have
to do anything here. */ to do anything here. */
if (usingBuildHook) { if (hook) {
bool allValid = true; bool allValid = true;
foreach (DerivationOutputs::iterator, i, drv.outputs) foreach (DerivationOutputs::iterator, i, drv.outputs)
if (!worker.store.isValidPath(i->second.path)) allValid = false; if (!worker.store.isValidPath(i->second.path)) allValid = false;
@ -1868,7 +1934,7 @@ void DerivationGoal::computeClosure()
if (ht == htUnknown) if (ht == htUnknown)
throw BuildError(format("unknown hash algorithm `%1%'") % algo); throw BuildError(format("unknown hash algorithm `%1%'") % algo);
Hash h = parseHash(ht, i->second.hash); Hash h = parseHash(ht, i->second.hash);
Hash h2 = recursive ? hashPath(ht, path) : hashFile(ht, path); Hash h2 = recursive ? hashPath(ht, path).first : hashFile(ht, path);
if (h != h2) if (h != h2)
throw BuildError( throw BuildError(
format("output path `%1%' should have %2% hash `%3%', instead has `%4%'") format("output path `%1%' should have %2% hash `%3%', instead has `%4%'")
@ -1882,7 +1948,7 @@ void DerivationGoal::computeClosure()
contained in it. Compute the SHA-256 NAR hash at the same contained in it. Compute the SHA-256 NAR hash at the same
time. The hash is stored in the database so that we can time. The hash is stored in the database so that we can
verify later on whether nobody has messed with the store. */ verify later on whether nobody has messed with the store. */
Hash hash; HashResult hash;
PathSet references = scanForReferences(path, allPaths, hash); PathSet references = scanForReferences(path, allPaths, hash);
contentHashes[path] = hash; contentHashes[path] = hash;
@ -1911,14 +1977,18 @@ void DerivationGoal::computeClosure()
} }
/* Register each output path as valid, and register the sets of /* Register each output path as valid, and register the sets of
paths referenced by each of them. !!! this should be paths referenced by each of them. */
atomic so that either all paths are registered as valid, or ValidPathInfos infos;
none are. */ foreach (DerivationOutputs::iterator, i, drv.outputs) {
foreach (DerivationOutputs::iterator, i, drv.outputs) ValidPathInfo info;
worker.store.registerValidPath(i->second.path, info.path = i->second.path;
contentHashes[i->second.path], info.hash = contentHashes[i->second.path].first;
allReferences[i->second.path], info.narSize = contentHashes[i->second.path].second;
drvPath); info.references = allReferences[i->second.path];
info.deriver = drvPath;
infos.push_back(info);
}
worker.store.registerValidPaths(infos);
/* It is now safe to delete the lock files, since all future /* It is now safe to delete the lock files, since all future
lockers will see that the output paths are valid; they will not lockers will see that the output paths are valid; they will not
@ -1944,32 +2014,10 @@ Path DerivationGoal::openLogFile()
if (fdLogFile == -1) if (fdLogFile == -1)
throw SysError(format("creating log file `%1%'") % logFileName); throw SysError(format("creating log file `%1%'") % logFileName);
/* Create a pipe to get the output of the child. */
logPipe.create();
return logFileName; return logFileName;
} }
void DerivationGoal::initChild()
{
commonChildInit(logPipe);
if (chdir(tmpDir.c_str()) == -1)
throw SysError(format("changing into `%1%'") % tmpDir);
/* When running a hook, dup the communication pipes. */
if (usingBuildHook) {
toHook.writeSide.close();
if (dup2(toHook.readSide, STDIN_FILENO) == -1)
throw SysError("dupping to-hook read side");
}
/* Close all other file descriptors. */
closeMostFDs(set<int>());
}
void DerivationGoal::deleteTmpDir(bool force) void DerivationGoal::deleteTmpDir(bool force)
{ {
if (tmpDir != "") { if (tmpDir != "") {
@ -1989,19 +2037,22 @@ void DerivationGoal::deleteTmpDir(bool force)
void DerivationGoal::handleChildOutput(int fd, const string & data) void DerivationGoal::handleChildOutput(int fd, const string & data)
{ {
if (fd == logPipe.readSide) { if ((hook && fd == hook->builderOut.readSide) ||
(!hook && fd == builderOut.readSide))
{
if (verbosity >= buildVerbosity) if (verbosity >= buildVerbosity)
writeToStderr((unsigned char *) data.c_str(), data.size()); writeToStderr((unsigned char *) data.c_str(), data.size());
writeFull(fdLogFile, (unsigned char *) data.c_str(), data.size()); writeFull(fdLogFile, (unsigned char *) data.c_str(), data.size());
} }
else abort(); if (hook && fd == hook->fromHook.readSide)
writeToStderr((unsigned char *) data.c_str(), data.size());
} }
void DerivationGoal::handleEOF(int fd) void DerivationGoal::handleEOF(int fd)
{ {
if (fd == logPipe.readSide) worker.wakeUp(shared_from_this()); worker.wakeUp(shared_from_this());
} }
@ -2345,10 +2396,15 @@ void SubstitutionGoal::finished()
canonicalisePathMetaData(storePath); canonicalisePathMetaData(storePath);
Hash contentHash = hashPath(htSHA256, storePath); HashResult hash = hashPath(htSHA256, storePath);
worker.store.registerValidPath(storePath, contentHash, ValidPathInfo info2;
info.references, info.deriver); info2.path = storePath;
info2.hash = hash.first;
info2.narSize = hash.second;
info2.references = info.references;
info2.deriver = info.deriver;
worker.store.registerValidPath(info2);
outputLock->setDeletion(true); outputLock->setDeletion(true);
@ -2395,6 +2451,7 @@ Worker::Worker(LocalStore & store)
nrLocalBuilds = 0; nrLocalBuilds = 0;
lastWokenUp = 0; lastWokenUp = 0;
cacheFailure = queryBoolSetting("build-cache-failure", false); cacheFailure = queryBoolSetting("build-cache-failure", false);
permanentFailure = false;
} }
@ -2721,6 +2778,11 @@ void Worker::waitForInput()
} }
unsigned int Worker::exitStatus()
{
return permanentFailure ? 100 : 1;
}
////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////
@ -2747,7 +2809,7 @@ void LocalStore::buildDerivations(const PathSet & drvPaths)
} }
if (!failed.empty()) if (!failed.empty())
throw Error(format("build of %1% failed") % showPaths(failed)); throw Error(format("build of %1% failed") % showPaths(failed), worker.exitStatus());
} }
@ -2763,7 +2825,7 @@ void LocalStore::ensurePath(const Path & path)
worker.run(goals); worker.run(goals);
if (goal->getExitCode() != Goal::ecSuccess) if (goal->getExitCode() != Goal::ecSuccess)
throw Error(format("path `%1%' does not exist and cannot be created") % path); throw Error(format("path `%1%' does not exist and cannot be created") % path, worker.exitStatus());
} }

View file

@ -1,10 +1,10 @@
#ifndef __DERIVATIONS_H #ifndef __DERIVATIONS_H
#define __DERIVATIONS_H #define __DERIVATIONS_H
#include "hash.hh"
#include <map> #include <map>
#include "types.hh"
namespace nix { namespace nix {

View file

@ -1,6 +1,5 @@
#include "globals.hh" #include "globals.hh"
#include "misc.hh" #include "misc.hh"
#include "pathlocks.hh"
#include "local-store.hh" #include "local-store.hh"
#include <boost/shared_ptr.hpp> #include <boost/shared_ptr.hpp>
@ -31,7 +30,7 @@ static const int defaultGcLevel = 1000;
read. To be precise: when they try to create a new temporary root read. To be precise: when they try to create a new temporary root
file, they will block until the garbage collector has finished / file, they will block until the garbage collector has finished /
yielded the GC lock. */ yielded the GC lock. */
static int openGCLock(LockType lockType) int LocalStore::openGCLock(LockType lockType)
{ {
Path fnGCLock = (format("%1%/%2%") Path fnGCLock = (format("%1%/%2%")
% nixStateDir % gcLockName).str(); % nixStateDir % gcLockName).str();
@ -127,7 +126,7 @@ Path addPermRoot(const Path & _storePath, const Path & _gcRoot,
Instead of reading all the roots, it would be more efficient to Instead of reading all the roots, it would be more efficient to
check if the root is in a directory in or linked from the check if the root is in a directory in or linked from the
gcroots directory. */ gcroots directory. */
if (queryBoolSetting("gc-check-reachability", true)) { if (queryBoolSetting("gc-check-reachability", false)) {
Roots roots = store->findRoots(); Roots roots = store->findRoots();
if (roots.find(gcRoot) == roots.end()) if (roots.find(gcRoot) == roots.end())
printMsg(lvlError, printMsg(lvlError,
@ -416,18 +415,13 @@ struct LocalStore::GCState
PathSet busy; PathSet busy;
bool gcKeepOutputs; bool gcKeepOutputs;
bool gcKeepDerivations; bool gcKeepDerivations;
GCState(GCResults & results_) : results(results_)
bool drvsIndexed;
typedef std::multimap<string, Path> DrvsByName;
DrvsByName drvsByName; // derivation paths hashed by name attribute
GCState(GCResults & results_) : results(results_), drvsIndexed(false)
{ {
} }
}; };
static bool doDelete(GCOptions::GCAction action) static bool shouldDelete(GCOptions::GCAction action)
{ {
return action == GCOptions::gcDeleteDead return action == GCOptions::gcDeleteDead
|| action == GCOptions::gcDeleteSpecific; || action == GCOptions::gcDeleteSpecific;
@ -442,44 +436,10 @@ bool LocalStore::isActiveTempFile(const GCState & state,
} }
/* Return all the derivations in the Nix store that have `path' as an
output. This function assumes that derivations have the same name
as their outputs. */
PathSet LocalStore::findDerivers(GCState & state, const Path & path)
{
PathSet derivers;
Path deriver = queryDeriver(path);
if (deriver != "") derivers.insert(deriver);
if (!state.drvsIndexed) {
Paths entries = readDirectory(nixStore);
foreach (Paths::iterator, i, entries)
if (isDerivation(*i))
state.drvsByName.insert(std::pair<string, Path>(
getNameOfStorePath(*i), nixStore + "/" + *i));
state.drvsIndexed = true;
}
string name = getNameOfStorePath(path);
// Urgh, I should have used Haskell...
std::pair<GCState::DrvsByName::iterator, GCState::DrvsByName::iterator> range =
state.drvsByName.equal_range(name);
for (GCState::DrvsByName::iterator i = range.first; i != range.second; ++i)
if (isValidPath(i->second)) {
Derivation drv = derivationFromPath(i->second);
foreach (DerivationOutputs::iterator, j, drv.outputs)
if (j->second.path == path) derivers.insert(i->second);
}
return derivers;
}
bool LocalStore::tryToDelete(GCState & state, const Path & path) bool LocalStore::tryToDelete(GCState & state, const Path & path)
{ {
checkInterrupt();
if (!pathExists(path)) return true; if (!pathExists(path)) return true;
if (state.deleted.find(path) != state.deleted.end()) return true; if (state.deleted.find(path) != state.deleted.end()) return true;
if (state.live.find(path) != state.live.end()) return false; if (state.live.find(path) != state.live.end()) return false;
@ -508,10 +468,10 @@ bool LocalStore::tryToDelete(GCState & state, const Path & path)
then don't delete the derivation if any of the outputs are then don't delete the derivation if any of the outputs are
live. */ live. */
if (state.gcKeepDerivations && isDerivation(path)) { if (state.gcKeepDerivations && isDerivation(path)) {
Derivation drv = derivationFromPath(path); PathSet outputs = queryDerivationOutputs(path);
foreach (DerivationOutputs::iterator, i, drv.outputs) foreach (PathSet::iterator, i, outputs)
if (!tryToDelete(state, i->second.path)) { if (!tryToDelete(state, *i)) {
printMsg(lvlDebug, format("cannot delete derivation `%1%' because its output is alive") % path); printMsg(lvlDebug, format("cannot delete derivation `%1%' because its output `%2%' is alive") % path % *i);
goto isLive; goto isLive;
} }
} }
@ -522,18 +482,9 @@ bool LocalStore::tryToDelete(GCState & state, const Path & path)
if (!pathExists(path)) return true; if (!pathExists(path)) return true;
/* If gc-keep-outputs is set, then don't delete this path if /* If gc-keep-outputs is set, then don't delete this path if
its deriver is not garbage. !!! Nix does not reliably there are derivers of this path that are not garbage. */
store derivers, so we have to look at all derivations to
determine which of them derive `path'. Since this makes
the garbage collector very slow to start on large Nix
stores, here we just look for all derivations that have the
same name as `path' (where the name is the part of the
filename after the hash, i.e. the `name' attribute of the
derivation). This is somewhat hacky: currently, the
deriver of a path always has the same name as the output,
but this might change in the future. */
if (state.gcKeepOutputs) { if (state.gcKeepOutputs) {
PathSet derivers = findDerivers(state, path); PathSet derivers = queryValidDerivers(path);
foreach (PathSet::iterator, deriver, derivers) { foreach (PathSet::iterator, deriver, derivers) {
/* Break an infinite recursion if gc-keep-derivations /* Break an infinite recursion if gc-keep-derivations
and gc-keep-outputs are both set by tentatively and gc-keep-outputs are both set by tentatively
@ -567,7 +518,7 @@ bool LocalStore::tryToDelete(GCState & state, const Path & path)
} }
/* The path is garbage, so delete it. */ /* The path is garbage, so delete it. */
if (doDelete(state.options.action)) { if (shouldDelete(state.options.action)) {
printMsg(lvlInfo, format("deleting `%1%'") % path); printMsg(lvlInfo, format("deleting `%1%'") % path);
unsigned long long bytesFreed, blocksFreed; unsigned long long bytesFreed, blocksFreed;
@ -614,6 +565,15 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
state.gcKeepOutputs = queryBoolSetting("gc-keep-outputs", false); state.gcKeepOutputs = queryBoolSetting("gc-keep-outputs", false);
state.gcKeepDerivations = queryBoolSetting("gc-keep-derivations", true); state.gcKeepDerivations = queryBoolSetting("gc-keep-derivations", true);
/* Using `--ignore-liveness' with `--delete' can have unintended
consequences if `gc-keep-outputs' or `gc-keep-derivations' are
true (the garbage collector will recurse into deleting the
outputs or derivers, respectively). So disable them. */
if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) {
state.gcKeepOutputs = false;
state.gcKeepDerivations = false;
}
/* Acquire the global GC root. This prevents /* Acquire the global GC root. This prevents
a) New roots from being added. a) New roots from being added.
b) Processes from creating new temporary root files. */ b) Processes from creating new temporary root files. */
@ -667,7 +627,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
vector<Path> entries_(entries.begin(), entries.end()); vector<Path> entries_(entries.begin(), entries.end());
random_shuffle(entries_.begin(), entries_.end()); random_shuffle(entries_.begin(), entries_.end());
if (doDelete(state.options.action)) if (shouldDelete(state.options.action))
printMsg(lvlError, format("deleting garbage...")); printMsg(lvlError, format("deleting garbage..."));
else else
printMsg(lvlError, format("determining live/dead paths...")); printMsg(lvlError, format("determining live/dead paths..."));

View file

@ -20,7 +20,7 @@ string nixBinDir = "/UNINIT";
bool keepFailed = false; bool keepFailed = false;
bool keepGoing = false; bool keepGoing = false;
bool tryFallback = false; bool tryFallback = false;
Verbosity buildVerbosity = lvlInfo; Verbosity buildVerbosity = lvlError;
unsigned int maxBuildJobs = 1; unsigned int maxBuildJobs = 1;
unsigned int buildCores = 1; unsigned int buildCores = 1;
bool readOnlyMode = false; bool readOnlyMode = false;

File diff suppressed because it is too large Load diff

View file

@ -5,6 +5,11 @@
#include "store-api.hh" #include "store-api.hh"
#include "util.hh" #include "util.hh"
#include "pathlocks.hh"
class sqlite3;
class sqlite3_stmt;
namespace nix { namespace nix {
@ -12,8 +17,9 @@ namespace nix {
/* Nix store and database schema version. Version 1 (or 0) was Nix <= /* Nix store and database schema version. Version 1 (or 0) was Nix <=
0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10. 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10.
Version 4 is Nix 0.11. Version 5 is Nix 0.12*/ Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is
const int nixSchemaVersion = 5; Nix 1.0. */
const int nixSchemaVersion = 6;
extern string drvsLogDir; extern string drvsLogDir;
@ -41,6 +47,34 @@ struct RunningSubstituter
}; };
/* Wrapper object to close the SQLite database automatically. */
struct SQLite
{
sqlite3 * db;
SQLite() { db = 0; }
~SQLite();
operator sqlite3 * () { return db; }
};
/* Wrapper object to create and destroy SQLite prepared statements. */
struct SQLiteStmt
{
sqlite3 * db;
sqlite3_stmt * stmt;
unsigned int curArg;
SQLiteStmt() { stmt = 0; }
void create(sqlite3 * db, const string & s);
void reset();
~SQLiteStmt();
operator sqlite3_stmt * () { return stmt; }
void bind(const string & value);
void bind(int value);
void bind64(long long value);
void bind();
};
class LocalStore : public StoreAPI class LocalStore : public StoreAPI
{ {
private: private:
@ -64,6 +98,8 @@ public:
PathSet queryValidPaths(); PathSet queryValidPaths();
ValidPathInfo queryPathInfo(const Path & path);
Hash queryPathHash(const Path & path); Hash queryPathHash(const Path & path);
void queryReferences(const Path & path, PathSet & references); void queryReferences(const Path & path, PathSet & references);
@ -72,6 +108,14 @@ public:
Path queryDeriver(const Path & path); Path queryDeriver(const Path & path);
/* Return all currently valid derivations that have `path' as an
output. (Note that the result of `queryDeriver()' is the
derivation that was actually used to produce `path', which may
not exist anymore.) */
PathSet queryValidDerivers(const Path & path);
PathSet queryDerivationOutputs(const Path & path);
PathSet querySubstitutablePaths(); PathSet querySubstitutablePaths();
bool hasSubstitutes(const Path & path); bool hasSubstitutes(const Path & path);
@ -132,8 +176,7 @@ public:
execution of the derivation (or something equivalent). Also execution of the derivation (or something equivalent). Also
register the hash of the file system contents of the path. The register the hash of the file system contents of the path. The
hash must be a SHA-256 hash. */ hash must be a SHA-256 hash. */
void registerValidPath(const Path & path, void registerValidPath(const ValidPathInfo & info);
const Hash & hash, const PathSet & references, const Path & deriver);
void registerValidPaths(const ValidPathInfos & infos); void registerValidPaths(const ValidPathInfos & infos);
@ -144,6 +187,10 @@ public:
/* Query whether `path' previously failed to build. */ /* Query whether `path' previously failed to build. */
bool hasPathFailed(const Path & path); bool hasPathFailed(const Path & path);
PathSet queryFailedPaths();
void clearFailedPaths(const PathSet & paths);
private: private:
Path schemaPath; Path schemaPath;
@ -151,45 +198,63 @@ private:
/* Lock file used for upgrading. */ /* Lock file used for upgrading. */
AutoCloseFD globalLock; AutoCloseFD globalLock;
/* !!! The cache can grow very big. Maybe it should be pruned /* The SQLite database object. */
every once in a while. */ SQLite db;
std::map<Path, ValidPathInfo> pathInfoCache;
/* Store paths for which the referrers file must be purged. */ /* Some precompiled SQLite statements. */
PathSet delayedUpdates; SQLiteStmt stmtRegisterValidPath;
SQLiteStmt stmtUpdatePathInfo;
/* Whether to do an fsync() after writing Nix metadata. */ SQLiteStmt stmtAddReference;
bool doFsync; SQLiteStmt stmtQueryPathInfo;
SQLiteStmt stmtQueryReferences;
SQLiteStmt stmtQueryReferrers;
SQLiteStmt stmtInvalidatePath;
SQLiteStmt stmtRegisterFailedPath;
SQLiteStmt stmtHasPathFailed;
SQLiteStmt stmtQueryFailedPaths;
SQLiteStmt stmtClearFailedPath;
SQLiteStmt stmtAddDerivationOutput;
SQLiteStmt stmtQueryValidDerivers;
SQLiteStmt stmtQueryDerivationOutputs;
int getSchema(); int getSchema();
void registerValidPath(const ValidPathInfo & info, bool ignoreValidity = false); void openDB(bool create);
ValidPathInfo queryPathInfo(const Path & path, bool ignoreErrors = false); unsigned long long queryValidPathId(const Path & path);
unsigned long long addValidPath(const ValidPathInfo & info);
void addReference(unsigned long long referrer, unsigned long long reference);
void appendReferrer(const Path & from, const Path & to, bool lock); void appendReferrer(const Path & from, const Path & to, bool lock);
void rewriteReferrers(const Path & path, bool purge, PathSet referrers); void rewriteReferrers(const Path & path, bool purge, PathSet referrers);
void flushDelayedUpdates();
bool queryReferrersInternal(const Path & path, PathSet & referrers);
void invalidatePath(const Path & path); void invalidatePath(const Path & path);
void upgradeStore12(); void verifyPath(const Path & path, const PathSet & store,
PathSet & done, PathSet & validPaths);
void updatePathInfo(const ValidPathInfo & info);
void upgradeStore6();
PathSet queryValidPathsOld();
ValidPathInfo queryPathInfoOld(const Path & path);
struct GCState; struct GCState;
bool tryToDelete(GCState & state, const Path & path); bool tryToDelete(GCState & state, const Path & path);
PathSet findDerivers(GCState & state, const Path & path);
bool isActiveTempFile(const GCState & state, bool isActiveTempFile(const GCState & state,
const Path & path, const string & suffix); const Path & path, const string & suffix);
int openGCLock(LockType lockType);
void startSubstituter(const Path & substituter, void startSubstituter(const Path & substituter,
RunningSubstituter & runningSubstituter); RunningSubstituter & runningSubstituter);
Path createTempDirInStore();
}; };

View file

@ -27,10 +27,10 @@ void computeFSClosure(const Path & storePath,
store->queryReferences(storePath, references); store->queryReferences(storePath, references);
if (includeOutputs && isDerivation(storePath)) { if (includeOutputs && isDerivation(storePath)) {
Derivation drv = derivationFromPath(storePath); PathSet outputs = store->queryDerivationOutputs(storePath);
foreach (DerivationOutputs::iterator, i, drv.outputs) foreach (PathSet::iterator, i, outputs)
if (store->isValidPath(i->second.path)) if (store->isValidPath(*i))
computeFSClosure(i->second.path, paths, flipDirection, true); computeFSClosure(*i, paths, flipDirection, true);
} }
foreach (PathSet::iterator, i, references) foreach (PathSet::iterator, i, references)
@ -48,9 +48,9 @@ Path findOutput(const Derivation & drv, string id)
void queryMissing(const PathSet & targets, void queryMissing(const PathSet & targets,
PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,
unsigned long long & downloadSize) unsigned long long & downloadSize, unsigned long long & narSize)
{ {
downloadSize = 0; downloadSize = narSize = 0;
PathSet todo(targets.begin(), targets.end()), done; PathSet todo(targets.begin(), targets.end()), done;
@ -88,6 +88,7 @@ void queryMissing(const PathSet & targets,
if (store->querySubstitutablePathInfo(p, info)) { if (store->querySubstitutablePathInfo(p, info)) {
willSubstitute.insert(p); willSubstitute.insert(p);
downloadSize += info.downloadSize; downloadSize += info.downloadSize;
narSize += info.narSize;
todo.insert(info.references.begin(), info.references.end()); todo.insert(info.references.begin(), info.references.end());
} else } else
unknown.insert(p); unknown.insert(p);

View file

@ -31,7 +31,7 @@ Path findOutput(const Derivation & drv, string id);
will be substituted. */ will be substituted. */
void queryMissing(const PathSet & targets, void queryMissing(const PathSet & targets,
PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,
unsigned long long & downloadSize); unsigned long long & downloadSize, unsigned long long & narSize);
} }

View file

@ -68,7 +68,7 @@ static void hashAndLink(bool dryRun, HashToPath & hashToPath,
the contents of the symlink (i.e. the result of the contents of the symlink (i.e. the result of
readlink()), not the contents of the target (which may not readlink()), not the contents of the target (which may not
even exist). */ even exist). */
Hash hash = hashPath(htSHA256, path); Hash hash = hashPath(htSHA256, path).first;
stats.totalFiles++; stats.totalFiles++;
printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash)); printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash));

View file

@ -81,7 +81,7 @@ void RefScanSink::operator () (const unsigned char * data, unsigned int len)
PathSet scanForReferences(const string & path, PathSet scanForReferences(const string & path,
const PathSet & refs, Hash & hash) const PathSet & refs, HashResult & hash)
{ {
RefScanSink sink; RefScanSink sink;
std::map<string, Path> backMap; std::map<string, Path> backMap;

View file

@ -7,7 +7,7 @@
namespace nix { namespace nix {
PathSet scanForReferences(const Path & path, const PathSet & refs, PathSet scanForReferences(const Path & path, const PathSet & refs,
Hash & hash); HashResult & hash);
} }

View file

@ -97,10 +97,6 @@ void RemoteStore::forkSlave()
if (worker == "") if (worker == "")
worker = nixBinDir + "/nix-worker"; worker = nixBinDir + "/nix-worker";
string verbosityArg = "-";
for (int i = 1; i < verbosity; ++i)
verbosityArg += "v";
child = fork(); child = fork();
switch (child) { switch (child) {
@ -120,10 +116,7 @@ void RemoteStore::forkSlave()
close(fdSocket); close(fdSocket);
close(fdChild); close(fdChild);
execlp(worker.c_str(), worker.c_str(), "--slave", execlp(worker.c_str(), worker.c_str(), "--slave", NULL);
/* hacky - must be at the end */
verbosityArg == "-" ? NULL : verbosityArg.c_str(),
NULL);
throw SysError(format("executing `%1%'") % worker); throw SysError(format("executing `%1%'") % worker);
@ -198,9 +191,8 @@ void RemoteStore::setOptions()
writeInt(logType, to); writeInt(logType, to);
writeInt(printBuildTrace, to); writeInt(printBuildTrace, to);
} }
if (GET_PROTOCOL_MINOR(daemonVersion) >= 6) { if (GET_PROTOCOL_MINOR(daemonVersion) >= 6)
writeInt(buildCores, to); writeInt(buildCores, to);
}
processStderr(); processStderr();
} }
@ -219,7 +211,9 @@ bool RemoteStore::isValidPath(const Path & path)
PathSet RemoteStore::queryValidPaths() PathSet RemoteStore::queryValidPaths()
{ {
openConnection(); openConnection();
throw Error("not implemented"); writeInt(wopQueryValidPaths, to);
processStderr();
return readStorePaths(from);
} }
@ -248,10 +242,29 @@ bool RemoteStore::querySubstitutablePathInfo(const Path & path,
if (info.deriver != "") assertStorePath(info.deriver); if (info.deriver != "") assertStorePath(info.deriver);
info.references = readStorePaths(from); info.references = readStorePaths(from);
info.downloadSize = readLongLong(from); info.downloadSize = readLongLong(from);
info.narSize = GET_PROTOCOL_MINOR(daemonVersion) >= 7 ? readLongLong(from) : 0;
return true; return true;
} }
ValidPathInfo RemoteStore::queryPathInfo(const Path & path)
{
openConnection();
writeInt(wopQueryPathInfo, to);
writeString(path, to);
processStderr();
ValidPathInfo info;
info.path = path;
info.deriver = readString(from);
if (info.deriver != "") assertStorePath(info.deriver);
info.hash = parseHash(htSHA256, readString(from));
info.references = readStorePaths(from);
info.registrationTime = readInt(from);
info.narSize = readLongLong(from);
return info;
}
Hash RemoteStore::queryPathHash(const Path & path) Hash RemoteStore::queryPathHash(const Path & path)
{ {
openConnection(); openConnection();
@ -299,6 +312,16 @@ Path RemoteStore::queryDeriver(const Path & path)
} }
PathSet RemoteStore::queryDerivationOutputs(const Path & path)
{
openConnection();
writeInt(wopQueryDerivationOutputs, to);
writeString(path, to);
processStderr();
return readStorePaths(from);
}
Path RemoteStore::addToStore(const Path & _srcPath, Path RemoteStore::addToStore(const Path & _srcPath,
bool recursive, HashType hashAlgo, PathFilter & filter) bool recursive, HashType hashAlgo, PathFilter & filter)
{ {
@ -444,6 +467,25 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
} }
PathSet RemoteStore::queryFailedPaths()
{
openConnection();
writeInt(wopQueryFailedPaths, to);
processStderr();
return readStorePaths(from);
}
void RemoteStore::clearFailedPaths(const PathSet & paths)
{
openConnection();
writeInt(wopClearFailedPaths, to);
writeStringSet(paths, to);
processStderr();
readInt(from);
}
void RemoteStore::processStderr(Sink * sink, Source * source) void RemoteStore::processStderr(Sink * sink, Source * source)
{ {
unsigned int msg; unsigned int msg;
@ -467,8 +509,11 @@ void RemoteStore::processStderr(Sink * sink, Source * source)
writeToStderr((const unsigned char *) s.c_str(), s.size()); writeToStderr((const unsigned char *) s.c_str(), s.size());
} }
} }
if (msg == STDERR_ERROR) if (msg == STDERR_ERROR) {
throw Error(readString(from)); string error = readString(from);
unsigned int status = GET_PROTOCOL_MINOR(daemonVersion) >= 8 ? readInt(from) : 1;
throw Error(error, status);
}
else if (msg != STDERR_LAST) else if (msg != STDERR_LAST)
throw Error("protocol error processing standard error"); throw Error("protocol error processing standard error");
} }

View file

@ -29,6 +29,8 @@ public:
PathSet queryValidPaths(); PathSet queryValidPaths();
ValidPathInfo queryPathInfo(const Path & path);
Hash queryPathHash(const Path & path); Hash queryPathHash(const Path & path);
void queryReferences(const Path & path, PathSet & references); void queryReferences(const Path & path, PathSet & references);
@ -37,6 +39,8 @@ public:
Path queryDeriver(const Path & path); Path queryDeriver(const Path & path);
PathSet queryDerivationOutputs(const Path & path);
bool hasSubstitutes(const Path & path); bool hasSubstitutes(const Path & path);
bool querySubstitutablePathInfo(const Path & path, bool querySubstitutablePathInfo(const Path & path,
@ -68,6 +72,10 @@ public:
void collectGarbage(const GCOptions & options, GCResults & results); void collectGarbage(const GCOptions & options, GCResults & results);
PathSet queryFailedPaths();
void clearFailedPaths(const PathSet & paths);
private: private:
AutoCloseFD fdSocket; AutoCloseFD fdSocket;
FdSink to; FdSink to;

44
src/libstore/schema.sql Normal file
View file

@ -0,0 +1,44 @@
create table if not exists ValidPaths (
id integer primary key autoincrement not null,
path text unique not null,
hash text not null,
registrationTime integer not null,
deriver text,
narSize integer
);
create table if not exists Refs (
referrer integer not null,
reference integer not null,
primary key (referrer, reference),
foreign key (referrer) references ValidPaths(id) on delete cascade,
foreign key (reference) references ValidPaths(id) on delete restrict
);
create index if not exists IndexReferrer on Refs(referrer);
create index if not exists IndexReference on Refs(reference);
-- Paths can refer to themselves, causing a tuple (N, N) in the Refs
-- table. This causes a deletion of the corresponding row in
-- ValidPaths to cause a foreign key constraint violation (due to `on
-- delete restrict' on the `reference' column). Therefore, explicitly
-- get rid of self-references.
create trigger if not exists DeleteSelfRefs before delete on ValidPaths
begin
delete from Refs where referrer = old.id and reference = old.id;
end;
create table if not exists DerivationOutputs (
drv integer not null,
id text not null, -- symbolic output id, usually "out"
path text not null,
primary key (drv, id),
foreign key (drv) references ValidPaths(id) on delete cascade
);
create index if not exists IndexDerivationOutputs on DerivationOutputs(path);
create table if not exists FailedPaths (
path text primary key not null,
time integer not null
);

View file

@ -1,7 +1,6 @@
#include "store-api.hh" #include "store-api.hh"
#include "globals.hh" #include "globals.hh"
#include "util.hh" #include "util.hh"
#include "derivations.hh"
#include <limits.h> #include <limits.h>
@ -53,18 +52,6 @@ Path toStorePath(const Path & path)
} }
string getNameOfStorePath(const Path & path)
{
Path::size_type slash = path.rfind('/');
string p = slash == Path::npos ? path : string(path, slash + 1);
Path::size_type dash = p.find('-');
assert(dash != Path::npos);
string p2 = string(p, dash + 1);
if (isDerivation(p2)) p2 = string(p2, 0, p2.size() - 4);
return p2;
}
Path followLinksToStore(const Path & _path) Path followLinksToStore(const Path & _path)
{ {
Path path = absPath(_path); Path path = absPath(_path);
@ -203,7 +190,7 @@ std::pair<Path, Hash> computeStorePathForPath(const Path & srcPath,
bool recursive, HashType hashAlgo, PathFilter & filter) bool recursive, HashType hashAlgo, PathFilter & filter)
{ {
HashType ht(hashAlgo); HashType ht(hashAlgo);
Hash h = recursive ? hashPath(ht, srcPath, filter) : hashFile(ht, srcPath); Hash h = recursive ? hashPath(ht, srcPath, filter).first : hashFile(ht, srcPath);
string name = baseNameOf(srcPath); string name = baseNameOf(srcPath);
Path dstPath = makeFixedOutputPath(recursive, hashAlgo, h, name); Path dstPath = makeFixedOutputPath(recursive, hashAlgo, h, name);
return std::pair<Path, Hash>(dstPath, h); return std::pair<Path, Hash>(dstPath, h);
@ -229,7 +216,7 @@ Path computeStorePathForText(const string & name, const string & s,
/* Return a string accepted by decodeValidPathInfo() that /* Return a string accepted by decodeValidPathInfo() that
registers the specified paths as valid. Note: it's the registers the specified paths as valid. Note: it's the
responsibility of the caller to provide a closure. */ responsibility of the caller to provide a closure. */
string makeValidityRegistration(const PathSet & paths, string StoreAPI::makeValidityRegistration(const PathSet & paths,
bool showDerivers, bool showHash) bool showDerivers, bool showHash)
{ {
string s = ""; string s = "";
@ -237,18 +224,19 @@ string makeValidityRegistration(const PathSet & paths,
foreach (PathSet::iterator, i, paths) { foreach (PathSet::iterator, i, paths) {
s += *i + "\n"; s += *i + "\n";
if (showHash) ValidPathInfo info = queryPathInfo(*i);
s += printHash(store->queryPathHash(*i)) + "\n";
Path deriver = showDerivers ? store->queryDeriver(*i) : ""; if (showHash) {
s += printHash(info.hash) + "\n";
s += (format("%1%\n") % info.narSize).str();
}
Path deriver = showDerivers ? info.deriver : "";
s += deriver + "\n"; s += deriver + "\n";
PathSet references; s += (format("%1%\n") % info.references.size()).str();
store->queryReferences(*i, references);
s += (format("%1%\n") % references.size()).str(); foreach (PathSet::iterator, j, info.references)
foreach (PathSet::iterator, j, references)
s += *j + "\n"; s += *j + "\n";
} }
@ -265,6 +253,8 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven)
string s; string s;
getline(str, s); getline(str, s);
info.hash = parseHash(htSHA256, s); info.hash = parseHash(htSHA256, s);
getline(str, s);
if (!string2Int(s, info.narSize)) throw Error("number expected");
} }
getline(str, info.deriver); getline(str, info.deriver);
string s; int n; string s; int n;

View file

@ -87,9 +87,25 @@ struct SubstitutablePathInfo
Path deriver; Path deriver;
PathSet references; PathSet references;
unsigned long long downloadSize; /* 0 = unknown or inapplicable */ unsigned long long downloadSize; /* 0 = unknown or inapplicable */
unsigned long long narSize; /* 0 = unknown */
}; };
struct ValidPathInfo
{
Path path;
Path deriver;
Hash hash;
PathSet references;
time_t registrationTime;
unsigned long long narSize; // 0 = unknown
unsigned long long id; // internal use only
ValidPathInfo() : registrationTime(0), narSize(0) { }
};
typedef list<ValidPathInfo> ValidPathInfos;
class StoreAPI class StoreAPI
{ {
public: public:
@ -102,6 +118,9 @@ public:
/* Query the set of valid paths. */ /* Query the set of valid paths. */
virtual PathSet queryValidPaths() = 0; virtual PathSet queryValidPaths() = 0;
/* Query information about a valid path. */
virtual ValidPathInfo queryPathInfo(const Path & path) = 0;
/* Queries the hash of a valid path. */ /* Queries the hash of a valid path. */
virtual Hash queryPathHash(const Path & path) = 0; virtual Hash queryPathHash(const Path & path) = 0;
@ -110,33 +129,18 @@ public:
virtual void queryReferences(const Path & path, virtual void queryReferences(const Path & path,
PathSet & references) = 0; PathSet & references) = 0;
/* Like queryReferences, but with self-references filtered out. */
PathSet queryReferencesNoSelf(const Path & path)
{
PathSet res;
queryReferences(path, res);
res.erase(path);
return res;
}
/* Queries the set of incoming FS references for a store path. /* Queries the set of incoming FS references for a store path.
The result is not cleared. */ The result is not cleared. */
virtual void queryReferrers(const Path & path, virtual void queryReferrers(const Path & path,
PathSet & referrers) = 0; PathSet & referrers) = 0;
/* Like queryReferrers, but with self-references filtered out. */
PathSet queryReferrersNoSelf(const Path & path)
{
PathSet res;
queryReferrers(path, res);
res.erase(path);
return res;
}
/* Query the deriver of a store path. Return the empty string if /* Query the deriver of a store path. Return the empty string if
no deriver has been set. */ no deriver has been set. */
virtual Path queryDeriver(const Path & path) = 0; virtual Path queryDeriver(const Path & path) = 0;
/* Query the outputs of the derivation denoted by `path'. */
virtual PathSet queryDerivationOutputs(const Path & path) = 0;
/* Query whether a path has substitutes. */ /* Query whether a path has substitutes. */
virtual bool hasSubstitutes(const Path & path) = 0; virtual bool hasSubstitutes(const Path & path) = 0;
@ -222,6 +226,19 @@ public:
/* Perform a garbage collection. */ /* Perform a garbage collection. */
virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0; virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0;
/* Return the set of paths that have failed to build.*/
virtual PathSet queryFailedPaths() = 0;
/* Clear the "failed" status of the given paths. The special
value `*' causes all failed paths to be cleared. */
virtual void clearFailedPaths(const PathSet & paths) = 0;
/* Return a string representing information about the path that
can be loaded into the database using `nix-store --load-db' or
`nix-store --register-validity'. */
string makeValidityRegistration(const PathSet & paths,
bool showDerivers, bool showHash);
}; };
@ -241,12 +258,6 @@ void checkStoreName(const string & name);
Path toStorePath(const Path & path); Path toStorePath(const Path & path);
/* Get the "name" part of a store path, that is, the part after the
hash and the dash, and with any ".drv" suffix removed
(e.g. /nix/store/<hash>-foo-1.2.3.drv => foo-1.2.3). */
string getNameOfStorePath(const Path & path);
/* Follow symlinks until we end up with a path in the Nix store. */ /* Follow symlinks until we end up with a path in the Nix store. */
Path followLinksToStore(const Path & path); Path followLinksToStore(const Path & path);
@ -321,21 +332,6 @@ boost::shared_ptr<StoreAPI> openStore();
string showPaths(const PathSet & paths); string showPaths(const PathSet & paths);
string makeValidityRegistration(const PathSet & paths,
bool showDerivers, bool showHash);
struct ValidPathInfo
{
Path path;
Path deriver;
Hash hash;
PathSet references;
time_t registrationTime;
ValidPathInfo() : registrationTime(0) { }
};
typedef list<ValidPathInfo> ValidPathInfos;
ValidPathInfo decodeValidPathInfo(std::istream & str, ValidPathInfo decodeValidPathInfo(std::istream & str,
bool hashGiven = false); bool hashGiven = false);

View file

@ -8,7 +8,7 @@ namespace nix {
#define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_1 0x6e697863
#define WORKER_MAGIC_2 0x6478696f #define WORKER_MAGIC_2 0x6478696f
#define PROTOCOL_VERSION 0x106 #define PROTOCOL_VERSION 0x108
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
@ -34,6 +34,11 @@ typedef enum {
wopSetOptions = 19, wopSetOptions = 19,
wopCollectGarbage = 20, wopCollectGarbage = 20,
wopQuerySubstitutablePathInfo = 21, wopQuerySubstitutablePathInfo = 21,
wopQueryDerivationOutputs = 22,
wopQueryValidPaths = 23,
wopQueryFailedPaths = 24,
wopClearFailedPaths = 25,
wopQueryPathInfo = 26,
} WorkerOp; } WorkerOp;

View file

@ -3,7 +3,7 @@ pkglib_LTLIBRARIES = libutil.la
libutil_la_SOURCES = util.cc hash.cc serialise.cc \ libutil_la_SOURCES = util.cc hash.cc serialise.cc \
archive.cc xml-writer.cc archive.cc xml-writer.cc
libutil_la_LIBADD = ../boost/format/libformat.la libutil_la_LIBADD = ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib}
pkginclude_HEADERS = util.hh hash.hh serialise.hh \ pkginclude_HEADERS = util.hh hash.hh serialise.hh \
archive.hh xml-writer.hh types.hh archive.hh xml-writer.hh types.hh

View file

@ -181,8 +181,6 @@ static void parseContents(ParseSink & sink, Source & source, const Path & path)
left -= n; left -= n;
} }
sink.finalizeContents(size);
readPadding(size, source); readPadding(size, source);
} }
@ -317,12 +315,6 @@ struct RestoreSink : ParseSink
writeFull(fd, data, len); writeFull(fd, data, len);
} }
void finalizeContents(unsigned long long size)
{
errno = ftruncate(fd, size);
if (errno) throw SysError(format("truncating file to its allocated length of %1% bytes") % size);
}
void createSymlink(const Path & path, const string & target) void createSymlink(const Path & path, const string & target)
{ {
Path p = dstPath + path; Path p = dstPath + path;

View file

@ -64,7 +64,6 @@ struct ParseSink
virtual void isExecutable() { }; virtual void isExecutable() { };
virtual void preallocateContents(unsigned long long size) { }; virtual void preallocateContents(unsigned long long size) { };
virtual void receiveContents(unsigned char * data, unsigned int len) { }; virtual void receiveContents(unsigned char * data, unsigned int len) { };
virtual void finalizeContents(unsigned long long size) { };
virtual void createSymlink(const Path & path, const string & target) { }; virtual void createSymlink(const Path & path, const string & target) { };
}; };

View file

@ -286,9 +286,18 @@ Hash hashFile(HashType ht, const Path & path)
HashSink::HashSink(HashType ht) : ht(ht) HashSink::HashSink(HashType ht) : ht(ht)
{ {
ctx = new Ctx; ctx = new Ctx;
bytes = 0;
start(ht, *ctx); start(ht, *ctx);
} }
HashSink::HashSink(const HashSink & h)
{
ht = h.ht;
bytes = h.bytes;
ctx = new Ctx;
*ctx = *h.ctx;
}
HashSink::~HashSink() HashSink::~HashSink()
{ {
delete ctx; delete ctx;
@ -297,18 +306,20 @@ HashSink::~HashSink()
void HashSink::operator () void HashSink::operator ()
(const unsigned char * data, unsigned int len) (const unsigned char * data, unsigned int len)
{ {
bytes += len;
update(ht, *ctx, data, len); update(ht, *ctx, data, len);
} }
Hash HashSink::finish() HashResult HashSink::finish()
{ {
Hash hash(ht); Hash hash(ht);
nix::finish(ht, *ctx, hash.hash); nix::finish(ht, *ctx, hash.hash);
return hash; return HashResult(hash, bytes);
} }
Hash hashPath(HashType ht, const Path & path, PathFilter & filter) HashResult hashPath(
HashType ht, const Path & path, PathFilter & filter)
{ {
HashSink sink(ht); HashSink sink(ht);
dumpPath(path, sink, filter); dumpPath(path, sink, filter);

View file

@ -40,7 +40,6 @@ struct Hash
/* For sorting. */ /* For sorting. */
bool operator < (const Hash & h) const; bool operator < (const Hash & h) const;
}; };
@ -72,7 +71,8 @@ Hash hashFile(HashType ht, const Path & path);
(essentially) hashString(ht, dumpPath(path)). */ (essentially) hashString(ht, dumpPath(path)). */
struct PathFilter; struct PathFilter;
extern PathFilter defaultPathFilter; extern PathFilter defaultPathFilter;
Hash hashPath(HashType ht, const Path & path, typedef std::pair<Hash, unsigned long long> HashResult;
HashResult hashPath(HashType ht, const Path & path,
PathFilter & filter = defaultPathFilter); PathFilter & filter = defaultPathFilter);
/* Compress a hash to the specified number of bytes by cyclically /* Compress a hash to the specified number of bytes by cyclically
@ -93,12 +93,14 @@ class HashSink : public Sink
private: private:
HashType ht; HashType ht;
Ctx * ctx; Ctx * ctx;
unsigned long long bytes;
public: public:
HashSink(HashType ht); HashSink(HashType ht);
HashSink(const HashSink & h);
~HashSink(); ~HashSink();
virtual void operator () (const unsigned char * data, unsigned int len); virtual void operator () (const unsigned char * data, unsigned int len);
Hash finish(); HashResult finish();
}; };

View file

@ -27,7 +27,8 @@ protected:
string prefix_; // used for location traces etc. string prefix_; // used for location traces etc.
string err; string err;
public: public:
BaseError(const format & f); unsigned int status; // exit status
BaseError(const format & f, unsigned int status = 1);
~BaseError() throw () { }; ~BaseError() throw () { };
const char * what() const throw () { return err.c_str(); } const char * what() const throw () { return err.c_str(); }
const string & msg() const throw () { return err; } const string & msg() const throw () { return err; }
@ -39,7 +40,7 @@ public:
class newClass : public superClass \ class newClass : public superClass \
{ \ { \
public: \ public: \
newClass(const format & f) : superClass(f) { }; \ newClass(const format & f, unsigned int status = 1) : superClass(f, status) { }; \
}; };
MakeError(Error, BaseError) MakeError(Error, BaseError)
@ -63,7 +64,7 @@ typedef set<Path> PathSet;
typedef enum { typedef enum {
lvlError, lvlError = 0,
lvlInfo, lvlInfo,
lvlTalkative, lvlTalkative,
lvlChatty, lvlChatty,

View file

@ -7,11 +7,8 @@
#include <sstream> #include <sstream>
#include <cstring> #include <cstring>
#include <sys/stat.h>
#include <sys/wait.h> #include <sys/wait.h>
#include <sys/types.h>
#include <fcntl.h> #include <fcntl.h>
#include <unistd.h>
#include <limits.h> #include <limits.h>
#include "util.hh" #include "util.hh"
@ -23,7 +20,8 @@ extern char * * environ;
namespace nix { namespace nix {
BaseError::BaseError(const format & f) BaseError::BaseError(const format & f, unsigned int status)
: status(status)
{ {
err = f.str(); err = f.str();
} }
@ -149,6 +147,15 @@ string baseNameOf(const Path & path)
} }
struct stat lstat(const Path & path)
{
struct stat st;
if (lstat(path.c_str(), &st))
throw SysError(format("getting status of `%1%'") % path);
return st;
}
bool pathExists(const Path & path) bool pathExists(const Path & path)
{ {
int res; int res;
@ -164,9 +171,7 @@ bool pathExists(const Path & path)
Path readLink(const Path & path) Path readLink(const Path & path)
{ {
checkInterrupt(); checkInterrupt();
struct stat st; struct stat st = lstat(path);
if (lstat(path.c_str(), &st))
throw SysError(format("getting status of `%1%'") % path);
if (!S_ISLNK(st.st_mode)) if (!S_ISLNK(st.st_mode))
throw Error(format("`%1%' is not a symlink") % path); throw Error(format("`%1%' is not a symlink") % path);
char buf[st.st_size]; char buf[st.st_size];
@ -178,9 +183,7 @@ Path readLink(const Path & path)
bool isLink(const Path & path) bool isLink(const Path & path)
{ {
struct stat st; struct stat st = lstat(path);
if (lstat(path.c_str(), &st))
throw SysError(format("getting status of `%1%'") % path);
return S_ISLNK(st.st_mode); return S_ISLNK(st.st_mode);
} }
@ -228,13 +231,12 @@ string readFile(const Path & path)
} }
void writeFile(const Path & path, const string & s, bool doFsync) void writeFile(const Path & path, const string & s)
{ {
AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0666); AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0666);
if (fd == -1) if (fd == -1)
throw SysError(format("opening file `%1%'") % path); throw SysError(format("opening file `%1%'") % path);
writeFull(fd, (unsigned char *) s.c_str(), s.size()); writeFull(fd, (unsigned char *) s.c_str(), s.size());
if (doFsync) fsync(fd);
} }
@ -270,9 +272,7 @@ static void _computePathSize(const Path & path,
{ {
checkInterrupt(); checkInterrupt();
struct stat st; struct stat st = lstat(path);
if (lstat(path.c_str(), &st))
throw SysError(format("getting attributes of path `%1%'") % path);
bytes += st.st_size; bytes += st.st_size;
blocks += st.st_blocks; blocks += st.st_blocks;
@ -302,9 +302,7 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed,
printMsg(lvlVomit, format("%1%") % path); printMsg(lvlVomit, format("%1%") % path);
struct stat st; struct stat st = lstat(path);
if (lstat(path.c_str(), &st))
throw SysError(format("getting attributes of path `%1%'") % path);
if (!S_ISDIR(st.st_mode) && st.st_nlink == 1) { if (!S_ISDIR(st.st_mode) && st.st_nlink == 1) {
bytesFreed += st.st_size; bytesFreed += st.st_size;
@ -351,9 +349,7 @@ void makePathReadOnly(const Path & path)
{ {
checkInterrupt(); checkInterrupt();
struct stat st; struct stat st = lstat(path);
if (lstat(path.c_str(), &st))
throw SysError(format("getting attributes of path `%1%'") % path);
if (!S_ISLNK(st.st_mode) && (st.st_mode & S_IWUSR)) { if (!S_ISLNK(st.st_mode) && (st.st_mode & S_IWUSR)) {
if (chmod(path.c_str(), st.st_mode & ~S_IWUSR) == -1) if (chmod(path.c_str(), st.st_mode & ~S_IWUSR) == -1)
@ -412,12 +408,18 @@ Paths createDirs(const Path & path)
{ {
Paths created; Paths created;
if (path == "/") return created; if (path == "/") return created;
if (!pathExists(path)) {
struct stat st;
if (lstat(path.c_str(), &st) == -1) {
created = createDirs(dirOf(path)); created = createDirs(dirOf(path));
if (mkdir(path.c_str(), 0777) == -1) if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST)
throw SysError(format("creating directory `%1%'") % path); throw SysError(format("creating directory `%1%'") % path);
st = lstat(path);
created.push_back(path); created.push_back(path);
} }
if (!S_ISDIR(st.st_mode)) throw Error(format("`%1%' is not a directory") % path);
return created; return created;
} }
@ -976,6 +978,17 @@ Strings tokenizeString(const string & s, const string & separators)
} }
string concatStringsSep(const string & sep, const Strings & ss)
{
string s;
foreach (Strings::const_iterator, i, ss) {
if (s.size() != 0) s += sep;
s += *i;
}
return s;
}
string statusToString(int status) string statusToString(int status)
{ {
if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {

View file

@ -4,6 +4,7 @@
#include "types.hh" #include "types.hh"
#include <sys/types.h> #include <sys/types.h>
#include <sys/stat.h>
#include <dirent.h> #include <dirent.h>
#include <unistd.h> #include <unistd.h>
#include <signal.h> #include <signal.h>
@ -42,6 +43,9 @@ Path dirOf(const Path & path);
following the final `/'. */ following the final `/'. */
string baseNameOf(const Path & path); string baseNameOf(const Path & path);
/* Get status of `path'. */
struct stat lstat(const Path & path);
/* Return true iff the given path exists. */ /* Return true iff the given path exists. */
bool pathExists(const Path & path); bool pathExists(const Path & path);
@ -60,7 +64,7 @@ string readFile(int fd);
string readFile(const Path & path); string readFile(const Path & path);
/* Write a string to a file. */ /* Write a string to a file. */
void writeFile(const Path & path, const string & s, bool doFsync = false); void writeFile(const Path & path, const string & s);
/* Read a line from a file descriptor. */ /* Read a line from a file descriptor. */
string readLine(int fd); string readLine(int fd);
@ -280,6 +284,11 @@ MakeError(Interrupted, BaseError)
Strings tokenizeString(const string & s, const string & separators = " \t\n\r"); Strings tokenizeString(const string & s, const string & separators = " \t\n\r");
/* Concatenate the given strings with a separator between the
elements. */
string concatStringsSep(const string & sep, const Strings & ss);
/* Convert the exit status of a child as returned by wait() into an /* Convert the exit status of a child as returned by wait() into an
error string. */ error string. */
string statusToString(int status); string statusToString(int status);

View file

@ -4,7 +4,7 @@ nix_env_SOURCES = nix-env.cc profiles.cc profiles.hh user-env.cc user-env.hh hel
nix_env_LDADD = ../libmain/libmain.la ../libexpr/libexpr.la \ nix_env_LDADD = ../libmain/libmain.la ../libexpr/libexpr.la \
../libstore/libstore.la ../libutil/libutil.la \ ../libstore/libstore.la ../libutil/libutil.la \
../boost/format/libformat.la @ADDITIONAL_NETWORK_LIBS@ ../boost/format/libformat.la
nix-env.o: help.txt.hh nix-env.o: help.txt.hh

View file

@ -69,7 +69,7 @@ typedef void (* Operation) (Globals & globals,
void printHelp() void printHelp()
{ {
cout << string((char *) helpText, sizeof helpText); cout << string((char *) helpText);
} }

View file

@ -2,7 +2,7 @@ bin_PROGRAMS = nix-hash
nix_hash_SOURCES = nix-hash.cc help.txt nix_hash_SOURCES = nix-hash.cc help.txt
nix_hash_LDADD = ../libmain/libmain.la ../libstore/libstore.la ../libutil/libutil.la \ nix_hash_LDADD = ../libmain/libmain.la ../libstore/libstore.la ../libutil/libutil.la \
../boost/format/libformat.la @ADDITIONAL_NETWORK_LIBS@ ../boost/format/libformat.la
nix-hash.o: help.txt.hh nix-hash.o: help.txt.hh

View file

@ -10,7 +10,7 @@ using namespace nix;
void printHelp() void printHelp()
{ {
std::cout << string((char *) helpText, sizeof helpText); std::cout << string((char *) helpText);
} }
@ -44,7 +44,7 @@ void run(Strings args)
if (op == opHash) { if (op == opHash) {
for (Strings::iterator i = ss.begin(); i != ss.end(); ++i) { for (Strings::iterator i = ss.begin(); i != ss.end(); ++i) {
Hash h = flat ? hashFile(ht, *i) : hashPath(ht, *i); Hash h = flat ? hashFile(ht, *i) : hashPath(ht, *i).first;
if (truncate && h.hashSize > 20) h = compressHash(h, 20); if (truncate && h.hashSize > 20) h = compressHash(h, 20);
std::cout << format("%1%\n") % std::cout << format("%1%\n") %
(base32 ? printHash32(h) : printHash(h)); (base32 ? printHash32(h) : printHash(h));

View file

@ -3,7 +3,7 @@ bin_PROGRAMS = nix-instantiate
nix_instantiate_SOURCES = nix-instantiate.cc help.txt nix_instantiate_SOURCES = nix-instantiate.cc help.txt
nix_instantiate_LDADD = ../libmain/libmain.la ../libexpr/libexpr.la \ nix_instantiate_LDADD = ../libmain/libmain.la ../libexpr/libexpr.la \
../libstore/libstore.la ../libutil/libutil.la \ ../libstore/libstore.la ../libutil/libutil.la \
../boost/format/libformat.la @ADDITIONAL_NETWORK_LIBS@ ../boost/format/libformat.la
nix-instantiate.o: help.txt.hh nix-instantiate.o: help.txt.hh

View file

@ -19,7 +19,7 @@ using namespace nix;
void printHelp() void printHelp()
{ {
std::cout << string((char *) helpText, sizeof helpText); std::cout << string((char *) helpText);
} }

View file

@ -18,6 +18,8 @@ struct Decoder
int priority; int priority;
bool ignoreLF; bool ignoreLF;
int lineNo, charNo; int lineNo, charNo;
bool warning;
bool error;
Decoder() Decoder()
{ {
@ -29,6 +31,8 @@ struct Decoder
ignoreLF = false; ignoreLF = false;
lineNo = 1; lineNo = 1;
charNo = 0; charNo = 0;
warning = false;
error = false;
} }
void pushChar(char c); void pushChar(char c);
@ -95,6 +99,12 @@ void Decoder::pushChar(char c)
case 'b': case 'b':
ignoreLF = false; ignoreLF = false;
break; break;
case 'e':
error = true;
break;
case 'w':
warning = true;
break;
} }
} else if (c >= '0' && c <= '9') { } else if (c >= '0' && c <= '9') {
int n = 0; int n = 0;
@ -118,6 +128,8 @@ void Decoder::finishLine()
string tag = inHeader ? "head" : "line"; string tag = inHeader ? "head" : "line";
cout << "<" << tag; cout << "<" << tag;
if (priority != 1) cout << " priority='" << priority << "'"; if (priority != 1) cout << " priority='" << priority << "'";
if (warning) cout << " warning='1'";
if (error) cout << " error='1'";
cout << ">"; cout << ">";
for (unsigned int i = 0; i < line.size(); i++) { for (unsigned int i = 0; i < line.size(); i++) {
@ -158,6 +170,8 @@ void Decoder::finishLine()
line = ""; line = "";
inHeader = false; inHeader = false;
priority = 1; priority = 1;
warning = false;
error = false;
} }

View file

@ -4,5 +4,4 @@ nix_setuid_helper_SOURCES = nix-setuid-helper.cc
nix_setuid_helper_LDADD = ../libutil/libutil.la \ nix_setuid_helper_LDADD = ../libutil/libutil.la \
../boost/format/libformat.la ../boost/format/libformat.la
AM_CXXFLAGS = \ AM_CXXFLAGS = -I$(srcdir)/.. -I$(srcdir)/../libutil
-I$(srcdir)/.. -I$(srcdir)/../libutil

View file

@ -5,7 +5,7 @@ nix_store_SOURCES = \
xmlgraph.cc xmlgraph.hh xmlgraph.cc xmlgraph.hh
nix_store_LDADD = ../libmain/libmain.la ../libstore/libstore.la ../libutil/libutil.la \ nix_store_LDADD = ../libmain/libmain.la ../libstore/libstore.la ../libutil/libutil.la \
../boost/format/libformat.la @ADDITIONAL_NETWORK_LIBS@ ../boost/format/libformat.la
nix-store.o: help.txt.hh nix-store.o: help.txt.hh

View file

@ -16,7 +16,7 @@ Operations:
--gc: run the garbage collector --gc: run the garbage collector
--dump: dump a path as a Nix archive, forgetting dependencies --dump: dump a path as a Nix archive (NAR), forgetting dependencies
--restore: restore a path from a Nix archive, without --restore: restore a path from a Nix archive, without
registering validity registering validity
@ -27,6 +27,9 @@ Operations:
--verify: verify Nix structures --verify: verify Nix structures
--optimise: optimise the Nix store by hard-linking identical files --optimise: optimise the Nix store by hard-linking identical files
--query-failed-paths: list paths that failed to build (if enabled)
--clear-failed-paths: clear the failed status of the given paths
--version: output version information --version: output version information
--help: display help --help: display help
@ -41,6 +44,7 @@ Query flags:
--graph: print a dot graph rooted at given path --graph: print a dot graph rooted at given path
--xml: emit an XML representation of the graph rooted at the given path --xml: emit an XML representation of the graph rooted at the given path
--hash: print the SHA-256 hash of the contents of the path --hash: print the SHA-256 hash of the contents of the path
--size: print the size of the NAR dump of the path
--roots: print the garbage collector roots that point to the path --roots: print the garbage collector roots that point to the path
Query switches (not applicable to all queries): Query switches (not applicable to all queries):

View file

@ -22,7 +22,7 @@ typedef void (* Operation) (Strings opFlags, Strings opArgs);
void printHelp() void printHelp()
{ {
cout << string((char *) helpText, sizeof helpText); cout << string((char *) helpText);
} }
@ -34,7 +34,7 @@ static bool indirectRoot = false;
LocalStore & ensureLocalStore() LocalStore & ensureLocalStore()
{ {
LocalStore * store2(dynamic_cast<LocalStore *>(store.get())); LocalStore * store2(dynamic_cast<LocalStore *>(store.get()));
if (!store2) throw Error("you don't have sufficient rights to use --verify"); if (!store2) throw Error("you don't have sufficient rights to use this command");
return *store2; return *store2;
} }
@ -226,7 +226,7 @@ static void printTree(const Path & path,
static void opQuery(Strings opFlags, Strings opArgs) static void opQuery(Strings opFlags, Strings opArgs)
{ {
enum { qOutputs, qRequisites, qReferences, qReferrers enum { qOutputs, qRequisites, qReferences, qReferrers
, qReferrersClosure, qDeriver, qBinding, qHash , qReferrersClosure, qDeriver, qBinding, qHash, qSize
, qTree, qGraph, qXml, qResolve, qRoots } query = qOutputs; , qTree, qGraph, qXml, qResolve, qRoots } query = qOutputs;
bool useOutput = false; bool useOutput = false;
bool includeOutputs = false; bool includeOutputs = false;
@ -248,6 +248,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
query = qBinding; query = qBinding;
} }
else if (*i == "--hash") query = qHash; else if (*i == "--hash") query = qHash;
else if (*i == "--size") query = qSize;
else if (*i == "--tree") query = qTree; else if (*i == "--tree") query = qTree;
else if (*i == "--graph") query = qGraph; else if (*i == "--graph") query = qGraph;
else if (*i == "--xml") query = qXml; else if (*i == "--xml") query = qXml;
@ -310,11 +311,15 @@ static void opQuery(Strings opFlags, Strings opArgs)
break; break;
case qHash: case qHash:
case qSize:
foreach (Strings::iterator, i, opArgs) { foreach (Strings::iterator, i, opArgs) {
Path path = maybeUseOutput(followLinksToStorePath(*i), useOutput, forceRealise); Path path = maybeUseOutput(followLinksToStorePath(*i), useOutput, forceRealise);
Hash hash = store->queryPathHash(path); ValidPathInfo info = store->queryPathInfo(path);
assert(hash.type == htSHA256); if (query == qHash) {
cout << format("sha256:%1%\n") % printHash32(hash); assert(info.hash.type == htSHA256);
cout << format("sha256:%1%\n") % printHash32(info.hash);
} else if (query == qSize)
cout << format("%1%\n") % info.narSize;
} }
break; break;
@ -393,9 +398,8 @@ static void opDumpDB(Strings opFlags, Strings opArgs)
if (!opArgs.empty()) if (!opArgs.empty())
throw UsageError("no arguments expected"); throw UsageError("no arguments expected");
PathSet validPaths = store->queryValidPaths(); PathSet validPaths = store->queryValidPaths();
foreach (PathSet::iterator, i, validPaths) { foreach (PathSet::iterator, i, validPaths)
cout << makeValidityRegistration(singleton<PathSet>(*i), true, true); cout << store->makeValidityRegistration(singleton<PathSet>(*i), true, true);
}
} }
@ -410,8 +414,11 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise)
/* !!! races */ /* !!! races */
if (canonicalise) if (canonicalise)
canonicalisePathMetaData(info.path); canonicalisePathMetaData(info.path);
if (!hashGiven) if (!hashGiven) {
info.hash = hashPath(htSHA256, info.path); HashResult hash = hashPath(htSHA256, info.path);
info.hash = hash.first;
info.narSize = hash.second;
}
infos.push_back(info); infos.push_back(info);
} }
} }
@ -661,8 +668,7 @@ static void opOptimise(Strings opFlags, Strings opArgs)
bool dryRun = false; bool dryRun = false;
for (Strings::iterator i = opFlags.begin(); foreach (Strings::iterator, i, opFlags)
i != opFlags.end(); ++i)
if (*i == "--dry-run") dryRun = true; if (*i == "--dry-run") dryRun = true;
else throw UsageError(format("unknown flag `%1%'") % *i); else throw UsageError(format("unknown flag `%1%'") % *i);
@ -677,6 +683,24 @@ static void opOptimise(Strings opFlags, Strings opArgs)
} }
static void opQueryFailedPaths(Strings opFlags, Strings opArgs)
{
if (!opArgs.empty() || !opFlags.empty())
throw UsageError("no arguments expected");
PathSet failed = store->queryFailedPaths();
foreach (PathSet::iterator, i, failed)
cout << format("%1%\n") % *i;
}
static void opClearFailedPaths(Strings opFlags, Strings opArgs)
{
if (!opFlags.empty())
throw UsageError("no flags expected");
store->clearFailedPaths(PathSet(opArgs.begin(), opArgs.end()));
}
/* Scan the arguments; find the operation, set global flags, put all /* Scan the arguments; find the operation, set global flags, put all
other flags in a list, and put all other arguments in another other flags in a list, and put all other arguments in another
list. */ list. */
@ -728,6 +752,10 @@ void run(Strings args)
op = opVerify; op = opVerify;
else if (arg == "--optimise") else if (arg == "--optimise")
op = opOptimise; op = opOptimise;
else if (arg == "--query-failed-paths")
op = opQueryFailedPaths;
else if (arg == "--clear-failed-paths")
op = opClearFailedPaths;
else if (arg == "--add-root") { else if (arg == "--add-root") {
if (i == args.end()) if (i == args.end())
throw UsageError("`--add-root requires an argument"); throw UsageError("`--add-root requires an argument");

View file

@ -2,7 +2,7 @@ bin_PROGRAMS = nix-worker
nix_worker_SOURCES = nix-worker.cc help.txt nix_worker_SOURCES = nix-worker.cc help.txt
nix_worker_LDADD = ../libmain/libmain.la ../libstore/libstore.la ../libutil/libutil.la \ nix_worker_LDADD = ../libmain/libmain.la ../libstore/libstore.la ../libutil/libutil.la \
../boost/format/libformat.la @ADDITIONAL_NETWORK_LIBS@ ../boost/format/libformat.la
nix-worker.o: help.txt.hh nix-worker.o: help.txt.hh

View file

@ -178,7 +178,7 @@ static void startWork()
/* stopWork() means that we're done; stop sending stderr to the /* stopWork() means that we're done; stop sending stderr to the
client. */ client. */
static void stopWork(bool success = true, const string & msg = "") static void stopWork(bool success = true, const string & msg = "", unsigned int status = 0)
{ {
/* Stop handling async client death; we're going to a state where /* Stop handling async client death; we're going to a state where
we're either sending or receiving from the client, so we'll be we're either sending or receiving from the client, so we'll be
@ -192,6 +192,7 @@ static void stopWork(bool success = true, const string & msg = "")
else { else {
writeInt(STDERR_ERROR, to); writeInt(STDERR_ERROR, to);
writeString(msg, to); writeString(msg, to);
if (status != 0) writeInt(status, to);
} }
} }
@ -315,14 +316,16 @@ static void performOp(unsigned int clientVersion,
} }
case wopQueryReferences: case wopQueryReferences:
case wopQueryReferrers: { case wopQueryReferrers:
case wopQueryDerivationOutputs: {
Path path = readStorePath(from); Path path = readStorePath(from);
startWork(); startWork();
PathSet paths; PathSet paths;
if (op == wopQueryReferences) if (op == wopQueryReferences)
store->queryReferences(path, paths); store->queryReferences(path, paths);
else else if (op == wopQueryReferrers)
store->queryReferrers(path, paths); store->queryReferrers(path, paths);
else paths = store->queryDerivationOutputs(path);
stopWork(); stopWork();
writeStringSet(paths, to); writeStringSet(paths, to);
break; break;
@ -519,10 +522,50 @@ static void performOp(unsigned int clientVersion,
writeString(info.deriver, to); writeString(info.deriver, to);
writeStringSet(info.references, to); writeStringSet(info.references, to);
writeLongLong(info.downloadSize, to); writeLongLong(info.downloadSize, to);
if (GET_PROTOCOL_MINOR(clientVersion) >= 7)
writeLongLong(info.narSize, to);
} }
break; break;
} }
case wopQueryValidPaths: {
startWork();
PathSet paths = store->queryValidPaths();
stopWork();
writeStringSet(paths, to);
break;
}
case wopQueryFailedPaths: {
startWork();
PathSet paths = store->queryFailedPaths();
stopWork();
writeStringSet(paths, to);
break;
}
case wopClearFailedPaths: {
PathSet paths = readStringSet(from);
startWork();
store->clearFailedPaths(paths);
stopWork();
writeInt(1, to);
break;
}
case wopQueryPathInfo: {
Path path = readStorePath(from);
startWork();
ValidPathInfo info = store->queryPathInfo(path);
stopWork();
writeString(info.deriver, to);
writeString(printHash(info.hash), to);
writeStringSet(info.references, to);
writeInt(info.registrationTime, to);
writeLongLong(info.narSize, to);
break;
}
default: default:
throw Error(format("invalid operation %1%") % op); throw Error(format("invalid operation %1%") % op);
} }
@ -595,7 +638,7 @@ static void processConnection()
try { try {
performOp(clientVersion, from, to, op); performOp(clientVersion, from, to, op);
} catch (Error & e) { } catch (Error & e) {
stopWork(false, e.msg()); stopWork(false, e.msg(), GET_PROTOCOL_MINOR(clientVersion) >= 8 ? e.status : 0);
} }
assert(!canSendStderr); assert(!canSendStderr);
@ -670,9 +713,8 @@ static void daemonLoop()
while (1) { while (1) {
try { try {
/* Important: the server process *cannot* open the /* Important: the server process *cannot* open the SQLite
Berkeley DB environment, because it doesn't like forks database, because it doesn't like forks very much. */
very much. */
assert(!store); assert(!store);
/* Accept a connection. */ /* Accept a connection. */
@ -770,7 +812,7 @@ void run(Strings args)
void printHelp() void printHelp()
{ {
std::cout << string((char *) helpText, sizeof helpText); std::cout << string((char *) helpText);
} }

View file

@ -24,6 +24,7 @@
-e "s^@xmllint\@^$(xmllint)^g" \ -e "s^@xmllint\@^$(xmllint)^g" \
-e "s^@xmlflags\@^$(xmlflags)^g" \ -e "s^@xmlflags\@^$(xmlflags)^g" \
-e "s^@xsltproc\@^$(xsltproc)^g" \ -e "s^@xsltproc\@^$(xsltproc)^g" \
-e "s^@sqlite_bin\@^$(sqlite_bin)^g" \
-e "s^@version\@^$(VERSION)^g" \ -e "s^@version\@^$(VERSION)^g" \
-e "s^@testPath\@^$(coreutils):$$(dirname $$(type -p expr))^g" \ -e "s^@testPath\@^$(coreutils):$$(dirname $$(type -p expr))^g" \
< $< > $@ || rm $@ < $< > $@ || rm $@

View file

@ -7,7 +7,8 @@ TESTS = init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \
fallback.sh nix-push.sh gc.sh gc-concurrent.sh verify.sh nix-pull.sh \ fallback.sh nix-push.sh gc.sh gc-concurrent.sh verify.sh nix-pull.sh \
referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \ referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \
gc-runtime.sh install-package.sh check-refs.sh filter-source.sh \ gc-runtime.sh install-package.sh check-refs.sh filter-source.sh \
remote-store.sh export.sh export-graph.sh negative-caching.sh remote-store.sh export.sh export-graph.sh negative-caching.sh \
binary-patching.sh
XFAIL_TESTS = XFAIL_TESTS =
@ -31,5 +32,6 @@ EXTRA_DIST = $(TESTS) \
filter-source.nix \ filter-source.nix \
export-graph.nix \ export-graph.nix \
negative-caching.nix \ negative-caching.nix \
binary-patching.nix \
$(wildcard lang/*.nix) $(wildcard lang/*.exp) $(wildcard lang/*.exp.xml) $(wildcard lang/*.flags) \ $(wildcard lang/*.nix) $(wildcard lang/*.exp) $(wildcard lang/*.exp.xml) $(wildcard lang/*.flags) \
common.sh.in common.sh.in

18
tests/binary-patching.nix Normal file
View file

@ -0,0 +1,18 @@
{ version }:
with import ./config.nix;
mkDerivation {
name = "foo-${toString version}";
builder = builtins.toFile "builder.sh"
''
mkdir $out
seq 1 1000000 > $out/foo
${if version != 1 then ''
seq 1000000 1010000 >> $out/foo
'' else ""}
${if version == 3 then ''
echo foobar >> $out/foo
'' else ""}
'';
}

58
tests/binary-patching.sh Normal file
View file

@ -0,0 +1,58 @@
source common.sh
clearManifests
mkdir -p $TEST_ROOT/cache2 $TEST_ROOT/patches
RESULT=$TEST_ROOT/result
# Build version 1 and 2 of the "foo" package.
$NIX_BIN_DIR/nix-push --copy $TEST_ROOT/cache2 $TEST_ROOT/manifest1 \
$($nixbuild -o $RESULT binary-patching.nix --arg version 1)
out2=$($nixbuild -o $RESULT binary-patching.nix --arg version 2)
$NIX_BIN_DIR/nix-push --copy $TEST_ROOT/cache2 $TEST_ROOT/manifest2 $out2
out3=$($nixbuild -o $RESULT binary-patching.nix --arg version 3)
$NIX_BIN_DIR/nix-push --copy $TEST_ROOT/cache2 $TEST_ROOT/manifest3 $out3
rm $RESULT
# Generate binary patches.
$NIX_BIN_DIR/nix-generate-patches $TEST_ROOT/cache2 $TEST_ROOT/patches \
file://$TEST_ROOT/patches $TEST_ROOT/manifest1 $TEST_ROOT/manifest2
$NIX_BIN_DIR/nix-generate-patches $TEST_ROOT/cache2 $TEST_ROOT/patches \
file://$TEST_ROOT/patches $TEST_ROOT/manifest2 $TEST_ROOT/manifest3
grep -q "patch {" $TEST_ROOT/manifest3
# Get rid of versions 2 and 3.
$nixstore --delete $out2 $out3
# Pull the manifest containing the patches.
clearManifests
$NIX_BIN_DIR/nix-pull file://$TEST_ROOT/manifest3
# Make sure that the download size prediction uses the patches rather
# than the full download.
$nixbuild -o $RESULT binary-patching.nix --arg version 3 --dry-run 2>&1 | grep -q "0.01 MiB"
# Now rebuild it. This should use the two patches generated above.
rm -f $TEST_ROOT/var/log/nix/downloads
$nixbuild -o $RESULT binary-patching.nix --arg version 3
rm $RESULT
[ "$(grep ' patch ' $TEST_ROOT/var/log/nix/downloads | wc -l)" -eq 2 ]
# Add a patch from version 1 directly to version 3.
$NIX_BIN_DIR/nix-generate-patches $TEST_ROOT/cache2 $TEST_ROOT/patches \
file://$TEST_ROOT/patches $TEST_ROOT/manifest1 $TEST_ROOT/manifest3
# Rebuild version 3. This should use the direct patch rather than the
# sequence of two patches.
$nixstore --delete $out2 $out3
clearManifests
rm $TEST_ROOT/var/log/nix/downloads
$NIX_BIN_DIR/nix-pull file://$TEST_ROOT/manifest3
$nixbuild -o $RESULT binary-patching.nix --arg version 3
[ "$(grep ' patch ' $TEST_ROOT/var/log/nix/downloads | wc -l)" -eq 1 ]

View file

@ -2,7 +2,7 @@
#set -x #set -x
drv=$4 while read x y drv rest; do
echo "HOOK for $drv" >&2 echo "HOOK for $drv" >&2
@ -12,10 +12,12 @@ echo "output path is $outPath" >&2
if `echo $outPath | grep -q input-1`; then if `echo $outPath | grep -q input-1`; then
echo "# accept" >&2 echo "# accept" >&2
read x read inputs
echo "got $x" read outputs
mkdir $outPath mkdir $outPath
echo "BAR" > $outPath/foo echo "BAR" > $outPath/foo
else else
echo "# decline" >&2 echo "# decline" >&2
fi fi
done

View file

@ -38,6 +38,7 @@ export dot=@dot@
export xmllint="@xmllint@" export xmllint="@xmllint@"
export xmlflags="@xmlflags@" export xmlflags="@xmlflags@"
export xsltproc="@xsltproc@" export xsltproc="@xsltproc@"
export sqlite3="@sqlite_bin@/bin/sqlite3"
export SHELL="@shell@" export SHELL="@shell@"
export version=@version@ export version=@version@

View file

@ -23,28 +23,33 @@ ln -s $nixinstantiate $NIX_BIN_DIR/
ln -s $nixhash $NIX_BIN_DIR/ ln -s $nixhash $NIX_BIN_DIR/
ln -s $nixenv $NIX_BIN_DIR/ ln -s $nixenv $NIX_BIN_DIR/
ln -s $nixworker $NIX_BIN_DIR/ ln -s $nixworker $NIX_BIN_DIR/
ln -s $TOP/src/bsdiff-*/bsdiff $NIX_BIN_DIR/
ln -s $TOP/src/bsdiff-*/bspatch $NIX_BIN_DIR/
ln -s $TOP/scripts/nix-prefetch-url $NIX_BIN_DIR/ ln -s $TOP/scripts/nix-prefetch-url $NIX_BIN_DIR/
ln -s $TOP/scripts/nix-collect-garbage $NIX_BIN_DIR/ ln -s $TOP/scripts/nix-collect-garbage $NIX_BIN_DIR/
ln -s $TOP/scripts/nix-build $NIX_BIN_DIR/ ln -s $TOP/scripts/nix-build $NIX_BIN_DIR/
ln -s $TOP/scripts/nix-install-package $NIX_BIN_DIR/ ln -s $TOP/scripts/nix-install-package $NIX_BIN_DIR/
ln -s $TOP/scripts/nix-push $NIX_BIN_DIR/ ln -s $TOP/scripts/nix-push $NIX_BIN_DIR/
ln -s $TOP/scripts/nix-pull $NIX_BIN_DIR/ ln -s $TOP/scripts/nix-pull $NIX_BIN_DIR/
ln -s $TOP/scripts/nix-generate-patches $NIX_BIN_DIR/
mkdir $NIX_BIN_DIR/nix mkdir $NIX_BIN_DIR/nix
ln -s $bzip2_bin_test/bzip2 $NIX_BIN_DIR/nix/ ln -s $bzip2_bin_test/bzip2 $NIX_BIN_DIR/nix/
ln -s $bzip2_bin_test/bunzip2 $NIX_BIN_DIR/nix/ ln -s $bzip2_bin_test/bunzip2 $NIX_BIN_DIR/nix/
ln -s $TOP/scripts/copy-from-other-stores.pl $NIX_BIN_DIR/nix/ ln -s $TOP/scripts/copy-from-other-stores.pl $NIX_BIN_DIR/nix/
ln -s $TOP/scripts/download-using-manifests.pl $NIX_BIN_DIR/nix/ ln -s $TOP/scripts/download-using-manifests.pl $NIX_BIN_DIR/nix/
ln -s $TOP/scripts/readmanifest.pm $NIX_BIN_DIR/nix/ ln -s $TOP/scripts/GeneratePatches.pm $NIX_BIN_DIR/nix/
ln -s $TOP/scripts/NixManifest.pm $NIX_BIN_DIR/nix/
cat > "$NIX_CONF_DIR"/nix.conf <<EOF cat > "$NIX_CONF_DIR"/nix.conf <<EOF
gc-keep-outputs = false gc-keep-outputs = false
gc-keep-derivations = false gc-keep-derivations = false
env-keep-derivations = false env-keep-derivations = false
fsync-metadata = false
EOF EOF
mkdir $NIX_DATA_DIR/nix mkdir $NIX_DATA_DIR/nix
cp -pr $TOP/corepkgs $NIX_DATA_DIR/nix/ cp -pr $TOP/corepkgs $NIX_DATA_DIR/nix/
# Bah, script has the prefix hard-coded. This is really messy stuff # Bah, scripts have the prefix hard-coded. This is really messy stuff
# (and likely to fail). # (and likely to fail).
for i in \ for i in \
$NIX_DATA_DIR/nix/corepkgs/nar/nar.sh \ $NIX_DATA_DIR/nix/corepkgs/nar/nar.sh \
@ -56,7 +61,9 @@ for i in \
$NIX_BIN_DIR/nix-install-package \ $NIX_BIN_DIR/nix-install-package \
$NIX_BIN_DIR/nix-push \ $NIX_BIN_DIR/nix-push \
$NIX_BIN_DIR/nix-pull \ $NIX_BIN_DIR/nix-pull \
$NIX_BIN_DIR/nix/readmanifest.pm \ $NIX_BIN_DIR/nix-generate-patches \
$NIX_BIN_DIR/nix/NixManifest.pm \
$NIX_BIN_DIR/nix/GeneratePatches.pm \
; do ; do
sed < $i > $i.tmp \ sed < $i > $i.tmp \
-e "s^$REAL_BIN_DIR/nix-store^$NIX_BIN_DIR/nix-store^" \ -e "s^$REAL_BIN_DIR/nix-store^$NIX_BIN_DIR/nix-store^" \
@ -96,7 +103,6 @@ mv $NIX_BIN_DIR/nix/download-using-manifests.pl $NIX_BIN_DIR/nix/substituters/do
$nixstore --init $nixstore --init
# Did anything happen? # Did anything happen?
test -e "$NIX_DB_DIR"/info test -e "$NIX_DB_DIR"/db.sqlite
test -e "$NIX_DB_DIR"/referrer
echo 'Hello World' > ./dummy echo 'Hello World' > ./dummy

View file

@ -1 +1 @@
"ooxfoobarybarzobaabb" "ooxfoobarybarzobaabbc"

View file

@ -17,3 +17,5 @@ substring 1 2 s
+ substring 3 0 s + substring 3 0 s
+ "b" + "b"
+ substring 3 1 s + substring 3 1 s
+ "c"
+ substring 5 10 "perl"

View file

@ -5,7 +5,7 @@ outPath=$($nixstore -r $drvPath)
echo "pushing $drvPath" echo "pushing $drvPath"
mkdir $TEST_ROOT/cache mkdir -p $TEST_ROOT/cache
$NIX_BIN_DIR/nix-push \ $NIX_BIN_DIR/nix-push \
--copy $TEST_ROOT/cache $TEST_ROOT/manifest $drvPath --copy $TEST_ROOT/cache $TEST_ROOT/manifest $drvPath

View file

@ -1,9 +1,8 @@
source common.sh source common.sh
# This takes way to long on Cygwin (because process creation is so slow...). clearStore
if test "$system" = i686-cygwin; then exit 0; fi
max=1000 max=500
reference=$NIX_STORE_DIR/abcdef reference=$NIX_STORE_DIR/abcdef
touch $reference touch $reference
@ -13,46 +12,23 @@ echo "making registration..."
for ((n = 0; n < $max; n++)); do for ((n = 0; n < $max; n++)); do
storePath=$NIX_STORE_DIR/$n storePath=$NIX_STORE_DIR/$n
touch $storePath echo -n > $storePath
ref2=$NIX_STORE_DIR/$((n+1)) ref2=$NIX_STORE_DIR/$((n+1))
if test $((n+1)) = $max; then if test $((n+1)) = $max; then
ref2=$reference ref2=$reference
fi fi
(echo $storePath && echo && echo 2 && echo $reference && echo $ref2) echo $storePath; echo; echo 2; echo $reference; echo $ref2
done > $TEST_ROOT/reg_info done > $TEST_ROOT/reg_info
echo "registering..." echo "registering..."
time $nixstore --register-validity < $TEST_ROOT/reg_info $nixstore --register-validity < $TEST_ROOT/reg_info
oldTime=$(cat test-tmp/db/info/1 | grep Registered-At)
echo "sleeping..."
sleep 2
echo "reregistering..."
time $nixstore --register-validity --reregister < $TEST_ROOT/reg_info
newTime=$(cat test-tmp/db/info/1 | grep Registered-At)
if test "$newTime" != "$oldTime"; then
echo "reregistration changed original registration time"
exit 1
fi
if test "$(cat test-tmp/db/referrer/1 | wc -w)" -ne 1; then
echo "reregistration duplicated referrers"
exit 1
fi
echo "collecting garbage..." echo "collecting garbage..."
ln -sfn $reference "$NIX_STATE_DIR"/gcroots/ref ln -sfn $reference "$NIX_STATE_DIR"/gcroots/ref
time $nixstore --gc $nixstore --gc
if test "$(cat test-tmp/db/referrer/abcdef | wc -w)" -ne 0; then if test "$(sqlite3 ./test-tmp/db/db.sqlite 'select count(*) from Refs')" -ne 0; then
echo "referrers not cleaned up" echo "referrers not cleaned up"
exit 1 exit 1
fi fi

View file

@ -16,7 +16,8 @@ if test $1 = "--query"; then
echo 1 echo 1
echo "" # deriver echo "" # deriver
echo 0 # nr of refs echo 0 # nr of refs
echo 0 # download size echo $((1 * 1024 * 1024)) # download size
echo $((2 * 1024 * 1024)) # nar size
else else
echo "bad command $cmd" echo "bad command $cmd"
exit 1 exit 1

View file

@ -16,6 +16,7 @@ if test $1 = "--query"; then
echo "" # deriver echo "" # deriver
echo 0 # nr of refs echo 0 # nr of refs
echo 0 # download size echo 0 # download size
echo 0 # nar size
else else
echo "bad command $cmd" echo "bad command $cmd"
exit 1 exit 1

View file

@ -14,6 +14,8 @@ echo $outPath > $TEST_ROOT/sub-paths
export NIX_SUBSTITUTERS=$(pwd)/substituter.sh export NIX_SUBSTITUTERS=$(pwd)/substituter.sh
$nixstore -r "$drvPath" --dry-run 2>&1 | grep -q "1.00 MiB.*2.00 MiB"
$nixstore -rvv "$drvPath" $nixstore -rvv "$drvPath"
text=$(cat "$outPath"/hello) text=$(cat "$outPath"/hello)