nix-copy-closure: Add flag ‘--use-substitutes’

This commit is contained in:
Eelco Dolstra 2012-11-23 16:20:16 +01:00
parent 9de6bc5d05
commit a3d6585c5a
4 changed files with 48 additions and 17 deletions

View file

@ -28,6 +28,8 @@
<arg><option>--xz</option></arg> <arg><option>--xz</option></arg>
<arg><option>--show-progress</option></arg> <arg><option>--show-progress</option></arg>
<arg><option>--include-outputs</option></arg> <arg><option>--include-outputs</option></arg>
<arg><option>--use-substitutes</option></arg>
<arg><option>-s</option></arg>
<arg choice='plain'> <arg choice='plain'>
<replaceable>user@</replaceable><replaceable>machine</replaceable> <replaceable>user@</replaceable><replaceable>machine</replaceable>
</arg> </arg>
@ -120,8 +122,20 @@ those paths. If this bothers you, use
<varlistentry><term><option>--include-outputs</option></term> <varlistentry><term><option>--include-outputs</option></term>
<listitem><para>Also copy the outputs of store derivations included <listitem><para>Also copy the outputs of store derivations
in the closure.</para></listitem> included in the closure.</para></listitem>
</varlistentry>
<varlistentry><term><option>--use-substitutes</option> / <option>-s</option></term>
<listitem><para>Attempt to download missing paths on the target
machine using Nixs substitute mechanism. Any paths that cannot
be substituted on the target are still copied normally from the
source. This is useful, for instance, if the connection between
the source and target machine is slow, but the connection between
the target machine and <literal>nixos.org</literal> (the default
binary cache server) is fast.</para></listitem>
</varlistentry> </varlistentry>

View file

@ -81,6 +81,13 @@ $ mount -o remount,ro,bind /nix/store
modifications.</para> modifications.</para>
</listitem> </listitem>
<listitem>
<para>The command <command>nix-copy-closure</command> has a new
flag <option>--use-substitutes</option> (<option>-s</option>) to
download missing paths on the target machine using the substitute
mechanism.</para>
</listitem>
<listitem> <listitem>
<para>The command <command>nix-worker</command> has been renamed <para>The command <command>nix-worker</command> has been renamed
to <command>nix-daemon</command>. Support for running the Nix to <command>nix-daemon</command>. Support for running the Nix

View file

@ -6,7 +6,8 @@ use Nix::Store;
sub copyTo { sub copyTo {
my ($sshHost, $sshOpts, $storePaths, $compressor, $decompressor, $includeOutputs, $dryRun, $sign, $progressViewer) = @_; my ($sshHost, $sshOpts, $storePaths, $compressor, $decompressor,
$includeOutputs, $dryRun, $sign, $progressViewer, $useSubstitutes) = @_;
$compressor = "$compressor |" if $compressor ne ""; $compressor = "$compressor |" if $compressor ne "";
$decompressor = "$decompressor |" if $decompressor ne ""; $decompressor = "$decompressor |" if $decompressor ne "";
@ -16,6 +17,12 @@ sub copyTo {
my @closure = reverse(topoSortPaths(computeFSClosure(0, $includeOutputs, my @closure = reverse(topoSortPaths(computeFSClosure(0, $includeOutputs,
map { followLinksToStorePath $_ } @{$storePaths}))); map { followLinksToStorePath $_ } @{$storePaths})));
# Optionally use substitutes on the remote host.
if (!$dryRun && $useSubstitutes) {
system "ssh $sshHost @{$sshOpts} nix-store -r --ignore-unknown @closure";
# Ignore exit status because this is just an optimisation.
}
# Ask the remote host which paths are invalid. Because of limits # Ask the remote host which paths are invalid. Because of limits
# to the command line length, do this in chunks. Eventually, # to the command line length, do this in chunks. Eventually,
# we'll want to use --from-stdin, but we can't rely on the # we'll want to use --from-stdin, but we can't rely on the

View file

@ -17,19 +17,14 @@ EOF
# Get the target host. # Get the target host.
my $sshHost; my $sshHost;
my $sign = 0; my $sign = 0;
my $compressor = ""; my $compressor = "";
my $decompressor = ""; my $decompressor = "";
my $progressViewer = ""; my $progressViewer = "";
my $toMode = 1; my $toMode = 1;
my $includeOutputs = 0; my $includeOutputs = 0;
my $dryRun = 0; my $dryRun = 0;
my $useSubstitutes = 0;
# !!! Copied from nix-pack-closure, should put this in a module. # !!! Copied from nix-pack-closure, should put this in a module.
@ -71,6 +66,9 @@ while (@ARGV) {
elsif ($arg eq "--dry-run") { elsif ($arg eq "--dry-run") {
$dryRun = 1; $dryRun = 1;
} }
elsif ($arg eq "--use-substitutes" || $arg eq "-s") {
$useSubstitutes = 1;
}
elsif (!defined $sshHost) { elsif (!defined $sshHost) {
$sshHost = $arg; $sshHost = $arg;
} }
@ -84,7 +82,9 @@ openSSHConnection $sshHost or die "$0: unable to start SSH\n";
if ($toMode) { # Copy TO the remote machine. if ($toMode) { # Copy TO the remote machine.
Nix::CopyClosure::copyTo($sshHost, [ @sshOpts ], [ @storePaths ], $compressor, $decompressor, $includeOutputs, $dryRun, $sign, $progressViewer); Nix::CopyClosure::copyTo(
$sshHost, [ @sshOpts ], [ @storePaths ], $compressor, $decompressor,
$includeOutputs, $dryRun, $sign, $progressViewer, $useSubstitutes);
} }
else { # Copy FROM the remote machine. else { # Copy FROM the remote machine.
@ -107,10 +107,13 @@ else { # Copy FROM the remote machine.
# Export the store paths on the remote machine and import them locally. # Export the store paths on the remote machine and import them locally.
if (scalar @missing > 0) { if (scalar @missing > 0) {
print STDERR "copying ", scalar @missing, " missing paths from $sshHost...\n"; print STDERR "copying ", scalar @missing, " missing paths from $sshHost...\n";
$compressor = "| $compressor" if $compressor ne "";
$decompressor = "$decompressor |" if $decompressor ne "";
$progressViewer = "$progressViewer |" if $progressViewer ne "";
unless ($dryRun) { unless ($dryRun) {
if ($useSubstitutes) {
system "$Nix::Config::binDir/nix-store -r --ignore-unknown @missing";
}
$compressor = "| $compressor" if $compressor ne "";
$decompressor = "$decompressor |" if $decompressor ne "";
$progressViewer = "$progressViewer |" if $progressViewer ne "";
my $extraOpts = $sign ? "--sign" : ""; my $extraOpts = $sign ? "--sign" : "";
system("set -f; ssh $sshHost @sshOpts 'nix-store --export $extraOpts @missing $compressor' | $progressViewer $decompressor $Nix::Config::binDir/nix-store --import > /dev/null") == 0 system("set -f; ssh $sshHost @sshOpts 'nix-store --export $extraOpts @missing $compressor' | $progressViewer $decompressor $Nix::Config::binDir/nix-store --import > /dev/null") == 0
or die "copying store paths from remote machine `$sshHost' failed: $?"; or die "copying store paths from remote machine `$sshHost' failed: $?";