lix-project/lix
20
57
Fork 31
Code Issues 347 Code Review (Gerrit) Projects 3 Releases Wiki Activity

nix-copy-closure: Drop --bzip2, --xz, --show-progress

Browse source 
These are too difficult to implement via nix-store --serve.

‘--show-progress’ could be re-implemented fairly easily via a
sink/source wrapper class.
This commit is contained in:
Eelco Dolstra 2014-07-24 17:11:54 +02:00
parent 7f7d4ab686
commit 1a44f1cb65
5 changed files with 23 additions and 92 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
doc/manual/nix-copy-closure.xml
View file

@ -24,9 +24,9 @@
</group>
<arg><option>--sign</option></arg>
<arg><option>--gzip</option></arg>
<arg><option>--bzip2</option></arg>
<arg><option>--xz</option></arg>
<arg><option>--show-progress</option></arg>
<!--
<arg><option>- -show-progress</option></arg>
-->
<arg><option>--include-outputs</option></arg>
<arg><option>--use-substitutes</option></arg>
<arg><option>-s</option></arg>
@ -103,22 +103,21 @@ those paths. If this bothers you, use
</varlistentry>
<varlistentry><term><option>--gzip</option> / <option>--bzip2</option> / <option>--xz</option></term>
<varlistentry><term><option>--gzip</option></term>
<listitem><para>Compress the dump of each path with respectively
<command>gzip</command>, <command>bzip2</command> or
<command>xz</command> before sending it. The corresponding
decompression program must be installed on the target
machine.</para></listitem>
<listitem><para>Enable compression of the SSH
connection.</para></listitem>
</varlistentry>
<varlistentry><term><option>--show-progress</option></term>
<!--
<varlistentry><term><option>- -show-progress</option></term>
<listitem><para>Show the progress of each path's transfer as it's made.
This requires the <command>pv</command> utility to be in <envar>PATH</envar>.</para></listitem>
</varlistentry>
-->
<varlistentry><term><option>--include-outputs</option></term>

41
perl/lib/Nix/CopyClosure.pm
View file

@ -9,8 +9,7 @@ use IPC::Open2;
sub copyToOpen {
my ($from, $to, $sshHost, $storePaths, $compressor, $decompressor,
$includeOutputs, $dryRun, $sign, $progressViewer, $useSubstitutes) = @_;
my ($from, $to, $sshHost, $storePaths, $includeOutputs, $dryRun, $sign, $useSubstitutes) = @_;
$useSubstitutes = 0 if $dryRun || !defined $useSubstitutes;
@ -41,34 +40,13 @@ sub copyToOpen {
# Send the "import paths" command.
syswrite($to, pack("L<x4", 4)) or die;
writeString($compressor, $to);
if ($compressor || $progressViewer) {
# Compute the size of the closure for the progress viewer.
$progressViewer = "$progressViewer -s $missingSize" if $progressViewer;
# Start the compressor and/or progress viewer in between us
# and the remote host.
my $to_;
my $pid2 = open2(">&" . fileno($to), $to_,
$progressViewer && $compressor ? "$progressViewer | $compressor" : $progressViewer || $compressor);
close $to;
exportPaths(fileno($to_), $sign, @missing);
close $to_;
waitpid $pid2, 0;
} else {
exportPaths(fileno($to), $sign, @missing);
}
exportPaths(fileno($to), $sign, @missing);
readInt($from) == 1 or die "remote machine \`$sshHost' failed to import closure\n";
}
sub copyTo {
my ($sshHost, $sshOpts, $storePaths, $compressor, $decompressor,
$includeOutputs, $dryRun, $sign, $progressViewer, $useSubstitutes) = @_;
my ($sshHost, $sshOpts, $storePaths, $includeOutputs, $dryRun, $sign, $useSubstitutes) = @_;
# Connect to the remote host.
my ($from, $to);
@ -81,8 +59,7 @@ sub copyTo {
return oldCopyTo(@_);
}
copyToOpen($from, $to, $sshHost, $storePaths, $compressor, $decompressor,
$includeOutputs, $dryRun, $sign, $progressViewer, $useSubstitutes);
copyToOpen($from, $to, $sshHost, $storePaths, $includeOutputs, $dryRun, $sign, $useSubstitutes);
close $to;
}
@ -91,8 +68,7 @@ sub copyTo {
# For backwards compatibility with Nix <= 1.7. Will be removed
# eventually.
sub oldCopyTo {
my ($sshHost, $sshOpts, $storePaths, $compressor, $decompressor,
$includeOutputs, $dryRun, $sign, $progressViewer, $useSubstitutes) = @_;
my ($sshHost, $sshOpts, $storePaths, $includeOutputs, $dryRun, $sign, $useSubstitutes) = @_;
# Get the closure of this path.
my @closure = reverse(topoSortPaths(computeFSClosure(0, $includeOutputs,
@ -122,15 +98,12 @@ sub oldCopyTo {
close READ or die;
}
$compressor = "$compressor |" if $compressor ne "";
$decompressor = "$decompressor |" if $decompressor ne "";
$progressViewer = "$progressViewer -s $missingSize |" if $progressViewer ne "";
# Export the store paths and import them on the remote machine.
if (scalar @missing > 0) {
print STDERR "copying ", scalar @missing, " missing paths to ‘$sshHost’...\n";
print STDERR "@missing\n";
unless ($dryRun) {
open SSH, "| $progressViewer $compressor ssh $sshHost @{$sshOpts} @globalSshOpts '$decompressor nix-store --import' > /dev/null" or die;
open SSH, "| ssh $sshHost @{$sshOpts} @globalSshOpts 'nix-store --import' > /dev/null" or die;
exportPaths(fileno(SSH), $sign, @missing);
close SSH or die "copying store paths to remote machine `$sshHost' failed: $?";
}

2
scripts/build-remote.pl.in
View file

@ -246,7 +246,7 @@ if ($@) {
print STDERR "somebody is hogging $uploadLock, continuing...\n";
unlink $uploadLock;
}
Nix::CopyClosure::copyToOpen($from, $to, $hostName, [ $drvPath, @inputs ], "", "", 0, 0, $maybeSign ne "", "");
Nix::CopyClosure::copyToOpen($from, $to, $hostName, [ $drvPath, @inputs ], 0, 0, $maybeSign ne "");
close UPLOADLOCK;

20
scripts/nix-copy-closure.in
View file

@ -19,9 +19,6 @@ EOF
# Get the target host.
my $sshHost;
my $sign = 0;
my $compressor = "";
my $decompressor = "";
my $progressViewer = "";
my $toMode = 1;
my $includeOutputs = 0;
my $dryRun = 0;
@ -40,17 +37,10 @@ while (@ARGV) {
elsif ($arg eq "--sign") {
$sign = 1;
}
elsif ($arg eq "--gzip") {
elsif ($arg eq "--gzip" || $arg eq "--bzip2" || $arg eq "--xz") {
warn "$0: `$arg' is not implemented\n" if $arg ne "--gzip";
push @globalSshOpts, "-C";
}
elsif ($arg eq "--bzip2") {
$compressor = "bzip2";
$decompressor = "bzip2 -d";
}
elsif ($arg eq "--xz") {
$compressor = "xz";
$decompressor = "xz -d";
}
elsif ($arg eq "--from") {
$toMode = 0;
}
@ -61,7 +51,7 @@ while (@ARGV) {
$includeOutputs = 1;
}
elsif ($arg eq "--show-progress") {
$progressViewer = "@pv@";
warn "$0: `$arg' is not implemented\n";
}
elsif ($arg eq "--dry-run") {
$dryRun = 1;
@ -82,8 +72,8 @@ die "$0: you did not specify a host name\n" unless defined $sshHost;
if ($toMode) { # Copy TO the remote machine.
Nix::CopyClosure::copyTo(
$sshHost, [ @sshOpts ], [ @storePaths ], $compressor, $decompressor,
$includeOutputs, $dryRun, $sign, $progressViewer, $useSubstitutes);
$sshHost, [ @sshOpts ], [ @storePaths ],
$includeOutputs, $dryRun, $sign, $useSubstitutes);
}
else { # Copy FROM the remote machine.

33
src/nix-store/nix-store.cc
View file

@ -956,39 +956,8 @@ static void opServe(Strings opFlags, Strings opArgs)
case cmdImportPaths: {
if (!writeAllowed) throw Error("importing paths is not allowed");
string compression = readString(in);
if (compression != "") {
if (compression != "gzip" && compression != "bzip2" && compression != "xz")
throw Error(format("unsupported compression method `%1%'") % compression);
Pipe fromDecompressor;
fromDecompressor.create();
Pid pid = startProcess([&]() {
fromDecompressor.readSide.close();
if (dup2(fromDecompressor.writeSide, STDOUT_FILENO) == -1)
throw SysError("dupping stdout");
// FIXME: use absolute path.
execlp(compression.c_str(), compression.c_str(), "-d", NULL);
throw SysError(format("executing `%1%'") % compression);
});
fromDecompressor.writeSide.close();
FdSource fromDecompressor_(fromDecompressor.readSide);
store->importPaths(false, fromDecompressor_);
pid.wait(true);
} else
store->importPaths(false, in);
store->importPaths(false, in);
writeInt(1, out); // indicate success
/* The decompressor will have left stdin in an
undefined state, so we can't continue. */
if (compression != "") return;
break;
}