puck/lix
0
0
Fork 0
forked from lix-project/lix
Code Pull requests Activity

nix-copy-closure: Drop --bzip2, --xz, --show-progress

Browse source 
These are too difficult to implement via nix-store --serve.

‘--show-progress’ could be re-implemented fairly easily via a
sink/source wrapper class.
This commit is contained in:
Eelco Dolstra 2014-07-24 17:11:54 +02:00
parent 7f7d4ab686
commit 1a44f1cb65
5 changed files with 23 additions and 92 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
doc/manual/nix-copy-closure.xml
View file

@ -24,9 +24,9 @@
</group> </group>
<arg><option>--sign</option></arg> <arg><option>--sign</option></arg>
<arg><option>--gzip</option></arg> <arg><option>--gzip</option></arg>
<arg><option>--bzip2</option></arg> <!--
<arg><option>--xz</option></arg> <arg><option>- -show-progress</option></arg>
<arg><option>--show-progress</option></arg> -->
<arg><option>--include-outputs</option></arg> <arg><option>--include-outputs</option></arg>
<arg><option>--use-substitutes</option></arg> <arg><option>--use-substitutes</option></arg>
<arg><option>-s</option></arg> <arg><option>-s</option></arg>
@ -103,22 +103,21 @@ those paths. If this bothers you, use
</varlistentry> </varlistentry>
<varlistentry><term><option>--gzip</option> / <option>--bzip2</option> / <option>--xz</option></term> <varlistentry><term><option>--gzip</option></term>
<listitem><para>Compress the dump of each path with respectively <listitem><para>Enable compression of the SSH
<command>gzip</command>, <command>bzip2</command> or connection.</para></listitem>
<command>xz</command> before sending it. The corresponding
decompression program must be installed on the target
machine.</para></listitem>
</varlistentry> </varlistentry>
<varlistentry><term><option>--show-progress</option></term> <!--
<varlistentry><term><option>- -show-progress</option></term>
<listitem><para>Show the progress of each path's transfer as it's made. <listitem><para>Show the progress of each path's transfer as it's made.
This requires the <command>pv</command> utility to be in <envar>PATH</envar>.</para></listitem> This requires the <command>pv</command> utility to be in <envar>PATH</envar>.</para></listitem>
</varlistentry> </varlistentry>
-->
<varlistentry><term><option>--include-outputs</option></term> <varlistentry><term><option>--include-outputs</option></term>

41
perl/lib/Nix/CopyClosure.pm
View file

@ -9,8 +9,7 @@ use IPC::Open2;
sub copyToOpen { sub copyToOpen {
my ($from, $to, $sshHost, $storePaths, $compressor, $decompressor, my ($from, $to, $sshHost, $storePaths, $includeOutputs, $dryRun, $sign, $useSubstitutes) = @_;
$includeOutputs, $dryRun, $sign, $progressViewer, $useSubstitutes) = @_;
$useSubstitutes = 0 if $dryRun || !defined $useSubstitutes; $useSubstitutes = 0 if $dryRun || !defined $useSubstitutes;
@ -41,34 +40,13 @@ sub copyToOpen {
# Send the "import paths" command. # Send the "import paths" command.
syswrite($to, pack("L<x4", 4)) or die; syswrite($to, pack("L<x4", 4)) or die;
writeString($compressor, $to); exportPaths(fileno($to), $sign, @missing);
if ($compressor || $progressViewer) {
# Compute the size of the closure for the progress viewer.
$progressViewer = "$progressViewer -s $missingSize" if $progressViewer;
# Start the compressor and/or progress viewer in between us
# and the remote host.
my $to_;
my $pid2 = open2(">&" . fileno($to), $to_,
$progressViewer && $compressor ? "$progressViewer | $compressor" : $progressViewer || $compressor);
close $to;
exportPaths(fileno($to_), $sign, @missing);
close $to_;
waitpid $pid2, 0;
} else {
exportPaths(fileno($to), $sign, @missing);
}
readInt($from) == 1 or die "remote machine \`$sshHost' failed to import closure\n"; readInt($from) == 1 or die "remote machine \`$sshHost' failed to import closure\n";
} }
sub copyTo { sub copyTo {
my ($sshHost, $sshOpts, $storePaths, $compressor, $decompressor, my ($sshHost, $sshOpts, $storePaths, $includeOutputs, $dryRun, $sign, $useSubstitutes) = @_;
$includeOutputs, $dryRun, $sign, $progressViewer, $useSubstitutes) = @_;
# Connect to the remote host. # Connect to the remote host.
my ($from, $to); my ($from, $to);
@ -81,8 +59,7 @@ sub copyTo {
return oldCopyTo(@_); return oldCopyTo(@_);
} }
copyToOpen($from, $to, $sshHost, $storePaths, $compressor, $decompressor, copyToOpen($from, $to, $sshHost, $storePaths, $includeOutputs, $dryRun, $sign, $useSubstitutes);
$includeOutputs, $dryRun, $sign, $progressViewer, $useSubstitutes);
close $to; close $to;
} }
@ -91,8 +68,7 @@ sub copyTo {
# For backwards compatibility with Nix <= 1.7. Will be removed # For backwards compatibility with Nix <= 1.7. Will be removed
# eventually. # eventually.
sub oldCopyTo { sub oldCopyTo {
my ($sshHost, $sshOpts, $storePaths, $compressor, $decompressor, my ($sshHost, $sshOpts, $storePaths, $includeOutputs, $dryRun, $sign, $useSubstitutes) = @_;
$includeOutputs, $dryRun, $sign, $progressViewer, $useSubstitutes) = @_;
# Get the closure of this path. # Get the closure of this path.
my @closure = reverse(topoSortPaths(computeFSClosure(0, $includeOutputs, my @closure = reverse(topoSortPaths(computeFSClosure(0, $includeOutputs,
@ -122,15 +98,12 @@ sub oldCopyTo {
close READ or die; close READ or die;
} }
$compressor = "$compressor |" if $compressor ne "";
$decompressor = "$decompressor |" if $decompressor ne "";
$progressViewer = "$progressViewer -s $missingSize |" if $progressViewer ne "";
# Export the store paths and import them on the remote machine. # Export the store paths and import them on the remote machine.
if (scalar @missing > 0) { if (scalar @missing > 0) {
print STDERR "copying ", scalar @missing, " missing paths to ‘$sshHost’...\n"; print STDERR "copying ", scalar @missing, " missing paths to ‘$sshHost’...\n";
print STDERR "@missing\n";
unless ($dryRun) { unless ($dryRun) {
open SSH, "| $progressViewer $compressor ssh $sshHost @{$sshOpts} @globalSshOpts '$decompressor nix-store --import' > /dev/null" or die; open SSH, "| ssh $sshHost @{$sshOpts} @globalSshOpts 'nix-store --import' > /dev/null" or die;
exportPaths(fileno(SSH), $sign, @missing); exportPaths(fileno(SSH), $sign, @missing);
close SSH or die "copying store paths to remote machine `$sshHost' failed: $?"; close SSH or die "copying store paths to remote machine `$sshHost' failed: $?";
} }

2
scripts/build-remote.pl.in
View file

@ -246,7 +246,7 @@ if ($@) {
print STDERR "somebody is hogging $uploadLock, continuing...\n"; print STDERR "somebody is hogging $uploadLock, continuing...\n";
unlink $uploadLock; unlink $uploadLock;
} }
Nix::CopyClosure::copyToOpen($from, $to, $hostName, [ $drvPath, @inputs ], "", "", 0, 0, $maybeSign ne "", ""); Nix::CopyClosure::copyToOpen($from, $to, $hostName, [ $drvPath, @inputs ], 0, 0, $maybeSign ne "");
close UPLOADLOCK; close UPLOADLOCK;

20
scripts/nix-copy-closure.in
View file

@ -19,9 +19,6 @@ EOF
# Get the target host. # Get the target host.
my $sshHost; my $sshHost;
my $sign = 0; my $sign = 0;
my $compressor = "";
my $decompressor = "";
my $progressViewer = "";
my $toMode = 1; my $toMode = 1;
my $includeOutputs = 0; my $includeOutputs = 0;
my $dryRun = 0; my $dryRun = 0;
@ -40,17 +37,10 @@ while (@ARGV) {
elsif ($arg eq "--sign") { elsif ($arg eq "--sign") {
$sign = 1; $sign = 1;
} }
elsif ($arg eq "--gzip") { elsif ($arg eq "--gzip" || $arg eq "--bzip2" || $arg eq "--xz") {
warn "$0: `$arg' is not implemented\n" if $arg ne "--gzip";
push @globalSshOpts, "-C"; push @globalSshOpts, "-C";
} }
elsif ($arg eq "--bzip2") {
$compressor = "bzip2";
$decompressor = "bzip2 -d";
}
elsif ($arg eq "--xz") {
$compressor = "xz";
$decompressor = "xz -d";
}
elsif ($arg eq "--from") { elsif ($arg eq "--from") {
$toMode = 0; $toMode = 0;
} }
@ -61,7 +51,7 @@ while (@ARGV) {
$includeOutputs = 1; $includeOutputs = 1;
} }
elsif ($arg eq "--show-progress") { elsif ($arg eq "--show-progress") {
$progressViewer = "@pv@"; warn "$0: `$arg' is not implemented\n";
} }
elsif ($arg eq "--dry-run") { elsif ($arg eq "--dry-run") {
$dryRun = 1; $dryRun = 1;
@ -82,8 +72,8 @@ die "$0: you did not specify a host name\n" unless defined $sshHost;
if ($toMode) { # Copy TO the remote machine. if ($toMode) { # Copy TO the remote machine.
Nix::CopyClosure::copyTo( Nix::CopyClosure::copyTo(
$sshHost, [ @sshOpts ], [ @storePaths ], $compressor, $decompressor, $sshHost, [ @sshOpts ], [ @storePaths ],
$includeOutputs, $dryRun, $sign, $progressViewer, $useSubstitutes); $includeOutputs, $dryRun, $sign, $useSubstitutes);
} }
else { # Copy FROM the remote machine. else { # Copy FROM the remote machine.

33
src/nix-store/nix-store.cc
View file

@ -956,39 +956,8 @@ static void opServe(Strings opFlags, Strings opArgs)
case cmdImportPaths: { case cmdImportPaths: {
if (!writeAllowed) throw Error("importing paths is not allowed"); if (!writeAllowed) throw Error("importing paths is not allowed");
string compression = readString(in); store->importPaths(false, in);
if (compression != "") {
if (compression != "gzip" && compression != "bzip2" && compression != "xz")
throw Error(format("unsupported compression method `%1%'") % compression);
Pipe fromDecompressor;
fromDecompressor.create();
Pid pid = startProcess([&]() {
fromDecompressor.readSide.close();
if (dup2(fromDecompressor.writeSide, STDOUT_FILENO) == -1)
throw SysError("dupping stdout");
// FIXME: use absolute path.
execlp(compression.c_str(), compression.c_str(), "-d", NULL);
throw SysError(format("executing `%1%'") % compression);
});
fromDecompressor.writeSide.close();
FdSource fromDecompressor_(fromDecompressor.readSide);
store->importPaths(false, fromDecompressor_);
pid.wait(true);
} else
store->importPaths(false, in);
writeInt(1, out); // indicate success writeInt(1, out); // indicate success
/* The decompressor will have left stdin in an
undefined state, so we can't continue. */
if (compression != "") return;
break; break;
} }