Merge branch 'nix-copy-closure-c++' of https://github.com/shlevy/nix

.gitignore

@@ -77,6 +77,8 @@ Makefile.config
 # /src/nix-build/
 /src/nix-build/nix-build
 
+/src/nix-copy-closure/nix-copy-closure
+
 /src/build-remote/build-remote
 
 # /tests/

Makefile

@@ -11,6 +11,7 @@ makefiles = \
   src/nix-env/local.mk \
   src/nix-daemon/local.mk \
   src/nix-collect-garbage/local.mk \
+  src/nix-copy-closure/local.mk \
   src/nix-prefetch-url/local.mk \
   src/buildenv/local.mk \
   src/resolve-system-dependencies/local.mk \

scripts/local.mk

@@ -1,8 +1,3 @@
-nix_bin_scripts := \
-  $(d)/nix-copy-closure \
-
-bin-scripts += $(nix_bin_scripts)
-
 nix_noinst_scripts := \
   $(d)/nix-http-export.cgi \
   $(d)/nix-profile.sh \
@@ -14,4 +9,4 @@ profiledir = $(sysconfdir)/profile.d
 
 $(eval $(call install-file-as, $(d)/nix-profile.sh, $(profiledir)/nix.sh, 0644))
 
-clean-files += $(nix_bin_scripts) $(nix_noinst_scripts)
+clean-files += $(nix_noinst_scripts)

scripts/nix-copy-closure.in

@@ -1,103 +0,0 @@
-#! @perl@ -w @perlFlags@
-
-use utf8;
-use strict;
-use Nix::SSH;
-use Nix::Config;
-use Nix::Store;
-use Nix::CopyClosure;
-use List::Util qw(sum);
-
-binmode STDERR, ":encoding(utf8)";
-
-if (scalar @ARGV < 1) {
-    print STDERR <<EOF
-Usage: nix-copy-closure [--from | --to] HOSTNAME [--gzip] [--bzip2] [--xz] PATHS...
-EOF
-    ;
-    exit 1;
-}
-
-
-# Get the target host.
-my $sshHost;
-my $toMode = 1;
-my $includeOutputs = 0;
-my $dryRun = 0;
-my $useSubstitutes = 0;
-my $verbosity = 1;
-
-
-# !!! Copied from nix-pack-closure, should put this in a module.
-my @storePaths = ();
-
-while (@ARGV) {
-    my $arg = shift @ARGV;
-
-    if ($arg eq "--help") {
-        exec "man nix-copy-closure" or die;
-    }
-    elsif ($arg eq "--gzip" || $arg eq "--bzip2" || $arg eq "--xz") {
-        warn "$0: ‘$arg’ is not implemented\n" if $arg ne "--gzip";
-        push @globalSshOpts, "-C";
-    }
-    elsif ($arg eq "--from") {
-        $toMode = 0;
-    }
-    elsif ($arg eq "--to") {
-        $toMode = 1;
-    }
-    elsif ($arg eq "--include-outputs") {
-        $includeOutputs = 1;
-    }
-    elsif ($arg eq "--show-progress") {
-        warn "$0: ‘$arg’ is not implemented\n";
-    }
-    elsif ($arg eq "--dry-run") {
-        $dryRun = 1;
-    }
-    elsif ($arg eq "--use-substitutes" || $arg eq "-s") {
-        $useSubstitutes = 1;
-    }
-    elsif ($arg eq "-v") {
-        $verbosity++;
-        setVerbosity($verbosity);
-    }
-    elsif (!defined $sshHost) {
-        $sshHost = $arg;
-    }
-    else {
-        push @storePaths, $arg;
-    }
-}
-
-die "$0: you did not specify a host name\n" unless defined $sshHost;
-
-
-if ($toMode) { # Copy TO the remote machine.
-    Nix::CopyClosure::copyTo(
-        $sshHost, [ @storePaths ],
-        $includeOutputs, $dryRun, $useSubstitutes);
-}
-
-else { # Copy FROM the remote machine.
-
-    my ($from, $to) = connectToRemoteNix($sshHost, []);
-
-    # Query the closure of the given store paths on the remote
-    # machine.  Paths are assumed to be store paths; there is no
-    # resolution (following of symlinks).
-    syswrite($to, pack("L<x4L<x4", 7, $includeOutputs ? 1 : 0)) or die;
-    writeStrings(\@storePaths, $to);
-    my @missing = grep { !isValidPath($_) } readStrings($from);
-
-    # Export the store paths on the remote machine and import them locally.
-    if (scalar @missing > 0) {
-        print STDERR "copying ", scalar @missing, " missing paths from ‘$sshHost’...\n";
-        writeInt(5, $to); # == cmdExportPaths
-        writeInt(0, $to); # obsolete
-        writeStrings(\@missing, $to);
-        importPaths(fileno($from), 1);
-    }
-
-}

src/build-remote/build-remote.cc

@@ -12,7 +12,6 @@
 #include "shared.hh"
 #include "pathlocks.hh"
 #include "globals.hh"
-#include "serve-protocol.hh"
 #include "serialise.hh"
 #include "store-api.hh"
 #include "derivations.hh"

src/build-remote/local.mk

@@ -8,4 +8,4 @@ build-remote_LIBS = libmain libutil libformat libstore
 
 build-remote_SOURCES := $(d)/build-remote.cc
 
-build-remote_CXXFLAGS = -DSYSCONFDIR="\"$(sysconfdir)\"" -Isrc/nix-store
+build-remote_CXXFLAGS = -DSYSCONFDIR="\"$(sysconfdir)\""

src/libstore/ssh-store.cc

@@ -41,6 +41,8 @@ private:
     string host;
 
     Path key;
+
+    bool compress;
 };
 
 SSHStore::SSHStore(string host, const Params & params, size_t maxConnections)
@@ -50,6 +52,7 @@ SSHStore::SSHStore(string host, const Params & params, size_t maxConnections)
     , socketPath((Path) tmpDir + "/ssh.sock")
     , host(std::move(host))
     , key(get(params, "ssh-key", ""))
+    , compress(get(params, "compress", "") == "true")
 {
     /* open a connection and perform the handshake to verify all is well */
     connections->get();

src/libstore/store-api.cc

@@ -5,6 +5,7 @@
 #include "nar-info-disk-cache.hh"
 #include "thread-pool.hh"
 #include "json.hh"
+#include "derivations.hh"
 
 #include <future>
 
@@ -780,8 +781,27 @@ std::list<ref<Store>> getDefaultSubstituters()
 }
 
 
-void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths)
+void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths, bool substitute)
 {
+    if (substitute) {
+        /* Filter out .drv files (we don't want to build anything). */
+        PathSet paths2;
+        for (auto & path : storePaths)
+            if (!isDerivation(path)) paths2.insert(path);
+        unsigned long long downloadSize, narSize;
+        PathSet willBuild, willSubstitute, unknown;
+        to->queryMissing(PathSet(paths2.begin(), paths2.end()),
+            willBuild, willSubstitute, unknown, downloadSize, narSize);
+        /* FIXME: should use ensurePath(), but it only
+           does one path at a time. */
+        if (!willSubstitute.empty())
+            try {
+                to->buildPaths(willSubstitute);
+            } catch (Error & e) {
+                printMsg(lvlError, format("warning: %1%") % e.msg());
+            }
+    }
+
     std::string copiedLabel = "copied";
 
     logger->setExpected(copiedLabel, storePaths.size());

src/libstore/store-api.hh

@@ -643,7 +643,7 @@ void removeTempRoots();
 ref<Store> openStore(const std::string & uri = getEnv("NIX_REMOTE"));
 
 
-void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths);
+void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths, bool substitute = false);
 
 enum StoreType {
     tDaemon,

src/nix-copy-closure/local.mk

@@ -0,0 +1,7 @@
+programs += nix-copy-closure
+
+nix-copy-closure_DIR := $(d)
+
+nix-copy-closure_LIBS = libmain libutil libformat libstore
+
+nix-copy-closure_SOURCES := $(d)/nix-copy-closure.cc

src/nix-copy-closure/nix-copy-closure.cc

@@ -0,0 +1,60 @@
+#include "shared.hh"
+#include "store-api.hh"
+
+using namespace nix;
+
+int main(int argc, char ** argv)
+{
+    return handleExceptions(argv[0], [&]() {
+        initNix();
+        auto gzip = false;
+        auto toMode = true;
+        auto includeOutputs = false;
+        auto dryRun = false;
+        auto useSubstitutes = false;
+        auto sshHost = string{};
+        auto storePaths = PathSet{};
+        parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
+            if (*arg == "--help")
+                showManPage("nix-copy-closure");
+            else if (*arg == "--version")
+                printVersion("nix-copy-closure");
+            else if (*arg == "--gzip" || *arg == "--bzip2" || *arg == "--xz") {
+                if (*arg != "--gzip")
+                    printMsg(lvlError, format("Warning: ‘%1%’ is not implemented, falling back to gzip") % *arg);
+                gzip = true;
+            } else if (*arg == "--from")
+                toMode = false;
+            else if (*arg == "--to")
+                toMode = true;
+            else if (*arg == "--include-outputs")
+                includeOutputs = true;
+            else if (*arg == "--show-progress")
+                printMsg(lvlError, "Warning: ‘--show-progress’ is not implemented");
+            else if (*arg == "--dry-run")
+                dryRun = true;
+            else if (*arg == "--use-substitutes" || *arg == "-s")
+                useSubstitutes = true;
+            else if (sshHost.empty())
+                sshHost = *arg;
+            else
+                storePaths.insert(*arg);
+            return true;
+        });
+        if (sshHost.empty())
+            throw UsageError("no host name specified");
+
+        auto remoteUri = "ssh://" + sshHost + (gzip ? "?compress=true" : "");
+        auto to = toMode ? openStore(remoteUri) : openStore();
+        auto from = toMode ? openStore() : openStore(remoteUri);
+        if (includeOutputs) {
+            auto newPaths = PathSet{};
+            for (const auto & p : storePaths) {
+                auto outputs = from->queryDerivationOutputs(p);
+                newPaths.insert(outputs.begin(), outputs.end());
+            }
+            storePaths.insert(newPaths.begin(), newPaths.end());
+        }
+        copyPaths(from, to, Paths(storePaths.begin(), storePaths.end()), useSubstitutes);
+    });
+}