Merge branch 'nix-copy-closure-c++' of https://github.com/shlevy/nix

This commit is contained in:
Eelco Dolstra 2017-02-07 20:47:45 +01:00
commit 6f4682ad36
No known key found for this signature in database
GPG key ID: 8170B4726D7198DE
11 changed files with 97 additions and 113 deletions

2
.gitignore vendored
View file

@ -77,6 +77,8 @@ Makefile.config
# /src/nix-build/ # /src/nix-build/
/src/nix-build/nix-build /src/nix-build/nix-build
/src/nix-copy-closure/nix-copy-closure
/src/build-remote/build-remote /src/build-remote/build-remote
# /tests/ # /tests/

View file

@ -11,6 +11,7 @@ makefiles = \
src/nix-env/local.mk \ src/nix-env/local.mk \
src/nix-daemon/local.mk \ src/nix-daemon/local.mk \
src/nix-collect-garbage/local.mk \ src/nix-collect-garbage/local.mk \
src/nix-copy-closure/local.mk \
src/nix-prefetch-url/local.mk \ src/nix-prefetch-url/local.mk \
src/buildenv/local.mk \ src/buildenv/local.mk \
src/resolve-system-dependencies/local.mk \ src/resolve-system-dependencies/local.mk \

View file

@ -1,8 +1,3 @@
nix_bin_scripts := \
$(d)/nix-copy-closure \
bin-scripts += $(nix_bin_scripts)
nix_noinst_scripts := \ nix_noinst_scripts := \
$(d)/nix-http-export.cgi \ $(d)/nix-http-export.cgi \
$(d)/nix-profile.sh \ $(d)/nix-profile.sh \
@ -14,4 +9,4 @@ profiledir = $(sysconfdir)/profile.d
$(eval $(call install-file-as, $(d)/nix-profile.sh, $(profiledir)/nix.sh, 0644)) $(eval $(call install-file-as, $(d)/nix-profile.sh, $(profiledir)/nix.sh, 0644))
clean-files += $(nix_bin_scripts) $(nix_noinst_scripts) clean-files += $(nix_noinst_scripts)

View file

@ -1,103 +0,0 @@
#! @perl@ -w @perlFlags@
use utf8;
use strict;
use Nix::SSH;
use Nix::Config;
use Nix::Store;
use Nix::CopyClosure;
use List::Util qw(sum);
binmode STDERR, ":encoding(utf8)";
if (scalar @ARGV < 1) {
print STDERR <<EOF
Usage: nix-copy-closure [--from | --to] HOSTNAME [--gzip] [--bzip2] [--xz] PATHS...
EOF
;
exit 1;
}
# Get the target host.
my $sshHost;
my $toMode = 1;
my $includeOutputs = 0;
my $dryRun = 0;
my $useSubstitutes = 0;
my $verbosity = 1;
# !!! Copied from nix-pack-closure, should put this in a module.
my @storePaths = ();
while (@ARGV) {
my $arg = shift @ARGV;
if ($arg eq "--help") {
exec "man nix-copy-closure" or die;
}
elsif ($arg eq "--gzip" || $arg eq "--bzip2" || $arg eq "--xz") {
warn "$0: $arg is not implemented\n" if $arg ne "--gzip";
push @globalSshOpts, "-C";
}
elsif ($arg eq "--from") {
$toMode = 0;
}
elsif ($arg eq "--to") {
$toMode = 1;
}
elsif ($arg eq "--include-outputs") {
$includeOutputs = 1;
}
elsif ($arg eq "--show-progress") {
warn "$0: $arg is not implemented\n";
}
elsif ($arg eq "--dry-run") {
$dryRun = 1;
}
elsif ($arg eq "--use-substitutes" || $arg eq "-s") {
$useSubstitutes = 1;
}
elsif ($arg eq "-v") {
$verbosity++;
setVerbosity($verbosity);
}
elsif (!defined $sshHost) {
$sshHost = $arg;
}
else {
push @storePaths, $arg;
}
}
die "$0: you did not specify a host name\n" unless defined $sshHost;
if ($toMode) { # Copy TO the remote machine.
Nix::CopyClosure::copyTo(
$sshHost, [ @storePaths ],
$includeOutputs, $dryRun, $useSubstitutes);
}
else { # Copy FROM the remote machine.
my ($from, $to) = connectToRemoteNix($sshHost, []);
# Query the closure of the given store paths on the remote
# machine. Paths are assumed to be store paths; there is no
# resolution (following of symlinks).
syswrite($to, pack("L<x4L<x4", 7, $includeOutputs ? 1 : 0)) or die;
writeStrings(\@storePaths, $to);
my @missing = grep { !isValidPath($_) } readStrings($from);
# Export the store paths on the remote machine and import them locally.
if (scalar @missing > 0) {
print STDERR "copying ", scalar @missing, " missing paths from $sshHost...\n";
writeInt(5, $to); # == cmdExportPaths
writeInt(0, $to); # obsolete
writeStrings(\@missing, $to);
importPaths(fileno($from), 1);
}
}

View file

@ -12,7 +12,6 @@
#include "shared.hh" #include "shared.hh"
#include "pathlocks.hh" #include "pathlocks.hh"
#include "globals.hh" #include "globals.hh"
#include "serve-protocol.hh"
#include "serialise.hh" #include "serialise.hh"
#include "store-api.hh" #include "store-api.hh"
#include "derivations.hh" #include "derivations.hh"

View file

@ -8,4 +8,4 @@ build-remote_LIBS = libmain libutil libformat libstore
build-remote_SOURCES := $(d)/build-remote.cc build-remote_SOURCES := $(d)/build-remote.cc
build-remote_CXXFLAGS = -DSYSCONFDIR="\"$(sysconfdir)\"" -Isrc/nix-store build-remote_CXXFLAGS = -DSYSCONFDIR="\"$(sysconfdir)\""

View file

@ -41,6 +41,8 @@ private:
string host; string host;
Path key; Path key;
bool compress;
}; };
SSHStore::SSHStore(string host, const Params & params, size_t maxConnections) SSHStore::SSHStore(string host, const Params & params, size_t maxConnections)
@ -50,6 +52,7 @@ SSHStore::SSHStore(string host, const Params & params, size_t maxConnections)
, socketPath((Path) tmpDir + "/ssh.sock") , socketPath((Path) tmpDir + "/ssh.sock")
, host(std::move(host)) , host(std::move(host))
, key(get(params, "ssh-key", "")) , key(get(params, "ssh-key", ""))
, compress(get(params, "compress", "") == "true")
{ {
/* open a connection and perform the handshake to verify all is well */ /* open a connection and perform the handshake to verify all is well */
connections->get(); connections->get();

View file

@ -5,6 +5,7 @@
#include "nar-info-disk-cache.hh" #include "nar-info-disk-cache.hh"
#include "thread-pool.hh" #include "thread-pool.hh"
#include "json.hh" #include "json.hh"
#include "derivations.hh"
#include <future> #include <future>
@ -780,8 +781,27 @@ std::list<ref<Store>> getDefaultSubstituters()
} }
void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths) void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths, bool substitute)
{ {
if (substitute) {
/* Filter out .drv files (we don't want to build anything). */
PathSet paths2;
for (auto & path : storePaths)
if (!isDerivation(path)) paths2.insert(path);
unsigned long long downloadSize, narSize;
PathSet willBuild, willSubstitute, unknown;
to->queryMissing(PathSet(paths2.begin(), paths2.end()),
willBuild, willSubstitute, unknown, downloadSize, narSize);
/* FIXME: should use ensurePath(), but it only
does one path at a time. */
if (!willSubstitute.empty())
try {
to->buildPaths(willSubstitute);
} catch (Error & e) {
printMsg(lvlError, format("warning: %1%") % e.msg());
}
}
std::string copiedLabel = "copied"; std::string copiedLabel = "copied";
logger->setExpected(copiedLabel, storePaths.size()); logger->setExpected(copiedLabel, storePaths.size());

View file

@ -643,7 +643,7 @@ void removeTempRoots();
ref<Store> openStore(const std::string & uri = getEnv("NIX_REMOTE")); ref<Store> openStore(const std::string & uri = getEnv("NIX_REMOTE"));
void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths); void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths, bool substitute = false);
enum StoreType { enum StoreType {
tDaemon, tDaemon,

View file

@ -0,0 +1,7 @@
programs += nix-copy-closure
nix-copy-closure_DIR := $(d)
nix-copy-closure_LIBS = libmain libutil libformat libstore
nix-copy-closure_SOURCES := $(d)/nix-copy-closure.cc

View file

@ -0,0 +1,60 @@
#include "shared.hh"
#include "store-api.hh"
using namespace nix;
int main(int argc, char ** argv)
{
return handleExceptions(argv[0], [&]() {
initNix();
auto gzip = false;
auto toMode = true;
auto includeOutputs = false;
auto dryRun = false;
auto useSubstitutes = false;
auto sshHost = string{};
auto storePaths = PathSet{};
parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
if (*arg == "--help")
showManPage("nix-copy-closure");
else if (*arg == "--version")
printVersion("nix-copy-closure");
else if (*arg == "--gzip" || *arg == "--bzip2" || *arg == "--xz") {
if (*arg != "--gzip")
printMsg(lvlError, format("Warning: %1% is not implemented, falling back to gzip") % *arg);
gzip = true;
} else if (*arg == "--from")
toMode = false;
else if (*arg == "--to")
toMode = true;
else if (*arg == "--include-outputs")
includeOutputs = true;
else if (*arg == "--show-progress")
printMsg(lvlError, "Warning: --show-progress is not implemented");
else if (*arg == "--dry-run")
dryRun = true;
else if (*arg == "--use-substitutes" || *arg == "-s")
useSubstitutes = true;
else if (sshHost.empty())
sshHost = *arg;
else
storePaths.insert(*arg);
return true;
});
if (sshHost.empty())
throw UsageError("no host name specified");
auto remoteUri = "ssh://" + sshHost + (gzip ? "?compress=true" : "");
auto to = toMode ? openStore(remoteUri) : openStore();
auto from = toMode ? openStore() : openStore(remoteUri);
if (includeOutputs) {
auto newPaths = PathSet{};
for (const auto & p : storePaths) {
auto outputs = from->queryDerivationOutputs(p);
newPaths.insert(outputs.begin(), outputs.end());
}
storePaths.insert(newPaths.begin(), newPaths.end());
}
copyPaths(from, to, Paths(storePaths.begin(), storePaths.end()), useSubstitutes);
});
}