0
0
Fork 0
lix-releng-staging/scripts/nix-push.in
Eelco Dolstra f2b67fbf2a nix-push: Support -j
Fixes .
2015-06-08 14:16:06 +02:00

300 lines
9.5 KiB
Plaintext
Executable file
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#! @perl@ -w @perlFlags@
use utf8;
use strict;
use File::Basename;
use File::Path qw(mkpath);
use File::stat;
use File::Copy;
use MIME::Base64;
use Nix::Config;
use Nix::Store;
use Nix::Manifest;
use Nix::Utils;
binmode STDERR, ":encoding(utf8)";
my $tmpDir = mkTempDir("nix-push");
my $nixExpr = "$tmpDir/create-nars.nix";
# Parse the command line.
my $compressionType = "xz";
my $force = 0;
my $destDir;
my $writeManifest = 0;
my $manifestPath;
my $archivesURL;
my $link = 0;
my $secretKeyFile;
my @roots;
my @buildArgs;
for (my $n = 0; $n < scalar @ARGV; $n++) {
my $arg = $ARGV[$n];
if ($arg eq "--help") {
exec "man nix-push" or die;
} elsif ($arg eq "--bzip2") {
$compressionType = "bzip2";
} elsif ($arg eq "--none") {
$compressionType = "none";
} elsif ($arg eq "--force") {
$force = 1;
} elsif ($arg eq "--dest") {
$n++;
die "$0: $arg requires an argument\n" unless $n < scalar @ARGV;
$destDir = $ARGV[$n];
mkpath($destDir, 0, 0755);
} elsif ($arg eq "--manifest") {
$writeManifest = 1;
} elsif ($arg eq "--manifest-path") {
$n++;
die "$0: $arg requires an argument\n" unless $n < scalar @ARGV;
$manifestPath = $ARGV[$n];
$writeManifest = 1;
mkpath(dirname($manifestPath), 0, 0755);
} elsif ($arg eq "--url-prefix") {
$n++;
die "$0: $arg requires an argument\n" unless $n < scalar @ARGV;
$archivesURL = $ARGV[$n];
} elsif ($arg eq "--link") {
$link = 1;
} elsif ($arg eq "--key-file") {
$n++;
die "$0: $arg requires an argument\n" unless $n < scalar @ARGV;
$secretKeyFile = $ARGV[$n];
} elsif ($arg eq "--max-jobs" || $arg eq "-j") {
$n++;
die "$0: $arg requires an argument\n" unless $n < scalar @ARGV;
push @buildArgs, ($arg, $ARGV[$n]);
} elsif (substr($arg, 0, 1) eq "-") {
die "$0: unknown flag $arg\n";
} else {
push @roots, $arg;
}
}
die "$0: please specify a destination directory\n" if !defined $destDir;
$archivesURL = "file://$destDir" unless defined $archivesURL;
# From the given store paths, determine the set of requisite store
# paths, i.e, the paths required to realise them.
my %storePaths;
foreach my $path (@roots) {
# Get all paths referenced by the normalisation of the given
# Nix expression.
my $pid = open(READ,
"$Nix::Config::binDir/nix-store --query --requisites --force-realise " .
"--include-outputs '$path'|") or die;
while (<READ>) {
chomp;
die "bad: $_" unless /^\//;
$storePaths{$_} = "";
}
close READ or die "nix-store failed: $?";
}
my @storePaths = keys %storePaths;
# Don't create archives for files that are already in the binary cache.
my @storePaths2;
my %narFiles;
foreach my $storePath (@storePaths) {
my $pathHash = substr(basename($storePath), 0, 32);
my $narInfoFile = "$destDir/$pathHash.narinfo";
if (!$force && -e $narInfoFile) {
my $narInfo = parseNARInfo($storePath, readFile($narInfoFile), 0, $narInfoFile) or die "cannot read $narInfoFile\n";
my $narFile = "$destDir/$narInfo->{url}";
if (-e $narFile) {
print STDERR "skipping existing $storePath\n";
# Add the NAR info to $narFiles if we're writing a
# manifest.
$narFiles{$storePath} = [
{ url => ("$archivesURL/" . basename $narInfo->{url})
, hash => $narInfo->{fileHash}
, size => $narInfo->{fileSize}
, compressionType => $narInfo->{compression}
, narHash => $narInfo->{narHash}
, narSize => $narInfo->{narSize}
, references => join(" ", map { "$Nix::Config::storeDir/$_" } @{$narInfo->{refs}})
, deriver => $narInfo->{deriver} ? "$Nix::Config::storeDir/$narInfo->{deriver}" : undef
}
] if $writeManifest;
next;
}
}
push @storePaths2, $storePath;
}
# Create a list of Nix derivations that turn each path into a Nix
# archive.
open NIX, ">$nixExpr";
print NIX "[";
foreach my $storePath (@storePaths2) {
die unless ($storePath =~ /\/[0-9a-z]{32}[^\"\\\$]*$/);
# Construct a Nix expression that creates a Nix archive.
my $nixexpr =
"(import <nix/nar.nix> " .
"{ storePath = builtins.storePath \"$storePath\"; hashAlgo = \"sha256\"; compressionType = \"$compressionType\"; }) ";
print NIX $nixexpr;
}
print NIX "]";
close NIX;
# Build the Nix expression.
print STDERR "building compressed archives...\n";
my @narPaths;
my $pid = open(READ, "-|", "$Nix::Config::binDir/nix-build", $nixExpr, "-o", "$tmpDir/result", @buildArgs)
or die "cannot run nix-build";
while (<READ>) {
chomp;
die unless /^\//;
push @narPaths, $_;
}
close READ or die "nix-build failed: $?";
# Write the cache info file.
my $cacheInfoFile = "$destDir/nix-cache-info";
if (! -e $cacheInfoFile) {
open FILE, ">$cacheInfoFile" or die "cannot create $cacheInfoFile: $!";
print FILE "StoreDir: $Nix::Config::storeDir\n";
print FILE "WantMassQuery: 0\n"; # by default, don't hit this cache for "nix-env -qas"
close FILE;
}
# Copy the archives and the corresponding NAR info files.
print STDERR "copying archives...\n";
my $totalNarSize = 0;
my $totalCompressedSize = 0;
for (my $n = 0; $n < scalar @storePaths2; $n++) {
my $storePath = $storePaths2[$n];
my $narDir = $narPaths[$n];
my $baseName = basename $storePath;
# Get info about the store path.
my ($deriver, $narHash, $time, $narSize, $refs) = queryPathInfo($storePath, 1);
# In some exceptional cases (such as VM tests that use the Nix
# store of the host), the database doesn't contain the hash. So
# compute it.
if ($narHash =~ /^sha256:0*$/) {
my $nar = "$tmpDir/nar";
system("$Nix::Config::binDir/nix-store --dump $storePath > $nar") == 0
or die "cannot dump $storePath\n";
$narHash = `$Nix::Config::binDir/nix-hash --type sha256 --base32 --flat $nar`;
die "cannot hash $nar" if $? != 0;
chomp $narHash;
$narHash = "sha256:$narHash";
$narSize = stat("$nar")->size;
unlink $nar or die;
}
$totalNarSize += $narSize;
# Get info about the compressed NAR.
open HASH, "$narDir/nar-compressed-hash" or die "cannot open nar-compressed-hash";
my $compressedHash = <HASH>;
chomp $compressedHash;
$compressedHash =~ /^[0-9a-z]+$/ or die "invalid hash";
close HASH;
my $narName = "$compressedHash.nar" . ($compressionType eq "xz" ? ".xz" : $compressionType eq "bzip2" ? ".bz2" : "");
my $narFile = "$narDir/$narName";
(-f $narFile) or die "NAR file for $storePath not found";
my $compressedSize = stat($narFile)->size;
$totalCompressedSize += $compressedSize;
printf STDERR "%s [%.2f MiB, %.1f%%]\n", $storePath,
$compressedSize / (1024 * 1024), $compressedSize / $narSize * 100;
# Copy the compressed NAR.
my $dst = "$destDir/$narName";
if (! -f $dst) {
my $tmp = "$destDir/.tmp.$$.$narName";
if ($link) {
link($narFile, $tmp) or die "cannot link $tmp to $narFile: $!\n";
} else {
copy($narFile, $tmp) or die "cannot copy $narFile to $tmp: $!\n";
}
rename($tmp, $dst) or die "cannot rename $tmp to $dst: $!\n";
}
# Write the info file.
my $info;
$info .= "StorePath: $storePath\n";
$info .= "URL: $narName\n";
$info .= "Compression: $compressionType\n";
$info .= "FileHash: sha256:$compressedHash\n";
$info .= "FileSize: $compressedSize\n";
$info .= "NarHash: $narHash\n";
$info .= "NarSize: $narSize\n";
$info .= "References: " . join(" ", map { basename $_ } @{$refs}) . "\n";
if (defined $deriver) {
$info .= "Deriver: " . basename $deriver . "\n";
if (isValidPath($deriver)) {
my $drv = derivationFromPath($deriver);
$info .= "System: $drv->{platform}\n";
}
}
if (defined $secretKeyFile) {
my $s = readFile $secretKeyFile;
chomp $s;
my ($keyName, $secretKey) = split ":", $s;
die "invalid secret key file $secretKeyFile\n" unless defined $keyName && defined $secretKey;
my $fingerprint = fingerprintPath($storePath, $narHash, $narSize, $refs);
my $sig = encode_base64(signString(decode_base64($secretKey), $fingerprint), "");
$info .= "Sig: $keyName:$sig\n";
}
my $pathHash = substr(basename($storePath), 0, 32);
$dst = "$destDir/$pathHash.narinfo";
if ($force || ! -f $dst) {
my $tmp = "$destDir/.tmp.$$.$pathHash.narinfo";
open INFO, ">$tmp" or die;
print INFO "$info" or die;
close INFO or die;
rename($tmp, $dst) or die "cannot rename $tmp to $dst: $!\n";
}
$narFiles{$storePath} = [
{ url => "$archivesURL/$narName"
, hash => "sha256:$compressedHash"
, size => $compressedSize
, compressionType => $compressionType
, narHash => "$narHash"
, narSize => $narSize
, references => join(" ", @{$refs})
, deriver => $deriver
}
] if $writeManifest;
}
printf STDERR "total compressed size %.2f MiB, %.1f%%\n",
$totalCompressedSize / (1024 * 1024), $totalCompressedSize / ($totalNarSize || 1) * 100;
# Optionally write a manifest.
writeManifest($manifestPath // "$destDir/MANIFEST", \%narFiles, \()) if $writeManifest;