forked from lix-project/lix
c8daeba303
If ‘--link’ is given, nix-push will create hard links to the NAR files in the store, rather than copying them. This is faster and requires less disk space. However, it doesn't work if the store is on a different file system.
271 lines
8.2 KiB
Plaintext
Executable file
271 lines
8.2 KiB
Plaintext
Executable file
#! @perl@ -w @perlFlags@
|
|
|
|
use strict;
|
|
use File::Basename;
|
|
use File::Temp qw(tempdir);
|
|
use File::Path qw(mkpath);
|
|
use File::stat;
|
|
use File::Copy;
|
|
use Nix::Config;
|
|
use Nix::Store;
|
|
use Nix::Manifest;
|
|
use Nix::Utils;
|
|
|
|
my $tmpDir = tempdir("nix-push.XXXXXX", CLEANUP => 1, TMPDIR => 1)
|
|
or die "cannot create a temporary directory";
|
|
|
|
my $nixExpr = "$tmpDir/create-nars.nix";
|
|
|
|
|
|
# Parse the command line.
|
|
my $compressionType = "xz";
|
|
my $force = 0;
|
|
my $destDir;
|
|
my $writeManifest = 0;
|
|
my $archivesURL;
|
|
my $link = 0;
|
|
my @roots;
|
|
|
|
for (my $n = 0; $n < scalar @ARGV; $n++) {
|
|
my $arg = $ARGV[$n];
|
|
|
|
if ($arg eq "--help") {
|
|
exec "man nix-push" or die;
|
|
} elsif ($arg eq "--bzip2") {
|
|
$compressionType = "bzip2";
|
|
} elsif ($arg eq "--force") {
|
|
$force = 1;
|
|
} elsif ($arg eq "--dest") {
|
|
$n++;
|
|
die "$0: `$arg' requires an argument\n" unless $n < scalar @ARGV;
|
|
$destDir = $ARGV[$n];
|
|
mkpath($destDir, 0, 0755);
|
|
} elsif ($arg eq "--manifest") {
|
|
$writeManifest = 1;
|
|
} elsif ($arg eq "--url-prefix") {
|
|
$n++;
|
|
die "$0: `$arg' requires an argument\n" unless $n < scalar @ARGV;
|
|
$archivesURL = $ARGV[$n];
|
|
} elsif ($arg eq "--link") {
|
|
$link = 1;
|
|
} elsif (substr($arg, 0, 1) eq "-") {
|
|
die "$0: unknown flag `$arg'\n";
|
|
} else {
|
|
push @roots, $arg;
|
|
}
|
|
}
|
|
|
|
die "$0: please specify a destination directory\n" if !defined $destDir;
|
|
|
|
$archivesURL = "file://$destDir" unless defined $archivesURL;
|
|
|
|
|
|
# From the given store paths, determine the set of requisite store
|
|
# paths, i.e, the paths required to realise them.
|
|
my %storePaths;
|
|
|
|
foreach my $path (@roots) {
|
|
die unless $path =~ /^\//;
|
|
|
|
# Get all paths referenced by the normalisation of the given
|
|
# Nix expression.
|
|
my $pid = open(READ,
|
|
"$Nix::Config::binDir/nix-store --query --requisites --force-realise " .
|
|
"--include-outputs '$path'|") or die;
|
|
|
|
while (<READ>) {
|
|
chomp;
|
|
die "bad: $_" unless /^\//;
|
|
$storePaths{$_} = "";
|
|
}
|
|
|
|
close READ or die "nix-store failed: $?";
|
|
}
|
|
|
|
my @storePaths = keys %storePaths;
|
|
|
|
|
|
# Don't create archives for files that are already in the binary cache.
|
|
my @storePaths2;
|
|
my %narFiles;
|
|
foreach my $storePath (@storePaths) {
|
|
my $pathHash = substr(basename($storePath), 0, 32);
|
|
my $narInfoFile = "$destDir/$pathHash.narinfo";
|
|
if (-e $narInfoFile) {
|
|
my $narInfo = parseNARInfo($storePath, readFile($narInfoFile));
|
|
my $narFile = "$destDir/$narInfo->{url}";
|
|
if (-e $narFile) {
|
|
print STDERR "skipping existing $storePath\n";
|
|
# Add the NAR info to $narFiles if we're writing a
|
|
# manifest.
|
|
$narFiles{$storePath} = [
|
|
{ url => ("$archivesURL/" . basename $narInfo->{url})
|
|
, hash => $narInfo->{fileHash}
|
|
, size => $narInfo->{fileSize}
|
|
, compressionType => $narInfo->{compression}
|
|
, narHash => $narInfo->{narHash}
|
|
, narSize => $narInfo->{narSize}
|
|
, references => join(" ", map { "$Nix::Config::storeDir/$_" } @{$narInfo->{refs}})
|
|
, deriver => $narInfo->{deriver} ? "$Nix::Config::storeDir/$narInfo->{deriver}" : undef
|
|
}
|
|
] if $writeManifest;
|
|
next;
|
|
}
|
|
}
|
|
push @storePaths2, $storePath;
|
|
}
|
|
|
|
|
|
# Create a list of Nix derivations that turn each path into a Nix
|
|
# archive.
|
|
open NIX, ">$nixExpr";
|
|
print NIX "[";
|
|
|
|
foreach my $storePath (@storePaths2) {
|
|
die unless ($storePath =~ /\/[0-9a-z]{32}[^\"\\\$]*$/);
|
|
|
|
# Construct a Nix expression that creates a Nix archive.
|
|
my $nixexpr =
|
|
"(import <nix/nar.nix> " .
|
|
"{ storePath = builtins.storePath \"$storePath\"; hashAlgo = \"sha256\"; compressionType = \"$compressionType\"; }) ";
|
|
|
|
print NIX $nixexpr;
|
|
}
|
|
|
|
print NIX "]";
|
|
close NIX;
|
|
|
|
|
|
# Build the Nix expression.
|
|
print STDERR "building compressed archives...\n";
|
|
my @narPaths;
|
|
my $pid = open(READ, "$Nix::Config::binDir/nix-build $nixExpr -o $tmpDir/result |")
|
|
or die "cannot run nix-build";
|
|
while (<READ>) {
|
|
chomp;
|
|
die unless /^\//;
|
|
push @narPaths, $_;
|
|
}
|
|
close READ or die "nix-build failed: $?";
|
|
|
|
|
|
# Write the cache info file.
|
|
my $cacheInfoFile = "$destDir/nix-cache-info";
|
|
if (! -e $cacheInfoFile) {
|
|
open FILE, ">$cacheInfoFile" or die "cannot create $cacheInfoFile: $!";
|
|
print FILE "StoreDir: $Nix::Config::storeDir\n";
|
|
print FILE "WantMassQuery: 0\n"; # by default, don't hit this cache for "nix-env -qas"
|
|
close FILE;
|
|
}
|
|
|
|
|
|
# Copy the archives and the corresponding NAR info files.
|
|
print STDERR "copying archives...\n";
|
|
|
|
my $totalNarSize = 0;
|
|
my $totalCompressedSize = 0;
|
|
|
|
for (my $n = 0; $n < scalar @storePaths2; $n++) {
|
|
my $storePath = $storePaths2[$n];
|
|
my $narDir = $narPaths[$n];
|
|
my $baseName = basename $storePath;
|
|
|
|
# Get info about the store path.
|
|
my ($deriver, $narHash, $time, $narSize, $refs) = queryPathInfo($storePath, 1);
|
|
|
|
# In some exceptional cases (such as VM tests that use the Nix
|
|
# store of the host), the database doesn't contain the hash. So
|
|
# compute it.
|
|
if ($narHash =~ /^sha256:0*$/) {
|
|
my $nar = "$tmpDir/nar";
|
|
system("$Nix::Config::binDir/nix-store --dump $storePath > $nar") == 0
|
|
or die "cannot dump $storePath\n";
|
|
$narHash = `$Nix::Config::binDir/nix-hash --type sha256 --base32 --flat $nar`;
|
|
die "cannot hash `$nar'" if $? != 0;
|
|
chomp $narHash;
|
|
$narHash = "sha256:$narHash";
|
|
$narSize = stat("$nar")->size;
|
|
unlink $nar or die;
|
|
}
|
|
|
|
$totalNarSize += $narSize;
|
|
|
|
# Get info about the compressed NAR.
|
|
open HASH, "$narDir/nar-compressed-hash" or die "cannot open nar-compressed-hash";
|
|
my $compressedHash = <HASH>;
|
|
chomp $compressedHash;
|
|
$compressedHash =~ /^[0-9a-z]+$/ or die "invalid hash";
|
|
close HASH;
|
|
|
|
my $narName = "$compressedHash.nar." . ($compressionType eq "xz" ? "xz" : "bz2");
|
|
|
|
my $narFile = "$narDir/$narName";
|
|
(-f $narFile) or die "NAR file for $storePath not found";
|
|
|
|
my $compressedSize = stat($narFile)->size;
|
|
$totalCompressedSize += $compressedSize;
|
|
|
|
printf STDERR "%s [%.2f MiB, %.1f%%]\n", $storePath,
|
|
$compressedSize / (1024 * 1024), $compressedSize / $narSize * 100;
|
|
|
|
# Copy the compressed NAR.
|
|
my $dst = "$destDir/$narName";
|
|
if (! -f $dst) {
|
|
my $tmp = "$destDir/.tmp.$$.$narName";
|
|
if ($link) {
|
|
link($narFile, $tmp) or die "cannot link $tmp to $narFile: $!\n";
|
|
} else {
|
|
copy($narFile, $tmp) or die "cannot copy $narFile to $tmp: $!\n";
|
|
}
|
|
rename($tmp, $dst) or die "cannot rename $tmp to $dst: $!\n";
|
|
}
|
|
|
|
# Write the info file.
|
|
my $info;
|
|
$info .= "StorePath: $storePath\n";
|
|
$info .= "URL: $narName\n";
|
|
$info .= "Compression: $compressionType\n";
|
|
$info .= "FileHash: sha256:$compressedHash\n";
|
|
$info .= "FileSize: $compressedSize\n";
|
|
$info .= "NarHash: $narHash\n";
|
|
$info .= "NarSize: $narSize\n";
|
|
$info .= "References: " . join(" ", map { basename $_ } @{$refs}) . "\n";
|
|
if (defined $deriver) {
|
|
$info .= "Deriver: " . basename $deriver . "\n";
|
|
if (isValidPath($deriver)) {
|
|
my $drv = derivationFromPath($deriver);
|
|
$info .= "System: $drv->{platform}\n";
|
|
}
|
|
}
|
|
|
|
my $pathHash = substr(basename($storePath), 0, 32);
|
|
|
|
$dst = "$destDir/$pathHash.narinfo";
|
|
if ($force || ! -f $dst) {
|
|
my $tmp = "$destDir/.tmp.$$.$pathHash.narinfo";
|
|
open INFO, ">$tmp" or die;
|
|
print INFO "$info" or die;
|
|
close INFO or die;
|
|
rename($tmp, $dst) or die "cannot rename $tmp to $dst: $!\n";
|
|
}
|
|
|
|
$narFiles{$storePath} = [
|
|
{ url => "$archivesURL/$narName"
|
|
, hash => "sha256:$compressedHash"
|
|
, size => $compressedSize
|
|
, compressionType => $compressionType
|
|
, narHash => "$narHash"
|
|
, narSize => $narSize
|
|
, references => join(" ", @{$refs})
|
|
, deriver => $deriver
|
|
}
|
|
] if $writeManifest;
|
|
}
|
|
|
|
printf STDERR "total compressed size %.2f MiB, %.1f%%\n",
|
|
$totalCompressedSize / (1024 * 1024), $totalCompressedSize / ($totalNarSize || 1) * 100;
|
|
|
|
|
|
# Optionally write a manifest.
|
|
writeManifest "$destDir/MANIFEST", \%narFiles, \() if $writeManifest;
|