puck/lix
0
0
Fork 0
forked from lix-project/lix
Code Pull requests Activity

nix-push: create a manifest-less binary cache

Browse source 
Manifests are a huge pain, since users need to run nix-pull directly
or indirectly to obtain them.  They tend to be large and lag behind
the available binaries; also, the downloaded manifests in
/nix/var/nix/manifest need to be in sync with the Nixpkgs sources.  So
we want to get rid of them.

The idea of manifest-free operation works as follows.  Nix is
configured with a set of URIs of binary caches, e.g.

  http://nixos.org/binary-cache

Whenever Nix needs a store path X, it checks each binary cache for the
existence of a file <CACHE-URI>/<SHA-256 hash of X>.narinfo, e.g.

  http://nixos.org/binary-cache/bi1gh9...ia17.narinfo

The .narinfo file contains the necessary information about the store
path that was formerly kept in the manifest, i.e., (relative) URI of
the compressed NAR, references, size, hash, etc.  For example:

  StorePath: /nix/store/xqp4l88cr9bxv01jinkz861mnc9p7qfi-neon-0.29.6
  URL: 1bjxbg52l32wj8ww47sw9f4qz0r8n5vs71l93lcbgk2506v3cpfd.nar.bz2
  CompressedHash: sha256:1bjxbg52l32wj8ww47sw9f4qz0r8n5vs71l93lcbgk2506v3cpfd
  CompressedSize: 202542
  NarHash: sha256:1af26536781e6134ab84201b33408759fc59b36cc5530f57c0663f67b588e15f
  NarSize: 700440
  References: 043zrsanirjh8nbc5vqpjn93hhrf107f-bash-4.2-p24 cj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13 ...
  Deriver: 4idz1bgi58h3pazxr3akrw4fsr6zrf3r-neon-0.29.6.drv
  System: x86_64-linux

Nix then knows that it needs to download

  http://nixos.org/binary-cache/1bjxbg52l32wj8ww47sw9f4qz0r8n5vs71l93lcbgk2506v3cpfd.nar.bz2

to substitute the store path.

Note that the store directory is omitted from the References and
Deriver fields to save space, and that the URL can be relative to the
binary cache prefix.

This patch just makes nix-push create binary caches in this format.
The next step is to make a substituter that supports them.
This commit is contained in:
Eelco Dolstra 2012-06-28 17:19:32 -04:00
parent 1aba0bf0fa
commit 49cd7387ad
1 changed files with 94 additions and 160 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

252
scripts/nix-push.in
View file

@ -1,10 +1,13 @@
#! @perl@ -w @perlFlags@ #! @perl@ -w @perlFlags@
use strict; use strict;
use File::Basename;
use File::Temp qw(tempdir); use File::Temp qw(tempdir);
use File::Path qw(mkpath);
use File::stat; use File::stat;
use File::Copy;
use Nix::Config; use Nix::Config;
use Nix::Manifest; use Nix::Store;
my $hashAlgo = "sha256"; my $hashAlgo = "sha256";
@ -12,7 +15,6 @@ my $tmpDir = tempdir("nix-push.XXXXXX", CLEANUP => 1, TMPDIR => 1)
or die "cannot create a temporary directory"; or die "cannot create a temporary directory";
my $nixExpr = "$tmpDir/create-nars.nix"; my $nixExpr = "$tmpDir/create-nars.nix";
my $manifest = "$tmpDir/MANIFEST";
my $curl = "$Nix::Config::curl --fail --silent"; my $curl = "$Nix::Config::curl --fail --silent";
my $extraCurlFlags = ${ENV{'CURL_FLAGS'}}; my $extraCurlFlags = ${ENV{'CURL_FLAGS'}};
@ -22,18 +24,14 @@ $curl = "$curl $extraCurlFlags" if defined $extraCurlFlags;
# Parse the command line. # Parse the command line.
my $localCopy; my $localCopy;
my $localArchivesDir; my $localArchivesDir;
my $localManifestFile;
my $targetArchivesUrl;
my $archivesPutURL; my $archivesPutURL;
my $archivesGetURL; my $archivesGetURL;
my $manifestPutURL;
sub showSyntax { sub showSyntax {
print STDERR <<EOF print STDERR <<EOF
Usage: nix-push --copy ARCHIVES_DIR MANIFEST_FILE PATHS... Usage: nix-push --copy ARCHIVES_DIR PATHS...
or: nix-push ARCHIVES_PUT_URL ARCHIVES_GET_URL MANIFEST_PUT_URL PATHS... or: nix-push ARCHIVES_PUT_URL ARCHIVES_GET_URL PATHS...
`nix-push' copies or uploads the closure of PATHS to the given `nix-push' copies or uploads the closure of PATHS to the given
destination. destination.
@ -45,25 +43,16 @@ EOF
showSyntax if scalar @ARGV < 1; showSyntax if scalar @ARGV < 1;
if ($ARGV[0] eq "--copy") { if ($ARGV[0] eq "--copy") {
showSyntax if scalar @ARGV < 3; showSyntax if scalar @ARGV < 2;
$localCopy = 1; $localCopy = 1;
shift @ARGV; shift @ARGV;
$localArchivesDir = shift @ARGV; $localArchivesDir = shift @ARGV;
$localManifestFile = shift @ARGV; mkpath($localArchivesDir, 0, 0755);
if ($ARGV[0] eq "--target") { } else {
shift @ARGV; showSyntax if scalar @ARGV < 2;
$targetArchivesUrl = shift @ARGV;
}
else {
$targetArchivesUrl = "file://$localArchivesDir";
}
}
else {
showSyntax if scalar @ARGV < 3;
$localCopy = 0; $localCopy = 0;
$archivesPutURL = shift @ARGV; $archivesPutURL = shift @ARGV;
$archivesGetURL = shift @ARGV; $archivesGetURL = shift @ARGV;
$manifestPutURL = shift @ARGV;
} }
@ -92,8 +81,8 @@ foreach my $path (@ARGV) {
my @storePaths = keys %storePaths; my @storePaths = keys %storePaths;
# For each path, create a Nix expression that turns the path into # Create a list of Nix derivations that turn each path into a Nix
# a Nix archive. # archive.
open NIX, ">$nixExpr"; open NIX, ">$nixExpr";
print NIX "["; print NIX "[";
@ -112,172 +101,117 @@ print NIX "]";
close NIX; close NIX;
# Instantiate store derivations from the Nix expression. # Build the Nix expression.
my @storeExprs; print STDERR "building compressed archives...\n";
print STDERR "instantiating store derivations...\n"; my @narPaths;
my $pid = open(READ, "$Nix::Config::binDir/nix-instantiate $nixExpr|") my $pid = open(READ, "$Nix::Config::binDir/nix-build $nixExpr|")
or die "cannot run nix-instantiate"; or die "cannot run nix-build";
while (<READ>) { while (<READ>) {
chomp; chomp;
die unless /^\//; die unless /^\//;
push @storeExprs, $_; push @narPaths, $_;
} }
close READ or die "nix-instantiate failed: $?"; close READ or die "nix-build failed: $?";
# Build the derivations. # Upload the archives and the corresponding info files.
print STDERR "creating archives...\n"; print STDERR "uploading/copying archives...\n";
my @narPaths; my $totalNarSize = 0;
my $totalNarBz2Size = 0;
my @tmp = @storeExprs;
while (scalar @tmp > 0) {
my $n = scalar @tmp;
if ($n > 256) { $n = 256 };
my @tmp2 = @tmp[0..$n - 1];
@tmp = @tmp[$n..scalar @tmp - 1];
my $pid = open(READ, "$Nix::Config::binDir/nix-store --realise @tmp2|")
or die "cannot run nix-store";
while (<READ>) {
chomp;
die unless (/^\//);
push @narPaths, "$_";
}
close READ or die "nix-store failed: $?";
}
# Create the manifest.
print STDERR "creating manifest...\n";
my %narFiles;
my %patches;
my @narArchives;
for (my $n = 0; $n < scalar @storePaths; $n++) { for (my $n = 0; $n < scalar @storePaths; $n++) {
my $storePath = $storePaths[$n]; my $storePath = $storePaths[$n];
my $narDir = $narPaths[$n]; my $narDir = $narPaths[$n];
my $baseName = basename $storePath;
$storePath =~ /\/([^\/]*)$/; # Get info about the store path.
my $basename = $1; my ($deriver, $narHash, $time, $narSize, $refs) = queryPathInfo($storePath);
defined $basename or die;
open HASH, "$narDir/narbz2-hash" or die "cannot open narbz2-hash";
my $narbz2Hash = <HASH>;
chomp $narbz2Hash;
$narbz2Hash =~ /^[0-9a-z]+$/ or die "invalid hash";
close HASH;
my $narName = "$narbz2Hash.nar.bz2";
my $narFile = "$narDir/$narName";
(-f $narFile) or die "narfile for $storePath not found";
push @narArchives, $narFile;
my $narbz2Size = stat($narFile)->size;
my $references = `$Nix::Config::binDir/nix-store --query --references '$storePath'`;
die "cannot query references for `$storePath'" if $? != 0;
$references = join(" ", split(" ", $references));
my $deriver = `$Nix::Config::binDir/nix-store --query --deriver '$storePath'`;
die "cannot query deriver for `$storePath'" if $? != 0;
chomp $deriver;
$deriver = "" if $deriver eq "unknown-deriver";
my $narHash = `$Nix::Config::binDir/nix-store --query --hash '$storePath'`;
die "cannot query hash for `$storePath'" if $? != 0;
chomp $narHash;
# In some exceptional cases (such as VM tests that use the Nix # In some exceptional cases (such as VM tests that use the Nix
# store of the host), the database doesn't contain the hash. So # store of the host), the database doesn't contain the hash. So
# compute it. # compute it.
if ($narHash =~ /^sha256:0*$/) { if ($narHash =~ /^sha256:0*$/) {
$narHash = `$Nix::Config::binDir/nix-hash --type sha256 --base32 '$storePath'`; my $nar = "$tmpDir/nar";
die "cannot hash `$storePath'" if $? != 0; system("$Nix::Config::binDir/nix-store --dump $storePath > $nar") == 0
or die "cannot dump $storePath\n";
$narHash = `$Nix::Config::binDir/nix-hash --type sha256 --flat $nar`;
die "cannot hash `$nar'" if $? != 0;
chomp $narHash; chomp $narHash;
$narHash = "sha256:$narHash"; $narHash = "sha256:$narHash";
$narSize = stat("$nar")->size;
unlink $nar or die;
} }
my $narSize = `$Nix::Config::binDir/nix-store --query --size '$storePath'`; $totalNarSize += $narSize;
die "cannot query size for `$storePath'" if $? != 0;
chomp $narSize;
my $url; # Get info about the compressed NAR.
open HASH, "$narDir/narbz2-hash" or die "cannot open narbz2-hash";
my $narBz2Hash = <HASH>;
chomp $narBz2Hash;
$narBz2Hash =~ /^[0-9a-z]+$/ or die "invalid hash";
close HASH;
my $narName = "$narBz2Hash.nar.bz2";
my $narFile = "$narDir/$narName";
(-f $narFile) or die "NAR file for $storePath not found";
my $narBz2Size = stat($narFile)->size;
$totalNarBz2Size += $narBz2Size;
printf STDERR "%s [%.2f MiB, %.1f%%]\n", $storePath,
$narBz2Size / (1024 * 1024), $narBz2Size / $narSize * 100;
# Upload the compressed NAR.
if ($localCopy) { if ($localCopy) {
$url = "$targetArchivesUrl/$narName"; my $dst = "$localArchivesDir/$narName";
if (! -f $dst) {
my $tmp = "$localArchivesDir/.tmp.$$.$narName";
copy($narFile, $tmp) or die "cannot copy $narFile to $tmp: $!\n";
rename($tmp, $dst) or die "cannot rename $tmp to $dst: $!\n";
}
} else { } else {
$url = "$archivesGetURL/$narName"; die "unimplemented";
} #if (!archiveExists("$basename")) {
$narFiles{$storePath} = [ # system("$curl --show-error --upload-file " .
{ url => $url # "'$narArchive' '$archivesPutURL/$basename' > /dev/null") == 0 or
, hash => "$hashAlgo:$narbz2Hash" # die "curl failed on $narArchive: $?";
, size => $narbz2Size #}
, narHash => "$narHash"
, narSize => $narSize
, references => $references
, deriver => $deriver
}
];
} }
writeManifest $manifest, \%narFiles, \%patches; # Upload the info file.
my $info;
$info .= "StorePath: $storePath\n";
sub copyFile { $info .= "URL: $narName\n";
my $src = shift; $info .= "CompressedHash: sha256:$narBz2Hash\n";
my $dst = shift; $info .= "CompressedSize: $narBz2Size\n";
my $tmp = "$dst.tmp.$$"; $info .= "NarHash: $narHash\n";
system("@coreutils@/cp", $src, $tmp) == 0 or die "cannot copy file"; $info .= "NarSize: $narSize\n";
rename($tmp, $dst) or die "cannot rename file: $!"; $info .= "References: " . join(" ", map { basename $_ } @{$refs}) . "\n";
if (defined $deriver) {
$info .= "Deriver: " . basename $deriver, "\n";
if (isValidPath($deriver)) {
my $drv = derivationFromPath($deriver);
$info .= "System: $drv->{platform}\n";
}
} }
my $infoName = hashString("sha256", 1, $storePath);
# Upload/copy the archives.
print STDERR "uploading/copying archives...\n";
sub archiveExists {
my $name = shift;
print STDERR " HEAD on $archivesGetURL/$name\n";
return system("$curl --head $archivesGetURL/$name > /dev/null") == 0;
}
foreach my $narArchive (@narArchives) {
$narArchive =~ /\/([^\/]*)$/;
my $basename = $1;
if ($localCopy) { if ($localCopy) {
# Since nix-push creates $dst atomically, if it exists we my $dst = "$localArchivesDir/$infoName.narinfo";
# don't have to copy again. if (! -f $dst) {
my $dst = "$localArchivesDir/$basename"; my $tmp = "$localArchivesDir/.tmp.$$.$infoName";
if (! -f "$localArchivesDir/$basename") { open INFO, ">$tmp" or die;
print STDERR " $narArchive\n"; print INFO "$info" or die;
copyFile $narArchive, $dst; close INFO or die;
rename($tmp, $dst) or die "cannot rename $tmp to $dst: $!\n";
} }
}
else {
if (!archiveExists("$basename")) {
print STDERR " $narArchive\n";
system("$curl --show-error --upload-file " .
"'$narArchive' '$archivesPutURL/$basename' > /dev/null") == 0 or
die "curl failed on $narArchive: $?";
}
}
}
# Upload the manifest.
print STDERR "uploading manifest...\n";
if ($localCopy) {
copyFile $manifest, $localManifestFile;
copyFile "$manifest.bz2", "$localManifestFile.bz2";
} else { } else {
system("$curl --show-error --upload-file " . die "unimplemented";
"'$manifest' '$manifestPutURL' > /dev/null") == 0 or
die "curl failed on $manifest: $?";
system("$curl --show-error --upload-file " .
"'$manifest'.bz2 '$manifestPutURL'.bz2 > /dev/null") == 0 or
die "curl failed on $manifest: $?";
} }
}
printf STDERR "total compressed size %.2f MiB, %.1f%%\n",
$totalNarBz2Size / (1024 * 1024), $totalNarBz2Size / $totalNarSize * 100;