#! @perl@ -w @perlFlags@ use utf8; use strict; use File::Basename; use File::Path qw(mkpath); use File::stat; use File::Copy; use MIME::Base64; use Nix::Config; use Nix::Store; use Nix::Manifest; use Nix::Utils; binmode STDERR, ":encoding(utf8)"; my $tmpDir = mkTempDir("nix-push"); my $nixExpr = "$tmpDir/create-nars.nix"; # Parse the command line. my $compressionType = "xz"; my $force = 0; my $destDir; my $writeManifest = 0; my $manifestPath; my $archivesURL; my $link = 0; my $secretKeyFile; my @roots; for (my $n = 0; $n < scalar @ARGV; $n++) { my $arg = $ARGV[$n]; if ($arg eq "--help") { exec "man nix-push" or die; } elsif ($arg eq "--bzip2") { $compressionType = "bzip2"; } elsif ($arg eq "--none") { $compressionType = "none"; } elsif ($arg eq "--force") { $force = 1; } elsif ($arg eq "--dest") { $n++; die "$0: ‘$arg’ requires an argument\n" unless $n < scalar @ARGV; $destDir = $ARGV[$n]; mkpath($destDir, 0, 0755); } elsif ($arg eq "--manifest") { $writeManifest = 1; } elsif ($arg eq "--manifest-path") { $n++; die "$0: ‘$arg’ requires an argument\n" unless $n < scalar @ARGV; $manifestPath = $ARGV[$n]; $writeManifest = 1; mkpath(dirname($manifestPath), 0, 0755); } elsif ($arg eq "--url-prefix") { $n++; die "$0: ‘$arg’ requires an argument\n" unless $n < scalar @ARGV; $archivesURL = $ARGV[$n]; } elsif ($arg eq "--link") { $link = 1; } elsif ($arg eq "--key-file") { $n++; die "$0: ‘$arg’ requires an argument\n" unless $n < scalar @ARGV; $secretKeyFile = $ARGV[$n]; } elsif (substr($arg, 0, 1) eq "-") { die "$0: unknown flag ‘$arg’\n"; } else { push @roots, $arg; } } die "$0: please specify a destination directory\n" if !defined $destDir; $archivesURL = "file://$destDir" unless defined $archivesURL; # From the given store paths, determine the set of requisite store # paths, i.e, the paths required to realise them. my %storePaths; foreach my $path (@roots) { # Get all paths referenced by the normalisation of the given # Nix expression. my $pid = open(READ, "$Nix::Config::binDir/nix-store --query --requisites --force-realise " . "--include-outputs '$path'|") or die; while (<READ>) { chomp; die "bad: $_" unless /^\//; $storePaths{$_} = ""; } close READ or die "nix-store failed: $?"; } my @storePaths = keys %storePaths; # Don't create archives for files that are already in the binary cache. my @storePaths2; my %narFiles; foreach my $storePath (@storePaths) { my $pathHash = substr(basename($storePath), 0, 32); my $narInfoFile = "$destDir/$pathHash.narinfo"; if (!$force && -e $narInfoFile) { my $narInfo = parseNARInfo($storePath, readFile($narInfoFile), 0, $narInfoFile) or die "cannot read ‘$narInfoFile’\n"; my $narFile = "$destDir/$narInfo->{url}"; if (-e $narFile) { print STDERR "skipping existing $storePath\n"; # Add the NAR info to $narFiles if we're writing a # manifest. $narFiles{$storePath} = [ { url => ("$archivesURL/" . basename $narInfo->{url}) , hash => $narInfo->{fileHash} , size => $narInfo->{fileSize} , compressionType => $narInfo->{compression} , narHash => $narInfo->{narHash} , narSize => $narInfo->{narSize} , references => join(" ", map { "$Nix::Config::storeDir/$_" } @{$narInfo->{refs}}) , deriver => $narInfo->{deriver} ? "$Nix::Config::storeDir/$narInfo->{deriver}" : undef } ] if $writeManifest; next; } } push @storePaths2, $storePath; } # Create a list of Nix derivations that turn each path into a Nix # archive. open NIX, ">$nixExpr"; print NIX "["; foreach my $storePath (@storePaths2) { die unless ($storePath =~ /\/[0-9a-z]{32}[^\"\\\$]*$/); # Construct a Nix expression that creates a Nix archive. my $nixexpr = "(import <nix/nar.nix> " . "{ storePath = builtins.storePath \"$storePath\"; hashAlgo = \"sha256\"; compressionType = \"$compressionType\"; }) "; print NIX $nixexpr; } print NIX "]"; close NIX; # Build the Nix expression. print STDERR "building compressed archives...\n"; my @narPaths; my $pid = open(READ, "$Nix::Config::binDir/nix-build $nixExpr -o $tmpDir/result |") or die "cannot run nix-build"; while (<READ>) { chomp; die unless /^\//; push @narPaths, $_; } close READ or die "nix-build failed: $?"; # Write the cache info file. my $cacheInfoFile = "$destDir/nix-cache-info"; if (! -e $cacheInfoFile) { open FILE, ">$cacheInfoFile" or die "cannot create $cacheInfoFile: $!"; print FILE "StoreDir: $Nix::Config::storeDir\n"; print FILE "WantMassQuery: 0\n"; # by default, don't hit this cache for "nix-env -qas" close FILE; } # Copy the archives and the corresponding NAR info files. print STDERR "copying archives...\n"; my $totalNarSize = 0; my $totalCompressedSize = 0; for (my $n = 0; $n < scalar @storePaths2; $n++) { my $storePath = $storePaths2[$n]; my $narDir = $narPaths[$n]; my $baseName = basename $storePath; # Get info about the store path. my ($deriver, $narHash, $time, $narSize, $refs) = queryPathInfo($storePath, 1); # In some exceptional cases (such as VM tests that use the Nix # store of the host), the database doesn't contain the hash. So # compute it. if ($narHash =~ /^sha256:0*$/) { my $nar = "$tmpDir/nar"; system("$Nix::Config::binDir/nix-store --dump $storePath > $nar") == 0 or die "cannot dump $storePath\n"; $narHash = `$Nix::Config::binDir/nix-hash --type sha256 --base32 --flat $nar`; die "cannot hash ‘$nar’" if $? != 0; chomp $narHash; $narHash = "sha256:$narHash"; $narSize = stat("$nar")->size; unlink $nar or die; } $totalNarSize += $narSize; # Get info about the compressed NAR. open HASH, "$narDir/nar-compressed-hash" or die "cannot open nar-compressed-hash"; my $compressedHash = <HASH>; chomp $compressedHash; $compressedHash =~ /^[0-9a-z]+$/ or die "invalid hash"; close HASH; my $narName = "$compressedHash.nar" . ($compressionType eq "xz" ? ".xz" : $compressionType eq "bzip2" ? ".bz2" : ""); my $narFile = "$narDir/$narName"; (-f $narFile) or die "NAR file for $storePath not found"; my $compressedSize = stat($narFile)->size; $totalCompressedSize += $compressedSize; printf STDERR "%s [%.2f MiB, %.1f%%]\n", $storePath, $compressedSize / (1024 * 1024), $compressedSize / $narSize * 100; # Copy the compressed NAR. my $dst = "$destDir/$narName"; if (! -f $dst) { my $tmp = "$destDir/.tmp.$$.$narName"; if ($link) { link($narFile, $tmp) or die "cannot link $tmp to $narFile: $!\n"; } else { copy($narFile, $tmp) or die "cannot copy $narFile to $tmp: $!\n"; } rename($tmp, $dst) or die "cannot rename $tmp to $dst: $!\n"; } # Write the info file. my $info; $info .= "StorePath: $storePath\n"; $info .= "URL: $narName\n"; $info .= "Compression: $compressionType\n"; $info .= "FileHash: sha256:$compressedHash\n"; $info .= "FileSize: $compressedSize\n"; $info .= "NarHash: $narHash\n"; $info .= "NarSize: $narSize\n"; $info .= "References: " . join(" ", map { basename $_ } @{$refs}) . "\n"; if (defined $deriver) { $info .= "Deriver: " . basename $deriver . "\n"; if (isValidPath($deriver)) { my $drv = derivationFromPath($deriver); $info .= "System: $drv->{platform}\n"; } } if (defined $secretKeyFile) { my $s = readFile $secretKeyFile; chomp $s; my ($keyName, $secretKey) = split ":", $s; die "invalid secret key file ‘$secretKeyFile’\n" unless defined $keyName && defined $secretKey; my $fingerprint = fingerprintPath($storePath, $narHash, $narSize, $refs); my $sig = encode_base64(signString(decode_base64($secretKey), $fingerprint), ""); $info .= "Sig: $keyName:$sig\n"; } my $pathHash = substr(basename($storePath), 0, 32); $dst = "$destDir/$pathHash.narinfo"; if ($force || ! -f $dst) { my $tmp = "$destDir/.tmp.$$.$pathHash.narinfo"; open INFO, ">$tmp" or die; print INFO "$info" or die; close INFO or die; rename($tmp, $dst) or die "cannot rename $tmp to $dst: $!\n"; } $narFiles{$storePath} = [ { url => "$archivesURL/$narName" , hash => "sha256:$compressedHash" , size => $compressedSize , compressionType => $compressionType , narHash => "$narHash" , narSize => $narSize , references => join(" ", @{$refs}) , deriver => $deriver } ] if $writeManifest; } printf STDERR "total compressed size %.2f MiB, %.1f%%\n", $totalCompressedSize / (1024 * 1024), $totalCompressedSize / ($totalNarSize || 1) * 100; # Optionally write a manifest. writeManifest($manifestPath // "$destDir/MANIFEST", \%narFiles, \()) if $writeManifest;