lix-releng-staging/scripts/nix-push.in

#! @perl@ -w @perlFlags@

use utf8;
use strict;
use File::Basename;
use File::Path qw(mkpath);
use File::stat;
use File::Copy;
use MIME::Base64;
use Nix::Config;
use Nix::Store;
use Nix::Manifest;
use Nix::Utils;

binmode STDERR, ":encoding(utf8)";

my $tmpDir = mkTempDir("nix-push");

my $nixExpr = "$tmpDir/create-nars.nix";


# Parse the command line.
my $compressionType = "xz";
my $force = 0;
my $destDir;
my $writeManifest = 0;
my $manifestPath;
my $archivesURL;
my $link = 0;
my $secretKeyFile;
my @roots;

for (my $n = 0; $n < scalar @ARGV; $n++) {
    my $arg = $ARGV[$n];

    if ($arg eq "--help") {
        exec "man nix-push" or die;
    } elsif ($arg eq "--bzip2") {
        $compressionType = "bzip2";
    } elsif ($arg eq "--none") {
        $compressionType = "none";
    } elsif ($arg eq "--force") {
        $force = 1;
    } elsif ($arg eq "--dest") {
        $n++;
        die "$0: ‘$arg’ requires an argument\n" unless $n < scalar @ARGV;
        $destDir = $ARGV[$n];
        mkpath($destDir, 0, 0755);
    } elsif ($arg eq "--manifest") {
        $writeManifest = 1;
    } elsif ($arg eq "--manifest-path") {
        $n++;
        die "$0: ‘$arg’ requires an argument\n" unless $n < scalar @ARGV;
        $manifestPath = $ARGV[$n];
        $writeManifest = 1;
        mkpath(dirname($manifestPath), 0, 0755);
    } elsif ($arg eq "--url-prefix") {
        $n++;
        die "$0: ‘$arg’ requires an argument\n" unless $n < scalar @ARGV;
        $archivesURL = $ARGV[$n];
    } elsif ($arg eq "--link") {
        $link = 1;
    } elsif ($arg eq "--key-file") {
        $n++;
        die "$0: ‘$arg’ requires an argument\n" unless $n < scalar @ARGV;
        $secretKeyFile = $ARGV[$n];
    } elsif (substr($arg, 0, 1) eq "-") {
        die "$0: unknown flag ‘$arg’\n";
    } else {
        push @roots, $arg;
    }
}

die "$0: please specify a destination directory\n" if !defined $destDir;

$archivesURL = "file://$destDir" unless defined $archivesURL;


# From the given store paths, determine the set of requisite store
# paths, i.e, the paths required to realise them.
my %storePaths;

foreach my $path (@roots) {
    # Get all paths referenced by the normalisation of the given
    # Nix expression.
    my $pid = open(READ,
        "$Nix::Config::binDir/nix-store --query --requisites --force-realise " .
        "--include-outputs '$path'|") or die;

    while (<READ>) {
        chomp;
        die "bad: $_" unless /^\//;
        $storePaths{$_} = "";
    }

    close READ or die "nix-store failed: $?";
}

my @storePaths = keys %storePaths;


# Don't create archives for files that are already in the binary cache.
my @storePaths2;
my %narFiles;
foreach my $storePath (@storePaths) {
    my $pathHash = substr(basename($storePath), 0, 32);
    my $narInfoFile = "$destDir/$pathHash.narinfo";
    if (!$force && -e $narInfoFile) {
        my $narInfo = parseNARInfo($storePath, readFile($narInfoFile), 0, $narInfoFile) or die "cannot read ‘$narInfoFile’\n";
        my $narFile = "$destDir/$narInfo->{url}";
        if (-e $narFile) {
            print STDERR "skipping existing $storePath\n";
            # Add the NAR info to $narFiles if we're writing a
            # manifest.
            $narFiles{$storePath} = [
                { url => ("$archivesURL/" . basename $narInfo->{url})
                  , hash => $narInfo->{fileHash}
                  , size => $narInfo->{fileSize}
                  , compressionType => $narInfo->{compression}
                  , narHash => $narInfo->{narHash}
                  , narSize => $narInfo->{narSize}
                  , references => join(" ", map { "$Nix::Config::storeDir/$_" } @{$narInfo->{refs}})
                  , deriver => $narInfo->{deriver} ? "$Nix::Config::storeDir/$narInfo->{deriver}" : undef
                  }
            ] if $writeManifest;
            next;
        }
    }
    push @storePaths2, $storePath;
}


# Create a list of Nix derivations that turn each path into a Nix
# archive.
open NIX, ">$nixExpr";
print NIX "[";

foreach my $storePath (@storePaths2) {
    die unless ($storePath =~ /\/[0-9a-z]{32}[^\"\\\$]*$/);

    # Construct a Nix expression that creates a Nix archive.
    my $nixexpr =
        "(import <nix/nar.nix> " .
        "{ storePath = builtins.storePath \"$storePath\"; hashAlgo = \"sha256\"; compressionType = \"$compressionType\"; }) ";

    print NIX $nixexpr;
}

print NIX "]";
close NIX;


# Build the Nix expression.
print STDERR "building compressed archives...\n";
my @narPaths;
my $pid = open(READ, "$Nix::Config::binDir/nix-build $nixExpr -o $tmpDir/result |")
    or die "cannot run nix-build";
while (<READ>) {
    chomp;
    die unless /^\//;
    push @narPaths, $_;
}
close READ or die "nix-build failed: $?";


# Write the cache info file.
my $cacheInfoFile = "$destDir/nix-cache-info";
if (! -e $cacheInfoFile) {
    open FILE, ">$cacheInfoFile" or die "cannot create $cacheInfoFile: $!";
    print FILE "StoreDir: $Nix::Config::storeDir\n";
    print FILE "WantMassQuery: 0\n"; # by default, don't hit this cache for "nix-env -qas"
    close FILE;
}


# Copy the archives and the corresponding NAR info files.
print STDERR "copying archives...\n";

my $totalNarSize = 0;
my $totalCompressedSize = 0;

for (my $n = 0; $n < scalar @storePaths2; $n++) {
    my $storePath = $storePaths2[$n];
    my $narDir = $narPaths[$n];
    my $baseName = basename $storePath;

    # Get info about the store path.
    my ($deriver, $narHash, $time, $narSize, $refs) = queryPathInfo($storePath, 1);

    # In some exceptional cases (such as VM tests that use the Nix
    # store of the host), the database doesn't contain the hash.  So
    # compute it.
    if ($narHash =~ /^sha256:0*$/) {
        my $nar = "$tmpDir/nar";
        system("$Nix::Config::binDir/nix-store --dump $storePath > $nar") == 0
            or die "cannot dump $storePath\n";
        $narHash = `$Nix::Config::binDir/nix-hash --type sha256 --base32 --flat $nar`;
        die "cannot hash ‘$nar’" if $? != 0;
        chomp $narHash;
        $narHash = "sha256:$narHash";
        $narSize = stat("$nar")->size;
        unlink $nar or die;
    }

    $totalNarSize += $narSize;

    # Get info about the compressed NAR.
    open HASH, "$narDir/nar-compressed-hash" or die "cannot open nar-compressed-hash";
    my $compressedHash = <HASH>;
    chomp $compressedHash;
    $compressedHash =~ /^[0-9a-z]+$/ or die "invalid hash";
    close HASH;

    my $narName = "$compressedHash.nar" . ($compressionType eq "xz" ? ".xz" : $compressionType eq "bzip2" ? ".bz2" : "");

    my $narFile = "$narDir/$narName";
    (-f $narFile) or die "NAR file for $storePath not found";

    my $compressedSize = stat($narFile)->size;
    $totalCompressedSize += $compressedSize;

    printf STDERR "%s [%.2f MiB, %.1f%%]\n", $storePath,
        $compressedSize / (1024 * 1024), $compressedSize / $narSize * 100;

    # Copy the compressed NAR.
    my $dst = "$destDir/$narName";
    if (! -f $dst) {
        my $tmp = "$destDir/.tmp.$$.$narName";
        if ($link) {
            link($narFile, $tmp) or die "cannot link $tmp to $narFile: $!\n";
        } else {
            copy($narFile, $tmp) or die "cannot copy $narFile to $tmp: $!\n";
        }
        rename($tmp, $dst) or die "cannot rename $tmp to $dst: $!\n";
    }

    # Write the info file.
    my $info;
    $info .= "StorePath: $storePath\n";
    $info .= "URL: $narName\n";
    $info .= "Compression: $compressionType\n";
    $info .= "FileHash: sha256:$compressedHash\n";
    $info .= "FileSize: $compressedSize\n";
    $info .= "NarHash: $narHash\n";
    $info .= "NarSize: $narSize\n";
    $info .= "References: " . join(" ", map { basename $_ } @{$refs}) . "\n";
    if (defined $deriver) {
        $info .= "Deriver: " . basename $deriver . "\n";
        if (isValidPath($deriver)) {
            my $drv = derivationFromPath($deriver);
            $info .= "System: $drv->{platform}\n";
        }
    }

    if (defined $secretKeyFile) {
        my $s = readFile $secretKeyFile;
        chomp $s;
        my ($keyName, $secretKey) = split ":", $s;
        die "invalid secret key file ‘$secretKeyFile’\n" unless defined $keyName && defined $secretKey;
        my $fingerprint = fingerprintPath($storePath, $narHash, $narSize, $refs);
        my $sig = encode_base64(signString(decode_base64($secretKey), $fingerprint), "");
        $info .= "Sig: $keyName:$sig\n";
    }

    my $pathHash = substr(basename($storePath), 0, 32);

    $dst = "$destDir/$pathHash.narinfo";
    if ($force || ! -f $dst) {
        my $tmp = "$destDir/.tmp.$$.$pathHash.narinfo";
        open INFO, ">$tmp" or die;
        print INFO "$info" or die;
        close INFO or die;
        rename($tmp, $dst) or die "cannot rename $tmp to $dst: $!\n";
    }

    $narFiles{$storePath} = [
        { url => "$archivesURL/$narName"
        , hash => "sha256:$compressedHash"
        , size => $compressedSize
        , compressionType => $compressionType
        , narHash => "$narHash"
        , narSize => $narSize
        , references => join(" ", @{$refs})
        , deriver => $deriver
        }
    ] if $writeManifest;
}

printf STDERR "total compressed size %.2f MiB, %.1f%%\n",
    $totalCompressedSize / (1024 * 1024), $totalCompressedSize / ($totalNarSize || 1) * 100;


# Optionally write a manifest.
writeManifest($manifestPath // "$destDir/MANIFEST", \%narFiles, \()) if $writeManifest;