lix/scripts/download-using-manifests.pl.in

353 lines
10 KiB
Perl
Raw Normal View History

#! @perl@ -w -I@libexecdir@/nix
use strict;
use readmanifest;
2006-10-04 18:58:11 +00:00
use POSIX qw(strftime);
use File::Temp qw(tempdir);
2008-11-20 15:44:59 +00:00
my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
STDOUT->autoflush(1);
my $manifestDir = "@localstatedir@/nix/manifests";
2004-12-30 16:34:54 +00:00
my $logFile = "@localstatedir@/log/nix/downloads";
# Load all manifests.
my %narFiles;
my %localPaths;
my %patches;
for my $manifest (glob "$manifestDir/*.nixmanifest") {
if (readManifest($manifest, \%narFiles, \%localPaths, \%patches) < 3) {
2005-02-25 16:12:52 +00:00
print STDERR "you have an old-style manifest `$manifest'; please delete it\n";
exit 1;
}
}
# Parse the arguments.
if ($ARGV[0] eq "--query") {
while (<STDIN>) {
my $cmd = $_; chomp $cmd;
if ($cmd eq "have") {
my $storePath = <STDIN>; chomp $storePath;
print STDOUT ((defined $narFiles{$storePath} or defined $localPaths{$storePath})
? "1\n" : "0\n");
}
elsif ($cmd eq "info") {
my $storePath = <STDIN>; chomp $storePath;
my $info;
if (defined $narFiles{$storePath}) {
$info = @{$narFiles{$storePath}}[0];
}
elsif (defined $localPaths{$storePath}) {
$info = @{$localPaths{$storePath}}[0];
}
else {
print "0\n";
next; # not an error
}
print "1\n";
print "$info->{deriver}\n";
my @references = split " ", $info->{references};
print scalar @references, "\n";
print "$_\n" foreach @references;
2008-09-08 11:02:15 +00:00
my $size = $info->{size} || 0;
2008-08-04 13:15:47 +00:00
print "$size\n";
}
else { die "unknown command `$cmd'"; }
}
exit 0;
}
elsif ($ARGV[0] ne "--substitute") {
die "syntax: $0 [--query-paths | --query-info PATHS... | --substitute PATH]\n";
}
die unless scalar @ARGV == 2;
my $targetPath = $ARGV[1];
# Create a temporary directory.
my $tmpDir = tempdir("nix-download.XXXXXX", CLEANUP => 1, TMPDIR => 1)
or die "cannot create a temporary directory";
chdir $tmpDir or die "cannot change to `$tmpDir': $!";
my $tmpNar = "$tmpDir/nar";
my $tmpNar2 = "$tmpDir/nar2";
open LOGFILE, ">>$logFile" or die "cannot open log file $logFile";
my $date = strftime ("%F %H:%M:%S UTC", gmtime (time));
print LOGFILE "$$ get $targetPath $date\n";
print "\n*** Trying to download/patch `$targetPath'\n";
# If we can copy from a local path, do that.
my $localPathList = $localPaths{$targetPath};
foreach my $localPath (@{$localPathList}) {
my $sourcePath = $localPath->{copyFrom};
if (-e $sourcePath) {
print "\n*** Step 1/1: copying from $sourcePath\n";
2008-11-20 15:44:59 +00:00
system("$binDir/nix-store --dump $sourcePath | $binDir/nix-store --restore $targetPath") == 0
or die "cannot copy `$sourcePath' to `$targetPath'";
exit 0;
}
}
# Build a graph of all store paths that might contribute to the
# construction of $targetPath, and the special node "start". The
# edges are either patch operations, or downloads of full NAR files.
# The latter edges only occur between "start" and a store path.
my %graph;
$graph{"start"} = {d => 0, pred => undef, edges => []};
my @queue = ();
my $queueFront = 0;
my %done;
sub addToQueue {
my $v = shift;
return if defined $done{$v};
$done{$v} = 1;
push @queue, $v;
}
sub addNode {
my $u = shift;
$graph{$u} = {d => 999999999999, pred => undef, edges => []}
unless defined $graph{$u};
}
sub addEdge {
my $u = shift;
my $v = shift;
my $w = shift;
my $type = shift;
my $info = shift;
addNode $u;
push @{$graph{$u}->{edges}},
{weight => $w, start => $u, end => $v, type => $type, info => $info};
my $n = scalar @{$graph{$u}->{edges}};
}
addToQueue $targetPath;
sub isValidPath {
my $p = shift;
2008-11-20 15:44:59 +00:00
return system("$binDir/nix-store --check-validity '$p' 2> /dev/null") == 0;
}
2005-03-14 17:05:42 +00:00
sub parseHash {
my $hash = shift;
if ($hash =~ /^(.+):(.+)$/) {
return ($1, $2);
} else {
return ("md5", $hash);
}
}
while ($queueFront < scalar @queue) {
my $u = $queue[$queueFront++];
2005-01-12 10:37:18 +00:00
# print "$u\n";
addNode $u;
# If the path already exists, it has distance 0 from the "start"
# node.
if (isValidPath($u)) {
addEdge "start", $u, 0, "present", undef;
}
else {
# Add patch edges.
my $patchList = $patches{$u};
foreach my $patch (@{$patchList}) {
if (isValidPath($patch->{basePath})) {
# !!! this should be cached
2005-03-14 17:05:42 +00:00
my ($baseHashAlgo, $baseHash) = parseHash $patch->{baseHash};
2005-03-14 18:55:29 +00:00
my $format = "--base32";
$format = "" if $baseHashAlgo eq "md5";
2008-11-20 15:44:59 +00:00
my $hash = `$binDir/nix-hash --type '$baseHashAlgo' $format "$patch->{basePath}"`;
chomp $hash;
2005-01-12 10:37:18 +00:00
# print " MY HASH is $hash\n";
2005-03-14 17:05:42 +00:00
if ($hash ne $baseHash) {
2005-01-12 10:37:18 +00:00
print LOGFILE "$$ rejecting $patch->{basePath}\n";
next;
}
}
2005-01-12 10:37:18 +00:00
# print " PATCH from $patch->{basePath}\n";
addToQueue $patch->{basePath};
addEdge $patch->{basePath}, $u, $patch->{size}, "patch", $patch;
}
# Add NAR file edges to the start node.
my $narFileList = $narFiles{$u};
foreach my $narFile (@{$narFileList}) {
2005-01-12 10:37:18 +00:00
# print " NAR from $narFile->{url}\n";
addEdge "start", $u, $narFile->{size}, "narfile", $narFile;
if ($u eq $targetPath) {
print LOGFILE "$$ full-download-would-be $narFile->{size}\n";
}
}
}
}
# Run Dijkstra's shortest path algorithm to determine the shortest
# sequence of download and/or patch actions that will produce
# $targetPath.
sub byDistance { # sort by distance, reversed
return -($graph{$a}->{d} <=> $graph{$b}->{d});
}
my @todo = keys %graph;
while (scalar @todo > 0) {
# Remove the closest element from the todo list.
@todo = sort byDistance @todo;
my $u = pop @todo;
my $u_ = $graph{$u};
2005-01-12 10:37:18 +00:00
# print "IN $u $u_->{d}\n";
foreach my $edge (@{$u_->{edges}}) {
my $v_ = $graph{$edge->{end}};
if ($v_->{d} > $u_->{d} + $edge->{weight}) {
$v_->{d} = $u_->{d} + $edge->{weight};
# Store the edge; to edge->start is actually the
# predecessor.
$v_->{pred} = $edge;
2005-01-12 10:37:18 +00:00
# print " RELAX $edge->{end} $v_->{d}\n";
}
}
}
# Retrieve the shortest path from "start" to $targetPath.
my @path = ();
my $cur = $targetPath;
die "don't know how to produce $targetPath\n"
unless defined $graph{$targetPath}->{pred};
while ($cur ne "start") {
push @path, $graph{$cur}->{pred};
$cur = $graph{$cur}->{pred}->{start};
}
# Traverse the shortest path, perform the actions described by the
# edges.
my $curStep = 1;
my $maxStep = scalar @path;
sub downloadFile {
my $url = shift;
2005-03-14 17:05:42 +00:00
my ($hashAlgo, $hash) = parseHash(shift);
$ENV{"PRINT_PATH"} = 1;
$ENV{"QUIET"} = 1;
$ENV{"NIX_HASH_ALGO"} = $hashAlgo;
2008-11-20 15:44:59 +00:00
my ($hash2, $path) = `$binDir/nix-prefetch-url '$url' '$hash'`;
2005-03-25 14:30:01 +00:00
die "download of `$url' failed" unless $? == 0;
chomp $hash2;
chomp $path;
die "hash mismatch, expected $hash, got $hash2" if $hash ne $hash2;
return $path;
}
while (scalar @path > 0) {
my $edge = pop @path;
my $u = $edge->{start};
my $v = $edge->{end};
print "\n*** Step $curStep/$maxStep: ";
if ($edge->{type} eq "present") {
print "using already present path `$v'\n";
2004-12-30 16:34:54 +00:00
print LOGFILE "$$ present $v\n";
if ($curStep < $maxStep) {
# Since this is not the last step, the path will be used
# as a base to one or more patches. So turn the base path
# into a NAR archive, to which we can apply the patch.
print " packing base path...\n";
2008-11-20 15:44:59 +00:00
system("$binDir/nix-store --dump $v > $tmpNar") == 0
2006-09-25 10:44:27 +00:00
or die "cannot dump `$v'";
}
}
elsif ($edge->{type} eq "patch") {
my $patch = $edge->{info};
print "applying patch `$patch->{url}' to `$u' to create `$v'\n";
2004-12-30 16:34:54 +00:00
print LOGFILE "$$ patch $patch->{url} $patch->{size} $patch->{baseHash} $u $v\n";
# Download the patch.
print " downloading patch...\n";
2005-03-14 17:05:42 +00:00
my $patchPath = downloadFile "$patch->{url}", "$patch->{hash}";
# Apply the patch to the NAR archive produced in step 1 (for
# the already present path) or a later step (for patch sequences).
print " applying patch...\n";
2006-09-25 10:44:27 +00:00
system("@libexecdir@/bspatch $tmpNar $tmpNar2 $patchPath") == 0
or die "cannot apply patch `$patchPath' to $tmpNar";
if ($curStep < $maxStep) {
# The archive will be used as the base of the next patch.
2005-09-15 15:21:35 +00:00
rename "$tmpNar2", "$tmpNar" or die "cannot rename NAR archive: $!";
} else {
# This was the last patch. Unpack the final NAR archive
# into the target path.
print " unpacking patched archive...\n";
2008-11-20 15:44:59 +00:00
system("$binDir/nix-store --restore $v < $tmpNar2") == 0
2006-09-25 10:44:27 +00:00
or die "cannot unpack $tmpNar2 into `$v'";
}
}
elsif ($edge->{type} eq "narfile") {
my $narFile = $edge->{info};
print "downloading `$narFile->{url}' into `$v'\n";
2004-12-30 16:34:54 +00:00
print LOGFILE "$$ narfile $narFile->{url} $narFile->{size} $v\n";
# Download the archive.
print " downloading archive...\n";
2005-03-14 17:05:42 +00:00
my $narFilePath = downloadFile "$narFile->{url}", "$narFile->{hash}";
if ($curStep < $maxStep) {
# The archive will be used a base to a patch.
2006-09-25 10:44:27 +00:00
system("@bunzip2@ < '$narFilePath' > $tmpNar") == 0
or die "cannot unpack `$narFilePath' into `$v'";
} else {
# Unpack the archive into the target path.
print " unpacking archive...\n";
2008-11-20 15:44:59 +00:00
system("@bunzip2@ < '$narFilePath' | $binDir/nix-store --restore '$v'") == 0
2006-09-25 10:44:27 +00:00
or die "cannot unpack `$narFilePath' into `$v'";
}
}
$curStep++;
}
2004-12-30 16:34:54 +00:00
print LOGFILE "$$ success\n";
close LOGFILE;