kloenk/lix
0
0
Fork 0
forked from lix-project/lix
Code Pull requests Activity

download-using-manifests: Don't use nix-prefetch-url

Browse source 
Instead call curl directly and pipe it into ‘nix-store --restore’.
This saves I/O and prevents creating garbage in the Nix store.
This commit is contained in:
Eelco Dolstra 2012-07-27 10:47:36 -04:00
parent b4ea83249b
commit 3a8f841612
2 changed files with 29 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
scripts/download-from-binary-cache.pl.in
View file

@ -428,7 +428,7 @@ sub downloadBinary {
} }
my $url = "$binaryCacheUrl/$info->{url}"; # FIXME: handle non-relative URLs my $url = "$binaryCacheUrl/$info->{url}"; # FIXME: handle non-relative URLs
print STDERR "\n*** Downloading ‘$url’ into ‘$storePath’...\n"; print STDERR "\n*** Downloading ‘$url’ into ‘$storePath’...\n";
if (system("$Nix::Config::curl --fail --location '$url' | $decompressor | $Nix::Config::binDir/nix-store --restore $storePath") != 0) { if (system("$Nix::Config::curl --fail --location --insecure '$url' | $decompressor | $Nix::Config::binDir/nix-store --restore $storePath") != 0) {
die "download of `$info->{url}' failed" . ($! ? ": $!" : "") . "\n" unless $? == 0; die "download of `$info->{url}' failed" . ($! ? ": $!" : "") . "\n" unless $? == 0;
next; next;
} }

68
scripts/download-using-manifests.pl.in
View file

@ -15,6 +15,9 @@ my $logFile = "$Nix::Config::logDir/downloads";
# estimating the expected download size. # estimating the expected download size.
my $fast = 1; my $fast = 1;
# ‘--insecure’ is fine because Nix verifies the hash of the result.
my $curl = "$Nix::Config::curl --fail --location --insecure";
# Open the manifest cache and update it if necessary. # Open the manifest cache and update it if necessary.
my $dbh = updateManifestDB(); my $dbh = updateManifestDB();
@ -38,7 +41,7 @@ sub parseHash {
# given path. # given path.
sub computeSmallestDownload { sub computeSmallestDownload {
my $targetPath = shift; my $targetPath = shift;
# Build a graph of all store paths that might contribute to the # Build a graph of all store paths that might contribute to the
# construction of $targetPath, and the special node "start". The # construction of $targetPath, and the special node "start". The
# edges are either patch operations, or downloads of full NAR # edges are either patch operations, or downloads of full NAR
@ -93,7 +96,7 @@ sub computeSmallestDownload {
my $patchList = $dbh->selectall_arrayref( my $patchList = $dbh->selectall_arrayref(
"select * from Patches where storePath = ?", "select * from Patches where storePath = ?",
{ Slice => {} }, $u); { Slice => {} }, $u);
foreach my $patch (@{$patchList}) { foreach my $patch (@{$patchList}) {
if (isValidPath($patch->{basePath})) { if (isValidPath($patch->{basePath})) {
my ($baseHashAlgo, $baseHash) = parseHash $patch->{baseHash}; my ($baseHashAlgo, $baseHash) = parseHash $patch->{baseHash};
@ -106,7 +109,7 @@ sub computeSmallestDownload {
$hash =~ s/.*://; $hash =~ s/.*://;
$hashCache->{$baseHashAlgo}->{$patch->{basePath}} = $hash; $hashCache->{$baseHashAlgo}->{$patch->{basePath}} = $hash;
} }
next if $hash ne $baseHash; next if $hash ne $baseHash;
} }
push @queue, $patch->{basePath}; push @queue, $patch->{basePath};
@ -117,7 +120,7 @@ sub computeSmallestDownload {
my $narFileList = $dbh->selectall_arrayref( my $narFileList = $dbh->selectall_arrayref(
"select * from NARs where storePath = ?", "select * from NARs where storePath = ?",
{ Slice => {} }, $u); { Slice => {} }, $u);
foreach my $narFile (@{$narFileList}) { foreach my $narFile (@{$narFileList}) {
# !!! how to handle files whose size is not known in advance? # !!! how to handle files whose size is not known in advance?
# For now, assume some arbitrary size (1 GB). # For now, assume some arbitrary size (1 GB).
@ -189,7 +192,7 @@ if ($ARGV[0] eq "--query") {
my $infos = $dbh->selectall_arrayref( my $infos = $dbh->selectall_arrayref(
"select * from NARs where storePath = ?", "select * from NARs where storePath = ?",
{ Slice => {} }, $storePath); { Slice => {} }, $storePath);
next unless scalar @{$infos} > 0; next unless scalar @{$infos} > 0;
my $info = @{$infos}[0]; my $info = @{$infos}[0];
@ -215,14 +218,14 @@ if ($ARGV[0] eq "--query") {
} }
print "$downloadSize\n"; print "$downloadSize\n";
my $narSize = $info->{narSize} || 0; my $narSize = $info->{narSize} || 0;
print "$narSize\n"; print "$narSize\n";
} }
print "\n"; print "\n";
} }
else { die "unknown command `$cmd'"; } else { die "unknown command `$cmd'"; }
} }
@ -271,16 +274,6 @@ $dbh->disconnect;
my $curStep = 1; my $curStep = 1;
my $maxStep = scalar @path; my $maxStep = scalar @path;
sub downloadFile {
my $url = shift;
$ENV{"PRINT_PATH"} = 1;
$ENV{"QUIET"} = 1;
my ($hash, $path) = `$Nix::Config::binDir/nix-prefetch-url '$url'`;
die "download of `$url' failed" . ($! ? ": $!" : "") . "\n" unless $? == 0;
chomp $path;
return $path;
}
my $finalNarHash; my $finalNarHash;
while (scalar @path > 0) { while (scalar @path > 0) {
@ -312,13 +305,15 @@ while (scalar @path > 0) {
# Download the patch. # Download the patch.
print STDERR " downloading patch...\n"; print STDERR " downloading patch...\n";
my $patchPath = downloadFile "$patch->{url}"; my $patchPath = "$tmpDir/patch";
system("$curl '$patch->{url}' -o $patchPath") == 0
or die "cannot download patch `$patch->{url}'\n";
# Apply the patch to the NAR archive produced in step 1 (for # Apply the patch to the NAR archive produced in step 1 (for
# the already present path) or a later step (for patch sequences). # the already present path) or a later step (for patch sequences).
print STDERR " applying patch...\n"; print STDERR " applying patch...\n";
system("$Nix::Config::libexecDir/bspatch $tmpNar $tmpNar2 $patchPath") == 0 system("$Nix::Config::libexecDir/bspatch $tmpNar $tmpNar2 $patchPath") == 0
or die "cannot apply patch `$patchPath' to $tmpNar"; or die "cannot apply patch `$patchPath' to $tmpNar\n";
if ($curStep < $maxStep) { if ($curStep < $maxStep) {
# The archive will be used as the base of the next patch. # The archive will be used as the base of the next patch.
@ -328,7 +323,7 @@ while (scalar @path > 0) {
# into the target path. # into the target path.
print STDERR " unpacking patched archive...\n"; print STDERR " unpacking patched archive...\n";
system("$Nix::Config::binDir/nix-store --restore $v < $tmpNar2") == 0 system("$Nix::Config::binDir/nix-store --restore $v < $tmpNar2") == 0
or die "cannot unpack $tmpNar2 into `$v'"; or die "cannot unpack $tmpNar2 into `$v'\n";
} }
$finalNarHash = $patch->{narHash}; $finalNarHash = $patch->{narHash};
@ -340,20 +335,15 @@ while (scalar @path > 0) {
my $size = $narFile->{size} || -1; my $size = $narFile->{size} || -1;
print LOGFILE "$$ narfile $narFile->{url} $size $v\n"; print LOGFILE "$$ narfile $narFile->{url} $size $v\n";
# Download the archive.
print STDERR " downloading archive...\n";
my $narFilePath = downloadFile "$narFile->{url}";
if ($curStep < $maxStep) { if ($curStep < $maxStep) {
# The archive will be used a base to a patch. # The archive will be used a base to a patch.
system("$Nix::Config::bzip2 -d < '$narFilePath' > $tmpNar") == 0 system("$curl '$narFile->{url}' | $Nix::Config::bzip2 -d > $tmpNar") == 0
or die "cannot unpack `$narFilePath' into `$v'"; or die "cannot download and unpack `$narFile->{url}' into `$v'\n";
} else { } else {
# Unpack the archive into the target path. # Unpack the archive into the target path.
print STDERR " unpacking archive...\n"; system("$curl '$narFile->{url}' | $Nix::Config::bzip2 -d | $Nix::Config::binDir/nix-store --restore '$v'") == 0
system("$Nix::Config::bzip2 -d < '$narFilePath' | $Nix::Config::binDir/nix-store --restore '$v'") == 0 or die "cannot download and unpack `$narFile->{url}' into `$v'\n";
or die "cannot unpack `$narFilePath' into `$v'";
} }
$finalNarHash = $narFile->{narHash}; $finalNarHash = $narFile->{narHash};
@ -365,19 +355,17 @@ while (scalar @path > 0) {
# Make sure that the hash declared in the manifest matches what we # Make sure that the hash declared in the manifest matches what we
# downloaded and unpacked. # downloaded and unpacked.
die "cannot check integrity of the downloaded path since its hash is not known\n"
unless defined $finalNarHash;
if (defined $finalNarHash) { my ($hashAlgo, $hash) = parseHash $finalNarHash;
my ($hashAlgo, $hash) = parseHash $finalNarHash;
# The hash in the manifest can be either in base-16 or base-32. # The hash in the manifest can be either in base-16 or base-32.
# Handle both. # Handle both.
my $hash2 = hashPath($hashAlgo, $hashAlgo eq "sha256" && length($hash) != 64, $targetPath); my $hash2 = hashPath($hashAlgo, $hashAlgo eq "sha256" && length($hash) != 64, $targetPath);
die "hash mismatch in downloaded path $targetPath; expected $hash, got $hash2\n" die "hash mismatch in downloaded path $targetPath; expected $hash, got $hash2\n"
if $hash ne $hash2; if $hash ne $hash2;
} else {
die "cannot check integrity of the downloaded path since its hash is not known\n";
}
print STDERR "\n"; print STDERR "\n";