download-from-binary-cache: in queries, preferred cached info

This commit is contained in:
Eelco Dolstra 2012-07-03 18:35:39 -04:00
parent 2a8e5c8b11
commit 89380c03e9

View file

@ -61,7 +61,8 @@ EOF
sub getInfoFrom { sub getInfoFrom {
my ($storePath, $pathHash, $binaryCacheUrl) = @_; my ($storePath, $pathHash, $binaryCacheUrl, $cacheId) = @_;
my $infoUrl = "$binaryCacheUrl/$pathHash.narinfo"; my $infoUrl = "$binaryCacheUrl/$pathHash.narinfo";
print STDERR "checking $infoUrl...\n"; print STDERR "checking $infoUrl...\n";
my $s = `$Nix::Config::curl --fail --silent --location $infoUrl`; my $s = `$Nix::Config::curl --fail --silent --location $infoUrl`;
@ -71,7 +72,9 @@ sub getInfoFrom {
if $status != 22 && $status != 37; if $status != 22 && $status != 37;
return undef; return undef;
} }
my ($storePath2, $url, $compression, $fileHash, $fileSize, $narHash, $narSize, $deriver);
my ($storePath2, $url, $fileHash, $fileSize, $narHash, $narSize, $deriver, $system);
my $compression = "bzip2";
my @refs; my @refs;
foreach my $line (split "\n", $s) { foreach my $line (split "\n", $s) {
$line =~ /^(.*): (.*)$/ or return undef; $line =~ /^(.*): (.*)$/ or return undef;
@ -84,21 +87,29 @@ sub getInfoFrom {
elsif ($1 eq "NarSize") { $narSize = int($2); } elsif ($1 eq "NarSize") { $narSize = int($2); }
elsif ($1 eq "References") { @refs = split / /, $2; } elsif ($1 eq "References") { @refs = split / /, $2; }
elsif ($1 eq "Deriver") { $deriver = $2; } elsif ($1 eq "Deriver") { $deriver = $2; }
elsif ($1 eq "System") { $system = $2; }
} }
return undef if $storePath ne $storePath2; return undef if $storePath ne $storePath2;
if ($storePath ne $storePath2 || !defined $url || !defined $narHash) { if ($storePath ne $storePath2 || !defined $url || !defined $narHash) {
print STDERR "bad NAR info file $infoUrl\n"; print STDERR "bad NAR info file $infoUrl\n";
return undef; return undef;
} }
# Cache the result.
$insertNAR->execute(
getCacheId($binaryCacheUrl), basename($storePath), $url, $compression, $fileHash, $fileSize,
$narHash, $narSize, join(" ", @refs), $deriver, $system, time());
return return
{ url => $url { url => $url
, compression => ($compression || "bzip2") , compression => $compression
, fileHash => $fileHash , fileHash => $fileHash
, fileSize => $fileSize , fileSize => $fileSize
, narHash => $narHash , narHash => $narHash
, narSize => $narSize , narSize => $narSize
, refs => [ @refs ] , refs => [ @refs ]
, deriver => $deriver , deriver => $deriver
, system => $system
}; };
} }
@ -127,11 +138,10 @@ sub getCacheId {
sub cachedGetInfoFrom { sub cachedGetInfoFrom {
my ($storePath, $pathHash, $binaryCacheUrl) = @_; my ($storePath, $pathHash, $binaryCacheUrl) = @_;
my $cacheId = getCacheId($binaryCacheUrl); $queryNAR->execute(getCacheId($binaryCacheUrl), basename($storePath));
# Look up $storePath in the SQLite cache.
$queryNAR->execute($cacheId, basename($storePath));
my $res = $queryNAR->fetchrow_hashref(); my $res = $queryNAR->fetchrow_hashref();
return undef unless defined $res;
return return
{ url => $res->{url} { url => $res->{url}
, compression => $res->{compression} , compression => $res->{compression}
@ -142,18 +152,6 @@ sub cachedGetInfoFrom {
, refs => [ split " ", $res->{refs} ] , refs => [ split " ", $res->{refs} ]
, deriver => $res->{deriver} , deriver => $res->{deriver}
} if defined $res; } if defined $res;
# Not found, so do an HTTP request to get the info.
my $info = getInfoFrom($storePath, $pathHash, $binaryCacheUrl);
# Cache the result.
$insertNAR->execute(
$cacheId, basename($storePath), $info->{url}, $info->{compression}, $info->{fileHash}, $info->{fileSize},
$info->{narHash}, $info->{narSize}, join(" ", @{$info->{refs}}),
$info->{deriver}, $info->{system}, time())
if defined $info;
return $info;
} }
@ -162,11 +160,18 @@ sub getInfo {
my $pathHash = substr(basename($storePath), 0, 32); my $pathHash = substr(basename($storePath), 0, 32);
cache: foreach my $binaryCacheUrl (@binaryCacheUrls) { # First look if we have cached info for one of the URLs.
foreach my $binaryCacheUrl (@binaryCacheUrls) {
my $info = cachedGetInfoFrom($storePath, $pathHash, $binaryCacheUrl); my $info = cachedGetInfoFrom($storePath, $pathHash, $binaryCacheUrl);
return $info if defined $info; return $info if defined $info;
} }
# No, so do an HTTP request until we get a hit.
foreach my $binaryCacheUrl (@binaryCacheUrls) {
my $info = getInfoFrom($storePath, $pathHash, $binaryCacheUrl);
return $info if defined $info;
}
return undef; return undef;
} }
@ -178,6 +183,7 @@ sub downloadBinary {
cache: foreach my $binaryCacheUrl (@binaryCacheUrls) { cache: foreach my $binaryCacheUrl (@binaryCacheUrls) {
my $info = cachedGetInfoFrom($storePath, $pathHash, $binaryCacheUrl); my $info = cachedGetInfoFrom($storePath, $pathHash, $binaryCacheUrl);
$info = getInfoFrom($storePath, $pathHash, $binaryCacheUrl) unless defined $info;
if (defined $info) { if (defined $info) {
my $decompressor; my $decompressor;
if ($info->{compression} eq "bzip2") { $decompressor = "$Nix::Config::bzip2 -d"; } if ($info->{compression} eq "bzip2") { $decompressor = "$Nix::Config::bzip2 -d"; }
@ -186,6 +192,7 @@ sub downloadBinary {
print STDERR "unknown compression method $info->{compression}\n"; print STDERR "unknown compression method $info->{compression}\n";
next; next;
} }
print STDERR "\n*** Downloading $info->{url} into $storePath...\n";
if (system("$Nix::Config::curl --fail --location $binaryCacheUrl/$info->{url} | $decompressor | $Nix::Config::binDir/nix-store --restore $storePath") != 0) { if (system("$Nix::Config::curl --fail --location $binaryCacheUrl/$info->{url} | $decompressor | $Nix::Config::binDir/nix-store --restore $storePath") != 0) {
die "download of `$info->{url}' failed" . ($! ? ": $!" : "") . "\n" unless $? == 0; die "download of `$info->{url}' failed" . ($! ? ": $!" : "") . "\n" unless $? == 0;
next; next;
@ -197,6 +204,7 @@ sub downloadBinary {
my $hash2 = hashPath("sha256", 1, $storePath); my $hash2 = hashPath("sha256", 1, $storePath);
die "hash mismatch in downloaded path $storePath; expected $hash, got $hash2\n" die "hash mismatch in downloaded path $storePath; expected $hash, got $hash2\n"
if $hash ne $hash2; if $hash ne $hash2;
print STDERR "\n";
return 1; return 1;
} }
} }