Allow a binary cache to declare that it doesn't support "nix-env -qas"
Querying all substitutable paths via "nix-env -qas" is potentially hard on a server, since it involves sending thousands of HEAD requests. So a binary cache must now have a meta-info file named "nix-cache-info" that specifies whether the server wants this. It also specifies the store prefix so that we don't send useless queries to a binary cache for a different store prefix.
This commit is contained in:
Eelco Dolstra
parent
6ecf4f13f6
commit
66a3ac6a56
3 changed files with 135 additions and 86 deletions
|
|
@ -12,18 +12,15 @@ use strict;
|
|||
|
||||
Nix::Config::readConfig;
|
||||
|
||||
my @binaryCacheUrls = map { s/\/+$//; $_ } split(/ /,
|
||||
($ENV{"NIX_BINARY_CACHES"}
|
||||
// $Nix::Config::config{"binary-caches"}
|
||||
// ($Nix::Config::storeDir eq "/nix/store" ? "http://nixos.org/binary-cache" : "")));
|
||||
my @caches;
|
||||
my $gotCaches = 0;
|
||||
|
||||
my $maxParallelRequests = int($Nix::Config::config{"binary-caches-parallel-connections"} // 150);
|
||||
$maxParallelRequests = 1 if $maxParallelRequests < 1;
|
||||
|
||||
my $debug = ($ENV{"NIX_DEBUG_SUBST"} // "") eq 1;
|
||||
|
||||
my ($dbh, $insertNAR, $queryNAR, $insertNARExistence, $queryNARExistence);
|
||||
my %cacheIds;
|
||||
my ($dbh, $queryCache, $insertNAR, $queryNAR, $insertNARExistence, $queryNARExistence);
|
||||
|
||||
my $curlm = WWW::Curl::Multi->new;
|
||||
my $activeRequests = 0;
|
||||
|
|
@ -112,7 +109,10 @@ sub initCache {
|
|||
$dbh->do(<<EOF);
|
||||
create table if not exists BinaryCaches (
|
||||
id integer primary key autoincrement not null,
|
||||
url text unique not null
|
||||
url text unique not null,
|
||||
timestamp integer not null,
|
||||
storeDir text not null,
|
||||
wantMassQuery integer not null
|
||||
);
|
||||
EOF
|
||||
|
||||
|
|
@ -146,6 +146,8 @@ EOF
|
|||
);
|
||||
EOF
|
||||
|
||||
$queryCache = $dbh->prepare("select id, storeDir, wantMassQuery from BinaryCaches where url = ?") or die;
|
||||
|
||||
$insertNAR = $dbh->prepare(
|
||||
"insert or replace into NARs(cache, storePath, url, compression, fileHash, fileSize, narHash, " .
|
||||
"narSize, refs, deriver, system, timestamp) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") or die;
|
||||
|
|
@ -159,35 +161,65 @@ EOF
|
|||
}
|
||||
|
||||
|
||||
sub getAvailableCaches {
|
||||
return if $gotCaches;
|
||||
$gotCaches = 1;
|
||||
|
||||
sub negativeHit {
|
||||
my ($storePath, $binaryCacheUrl) = @_;
|
||||
$queryNARExistence->execute(getCacheId($binaryCacheUrl), basename($storePath));
|
||||
my $res = $queryNARExistence->fetchrow_hashref();
|
||||
return defined $res && $res->{exist} == 0;
|
||||
}
|
||||
my @urls = map { s/\/+$//; $_ } split(/ /,
|
||||
($ENV{"NIX_BINARY_CACHES"}
|
||||
// $Nix::Config::config{"binary-caches"}
|
||||
// ($Nix::Config::storeDir eq "/nix/store" ? "http://nixos.org/binary-cache" : "")));
|
||||
|
||||
foreach my $url (@urls) {
|
||||
|
||||
sub positiveHit {
|
||||
my ($storePath, $binaryCacheUrl) = @_;
|
||||
return 1 if defined getCachedInfoFrom($storePath, $binaryCacheUrl);
|
||||
$queryNARExistence->execute(getCacheId($binaryCacheUrl), basename($storePath));
|
||||
my $res = $queryNARExistence->fetchrow_hashref();
|
||||
return defined $res && $res->{exist} == 1;
|
||||
# FIXME: not atomic.
|
||||
$queryCache->execute($url);
|
||||
my $res = $queryCache->fetchrow_hashref();
|
||||
if (defined $res) {
|
||||
next if $res->{storeDir} ne $Nix::Config::storeDir;
|
||||
push @caches, { id => $res->{id}, url => $url, wantMassQuery => $res->{wantMassQuery} };
|
||||
next;
|
||||
}
|
||||
|
||||
# Get the cache info file.
|
||||
my $request = addRequest(undef, $url . "/nix-cache-info");
|
||||
processRequests;
|
||||
|
||||
if ($request->{result} != 0) {
|
||||
print STDERR "could not download ‘$request->{url}’ (" .
|
||||
($request->{result} != 0 ? "Curl error $request->{result}" : "HTTP status $request->{httpStatus}") . ")\n";
|
||||
next;
|
||||
}
|
||||
|
||||
my $storeDir = "/nix/store";
|
||||
my $wantMassQuery = 0;
|
||||
foreach my $line (split "\n", $request->{content}) {
|
||||
unless ($line =~ /^(.*): (.*)$/) {
|
||||
print STDERR "bad cache info file ‘$request->{url}’\n";
|
||||
return undef;
|
||||
}
|
||||
if ($1 eq "StoreDir") { $storeDir = $2; }
|
||||
elsif ($1 eq "WantMassQuery") { $wantMassQuery = int($2); }
|
||||
}
|
||||
|
||||
$dbh->do("insert into BinaryCaches(url, timestamp, storeDir, wantMassQuery) values (?, ?, ?, ?)",
|
||||
{}, $url, time(), $storeDir, $wantMassQuery);
|
||||
my $id = $dbh->last_insert_id("", "", "", "");
|
||||
next if $storeDir ne $Nix::Config::storeDir;
|
||||
push @caches, { id => $id, url => $url, wantMassQuery => $wantMassQuery };
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
sub processNARInfo {
|
||||
my ($storePath, $binaryCacheUrl, $request) = @_;
|
||||
|
||||
my $cacheId = getCacheId($binaryCacheUrl);
|
||||
my ($storePath, $cache, $request) = @_;
|
||||
|
||||
if ($request->{result} != 0) {
|
||||
if ($request->{result} != 37 && $request->{httpStatus} != 404) {
|
||||
print STDERR "could not download ‘$request->{url}’ (" .
|
||||
($request->{result} != 0 ? "Curl error $request->{result}" : "HTTP status $request->{httpStatus}") . ")\n";
|
||||
} else {
|
||||
$insertNARExistence->execute($cacheId, basename($storePath), 0, time())
|
||||
$insertNARExistence->execute($cache->{id}, basename($storePath), 0, time())
|
||||
unless $request->{url} =~ /^file:/;
|
||||
}
|
||||
return undef;
|
||||
|
|
@ -222,7 +254,7 @@ sub processNARInfo {
|
|||
|
||||
# Cache the result.
|
||||
$insertNAR->execute(
|
||||
$cacheId, basename($storePath), $url, $compression, $fileHash, $fileSize,
|
||||
$cache->{id}, basename($storePath), $url, $compression, $fileHash, $fileSize,
|
||||
$narHash, $narSize, join(" ", @refs), $deriver, $system, time())
|
||||
unless $request->{url} =~ /^file:/;
|
||||
|
||||
|
|
@ -240,31 +272,10 @@ sub processNARInfo {
|
|||
}
|
||||
|
||||
|
||||
sub getCacheId {
|
||||
my ($binaryCacheUrl) = @_;
|
||||
|
||||
my $cacheId = $cacheIds{$binaryCacheUrl};
|
||||
return $cacheId if defined $cacheId;
|
||||
|
||||
# FIXME: not atomic.
|
||||
my @res = @{$dbh->selectcol_arrayref("select id from BinaryCaches where url = ?", {}, $binaryCacheUrl)};
|
||||
if (scalar @res == 1) {
|
||||
$cacheId = $res[0];
|
||||
} else {
|
||||
$dbh->do("insert into BinaryCaches(url) values (?)",
|
||||
{}, $binaryCacheUrl);
|
||||
$cacheId = $dbh->last_insert_id("", "", "", "");
|
||||
}
|
||||
|
||||
$cacheIds{$binaryCacheUrl} = $cacheId;
|
||||
return $cacheId;
|
||||
}
|
||||
|
||||
|
||||
sub getCachedInfoFrom {
|
||||
my ($storePath, $binaryCacheUrl) = @_;
|
||||
my ($storePath, $cache) = @_;
|
||||
|
||||
$queryNAR->execute(getCacheId($binaryCacheUrl), basename($storePath));
|
||||
$queryNAR->execute($cache->{id}, basename($storePath));
|
||||
my $res = $queryNAR->fetchrow_hashref();
|
||||
return undef unless defined $res;
|
||||
|
||||
|
|
@ -281,6 +292,23 @@ sub getCachedInfoFrom {
|
|||
}
|
||||
|
||||
|
||||
sub negativeHit {
|
||||
my ($storePath, $cache) = @_;
|
||||
$queryNARExistence->execute($cache->{id}, basename($storePath));
|
||||
my $res = $queryNARExistence->fetchrow_hashref();
|
||||
return defined $res && $res->{exist} == 0;
|
||||
}
|
||||
|
||||
|
||||
sub positiveHit {
|
||||
my ($storePath, $cache) = @_;
|
||||
return 1 if defined getCachedInfoFrom($storePath, $cache);
|
||||
$queryNARExistence->execute($cache->{id}, basename($storePath));
|
||||
my $res = $queryNARExistence->fetchrow_hashref();
|
||||
return defined $res && $res->{exist} == 1;
|
||||
}
|
||||
|
||||
|
||||
sub printInfo {
|
||||
my ($storePath, $info) = @_;
|
||||
print "$storePath\n";
|
||||
|
|
@ -306,8 +334,8 @@ sub printInfoParallel {
|
|||
my @left;
|
||||
foreach my $storePath (@paths) {
|
||||
my $found = 0;
|
||||
foreach my $binaryCacheUrl (@binaryCacheUrls) {
|
||||
my $info = getCachedInfoFrom($storePath, $binaryCacheUrl);
|
||||
foreach my $cache (@caches) {
|
||||
my $info = getCachedInfoFrom($storePath, $cache);
|
||||
if (defined $info) {
|
||||
printInfo($storePath, $info);
|
||||
$found = 1;
|
||||
|
|
@ -319,22 +347,22 @@ sub printInfoParallel {
|
|||
|
||||
return if scalar @left == 0;
|
||||
|
||||
foreach my $binaryCacheUrl (@binaryCacheUrls) {
|
||||
foreach my $cache (@caches) {
|
||||
|
||||
my @left2;
|
||||
%requests = ();
|
||||
foreach my $storePath (@left) {
|
||||
if (negativeHit($storePath, $binaryCacheUrl)) {
|
||||
if (negativeHit($storePath, $cache)) {
|
||||
push @left2, $storePath;
|
||||
next;
|
||||
}
|
||||
addRequest($storePath, infoUrl($binaryCacheUrl, $storePath));
|
||||
addRequest($storePath, infoUrl($cache->{url}, $storePath));
|
||||
}
|
||||
|
||||
processRequests;
|
||||
|
||||
foreach my $request (values %requests) {
|
||||
my $info = processNARInfo($request->{storePath}, $binaryCacheUrl, $request);
|
||||
my $info = processNARInfo($request->{storePath}, $cache, $request);
|
||||
if (defined $info) {
|
||||
printInfo($request->{storePath}, $info);
|
||||
} else {
|
||||
|
|
@ -354,8 +382,9 @@ sub printSubstitutablePaths {
|
|||
my @left;
|
||||
foreach my $storePath (@paths) {
|
||||
my $found = 0;
|
||||
foreach my $binaryCacheUrl (@binaryCacheUrls) {
|
||||
if (positiveHit($storePath, $binaryCacheUrl)) {
|
||||
foreach my $cache (@caches) {
|
||||
next unless $cache->{wantMassQuery};
|
||||
if (positiveHit($storePath, $cache)) {
|
||||
print "$storePath\n";
|
||||
$found = 1;
|
||||
last;
|
||||
|
|
@ -367,17 +396,16 @@ sub printSubstitutablePaths {
|
|||
return if scalar @left == 0;
|
||||
|
||||
# For remaining paths, do HEAD requests.
|
||||
foreach my $binaryCacheUrl (@binaryCacheUrls) {
|
||||
my $cacheId = getCacheId($binaryCacheUrl);
|
||||
|
||||
foreach my $cache (@caches) {
|
||||
next unless $cache->{wantMassQuery};
|
||||
my @left2;
|
||||
%requests = ();
|
||||
foreach my $storePath (@left) {
|
||||
if (negativeHit($storePath, $binaryCacheUrl)) {
|
||||
if (negativeHit($storePath, $cache)) {
|
||||
push @left2, $storePath;
|
||||
next;
|
||||
}
|
||||
addRequest($storePath, infoUrl($binaryCacheUrl, $storePath), 1);
|
||||
addRequest($storePath, infoUrl($cache->{url}, $storePath), 1);
|
||||
}
|
||||
|
||||
processRequests;
|
||||
|
|
@ -388,12 +416,12 @@ sub printSubstitutablePaths {
|
|||
print STDERR "could not check ‘$request->{url}’ (" .
|
||||
($request->{result} != 0 ? "Curl error $request->{result}" : "HTTP status $request->{httpStatus}") . ")\n";
|
||||
} else {
|
||||
$insertNARExistence->execute($cacheId, basename($request->{storePath}), 0, time())
|
||||
$insertNARExistence->execute($cache->{id}, basename($request->{storePath}), 0, time())
|
||||
unless $request->{url} =~ /^file:/;
|
||||
}
|
||||
push @left2, $request->{storePath};
|
||||
} else {
|
||||
$insertNARExistence->execute($cacheId, basename($request->{storePath}), 1, time())
|
||||
$insertNARExistence->execute($cache->{id}, basename($request->{storePath}), 1, time())
|
||||
unless $request->{url} =~ /^file:/;
|
||||
print "$request->{storePath}\n";
|
||||
}
|
||||
|
|
@ -407,14 +435,14 @@ sub printSubstitutablePaths {
|
|||
sub downloadBinary {
|
||||
my ($storePath) = @_;
|
||||
|
||||
foreach my $binaryCacheUrl (@binaryCacheUrls) {
|
||||
my $info = getCachedInfoFrom($storePath, $binaryCacheUrl);
|
||||
foreach my $cache (@caches) {
|
||||
my $info = getCachedInfoFrom($storePath, $cache);
|
||||
|
||||
unless (defined $info) {
|
||||
next if negativeHit($storePath, $binaryCacheUrl);
|
||||
my $request = addRequest($storePath, infoUrl($binaryCacheUrl, $storePath));
|
||||
next if negativeHit($storePath, $cache);
|
||||
my $request = addRequest($storePath, infoUrl($cache->{url}, $storePath));
|
||||
processRequests;
|
||||
$info = processNARInfo($storePath, $binaryCacheUrl, $request);
|
||||
$info = processNARInfo($storePath, $cache, $request);
|
||||
}
|
||||
|
||||
next unless defined $info;
|
||||
|
|
@ -426,7 +454,7 @@ sub downloadBinary {
|
|||
print STDERR "unknown compression method ‘$info->{compression}’\n";
|
||||
next;
|
||||
}
|
||||
my $url = "$binaryCacheUrl/$info->{url}"; # FIXME: handle non-relative URLs
|
||||
my $url = "$cache->{url}/$info->{url}"; # FIXME: handle non-relative URLs
|
||||
print STDERR "\n*** Downloading ‘$url’ into ‘$storePath’...\n";
|
||||
if (system("$Nix::Config::curl --fail --location --insecure '$url' | $decompressor | $Nix::Config::binDir/nix-store --restore $storePath") != 0) {
|
||||
die "download of `$info->{url}' failed" . ($! ? ": $!" : "") . "\n" unless $? == 0;
|
||||
|
|
@ -437,10 +465,10 @@ sub downloadBinary {
|
|||
print "$info->{narHash}\n";
|
||||
|
||||
print STDERR "\n";
|
||||
return 1;
|
||||
return;
|
||||
}
|
||||
|
||||
return 0;
|
||||
print STDERR "could not download ‘$storePath’ from any binary cache\n";
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -450,6 +478,7 @@ initCache();
|
|||
if ($ARGV[0] eq "--query") {
|
||||
|
||||
while (<STDIN>) {
|
||||
getAvailableCaches;
|
||||
chomp;
|
||||
my ($cmd, @args) = split " ", $_;
|
||||
|
||||
|
|
@ -472,9 +501,8 @@ if ($ARGV[0] eq "--query") {
|
|||
|
||||
elsif ($ARGV[0] eq "--substitute") {
|
||||
my $storePath = $ARGV[1] or die;
|
||||
if (!downloadBinary($storePath)) {
|
||||
print STDERR "could not download ‘$storePath’ from any binary cache\n";
|
||||
}
|
||||
getAvailableCaches;
|
||||
downloadBinary($storePath);
|
||||
}
|
||||
|
||||
else {
|
||||
|
|
|
|||
|
|
@ -61,7 +61,7 @@ for (my $n = 0; $n < scalar @ARGV; $n++) {
|
|||
push @roots, $arg;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
showSyntax if !defined $destDir;
|
||||
|
||||
$archivesURL = "file://$destDir" unless defined $archivesURL;
|
||||
|
|
@ -74,12 +74,12 @@ my %storePaths;
|
|||
foreach my $path (@roots) {
|
||||
die unless $path =~ /^\//;
|
||||
|
||||
# Get all paths referenced by the normalisation of the given
|
||||
# Get all paths referenced by the normalisation of the given
|
||||
# Nix expression.
|
||||
my $pid = open(READ,
|
||||
"$Nix::Config::binDir/nix-store --query --requisites --force-realise " .
|
||||
"--include-outputs '$path'|") or die;
|
||||
|
||||
|
||||
while (<READ>) {
|
||||
chomp;
|
||||
die "bad: $_" unless /^\//;
|
||||
|
|
@ -101,10 +101,10 @@ foreach my $storePath (@storePaths) {
|
|||
die unless ($storePath =~ /\/[0-9a-z]{32}[^\"\\\$]*$/);
|
||||
|
||||
# Construct a Nix expression that creates a Nix archive.
|
||||
my $nixexpr =
|
||||
my $nixexpr =
|
||||
"(import <nix/nar.nix> " .
|
||||
"{ storePath = builtins.storePath \"$storePath\"; hashAlgo = \"sha256\"; compressionType = \"$compressionType\"; }) ";
|
||||
|
||||
|
||||
print NIX $nixexpr;
|
||||
}
|
||||
|
||||
|
|
@ -125,7 +125,17 @@ while (<READ>) {
|
|||
close READ or die "nix-build failed: $?";
|
||||
|
||||
|
||||
# Copy the archives and the corresponding info files.
|
||||
# Write the cache info file.
|
||||
my $cacheInfoFile = "$destDir/nix-cache-info";
|
||||
if (! -e $cacheInfoFile) {
|
||||
open FILE, ">$cacheInfoFile" or die "cannot create $cacheInfoFile: $!";
|
||||
print FILE "StoreDir: $Nix::Config::storeDir\n";
|
||||
print FILE "WantMassQuery: 0\n"; # by default, don't hit this cache for "nix-env -qas"
|
||||
close FILE;
|
||||
}
|
||||
|
||||
|
||||
# Copy the archives and the corresponding NAR info files.
|
||||
print STDERR "copying archives...\n";
|
||||
|
||||
my $totalNarSize = 0;
|
||||
|
|
@ -157,7 +167,7 @@ for (my $n = 0; $n < scalar @storePaths; $n++) {
|
|||
}
|
||||
|
||||
$totalNarSize += $narSize;
|
||||
|
||||
|
||||
# Get info about the compressed NAR.
|
||||
open HASH, "$narDir/nar-compressed-hash" or die "cannot open nar-compressed-hash";
|
||||
my $compressedHash = <HASH>;
|
||||
|
|
@ -170,7 +180,7 @@ for (my $n = 0; $n < scalar @storePaths; $n++) {
|
|||
my $narFile = "$narDir/$narName";
|
||||
(-f $narFile) or die "NAR file for $storePath not found";
|
||||
|
||||
my $compressedSize = stat($narFile)->size;
|
||||
my $compressedSize = stat($narFile)->size;
|
||||
$totalCompressedSize += $compressedSize;
|
||||
|
||||
printf STDERR "%s [%.2f MiB, %.1f%%]\n", $storePath,
|
||||
|
|
@ -203,7 +213,7 @@ for (my $n = 0; $n < scalar @storePaths; $n++) {
|
|||
}
|
||||
|
||||
my $pathHash = substr(basename($storePath), 0, 32);
|
||||
|
||||
|
||||
$dst = "$destDir/$pathHash.narinfo";
|
||||
if ($force || ! -f $dst) {
|
||||
my $tmp = "$destDir/.tmp.$$.$pathHash.narinfo";
|
||||
|
|
@ -230,6 +240,4 @@ printf STDERR "total compressed size %.2f MiB, %.1f%%\n",
|
|||
|
||||
|
||||
# Optionally write a manifest.
|
||||
if ($writeManifest) {
|
||||
writeManifest "$destDir/MANIFEST", \%narFiles, \();
|
||||
}
|
||||
writeManifest "$destDir/MANIFEST", \%narFiles, \() if $writeManifest;
|
||||
|
|
|
|||
|
|
@ -10,13 +10,26 @@ outPath=$(nix-build dependencies.nix --no-out-link)
|
|||
|
||||
nix-push --dest $cacheDir $outPath
|
||||
|
||||
# Check that downloading works.
|
||||
|
||||
# By default, a binary cache doesn't support "nix-env -qas", but does
|
||||
# support installation.
|
||||
clearStore
|
||||
rm -f $NIX_STATE_DIR/binary-cache*
|
||||
|
||||
NIX_BINARY_CACHES="file://$cacheDir" nix-env -f dependencies.nix -qas \* | grep -- "---"
|
||||
|
||||
NIX_BINARY_CACHES="file://$cacheDir" nix-store -r $outPath
|
||||
|
||||
|
||||
# But with the right configuration, "nix-env -qas" should also work.
|
||||
clearStore
|
||||
rm -f $NIX_STATE_DIR/binary-cache*
|
||||
echo "WantMassQuery: 1" >> $cacheDir/nix-cache-info
|
||||
|
||||
NIX_BINARY_CACHES="file://$cacheDir" nix-env -f dependencies.nix -qas \* | grep -- "--S"
|
||||
|
||||
NIX_BINARY_CACHES="file://$cacheDir" nix-store -r $outPath
|
||||
|
||||
nix-store --check-validity $outPath
|
||||
nix-store -qR $outPath | grep input-2
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue