lehmanator/lix
0
0
Fork 0
forked from lix-project/lix
Code Pull requests Activity

Allow a binary cache to declare that it doesn't support "nix-env -qas"

Browse source 
Querying all substitutable paths via "nix-env -qas" is potentially
hard on a server, since it involves sending thousands of HEAD
requests.  So a binary cache must now have a meta-info file named
"nix-cache-info" that specifies whether the server wants this.  It
also specifies the store prefix so that we don't send useless queries
to a binary cache for a different store prefix.
This commit is contained in:
Eelco Dolstra 2012-07-27 18:16:05 -04:00
parent 6ecf4f13f6
commit 66a3ac6a56
3 changed files with 135 additions and 86 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

174
scripts/download-from-binary-cache.pl.in
View file

@ -12,18 +12,15 @@ use strict;
Nix::Config::readConfig; Nix::Config::readConfig;
my @binaryCacheUrls = map { s/\/+$//; $_ } split(/ /, my @caches;
($ENV{"NIX_BINARY_CACHES"} my $gotCaches = 0;
// $Nix::Config::config{"binary-caches"}
// ($Nix::Config::storeDir eq "/nix/store" ? "http://nixos.org/binary-cache" : "")));
my $maxParallelRequests = int($Nix::Config::config{"binary-caches-parallel-connections"} // 150); my $maxParallelRequests = int($Nix::Config::config{"binary-caches-parallel-connections"} // 150);
$maxParallelRequests = 1 if $maxParallelRequests < 1; $maxParallelRequests = 1 if $maxParallelRequests < 1;
my $debug = ($ENV{"NIX_DEBUG_SUBST"} // "") eq 1; my $debug = ($ENV{"NIX_DEBUG_SUBST"} // "") eq 1;
my ($dbh, $insertNAR, $queryNAR, $insertNARExistence, $queryNARExistence); my ($dbh, $queryCache, $insertNAR, $queryNAR, $insertNARExistence, $queryNARExistence);
my %cacheIds;
my $curlm = WWW::Curl::Multi->new; my $curlm = WWW::Curl::Multi->new;
my $activeRequests = 0; my $activeRequests = 0;
@ -112,7 +109,10 @@ sub initCache {
$dbh->do(<<EOF); $dbh->do(<<EOF);
create table if not exists BinaryCaches ( create table if not exists BinaryCaches (
id integer primary key autoincrement not null, id integer primary key autoincrement not null,
url text unique not null url text unique not null,
timestamp integer not null,
storeDir text not null,
wantMassQuery integer not null
); );
EOF EOF
@ -146,6 +146,8 @@ EOF
); );
EOF EOF
$queryCache = $dbh->prepare("select id, storeDir, wantMassQuery from BinaryCaches where url = ?") or die;
$insertNAR = $dbh->prepare( $insertNAR = $dbh->prepare(
"insert or replace into NARs(cache, storePath, url, compression, fileHash, fileSize, narHash, " . "insert or replace into NARs(cache, storePath, url, compression, fileHash, fileSize, narHash, " .
"narSize, refs, deriver, system, timestamp) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") or die; "narSize, refs, deriver, system, timestamp) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") or die;
@ -159,35 +161,65 @@ EOF
} }
sub getAvailableCaches {
return if $gotCaches;
$gotCaches = 1;
sub negativeHit { my @urls = map { s/\/+$//; $_ } split(/ /,
my ($storePath, $binaryCacheUrl) = @_; ($ENV{"NIX_BINARY_CACHES"}
$queryNARExistence->execute(getCacheId($binaryCacheUrl), basename($storePath)); // $Nix::Config::config{"binary-caches"}
my $res = $queryNARExistence->fetchrow_hashref(); // ($Nix::Config::storeDir eq "/nix/store" ? "http://nixos.org/binary-cache" : "")));
return defined $res && $res->{exist} == 0;
}
foreach my $url (@urls) {
sub positiveHit { # FIXME: not atomic.
my ($storePath, $binaryCacheUrl) = @_; $queryCache->execute($url);
return 1 if defined getCachedInfoFrom($storePath, $binaryCacheUrl); my $res = $queryCache->fetchrow_hashref();
$queryNARExistence->execute(getCacheId($binaryCacheUrl), basename($storePath)); if (defined $res) {
my $res = $queryNARExistence->fetchrow_hashref(); next if $res->{storeDir} ne $Nix::Config::storeDir;
return defined $res && $res->{exist} == 1; push @caches, { id => $res->{id}, url => $url, wantMassQuery => $res->{wantMassQuery} };
next;
}
# Get the cache info file.
my $request = addRequest(undef, $url . "/nix-cache-info");
processRequests;
if ($request->{result} != 0) {
print STDERR "could not download ‘$request->{url}’ (" .
($request->{result} != 0 ? "Curl error $request->{result}" : "HTTP status $request->{httpStatus}") . ")\n";
next;
}
my $storeDir = "/nix/store";
my $wantMassQuery = 0;
foreach my $line (split "\n", $request->{content}) {
unless ($line =~ /^(.*): (.*)$/) {
print STDERR "bad cache info file ‘$request->{url}’\n";
return undef;
}
if ($1 eq "StoreDir") { $storeDir = $2; }
elsif ($1 eq "WantMassQuery") { $wantMassQuery = int($2); }
}
$dbh->do("insert into BinaryCaches(url, timestamp, storeDir, wantMassQuery) values (?, ?, ?, ?)",
{}, $url, time(), $storeDir, $wantMassQuery);
my $id = $dbh->last_insert_id("", "", "", "");
next if $storeDir ne $Nix::Config::storeDir;
push @caches, { id => $id, url => $url, wantMassQuery => $wantMassQuery };
}
} }
sub processNARInfo { sub processNARInfo {
my ($storePath, $binaryCacheUrl, $request) = @_; my ($storePath, $cache, $request) = @_;
my $cacheId = getCacheId($binaryCacheUrl);
if ($request->{result} != 0) { if ($request->{result} != 0) {
if ($request->{result} != 37 && $request->{httpStatus} != 404) { if ($request->{result} != 37 && $request->{httpStatus} != 404) {
print STDERR "could not download ‘$request->{url}’ (" . print STDERR "could not download ‘$request->{url}’ (" .
($request->{result} != 0 ? "Curl error $request->{result}" : "HTTP status $request->{httpStatus}") . ")\n"; ($request->{result} != 0 ? "Curl error $request->{result}" : "HTTP status $request->{httpStatus}") . ")\n";
} else { } else {
$insertNARExistence->execute($cacheId, basename($storePath), 0, time()) $insertNARExistence->execute($cache->{id}, basename($storePath), 0, time())
unless $request->{url} =~ /^file:/; unless $request->{url} =~ /^file:/;
} }
return undef; return undef;
@ -222,7 +254,7 @@ sub processNARInfo {
# Cache the result. # Cache the result.
$insertNAR->execute( $insertNAR->execute(
$cacheId, basename($storePath), $url, $compression, $fileHash, $fileSize, $cache->{id}, basename($storePath), $url, $compression, $fileHash, $fileSize,
$narHash, $narSize, join(" ", @refs), $deriver, $system, time()) $narHash, $narSize, join(" ", @refs), $deriver, $system, time())
unless $request->{url} =~ /^file:/; unless $request->{url} =~ /^file:/;
@ -240,31 +272,10 @@ sub processNARInfo {
} }
sub getCacheId {
my ($binaryCacheUrl) = @_;
my $cacheId = $cacheIds{$binaryCacheUrl};
return $cacheId if defined $cacheId;
# FIXME: not atomic.
my @res = @{$dbh->selectcol_arrayref("select id from BinaryCaches where url = ?", {}, $binaryCacheUrl)};
if (scalar @res == 1) {
$cacheId = $res[0];
} else {
$dbh->do("insert into BinaryCaches(url) values (?)",
{}, $binaryCacheUrl);
$cacheId = $dbh->last_insert_id("", "", "", "");
}
$cacheIds{$binaryCacheUrl} = $cacheId;
return $cacheId;
}
sub getCachedInfoFrom { sub getCachedInfoFrom {
my ($storePath, $binaryCacheUrl) = @_; my ($storePath, $cache) = @_;
$queryNAR->execute(getCacheId($binaryCacheUrl), basename($storePath)); $queryNAR->execute($cache->{id}, basename($storePath));
my $res = $queryNAR->fetchrow_hashref(); my $res = $queryNAR->fetchrow_hashref();
return undef unless defined $res; return undef unless defined $res;
@ -281,6 +292,23 @@ sub getCachedInfoFrom {
} }
sub negativeHit {
my ($storePath, $cache) = @_;
$queryNARExistence->execute($cache->{id}, basename($storePath));
my $res = $queryNARExistence->fetchrow_hashref();
return defined $res && $res->{exist} == 0;
}
sub positiveHit {
my ($storePath, $cache) = @_;
return 1 if defined getCachedInfoFrom($storePath, $cache);
$queryNARExistence->execute($cache->{id}, basename($storePath));
my $res = $queryNARExistence->fetchrow_hashref();
return defined $res && $res->{exist} == 1;
}
sub printInfo { sub printInfo {
my ($storePath, $info) = @_; my ($storePath, $info) = @_;
print "$storePath\n"; print "$storePath\n";
@ -306,8 +334,8 @@ sub printInfoParallel {
my @left; my @left;
foreach my $storePath (@paths) { foreach my $storePath (@paths) {
my $found = 0; my $found = 0;
foreach my $binaryCacheUrl (@binaryCacheUrls) { foreach my $cache (@caches) {
my $info = getCachedInfoFrom($storePath, $binaryCacheUrl); my $info = getCachedInfoFrom($storePath, $cache);
if (defined $info) { if (defined $info) {
printInfo($storePath, $info); printInfo($storePath, $info);
$found = 1; $found = 1;
@ -319,22 +347,22 @@ sub printInfoParallel {
return if scalar @left == 0; return if scalar @left == 0;
foreach my $binaryCacheUrl (@binaryCacheUrls) { foreach my $cache (@caches) {
my @left2; my @left2;
%requests = (); %requests = ();
foreach my $storePath (@left) { foreach my $storePath (@left) {
if (negativeHit($storePath, $binaryCacheUrl)) { if (negativeHit($storePath, $cache)) {
push @left2, $storePath; push @left2, $storePath;
next; next;
} }
addRequest($storePath, infoUrl($binaryCacheUrl, $storePath)); addRequest($storePath, infoUrl($cache->{url}, $storePath));
} }
processRequests; processRequests;
foreach my $request (values %requests) { foreach my $request (values %requests) {
my $info = processNARInfo($request->{storePath}, $binaryCacheUrl, $request); my $info = processNARInfo($request->{storePath}, $cache, $request);
if (defined $info) { if (defined $info) {
printInfo($request->{storePath}, $info); printInfo($request->{storePath}, $info);
} else { } else {
@ -354,8 +382,9 @@ sub printSubstitutablePaths {
my @left; my @left;
foreach my $storePath (@paths) { foreach my $storePath (@paths) {
my $found = 0; my $found = 0;
foreach my $binaryCacheUrl (@binaryCacheUrls) { foreach my $cache (@caches) {
if (positiveHit($storePath, $binaryCacheUrl)) { next unless $cache->{wantMassQuery};
if (positiveHit($storePath, $cache)) {
print "$storePath\n"; print "$storePath\n";
$found = 1; $found = 1;
last; last;
@ -367,17 +396,16 @@ sub printSubstitutablePaths {
return if scalar @left == 0; return if scalar @left == 0;
# For remaining paths, do HEAD requests. # For remaining paths, do HEAD requests.
foreach my $binaryCacheUrl (@binaryCacheUrls) { foreach my $cache (@caches) {
my $cacheId = getCacheId($binaryCacheUrl); next unless $cache->{wantMassQuery};
my @left2; my @left2;
%requests = (); %requests = ();
foreach my $storePath (@left) { foreach my $storePath (@left) {
if (negativeHit($storePath, $binaryCacheUrl)) { if (negativeHit($storePath, $cache)) {
push @left2, $storePath; push @left2, $storePath;
next; next;
} }
addRequest($storePath, infoUrl($binaryCacheUrl, $storePath), 1); addRequest($storePath, infoUrl($cache->{url}, $storePath), 1);
} }
processRequests; processRequests;
@ -388,12 +416,12 @@ sub printSubstitutablePaths {
print STDERR "could not check ‘$request->{url}’ (" . print STDERR "could not check ‘$request->{url}’ (" .
($request->{result} != 0 ? "Curl error $request->{result}" : "HTTP status $request->{httpStatus}") . ")\n"; ($request->{result} != 0 ? "Curl error $request->{result}" : "HTTP status $request->{httpStatus}") . ")\n";
} else { } else {
$insertNARExistence->execute($cacheId, basename($request->{storePath}), 0, time()) $insertNARExistence->execute($cache->{id}, basename($request->{storePath}), 0, time())
unless $request->{url} =~ /^file:/; unless $request->{url} =~ /^file:/;
} }
push @left2, $request->{storePath}; push @left2, $request->{storePath};
} else { } else {
$insertNARExistence->execute($cacheId, basename($request->{storePath}), 1, time()) $insertNARExistence->execute($cache->{id}, basename($request->{storePath}), 1, time())
unless $request->{url} =~ /^file:/; unless $request->{url} =~ /^file:/;
print "$request->{storePath}\n"; print "$request->{storePath}\n";
} }
@ -407,14 +435,14 @@ sub printSubstitutablePaths {
sub downloadBinary { sub downloadBinary {
my ($storePath) = @_; my ($storePath) = @_;
foreach my $binaryCacheUrl (@binaryCacheUrls) { foreach my $cache (@caches) {
my $info = getCachedInfoFrom($storePath, $binaryCacheUrl); my $info = getCachedInfoFrom($storePath, $cache);
unless (defined $info) { unless (defined $info) {
next if negativeHit($storePath, $binaryCacheUrl); next if negativeHit($storePath, $cache);
my $request = addRequest($storePath, infoUrl($binaryCacheUrl, $storePath)); my $request = addRequest($storePath, infoUrl($cache->{url}, $storePath));
processRequests; processRequests;
$info = processNARInfo($storePath, $binaryCacheUrl, $request); $info = processNARInfo($storePath, $cache, $request);
} }
next unless defined $info; next unless defined $info;
@ -426,7 +454,7 @@ sub downloadBinary {
print STDERR "unknown compression method ‘$info->{compression}’\n"; print STDERR "unknown compression method ‘$info->{compression}’\n";
next; next;
} }
my $url = "$binaryCacheUrl/$info->{url}"; # FIXME: handle non-relative URLs my $url = "$cache->{url}/$info->{url}"; # FIXME: handle non-relative URLs
print STDERR "\n*** Downloading ‘$url’ into ‘$storePath’...\n"; print STDERR "\n*** Downloading ‘$url’ into ‘$storePath’...\n";
if (system("$Nix::Config::curl --fail --location --insecure '$url' | $decompressor | $Nix::Config::binDir/nix-store --restore $storePath") != 0) { if (system("$Nix::Config::curl --fail --location --insecure '$url' | $decompressor | $Nix::Config::binDir/nix-store --restore $storePath") != 0) {
die "download of `$info->{url}' failed" . ($! ? ": $!" : "") . "\n" unless $? == 0; die "download of `$info->{url}' failed" . ($! ? ": $!" : "") . "\n" unless $? == 0;
@ -437,10 +465,10 @@ sub downloadBinary {
print "$info->{narHash}\n"; print "$info->{narHash}\n";
print STDERR "\n"; print STDERR "\n";
return 1; return;
} }
return 0; print STDERR "could not download ‘$storePath’ from any binary cache\n";
} }
@ -450,6 +478,7 @@ initCache();
if ($ARGV[0] eq "--query") { if ($ARGV[0] eq "--query") {
while (<STDIN>) { while (<STDIN>) {
getAvailableCaches;
chomp; chomp;
my ($cmd, @args) = split " ", $_; my ($cmd, @args) = split " ", $_;
@ -472,9 +501,8 @@ if ($ARGV[0] eq "--query") {
elsif ($ARGV[0] eq "--substitute") { elsif ($ARGV[0] eq "--substitute") {
my $storePath = $ARGV[1] or die; my $storePath = $ARGV[1] or die;
if (!downloadBinary($storePath)) { getAvailableCaches;
print STDERR "could not download ‘$storePath’ from any binary cache\n"; downloadBinary($storePath);
}
} }
else { else {

32
scripts/nix-push.in
View file

@ -61,7 +61,7 @@ for (my $n = 0; $n < scalar @ARGV; $n++) {
push @roots, $arg; push @roots, $arg;
} }
} }
showSyntax if !defined $destDir; showSyntax if !defined $destDir;
$archivesURL = "file://$destDir" unless defined $archivesURL; $archivesURL = "file://$destDir" unless defined $archivesURL;
@ -74,12 +74,12 @@ my %storePaths;
foreach my $path (@roots) { foreach my $path (@roots) {
die unless $path =~ /^\//; die unless $path =~ /^\//;
# Get all paths referenced by the normalisation of the given # Get all paths referenced by the normalisation of the given
# Nix expression. # Nix expression.
my $pid = open(READ, my $pid = open(READ,
"$Nix::Config::binDir/nix-store --query --requisites --force-realise " . "$Nix::Config::binDir/nix-store --query --requisites --force-realise " .
"--include-outputs '$path'|") or die; "--include-outputs '$path'|") or die;
while (<READ>) { while (<READ>) {
chomp; chomp;
die "bad: $_" unless /^\//; die "bad: $_" unless /^\//;
@ -101,10 +101,10 @@ foreach my $storePath (@storePaths) {
die unless ($storePath =~ /\/[0-9a-z]{32}[^\"\\\$]*$/); die unless ($storePath =~ /\/[0-9a-z]{32}[^\"\\\$]*$/);
# Construct a Nix expression that creates a Nix archive. # Construct a Nix expression that creates a Nix archive.
my $nixexpr = my $nixexpr =
"(import <nix/nar.nix> " . "(import <nix/nar.nix> " .
"{ storePath = builtins.storePath \"$storePath\"; hashAlgo = \"sha256\"; compressionType = \"$compressionType\"; }) "; "{ storePath = builtins.storePath \"$storePath\"; hashAlgo = \"sha256\"; compressionType = \"$compressionType\"; }) ";
print NIX $nixexpr; print NIX $nixexpr;
} }
@ -125,7 +125,17 @@ while (<READ>) {
close READ or die "nix-build failed: $?"; close READ or die "nix-build failed: $?";
# Copy the archives and the corresponding info files. # Write the cache info file.
my $cacheInfoFile = "$destDir/nix-cache-info";
if (! -e $cacheInfoFile) {
open FILE, ">$cacheInfoFile" or die "cannot create $cacheInfoFile: $!";
print FILE "StoreDir: $Nix::Config::storeDir\n";
print FILE "WantMassQuery: 0\n"; # by default, don't hit this cache for "nix-env -qas"
close FILE;
}
# Copy the archives and the corresponding NAR info files.
print STDERR "copying archives...\n"; print STDERR "copying archives...\n";
my $totalNarSize = 0; my $totalNarSize = 0;
@ -157,7 +167,7 @@ for (my $n = 0; $n < scalar @storePaths; $n++) {
} }
$totalNarSize += $narSize; $totalNarSize += $narSize;
# Get info about the compressed NAR. # Get info about the compressed NAR.
open HASH, "$narDir/nar-compressed-hash" or die "cannot open nar-compressed-hash"; open HASH, "$narDir/nar-compressed-hash" or die "cannot open nar-compressed-hash";
my $compressedHash = <HASH>; my $compressedHash = <HASH>;
@ -170,7 +180,7 @@ for (my $n = 0; $n < scalar @storePaths; $n++) {
my $narFile = "$narDir/$narName"; my $narFile = "$narDir/$narName";
(-f $narFile) or die "NAR file for $storePath not found"; (-f $narFile) or die "NAR file for $storePath not found";
my $compressedSize = stat($narFile)->size; my $compressedSize = stat($narFile)->size;
$totalCompressedSize += $compressedSize; $totalCompressedSize += $compressedSize;
printf STDERR "%s [%.2f MiB, %.1f%%]\n", $storePath, printf STDERR "%s [%.2f MiB, %.1f%%]\n", $storePath,
@ -203,7 +213,7 @@ for (my $n = 0; $n < scalar @storePaths; $n++) {
} }
my $pathHash = substr(basename($storePath), 0, 32); my $pathHash = substr(basename($storePath), 0, 32);
$dst = "$destDir/$pathHash.narinfo"; $dst = "$destDir/$pathHash.narinfo";
if ($force || ! -f $dst) { if ($force || ! -f $dst) {
my $tmp = "$destDir/.tmp.$$.$pathHash.narinfo"; my $tmp = "$destDir/.tmp.$$.$pathHash.narinfo";
@ -230,6 +240,4 @@ printf STDERR "total compressed size %.2f MiB, %.1f%%\n",
# Optionally write a manifest. # Optionally write a manifest.
if ($writeManifest) { writeManifest "$destDir/MANIFEST", \%narFiles, \() if $writeManifest;
writeManifest "$destDir/MANIFEST", \%narFiles, \();
}

15
tests/binary-cache.sh
View file

@ -10,13 +10,26 @@ outPath=$(nix-build dependencies.nix --no-out-link)
nix-push --dest $cacheDir $outPath nix-push --dest $cacheDir $outPath
# Check that downloading works.
# By default, a binary cache doesn't support "nix-env -qas", but does
# support installation.
clearStore clearStore
rm -f $NIX_STATE_DIR/binary-cache* rm -f $NIX_STATE_DIR/binary-cache*
NIX_BINARY_CACHES="file://$cacheDir" nix-env -f dependencies.nix -qas \* | grep -- "---"
NIX_BINARY_CACHES="file://$cacheDir" nix-store -r $outPath
# But with the right configuration, "nix-env -qas" should also work.
clearStore
rm -f $NIX_STATE_DIR/binary-cache*
echo "WantMassQuery: 1" >> $cacheDir/nix-cache-info
NIX_BINARY_CACHES="file://$cacheDir" nix-env -f dependencies.nix -qas \* | grep -- "--S" NIX_BINARY_CACHES="file://$cacheDir" nix-env -f dependencies.nix -qas \* | grep -- "--S"
NIX_BINARY_CACHES="file://$cacheDir" nix-store -r $outPath NIX_BINARY_CACHES="file://$cacheDir" nix-store -r $outPath
nix-store --check-validity $outPath nix-store --check-validity $outPath
nix-store -qR $outPath | grep input-2 nix-store -qR $outPath | grep input-2