forked from lix-project/lix
75989bdca7
The fact that queryPathInfo() is synchronous meant that we needed a thread for every concurrent binary cache lookup, even though they end up being handled by the same download thread. Requiring hundreds of threads is not a good idea. So now there is an asynchronous version of queryPathInfo() that takes a callback function to process the result. Similarly, enqueueDownload() now takes a callback rather than returning a future. Thus, a command like nix path-info --store https://cache.nixos.org/ -r /nix/store/slljrzwmpygy1daay14kjszsr9xix063-nixos-16.09beta231.dccf8c5 that returns 4941 paths now takes 1.87s using only 2 threads (the main thread and the downloader thread). (This is with a prewarmed CloudFront.)
112 lines
3.1 KiB
C++
112 lines
3.1 KiB
C++
#include "binary-cache-store.hh"
|
||
#include "download.hh"
|
||
#include "globals.hh"
|
||
#include "nar-info-disk-cache.hh"
|
||
|
||
namespace nix {
|
||
|
||
MakeError(UploadToHTTP, Error);
|
||
|
||
class HttpBinaryCacheStore : public BinaryCacheStore
|
||
{
|
||
private:
|
||
|
||
Path cacheUri;
|
||
|
||
public:
|
||
|
||
HttpBinaryCacheStore(
|
||
const Params & params, const Path & _cacheUri)
|
||
: BinaryCacheStore(params)
|
||
, cacheUri(_cacheUri)
|
||
{
|
||
if (cacheUri.back() == '/')
|
||
cacheUri.pop_back();
|
||
|
||
diskCache = getNarInfoDiskCache();
|
||
}
|
||
|
||
std::string getUri() override
|
||
{
|
||
return cacheUri;
|
||
}
|
||
|
||
void init() override
|
||
{
|
||
// FIXME: do this lazily?
|
||
if (!diskCache->cacheExists(cacheUri, wantMassQuery_, priority)) {
|
||
try {
|
||
BinaryCacheStore::init();
|
||
} catch (UploadToHTTP &) {
|
||
throw Error(format("‘%s’ does not appear to be a binary cache") % cacheUri);
|
||
}
|
||
diskCache->createCache(cacheUri, storeDir, wantMassQuery_, priority);
|
||
}
|
||
}
|
||
|
||
protected:
|
||
|
||
bool fileExists(const std::string & path) override
|
||
{
|
||
try {
|
||
DownloadRequest request(cacheUri + "/" + path);
|
||
request.showProgress = DownloadRequest::no;
|
||
request.head = true;
|
||
request.tries = 5;
|
||
getDownloader()->download(request);
|
||
return true;
|
||
} catch (DownloadError & e) {
|
||
/* S3 buckets return 403 if a file doesn't exist and the
|
||
bucket is unlistable, so treat 403 as 404. */
|
||
if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
|
||
return false;
|
||
throw;
|
||
}
|
||
}
|
||
|
||
void upsertFile(const std::string & path, const std::string & data) override
|
||
{
|
||
throw UploadToHTTP("uploading to an HTTP binary cache is not supported");
|
||
}
|
||
|
||
void getFile(const std::string & path,
|
||
std::function<void(std::shared_ptr<std::string>)> success,
|
||
std::function<void(std::exception_ptr exc)> failure)
|
||
{
|
||
DownloadRequest request(cacheUri + "/" + path);
|
||
request.showProgress = DownloadRequest::no;
|
||
request.tries = 8;
|
||
|
||
getDownloader()->enqueueDownload(request,
|
||
[success](const DownloadResult & result) {
|
||
success(result.data);
|
||
},
|
||
[success, failure](std::exception_ptr exc) {
|
||
try {
|
||
std::rethrow_exception(exc);
|
||
} catch (DownloadError & e) {
|
||
if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
|
||
success(0);
|
||
failure(exc);
|
||
}
|
||
});
|
||
}
|
||
|
||
};
|
||
|
||
static RegisterStoreImplementation regStore([](
|
||
const std::string & uri, const Store::Params & params)
|
||
-> std::shared_ptr<Store>
|
||
{
|
||
if (std::string(uri, 0, 7) != "http://" &&
|
||
std::string(uri, 0, 8) != "https://" &&
|
||
(getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") != "1" || std::string(uri, 0, 7) != "file://")
|
||
) return 0;
|
||
auto store = std::make_shared<HttpBinaryCacheStore>(params, uri);
|
||
store->init();
|
||
return store;
|
||
});
|
||
|
||
}
|
||
|