HttpBinaryCacheStore: Retry on transient HTTP errors
This makes us more robust against 500 errors from CloudFront or S3 (assuming the 500 error isn't cached by CloudFront...).
This commit is contained in:
parent
9204ea7294
commit
66adbdfd97
|
@ -8,6 +8,7 @@
|
||||||
#include <curl/curl.h>
|
#include <curl/curl.h>
|
||||||
|
|
||||||
#include <iostream>
|
#include <iostream>
|
||||||
|
#include <thread>
|
||||||
|
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
@ -194,7 +195,11 @@ struct CurlDownloader : public Downloader
|
||||||
if (res != CURLE_OK) {
|
if (res != CURLE_OK) {
|
||||||
Error err =
|
Error err =
|
||||||
httpStatus == 404 ? NotFound :
|
httpStatus == 404 ? NotFound :
|
||||||
httpStatus == 403 ? Forbidden : Misc;
|
httpStatus == 403 ? Forbidden :
|
||||||
|
(httpStatus == 408 || httpStatus == 500 || httpStatus == 503
|
||||||
|
|| httpStatus == 504 || httpStatus == 522 || httpStatus == 524
|
||||||
|
|| res == CURLE_COULDNT_RESOLVE_HOST) ? Transient :
|
||||||
|
Misc;
|
||||||
if (res == CURLE_HTTP_RETURNED_ERROR && httpStatus != -1)
|
if (res == CURLE_HTTP_RETURNED_ERROR && httpStatus != -1)
|
||||||
throw DownloadError(err, format("unable to download ‘%s’: HTTP error %d")
|
throw DownloadError(err, format("unable to download ‘%s’: HTTP error %d")
|
||||||
% url % httpStatus);
|
% url % httpStatus);
|
||||||
|
@ -210,6 +215,10 @@ struct CurlDownloader : public Downloader
|
||||||
|
|
||||||
DownloadResult download(string url, const DownloadOptions & options) override
|
DownloadResult download(string url, const DownloadOptions & options) override
|
||||||
{
|
{
|
||||||
|
size_t attempt = 0;
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
try {
|
||||||
DownloadResult res;
|
DownloadResult res;
|
||||||
if (fetch(resolveUri(url), options)) {
|
if (fetch(resolveUri(url), options)) {
|
||||||
res.cached = false;
|
res.cached = false;
|
||||||
|
@ -218,6 +227,14 @@ struct CurlDownloader : public Downloader
|
||||||
res.cached = true;
|
res.cached = true;
|
||||||
res.etag = etag;
|
res.etag = etag;
|
||||||
return res;
|
return res;
|
||||||
|
} catch (DownloadError & e) {
|
||||||
|
attempt++;
|
||||||
|
if (e.error != Transient || attempt >= options.tries) throw;
|
||||||
|
auto ms = 25 * (1 << (attempt - 1));
|
||||||
|
printMsg(lvlError, format("warning: %s; retrying in %d ms") % e.what() % ms);
|
||||||
|
std::this_thread::sleep_for(std::chrono::milliseconds(ms));
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -9,10 +9,11 @@ namespace nix {
|
||||||
|
|
||||||
struct DownloadOptions
|
struct DownloadOptions
|
||||||
{
|
{
|
||||||
string expectedETag;
|
std::string expectedETag;
|
||||||
bool verifyTLS{true};
|
bool verifyTLS = true;
|
||||||
enum { yes, no, automatic } showProgress{yes};
|
enum { yes, no, automatic } showProgress = yes;
|
||||||
bool head{false};
|
bool head = false;
|
||||||
|
size_t tries = 1;
|
||||||
};
|
};
|
||||||
|
|
||||||
struct DownloadResult
|
struct DownloadResult
|
||||||
|
@ -31,7 +32,7 @@ struct Downloader
|
||||||
Path downloadCached(ref<Store> store, const string & url, bool unpack,
|
Path downloadCached(ref<Store> store, const string & url, bool unpack,
|
||||||
const Hash & expectedHash = Hash());
|
const Hash & expectedHash = Hash());
|
||||||
|
|
||||||
enum Error { NotFound, Forbidden, Misc };
|
enum Error { NotFound, Forbidden, Misc, Transient };
|
||||||
};
|
};
|
||||||
|
|
||||||
ref<Downloader> makeDownloader();
|
ref<Downloader> makeDownloader();
|
||||||
|
|
|
@ -58,6 +58,7 @@ protected:
|
||||||
DownloadOptions options;
|
DownloadOptions options;
|
||||||
options.showProgress = DownloadOptions::no;
|
options.showProgress = DownloadOptions::no;
|
||||||
options.head = true;
|
options.head = true;
|
||||||
|
options.tries = 5;
|
||||||
downloader->download(cacheUri + "/" + path, options);
|
downloader->download(cacheUri + "/" + path, options);
|
||||||
return true;
|
return true;
|
||||||
} catch (DownloadError & e) {
|
} catch (DownloadError & e) {
|
||||||
|
@ -79,6 +80,7 @@ protected:
|
||||||
auto downloader(downloaders.get());
|
auto downloader(downloaders.get());
|
||||||
DownloadOptions options;
|
DownloadOptions options;
|
||||||
options.showProgress = DownloadOptions::no;
|
options.showProgress = DownloadOptions::no;
|
||||||
|
options.tries = 3;
|
||||||
try {
|
try {
|
||||||
return downloader->download(cacheUri + "/" + path, options).data;
|
return downloader->download(cacheUri + "/" + path, options).data;
|
||||||
} catch (DownloadError & e) {
|
} catch (DownloadError & e) {
|
||||||
|
|
Loading…
Reference in a new issue