HttpBinaryCacheStore: Retry on transient HTTP errors

This makes us more robust against 500 errors from CloudFront or S3
(assuming the 500 error isn't cached by CloudFront...).
This commit is contained in:
Eelco Dolstra 2016-08-10 16:06:33 +02:00
parent 9204ea7294
commit 66adbdfd97
3 changed files with 34 additions and 14 deletions

View file

@ -8,6 +8,7 @@
#include <curl/curl.h> #include <curl/curl.h>
#include <iostream> #include <iostream>
#include <thread>
namespace nix { namespace nix {
@ -194,7 +195,11 @@ struct CurlDownloader : public Downloader
if (res != CURLE_OK) { if (res != CURLE_OK) {
Error err = Error err =
httpStatus == 404 ? NotFound : httpStatus == 404 ? NotFound :
httpStatus == 403 ? Forbidden : Misc; httpStatus == 403 ? Forbidden :
(httpStatus == 408 || httpStatus == 500 || httpStatus == 503
|| httpStatus == 504 || httpStatus == 522 || httpStatus == 524
|| res == CURLE_COULDNT_RESOLVE_HOST) ? Transient :
Misc;
if (res == CURLE_HTTP_RETURNED_ERROR && httpStatus != -1) if (res == CURLE_HTTP_RETURNED_ERROR && httpStatus != -1)
throw DownloadError(err, format("unable to download %s: HTTP error %d") throw DownloadError(err, format("unable to download %s: HTTP error %d")
% url % httpStatus); % url % httpStatus);
@ -210,6 +215,10 @@ struct CurlDownloader : public Downloader
DownloadResult download(string url, const DownloadOptions & options) override DownloadResult download(string url, const DownloadOptions & options) override
{ {
size_t attempt = 0;
while (true) {
try {
DownloadResult res; DownloadResult res;
if (fetch(resolveUri(url), options)) { if (fetch(resolveUri(url), options)) {
res.cached = false; res.cached = false;
@ -218,6 +227,14 @@ struct CurlDownloader : public Downloader
res.cached = true; res.cached = true;
res.etag = etag; res.etag = etag;
return res; return res;
} catch (DownloadError & e) {
attempt++;
if (e.error != Transient || attempt >= options.tries) throw;
auto ms = 25 * (1 << (attempt - 1));
printMsg(lvlError, format("warning: %s; retrying in %d ms") % e.what() % ms);
std::this_thread::sleep_for(std::chrono::milliseconds(ms));
}
}
} }
}; };

View file

@ -9,10 +9,11 @@ namespace nix {
struct DownloadOptions struct DownloadOptions
{ {
string expectedETag; std::string expectedETag;
bool verifyTLS{true}; bool verifyTLS = true;
enum { yes, no, automatic } showProgress{yes}; enum { yes, no, automatic } showProgress = yes;
bool head{false}; bool head = false;
size_t tries = 1;
}; };
struct DownloadResult struct DownloadResult
@ -31,7 +32,7 @@ struct Downloader
Path downloadCached(ref<Store> store, const string & url, bool unpack, Path downloadCached(ref<Store> store, const string & url, bool unpack,
const Hash & expectedHash = Hash()); const Hash & expectedHash = Hash());
enum Error { NotFound, Forbidden, Misc }; enum Error { NotFound, Forbidden, Misc, Transient };
}; };
ref<Downloader> makeDownloader(); ref<Downloader> makeDownloader();

View file

@ -58,6 +58,7 @@ protected:
DownloadOptions options; DownloadOptions options;
options.showProgress = DownloadOptions::no; options.showProgress = DownloadOptions::no;
options.head = true; options.head = true;
options.tries = 5;
downloader->download(cacheUri + "/" + path, options); downloader->download(cacheUri + "/" + path, options);
return true; return true;
} catch (DownloadError & e) { } catch (DownloadError & e) {
@ -79,6 +80,7 @@ protected:
auto downloader(downloaders.get()); auto downloader(downloaders.get());
DownloadOptions options; DownloadOptions options;
options.showProgress = DownloadOptions::no; options.showProgress = DownloadOptions::no;
options.tries = 3;
try { try {
return downloader->download(cacheUri + "/" + path, options).data; return downloader->download(cacheUri + "/" + path, options).data;
} catch (DownloadError & e) { } catch (DownloadError & e) {