If fallback is enabled, temporarily disable substituters after a failure

Otherwise, we just keep asking the substituter for other .narinfo
files, which can take a very long time due to retries/timeouts.
This commit is contained in:
Eelco Dolstra 2018-09-07 17:08:43 +02:00
parent 33c3f91885
commit bba3f0a308
No known key found for this signature in database
GPG key ID: 8170B4726D7198DE
4 changed files with 48 additions and 1 deletions

View file

@ -3682,6 +3682,12 @@ void SubstitutionGoal::tryNext()
} catch (InvalidPath &) {
tryNext();
return;
} catch (SubstituterDisabled &) {
if (settings.tryFallback) {
tryNext();
return;
}
throw;
} catch (Error & e) {
if (settings.tryFallback) {
printError(e.what());

View file

@ -13,6 +13,14 @@ private:
Path cacheUri;
struct State
{
bool enabled = true;
std::chrono::steady_clock::time_point disabledUntil;
};
Sync<State> _state;
public:
HttpBinaryCacheStore(
@ -46,8 +54,33 @@ public:
protected:
void maybeDisable()
{
auto state(_state.lock());
if (state->enabled && settings.tryFallback) {
int t = 60;
printError("disabling binary cache '%s' for %s seconds", getUri(), t);
state->enabled = false;
state->disabledUntil = std::chrono::steady_clock::now() + std::chrono::seconds(t);
}
}
void checkEnabled()
{
auto state(_state.lock());
if (state->enabled) return;
if (std::chrono::steady_clock::now() > state->disabledUntil) {
state->enabled = true;
debug("re-enabling binary cache '%s'", getUri());
return;
}
throw SubstituterDisabled("substituter '%s' is disabled", getUri());
}
bool fileExists(const std::string & path) override
{
checkEnabled();
try {
DownloadRequest request(cacheUri + "/" + path);
request.head = true;
@ -59,6 +92,7 @@ protected:
bucket is unlistable, so treat 403 as 404. */
if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
return false;
maybeDisable();
throw;
}
}
@ -86,12 +120,14 @@ protected:
void getFile(const std::string & path, Sink & sink) override
{
checkEnabled();
auto request(makeRequest(path));
try {
getDownloader()->download(std::move(request), sink);
} catch (DownloadError & e) {
if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
maybeDisable();
throw;
}
}
@ -99,15 +135,18 @@ protected:
void getFile(const std::string & path,
Callback<std::shared_ptr<std::string>> callback) override
{
checkEnabled();
auto request(makeRequest(path));
getDownloader()->enqueueDownload(request,
{[callback](std::future<DownloadResult> result) {
{[callback, this](std::future<DownloadResult> result) {
try {
callback(result.get().data);
} catch (DownloadError & e) {
if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
return callback(std::shared_ptr<std::string>());
maybeDisable();
callback.rethrow();
} catch (...) {
callback.rethrow();

View file

@ -880,6 +880,7 @@ void LocalStore::querySubstitutablePathInfos(const PathSet & paths,
narInfo ? narInfo->fileSize : 0,
info->narSize};
} catch (InvalidPath) {
} catch (SubstituterDisabled) {
} catch (Error & e) {
if (settings.tryFallback)
printError(e.what());

View file

@ -23,6 +23,7 @@ MakeError(BuildError, Error) /* denotes a permanent build failure */
MakeError(InvalidPath, Error)
MakeError(Unsupported, Error)
MakeError(SubstituteGone, Error)
MakeError(SubstituterDisabled, Error)
struct BasicDerivation;