2016-03-03 17:03:34 +00:00
|
|
|
#include "binary-cache-store.hh"
|
2020-04-06 21:57:28 +00:00
|
|
|
#include "filetransfer.hh"
|
2016-03-04 16:23:42 +00:00
|
|
|
#include "globals.hh"
|
2016-04-20 12:12:38 +00:00
|
|
|
#include "nar-info-disk-cache.hh"
|
2020-09-21 16:40:11 +00:00
|
|
|
#include "callback.hh"
|
2016-03-03 17:03:34 +00:00
|
|
|
|
|
|
|
namespace nix {
|
|
|
|
|
2016-05-30 11:33:05 +00:00
|
|
|
MakeError(UploadToHTTP, Error);
|
|
|
|
|
2020-09-10 08:55:51 +00:00
|
|
|
struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
|
|
|
|
{
|
|
|
|
using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
|
2020-09-14 12:04:02 +00:00
|
|
|
|
|
|
|
const std::string name() override { return "Http Binary Cache Store"; }
|
2020-09-10 08:55:51 +00:00
|
|
|
};
|
|
|
|
|
2020-12-20 15:33:12 +00:00
|
|
|
class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public virtual BinaryCacheStore
|
2016-03-03 17:03:34 +00:00
|
|
|
{
|
|
|
|
private:
|
|
|
|
|
|
|
|
Path cacheUri;
|
|
|
|
|
2018-09-07 15:08:43 +00:00
|
|
|
struct State
|
|
|
|
{
|
|
|
|
bool enabled = true;
|
|
|
|
std::chrono::steady_clock::time_point disabledUntil;
|
|
|
|
};
|
|
|
|
|
|
|
|
Sync<State> _state;
|
|
|
|
|
2016-03-03 17:03:34 +00:00
|
|
|
public:
|
|
|
|
|
2016-05-04 18:15:41 +00:00
|
|
|
HttpBinaryCacheStore(
|
2020-09-11 09:11:05 +00:00
|
|
|
const std::string & scheme,
|
2020-09-08 12:50:23 +00:00
|
|
|
const Path & _cacheUri,
|
|
|
|
const Params & params)
|
2020-09-11 09:06:18 +00:00
|
|
|
: StoreConfig(params)
|
2020-12-20 15:33:12 +00:00
|
|
|
, BinaryCacheStoreConfig(params)
|
|
|
|
, HttpBinaryCacheStoreConfig(params)
|
|
|
|
, Store(params)
|
2020-09-11 09:06:18 +00:00
|
|
|
, BinaryCacheStore(params)
|
2020-09-11 09:11:05 +00:00
|
|
|
, cacheUri(scheme + "://" + _cacheUri)
|
2016-03-03 17:03:34 +00:00
|
|
|
{
|
|
|
|
if (cacheUri.back() == '/')
|
|
|
|
cacheUri.pop_back();
|
2016-04-20 12:12:38 +00:00
|
|
|
|
|
|
|
diskCache = getNarInfoDiskCache();
|
|
|
|
}
|
|
|
|
|
|
|
|
std::string getUri() override
|
|
|
|
{
|
|
|
|
return cacheUri;
|
2016-03-03 17:03:34 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void init() override
|
|
|
|
{
|
|
|
|
// FIXME: do this lazily?
|
2019-12-17 16:17:53 +00:00
|
|
|
if (auto cacheInfo = diskCache->cacheExists(cacheUri)) {
|
2021-09-22 12:15:35 +00:00
|
|
|
wantMassQuery.setDefault(cacheInfo->wantMassQuery);
|
|
|
|
priority.setDefault(cacheInfo->priority);
|
2019-12-17 16:17:53 +00:00
|
|
|
} else {
|
2016-05-30 11:33:05 +00:00
|
|
|
try {
|
|
|
|
BinaryCacheStore::init();
|
|
|
|
} catch (UploadToHTTP &) {
|
2018-01-31 14:12:27 +00:00
|
|
|
throw Error("'%s' does not appear to be a binary cache", cacheUri);
|
2016-05-30 11:33:05 +00:00
|
|
|
}
|
2019-12-17 16:17:53 +00:00
|
|
|
diskCache->createCache(cacheUri, storeDir, wantMassQuery, priority);
|
2016-04-20 12:12:38 +00:00
|
|
|
}
|
2016-03-03 17:03:34 +00:00
|
|
|
}
|
|
|
|
|
2020-09-11 09:11:05 +00:00
|
|
|
static std::set<std::string> uriSchemes()
|
2020-09-08 12:50:23 +00:00
|
|
|
{
|
|
|
|
static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1";
|
2020-09-11 09:11:05 +00:00
|
|
|
auto ret = std::set<std::string>({"http", "https"});
|
|
|
|
if (forceHttp) ret.insert("file");
|
2020-09-08 12:50:23 +00:00
|
|
|
return ret;
|
|
|
|
}
|
2020-10-05 15:53:30 +00:00
|
|
|
|
2016-03-03 17:03:34 +00:00
|
|
|
protected:
|
|
|
|
|
2018-09-07 15:08:43 +00:00
|
|
|
void maybeDisable()
|
|
|
|
{
|
|
|
|
auto state(_state.lock());
|
|
|
|
if (state->enabled && settings.tryFallback) {
|
|
|
|
int t = 60;
|
|
|
|
printError("disabling binary cache '%s' for %s seconds", getUri(), t);
|
|
|
|
state->enabled = false;
|
|
|
|
state->disabledUntil = std::chrono::steady_clock::now() + std::chrono::seconds(t);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void checkEnabled()
|
|
|
|
{
|
|
|
|
auto state(_state.lock());
|
|
|
|
if (state->enabled) return;
|
|
|
|
if (std::chrono::steady_clock::now() > state->disabledUntil) {
|
|
|
|
state->enabled = true;
|
|
|
|
debug("re-enabling binary cache '%s'", getUri());
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
throw SubstituterDisabled("substituter '%s' is disabled", getUri());
|
|
|
|
}
|
|
|
|
|
2016-03-03 17:03:34 +00:00
|
|
|
bool fileExists(const std::string & path) override
|
|
|
|
{
|
2018-09-07 15:08:43 +00:00
|
|
|
checkEnabled();
|
|
|
|
|
2016-03-03 17:03:34 +00:00
|
|
|
try {
|
2020-09-01 14:41:42 +00:00
|
|
|
FileTransferRequest request(makeRequest(path));
|
2016-09-14 14:00:40 +00:00
|
|
|
request.head = true;
|
2020-04-06 21:43:43 +00:00
|
|
|
getFileTransfer()->download(request);
|
2016-03-03 17:03:34 +00:00
|
|
|
return true;
|
2020-04-06 21:43:43 +00:00
|
|
|
} catch (FileTransferError & e) {
|
2016-03-30 09:17:51 +00:00
|
|
|
/* S3 buckets return 403 if a file doesn't exist and the
|
|
|
|
bucket is unlistable, so treat 403 as 404. */
|
2020-04-06 21:43:43 +00:00
|
|
|
if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
|
2016-03-03 17:03:34 +00:00
|
|
|
return false;
|
2018-09-07 15:08:43 +00:00
|
|
|
maybeDisable();
|
2016-03-03 17:03:34 +00:00
|
|
|
throw;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-03-14 14:26:01 +00:00
|
|
|
void upsertFile(const std::string & path,
|
2020-07-13 18:07:19 +00:00
|
|
|
std::shared_ptr<std::basic_iostream<char>> istream,
|
2017-03-14 14:26:01 +00:00
|
|
|
const std::string & mimeType) override
|
2016-03-03 17:03:34 +00:00
|
|
|
{
|
2020-09-01 14:41:42 +00:00
|
|
|
auto req = makeRequest(path);
|
2022-01-17 21:20:05 +00:00
|
|
|
req.data = StreamToSourceAdapter(istream).drain();
|
2018-01-31 14:12:27 +00:00
|
|
|
req.mimeType = mimeType;
|
2018-01-26 19:12:30 +00:00
|
|
|
try {
|
2020-04-06 21:43:43 +00:00
|
|
|
getFileTransfer()->upload(req);
|
|
|
|
} catch (FileTransferError & e) {
|
2018-06-01 12:14:22 +00:00
|
|
|
throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg());
|
2018-01-26 19:12:30 +00:00
|
|
|
}
|
2016-03-03 17:03:34 +00:00
|
|
|
}
|
|
|
|
|
2020-04-06 21:43:43 +00:00
|
|
|
FileTransferRequest makeRequest(const std::string & path)
|
2016-03-03 17:03:34 +00:00
|
|
|
{
|
2020-09-01 14:41:42 +00:00
|
|
|
return FileTransferRequest(
|
|
|
|
hasPrefix(path, "https://") || hasPrefix(path, "http://") || hasPrefix(path, "file://")
|
|
|
|
? path
|
|
|
|
: cacheUri + "/" + path);
|
|
|
|
|
2018-03-27 22:01:47 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void getFile(const std::string & path, Sink & sink) override
|
|
|
|
{
|
2018-09-07 15:08:43 +00:00
|
|
|
checkEnabled();
|
2018-03-27 22:01:47 +00:00
|
|
|
auto request(makeRequest(path));
|
|
|
|
try {
|
2020-04-06 21:43:43 +00:00
|
|
|
getFileTransfer()->download(std::move(request), sink);
|
|
|
|
} catch (FileTransferError & e) {
|
|
|
|
if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
|
2018-03-27 22:01:47 +00:00
|
|
|
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
|
2018-09-07 15:08:43 +00:00
|
|
|
maybeDisable();
|
2018-03-27 22:01:47 +00:00
|
|
|
throw;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void getFile(const std::string & path,
|
2022-01-17 21:20:05 +00:00
|
|
|
Callback<std::optional<std::string>> callback) noexcept override
|
2018-03-27 22:01:47 +00:00
|
|
|
{
|
2022-05-30 11:24:04 +00:00
|
|
|
try {
|
|
|
|
checkEnabled();
|
|
|
|
} catch (...) {
|
|
|
|
callback.rethrow();
|
|
|
|
return;
|
|
|
|
}
|
2018-09-07 15:08:43 +00:00
|
|
|
|
2019-07-10 17:46:15 +00:00
|
|
|
auto request(makeRequest(path));
|
|
|
|
|
2019-09-03 10:51:35 +00:00
|
|
|
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
|
|
|
|
|
2020-04-06 21:43:43 +00:00
|
|
|
getFileTransfer()->enqueueFileTransfer(request,
|
|
|
|
{[callbackPtr, this](std::future<FileTransferResult> result) {
|
2019-07-10 17:46:15 +00:00
|
|
|
try {
|
2022-01-17 21:20:05 +00:00
|
|
|
(*callbackPtr)(std::move(result.get().data));
|
2020-04-06 21:43:43 +00:00
|
|
|
} catch (FileTransferError & e) {
|
|
|
|
if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
|
2022-01-17 21:20:05 +00:00
|
|
|
return (*callbackPtr)({});
|
2019-07-10 17:46:15 +00:00
|
|
|
maybeDisable();
|
2019-09-03 10:51:35 +00:00
|
|
|
callbackPtr->rethrow();
|
2019-07-10 17:46:15 +00:00
|
|
|
} catch (...) {
|
2019-09-03 10:51:35 +00:00
|
|
|
callbackPtr->rethrow();
|
2019-07-10 17:46:15 +00:00
|
|
|
}
|
|
|
|
}});
|
2016-03-03 17:03:34 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
};
|
|
|
|
|
2020-10-06 11:36:55 +00:00
|
|
|
static RegisterStoreImplementation<HttpBinaryCacheStore, HttpBinaryCacheStoreConfig> regHttpBinaryCacheStore;
|
2016-03-03 17:03:34 +00:00
|
|
|
|
|
|
|
}
|