raito/hydra
0
0
Fork 0
forked from lix-project/hydra
Code Pull requests Activity

S3BinaryCacheStore: Use disk cache

Browse source
This commit is contained in:
Eelco Dolstra 2016-04-20 15:29:40 +02:00
parent b2b978eda0
commit b50a105ca7
4 changed files with 83 additions and 71 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/hydra-queue-runner/build-result.cc
View file

@ -21,8 +21,8 @@ BuildOutput getBuildOutput(nix::ref<Store> store,
store->computeFSClosure(output, closure); store->computeFSClosure(output, closure);
for (auto & path : closure) { for (auto & path : closure) {
auto info = store->queryPathInfo(path); auto info = store->queryPathInfo(path);
res.closureSize += info.narSize; res.closureSize += info->narSize;
if (outputs.find(path) != outputs.end()) res.size += info.narSize; if (outputs.find(path) != outputs.end()) res.size += info->narSize;
} }
/* Get build products. */ /* Get build products. */

98
src/hydra-queue-runner/hydra-queue-runner.cc
View file

@ -630,60 +630,58 @@ void State::dumpStatus(Connection & conn, bool log)
} }
} }
auto store = dynamic_cast<S3BinaryCacheStore *>(&*getDestStore()); auto store = getDestStore();
if (store) { root.attr("store");
root.attr("store"); JSONObject nested(out);
JSONObject nested(out);
auto & stats = store->getStats(); auto & stats = store->getStats();
nested.attr("narInfoRead", stats.narInfoRead); nested.attr("narInfoRead", stats.narInfoRead);
nested.attr("narInfoReadAverted", stats.narInfoReadAverted); nested.attr("narInfoReadAverted", stats.narInfoReadAverted);
nested.attr("narInfoMissing", stats.narInfoMissing); nested.attr("narInfoMissing", stats.narInfoMissing);
nested.attr("narInfoWrite", stats.narInfoWrite); nested.attr("narInfoWrite", stats.narInfoWrite);
nested.attr("narInfoCacheSize", stats.narInfoCacheSize); nested.attr("narInfoCacheSize", stats.pathInfoCacheSize);
nested.attr("narRead", stats.narRead); nested.attr("narRead", stats.narRead);
nested.attr("narReadBytes", stats.narReadBytes); nested.attr("narReadBytes", stats.narReadBytes);
nested.attr("narReadCompressedBytes", stats.narReadCompressedBytes); nested.attr("narReadCompressedBytes", stats.narReadCompressedBytes);
nested.attr("narWrite", stats.narWrite); nested.attr("narWrite", stats.narWrite);
nested.attr("narWriteAverted", stats.narWriteAverted); nested.attr("narWriteAverted", stats.narWriteAverted);
nested.attr("narWriteBytes", stats.narWriteBytes); nested.attr("narWriteBytes", stats.narWriteBytes);
nested.attr("narWriteCompressedBytes", stats.narWriteCompressedBytes); nested.attr("narWriteCompressedBytes", stats.narWriteCompressedBytes);
nested.attr("narWriteCompressionTimeMs", stats.narWriteCompressionTimeMs); nested.attr("narWriteCompressionTimeMs", stats.narWriteCompressionTimeMs);
nested.attr("narCompressionSavings", nested.attr("narCompressionSavings",
stats.narWriteBytes stats.narWriteBytes
? 1.0 - (double) stats.narWriteCompressedBytes / stats.narWriteBytes ? 1.0 - (double) stats.narWriteCompressedBytes / stats.narWriteBytes
: 0.0);
nested.attr("narCompressionSpeed", // MiB/s
stats.narWriteCompressionTimeMs
? (double) stats.narWriteBytes / stats.narWriteCompressionTimeMs * 1000.0 / (1024.0 * 1024.0)
: 0.0);
auto s3Store = dynamic_cast<S3BinaryCacheStore *>(&*store);
if (s3Store) {
nested.attr("s3");
JSONObject nested2(out);
auto & s3Stats = s3Store->getS3Stats();
nested2.attr("put", s3Stats.put);
nested2.attr("putBytes", s3Stats.putBytes);
nested2.attr("putTimeMs", s3Stats.putTimeMs);
nested2.attr("putSpeed",
s3Stats.putTimeMs
? (double) s3Stats.putBytes / s3Stats.putTimeMs * 1000.0 / (1024.0 * 1024.0)
: 0.0); : 0.0);
nested.attr("narCompressionSpeed", // MiB/s nested2.attr("get", s3Stats.get);
stats.narWriteCompressionTimeMs nested2.attr("getBytes", s3Stats.getBytes);
? (double) stats.narWriteBytes / stats.narWriteCompressionTimeMs * 1000.0 / (1024.0 * 1024.0) nested2.attr("getTimeMs", s3Stats.getTimeMs);
nested2.attr("getSpeed",
s3Stats.getTimeMs
? (double) s3Stats.getBytes / s3Stats.getTimeMs * 1000.0 / (1024.0 * 1024.0)
: 0.0); : 0.0);
nested2.attr("head", s3Stats.head);
auto s3Store = dynamic_cast<S3BinaryCacheStore *>(&*store); nested2.attr("costDollarApprox",
if (s3Store) { (s3Stats.get + s3Stats.head) / 10000.0 * 0.004
nested.attr("s3"); + s3Stats.put / 1000.0 * 0.005 +
JSONObject nested2(out); + s3Stats.getBytes / (1024.0 * 1024.0 * 1024.0) * 0.09);
auto & s3Stats = s3Store->getS3Stats();
nested2.attr("put", s3Stats.put);
nested2.attr("putBytes", s3Stats.putBytes);
nested2.attr("putTimeMs", s3Stats.putTimeMs);
nested2.attr("putSpeed",
s3Stats.putTimeMs
? (double) s3Stats.putBytes / s3Stats.putTimeMs * 1000.0 / (1024.0 * 1024.0)
: 0.0);
nested2.attr("get", s3Stats.get);
nested2.attr("getBytes", s3Stats.getBytes);
nested2.attr("getTimeMs", s3Stats.getTimeMs);
nested2.attr("getSpeed",
s3Stats.getTimeMs
? (double) s3Stats.getBytes / s3Stats.getTimeMs * 1000.0 / (1024.0 * 1024.0)
: 0.0);
nested2.attr("head", s3Stats.head);
nested2.attr("costDollarApprox",
(s3Stats.get + s3Stats.head) / 10000.0 * 0.004
+ s3Stats.put / 1000.0 * 0.005 +
+ s3Stats.getBytes / (1024.0 * 1024.0 * 1024.0) * 0.09);
}
} }
} }

48
src/hydra-queue-runner/s3-binary-cache-store.cc
View file

@ -1,6 +1,7 @@
#include "s3-binary-cache-store.hh" #include "s3-binary-cache-store.hh"
#include "nar-info.hh" #include "nar-info.hh"
#include "nar-info-disk-cache.hh"
#include <aws/core/client/ClientConfiguration.h> #include <aws/core/client/ClientConfiguration.h>
#include <aws/s3/S3Client.h> #include <aws/s3/S3Client.h>
@ -38,6 +39,12 @@ S3BinaryCacheStore::S3BinaryCacheStore(std::shared_ptr<Store> localStore,
, config(makeConfig()) , config(makeConfig())
, client(make_ref<Aws::S3::S3Client>(*config)) , client(make_ref<Aws::S3::S3Client>(*config))
{ {
diskCache = getNarInfoDiskCache();
}
std::string S3BinaryCacheStore::getUri()
{
return "s3://" + bucketName;
} }
ref<Aws::Client::ClientConfiguration> S3BinaryCacheStore::makeConfig() ref<Aws::Client::ClientConfiguration> S3BinaryCacheStore::makeConfig()
@ -50,24 +57,29 @@ ref<Aws::Client::ClientConfiguration> S3BinaryCacheStore::makeConfig()
void S3BinaryCacheStore::init() void S3BinaryCacheStore::init()
{ {
/* Create the bucket if it doesn't already exists. */ if (!diskCache->cacheExists(getUri())) {
// FIXME: HeadBucket would be more appropriate, but doesn't return
// an easily parsed 404 message.
auto res = client->GetBucketLocation(
Aws::S3::Model::GetBucketLocationRequest().WithBucket(bucketName));
if (!res.IsSuccess()) { /* Create the bucket if it doesn't already exists. */
if (res.GetError().GetErrorType() != Aws::S3::S3Errors::NO_SUCH_BUCKET) // FIXME: HeadBucket would be more appropriate, but doesn't return
throw Error(format("AWS error checking bucket ‘%s’: %s") % bucketName % res.GetError().GetMessage()); // an easily parsed 404 message.
auto res = client->GetBucketLocation(
Aws::S3::Model::GetBucketLocationRequest().WithBucket(bucketName));
checkAws(format("AWS error creating bucket ‘%s’") % bucketName, if (!res.IsSuccess()) {
client->CreateBucket( if (res.GetError().GetErrorType() != Aws::S3::S3Errors::NO_SUCH_BUCKET)
Aws::S3::Model::CreateBucketRequest() throw Error(format("AWS error checking bucket ‘%s’: %s") % bucketName % res.GetError().GetMessage());
.WithBucket(bucketName)
.WithCreateBucketConfiguration( checkAws(format("AWS error creating bucket ‘%s’") % bucketName,
Aws::S3::Model::CreateBucketConfiguration() client->CreateBucket(
/* .WithLocationConstraint( Aws::S3::Model::CreateBucketRequest()
Aws::S3::Model::BucketLocationConstraint::US) */ ))); .WithBucket(bucketName)
.WithCreateBucketConfiguration(
Aws::S3::Model::CreateBucketConfiguration()
/* .WithLocationConstraint(
Aws::S3::Model::BucketLocationConstraint::US) */ )));
}
diskCache->createCache(getUri());
} }
BinaryCacheStore::init(); BinaryCacheStore::init();
@ -82,10 +94,10 @@ const S3BinaryCacheStore::Stats & S3BinaryCacheStore::getS3Stats()
fetches the .narinfo file, rather than first checking for its fetches the .narinfo file, rather than first checking for its
existence via a HEAD request. Since .narinfos are small, doing a existence via a HEAD request. Since .narinfos are small, doing a
GET is unlikely to be slower than HEAD. */ GET is unlikely to be slower than HEAD. */
bool S3BinaryCacheStore::isValidPath(const Path & storePath) bool S3BinaryCacheStore::isValidPathUncached(const Path & storePath)
{ {
try { try {
readNarInfo(storePath); queryPathInfo(storePath);
return true; return true;
} catch (InvalidPath & e) { } catch (InvalidPath & e) {
return false; return false;

4
src/hydra-queue-runner/s3-binary-cache-store.hh
View file

@ -25,6 +25,8 @@ public:
void init() override; void init() override;
std::string getUri();
struct Stats struct Stats
{ {
std::atomic<uint64_t> put{0}; std::atomic<uint64_t> put{0};
@ -38,7 +40,7 @@ public:
const Stats & getS3Stats(); const Stats & getS3Stats();
bool isValidPath(const Path & storePath) override; bool isValidPathUncached(const Path & storePath) override;
private: private: