From 2d40888e2e9a747dbaf32cc3b956454b17dd1bb4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 18 Feb 2016 16:18:50 +0100 Subject: [PATCH] Add an S3-backed binary cache store --- release.nix | 4 + src/hydra-queue-runner/Makefile.am | 5 +- src/hydra-queue-runner/binary-cache-store.cc | 12 +- src/hydra-queue-runner/hydra-queue-runner.cc | 11 +- .../local-binary-cache-store.cc | 3 +- .../local-binary-cache-store.hh | 5 +- .../s3-binary-cache-store.cc | 134 ++++++++++++++++++ .../s3-binary-cache-store.hh | 41 ++++++ 8 files changed, 205 insertions(+), 10 deletions(-) create mode 100644 src/hydra-queue-runner/s3-binary-cache-store.cc create mode 100644 src/hydra-queue-runner/s3-binary-cache-store.hh diff --git a/release.nix b/release.nix index 70f3505b..e482b2a1 100644 --- a/release.nix +++ b/release.nix @@ -159,6 +159,10 @@ rec { guile # optional, for Guile + Guix support perlDeps perl postgresql92 # for running the tests + (aws-sdk-cpp.override { + apis = ["s3"]; + customMemoryManagement = false; + }) ]; hydraPath = lib.makeSearchPath "bin" ( diff --git a/src/hydra-queue-runner/Makefile.am b/src/hydra-queue-runner/Makefile.am index 3b5b48a2..6d526c89 100644 --- a/src/hydra-queue-runner/Makefile.am +++ b/src/hydra-queue-runner/Makefile.am @@ -4,7 +4,8 @@ hydra_queue_runner_SOURCES = hydra-queue-runner.cc queue-monitor.cc dispatcher.c builder.cc build-result.cc build-remote.cc \ build-result.hh counter.hh pool.hh sync.hh token-server.hh state.hh db.hh \ binary-cache-store.hh binary-cache-store.cc \ - local-binary-cache-store.hh local-binary-cache-store.cc + local-binary-cache-store.hh local-binary-cache-store.cc \ + s3-binary-cache-store.hh s3-binary-cache-store.cc hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx -AM_CXXFLAGS = $(NIX_CFLAGS) -Wall +AM_CXXFLAGS = $(NIX_CFLAGS) -Wall -laws-cpp-sdk-s3 diff --git a/src/hydra-queue-runner/binary-cache-store.cc b/src/hydra-queue-runner/binary-cache-store.cc index 80720ba0..d9cd0a4e 100644 --- a/src/hydra-queue-runner/binary-cache-store.cc +++ b/src/hydra-queue-runner/binary-cache-store.cc @@ -7,6 +7,8 @@ #include "nar-info.hh" #include "worker-protocol.hh" +#include + namespace nix { BinaryCacheStore::BinaryCacheStore(ref localStore, @@ -50,15 +52,19 @@ void BinaryCacheStore::addToCache(const ValidPathInfo & info, if (info.narHash.type != htUnknown && info.narHash != narInfo.narHash) throw Error(format("refusing to copy corrupted path ‘%1%’ to binary cache") % info.path); - printMsg(lvlTalkative, format("copying path ‘%1%’ (%2% bytes) to binary cache") - % info.path % info.narSize); - /* Compress the NAR. */ narInfo.compression = "xz"; + auto now1 = std::chrono::steady_clock::now(); string narXz = compressXZ(nar); + auto now2 = std::chrono::steady_clock::now(); narInfo.fileHash = hashString(htSHA256, narXz); narInfo.fileSize = narXz.size(); + printMsg(lvlTalkative, format("copying path ‘%1%’ (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache") + % info.path % info.narSize + % ((1.0 - (double) narXz.size() / nar.size()) * 100.0) + % std::chrono::duration_cast(now2 - now1).count()); + /* Atomically write the NAR file. */ narInfo.url = "nar/" + printHash32(narInfo.fileHash) + ".nar.xz"; if (!fileExists(narInfo.url)) upsertFile(narInfo.url, narXz); diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index 8f363ef3..d95f9e94 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -8,6 +8,7 @@ #include "state.hh" #include "build-result.hh" #include "local-binary-cache-store.hh" +#include "s3-binary-cache-store.hh" #include "shared.hh" #include "globals.hh" @@ -33,10 +34,16 @@ ref State::getLocalStore() ref State::getDestStore() { +#if 0 auto store = make_ref(getLocalStore(), - "/tmp/binary-cache", "/home/eelco/Misc/Keys/test.nixos.org/secret", - "/home/eelco/Misc/Keys/test.nixos.org/public"); + "/home/eelco/Misc/Keys/test.nixos.org/public", + "/tmp/binary-cache"); +#endif + auto store = make_ref(getLocalStore(), + "/home/eelco/Misc/Keys/test.nixos.org/secret", + "/home/eelco/Misc/Keys/test.nixos.org/public", + "nix-test-cache-3"); store->init(); return store; } diff --git a/src/hydra-queue-runner/local-binary-cache-store.cc b/src/hydra-queue-runner/local-binary-cache-store.cc index f0b2637f..250eb3c1 100644 --- a/src/hydra-queue-runner/local-binary-cache-store.cc +++ b/src/hydra-queue-runner/local-binary-cache-store.cc @@ -3,7 +3,8 @@ namespace nix { LocalBinaryCacheStore::LocalBinaryCacheStore(ref localStore, - const Path & binaryCacheDir, const Path & secretKeyFile, const Path & publicKeyFile) + const Path & secretKeyFile, const Path & publicKeyFile, + const Path & binaryCacheDir) : BinaryCacheStore(localStore, secretKeyFile, publicKeyFile) , binaryCacheDir(binaryCacheDir) { diff --git a/src/hydra-queue-runner/local-binary-cache-store.hh b/src/hydra-queue-runner/local-binary-cache-store.hh index 91c56abc..26bac146 100644 --- a/src/hydra-queue-runner/local-binary-cache-store.hh +++ b/src/hydra-queue-runner/local-binary-cache-store.hh @@ -12,8 +12,9 @@ private: public: - LocalBinaryCacheStore(ref localStore, const Path & binaryCacheDir, - const Path & secretKeyFile, const Path & publicKeyFile); + LocalBinaryCacheStore(ref localStore, + const Path & secretKeyFile, const Path & publicKeyFile, + const Path & binaryCacheDir); void init() override; diff --git a/src/hydra-queue-runner/s3-binary-cache-store.cc b/src/hydra-queue-runner/s3-binary-cache-store.cc new file mode 100644 index 00000000..c00cf9a9 --- /dev/null +++ b/src/hydra-queue-runner/s3-binary-cache-store.cc @@ -0,0 +1,134 @@ +#include "s3-binary-cache-store.hh" + +#include +#include +#include +#include +#include +#include +#include + +namespace nix { + +/* Helper: given an Outcome, return R in case of success, or + throw an exception in case of an error. */ +template +R && checkAws(Aws::Utils::Outcome && outcome) +{ + if (!outcome.IsSuccess()) + throw Error(format("AWS error: %1%") % outcome.GetError().GetMessage()); + return outcome.GetResultWithOwnership(); +} + +S3BinaryCacheStore::S3BinaryCacheStore(ref localStore, + const Path & secretKeyFile, const Path & publicKeyFile, + const std::string & bucketName) + : BinaryCacheStore(localStore, secretKeyFile, publicKeyFile) + , bucketName(bucketName) + , config(makeConfig()) + , client(make_ref(*config)) +{ +} + +ref S3BinaryCacheStore::makeConfig() +{ + auto res = make_ref(); + res->region = Aws::Region::EU_WEST_1; + res->requestTimeoutMs = 600 * 1000; + return res; +} + +void S3BinaryCacheStore::init() +{ + /* Create the bucket if it doesn't already exists. */ + // FIXME: HeadBucket would be more appropriate, but doesn't return + // an easily parsed 404 message. + auto res = client->GetBucketLocation( + Aws::S3::Model::GetBucketLocationRequest().WithBucket(bucketName)); + + if (!res.IsSuccess()) { + if (res.GetError().GetErrorType() != Aws::S3::S3Errors::NO_SUCH_BUCKET) + throw Error(format("AWS error: %1%") % res.GetError().GetMessage()); + + checkAws(client->CreateBucket( + Aws::S3::Model::CreateBucketRequest() + .WithBucket(bucketName) + .WithCreateBucketConfiguration( + Aws::S3::Model::CreateBucketConfiguration() + .WithLocationConstraint( + Aws::S3::Model::BucketLocationConstraint::eu_west_1)))); + } + + BinaryCacheStore::init(); +} + +bool S3BinaryCacheStore::fileExists(const std::string & path) +{ + auto res = client->HeadObject( + Aws::S3::Model::HeadObjectRequest() + .WithBucket(bucketName) + .WithKey(path)); + + if (!res.IsSuccess()) { + auto & error = res.GetError(); + if (error.GetErrorType() == Aws::S3::S3Errors::UNKNOWN // FIXME + && error.GetMessage().find("404") != std::string::npos) + return false; + throw Error(format("AWS error: %1%") % error.GetMessage()); + } + + return true; +} + +void S3BinaryCacheStore::upsertFile(const std::string & path, const std::string & data) +{ + auto request = + Aws::S3::Model::PutObjectRequest() + .WithBucket(bucketName) + .WithKey(path); + + auto stream = std::make_shared(data); + + request.SetBody(stream); + + auto now1 = std::chrono::steady_clock::now(); + + auto result = checkAws(client->PutObject(request)); + + auto now2 = std::chrono::steady_clock::now(); + + printMsg(lvlError, format("uploaded ‘s3://%1%/%2%’ (%3% bytes) in %4% ms") + % bucketName % path + % data.size() + % std::chrono::duration_cast(now2 - now1).count()); +} + +std::string S3BinaryCacheStore::getFile(const std::string & path) +{ + auto request = + Aws::S3::Model::GetObjectRequest() + .WithBucket(bucketName) + .WithKey(path); + + request.SetResponseStreamFactory([&]() { + return Aws::New("STRINGSTREAM"); + }); + + auto now1 = std::chrono::steady_clock::now(); + + auto result = checkAws(client->GetObject(request)); + + auto now2 = std::chrono::steady_clock::now(); + + auto res = dynamic_cast(result.GetBody()).str(); + + printMsg(lvlError, format("downloaded ‘s3://%1%/%2%’ (%3%) in %4% ms") + % bucketName % path + % res.size() + % std::chrono::duration_cast(now2 - now1).count()); + + return res; +} + +} + diff --git a/src/hydra-queue-runner/s3-binary-cache-store.hh b/src/hydra-queue-runner/s3-binary-cache-store.hh new file mode 100644 index 00000000..2c11a164 --- /dev/null +++ b/src/hydra-queue-runner/s3-binary-cache-store.hh @@ -0,0 +1,41 @@ +#pragma once + +#include "binary-cache-store.hh" + +namespace Aws { namespace Client { class ClientConfiguration; } } +namespace Aws { namespace S3 { class S3Client; } } + +namespace nix { + +class S3BinaryCacheStore : public BinaryCacheStore +{ +private: + + std::string bucketName; + + ref config; + ref client; + +public: + + S3BinaryCacheStore(ref localStore, + const Path & secretKeyFile, const Path & publicKeyFile, + const std::string & bucketName); + + void init() override; + +private: + + ref makeConfig(); + +protected: + + bool fileExists(const std::string & path) override; + + void upsertFile(const std::string & path, const std::string & data) override; + + std::string getFile(const std::string & path) override; + +}; + +}