2016-02-24 13:48:16 +00:00
|
|
|
|
#include "archive.hh"
|
2016-02-25 16:43:19 +00:00
|
|
|
|
#include "binary-cache-store.hh"
|
2016-02-24 13:48:16 +00:00
|
|
|
|
#include "compression.hh"
|
|
|
|
|
#include "derivations.hh"
|
2016-02-25 16:43:19 +00:00
|
|
|
|
#include "fs-accessor.hh"
|
2016-02-24 13:48:16 +00:00
|
|
|
|
#include "globals.hh"
|
|
|
|
|
#include "nar-info.hh"
|
2016-02-25 16:43:19 +00:00
|
|
|
|
#include "sync.hh"
|
2016-09-02 18:24:34 +00:00
|
|
|
|
#include "remote-fs-accessor.hh"
|
2016-04-20 13:27:48 +00:00
|
|
|
|
#include "nar-info-disk-cache.hh"
|
2016-11-09 17:57:22 +00:00
|
|
|
|
#include "nar-accessor.hh"
|
2016-10-21 14:50:28 +00:00
|
|
|
|
#include "json.hh"
|
2019-09-04 17:24:35 +00:00
|
|
|
|
#include "thread-pool.hh"
|
2020-09-21 16:40:11 +00:00
|
|
|
|
#include "callback.hh"
|
2016-02-24 13:48:16 +00:00
|
|
|
|
|
|
|
|
|
#include <chrono>
|
2016-09-16 16:54:14 +00:00
|
|
|
|
#include <future>
|
2019-09-04 17:24:35 +00:00
|
|
|
|
#include <regex>
|
2020-07-13 18:07:19 +00:00
|
|
|
|
#include <fstream>
|
2019-09-04 17:24:35 +00:00
|
|
|
|
|
|
|
|
|
#include <nlohmann/json.hpp>
|
2016-09-16 16:54:14 +00:00
|
|
|
|
|
2016-02-24 13:48:16 +00:00
|
|
|
|
namespace nix {
|
|
|
|
|
|
2016-06-01 12:49:12 +00:00
|
|
|
|
BinaryCacheStore::BinaryCacheStore(const Params & params)
|
2020-09-10 08:55:51 +00:00
|
|
|
|
: BinaryCacheStoreConfig(params)
|
|
|
|
|
, Store(params)
|
2016-02-24 13:48:16 +00:00
|
|
|
|
{
|
2016-04-29 14:28:57 +00:00
|
|
|
|
if (secretKeyFile != "")
|
2016-02-24 13:48:16 +00:00
|
|
|
|
secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile)));
|
2016-02-24 15:52:28 +00:00
|
|
|
|
|
|
|
|
|
StringSink sink;
|
|
|
|
|
sink << narVersionMagic1;
|
2016-03-04 15:49:56 +00:00
|
|
|
|
narMagic = *sink.s;
|
2016-02-24 13:48:16 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void BinaryCacheStore::init()
|
|
|
|
|
{
|
|
|
|
|
std::string cacheInfoFile = "nix-cache-info";
|
2016-05-30 11:33:05 +00:00
|
|
|
|
|
|
|
|
|
auto cacheInfo = getFile(cacheInfoFile);
|
|
|
|
|
if (!cacheInfo) {
|
2017-03-14 14:26:01 +00:00
|
|
|
|
upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info");
|
2016-05-30 11:33:05 +00:00
|
|
|
|
} else {
|
|
|
|
|
for (auto & line : tokenizeString<Strings>(*cacheInfo, "\n")) {
|
2020-04-21 23:07:07 +00:00
|
|
|
|
size_t colon= line.find(':');
|
|
|
|
|
if (colon ==std::string::npos) continue;
|
2016-05-30 11:33:05 +00:00
|
|
|
|
auto name = line.substr(0, colon);
|
|
|
|
|
auto value = trim(line.substr(colon + 1, std::string::npos));
|
|
|
|
|
if (name == "StoreDir") {
|
2016-06-01 12:49:12 +00:00
|
|
|
|
if (value != storeDir)
|
2020-04-21 23:07:07 +00:00
|
|
|
|
throw Error("binary cache '%s' is for Nix stores with prefix '%s', not '%s'",
|
|
|
|
|
getUri(), value, storeDir);
|
2016-05-30 11:33:05 +00:00
|
|
|
|
} else if (name == "WantMassQuery") {
|
2019-12-17 16:17:53 +00:00
|
|
|
|
wantMassQuery.setDefault(value == "1" ? "true" : "false");
|
2016-05-30 11:33:05 +00:00
|
|
|
|
} else if (name == "Priority") {
|
2019-12-17 16:17:53 +00:00
|
|
|
|
priority.setDefault(fmt("%d", std::stoi(value)));
|
2016-05-30 11:33:05 +00:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
2016-02-24 13:48:16 +00:00
|
|
|
|
}
|
|
|
|
|
|
2020-07-10 18:58:02 +00:00
|
|
|
|
void BinaryCacheStore::upsertFile(const std::string & path,
|
2020-07-13 18:07:19 +00:00
|
|
|
|
std::string && data,
|
2020-07-10 18:58:02 +00:00
|
|
|
|
const std::string & mimeType)
|
|
|
|
|
{
|
2020-07-13 18:07:19 +00:00
|
|
|
|
upsertFile(path, std::make_shared<std::stringstream>(std::move(data)), mimeType);
|
2020-07-10 18:58:02 +00:00
|
|
|
|
}
|
|
|
|
|
|
2018-03-27 21:12:31 +00:00
|
|
|
|
void BinaryCacheStore::getFile(const std::string & path,
|
2019-09-03 11:00:55 +00:00
|
|
|
|
Callback<std::shared_ptr<std::string>> callback) noexcept
|
2018-03-27 21:12:31 +00:00
|
|
|
|
{
|
|
|
|
|
try {
|
|
|
|
|
callback(getFile(path));
|
|
|
|
|
} catch (...) { callback.rethrow(); }
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void BinaryCacheStore::getFile(const std::string & path, Sink & sink)
|
2016-09-16 16:54:14 +00:00
|
|
|
|
{
|
|
|
|
|
std::promise<std::shared_ptr<std::string>> promise;
|
|
|
|
|
getFile(path,
|
2018-03-27 20:16:01 +00:00
|
|
|
|
{[&](std::future<std::shared_ptr<std::string>> result) {
|
|
|
|
|
try {
|
|
|
|
|
promise.set_value(result.get());
|
|
|
|
|
} catch (...) {
|
|
|
|
|
promise.set_exception(std::current_exception());
|
|
|
|
|
}
|
|
|
|
|
}});
|
2020-12-02 13:00:43 +00:00
|
|
|
|
sink(*promise.get_future().get());
|
2018-03-27 21:12:31 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
|
|
|
|
|
{
|
2019-07-10 17:46:15 +00:00
|
|
|
|
StringSink sink;
|
|
|
|
|
try {
|
|
|
|
|
getFile(path, sink);
|
|
|
|
|
} catch (NoSuchBinaryCacheFile &) {
|
|
|
|
|
return nullptr;
|
|
|
|
|
}
|
|
|
|
|
return sink.s;
|
2016-09-16 16:54:14 +00:00
|
|
|
|
}
|
|
|
|
|
|
2019-12-05 18:11:09 +00:00
|
|
|
|
std::string BinaryCacheStore::narInfoFileFor(const StorePath & storePath)
|
2016-02-24 13:48:16 +00:00
|
|
|
|
{
|
2020-06-16 12:16:39 +00:00
|
|
|
|
return std::string(storePath.hashPart()) + ".narinfo";
|
2016-02-24 13:48:16 +00:00
|
|
|
|
}
|
|
|
|
|
|
2017-11-14 17:44:05 +00:00
|
|
|
|
void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
|
|
|
|
|
{
|
|
|
|
|
auto narInfoFile = narInfoFileFor(narInfo->path);
|
|
|
|
|
|
2019-12-05 18:11:09 +00:00
|
|
|
|
upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo");
|
2017-11-14 17:44:05 +00:00
|
|
|
|
|
2020-06-16 12:16:39 +00:00
|
|
|
|
std::string hashPart(narInfo->path.hashPart());
|
2017-11-14 17:44:05 +00:00
|
|
|
|
|
|
|
|
|
{
|
|
|
|
|
auto state_(state.lock());
|
2020-03-11 19:04:47 +00:00
|
|
|
|
state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = std::shared_ptr<NarInfo>(narInfo) });
|
2017-11-14 17:44:05 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (diskCache)
|
|
|
|
|
diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo));
|
|
|
|
|
}
|
|
|
|
|
|
2020-07-10 18:58:02 +00:00
|
|
|
|
AutoCloseFD openFile(const Path & path)
|
|
|
|
|
{
|
|
|
|
|
auto fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
|
|
|
|
|
if (!fd)
|
|
|
|
|
throw SysError("opening file '%1%'", path);
|
|
|
|
|
return fd;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
struct FileSource : FdSource
|
|
|
|
|
{
|
|
|
|
|
AutoCloseFD fd2;
|
|
|
|
|
|
|
|
|
|
FileSource(const Path & path)
|
|
|
|
|
: fd2(openFile(path))
|
|
|
|
|
{
|
|
|
|
|
fd = fd2.get();
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
2020-09-23 14:40:41 +00:00
|
|
|
|
ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
|
2020-08-15 16:41:28 +00:00
|
|
|
|
Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs,
|
|
|
|
|
std::function<ValidPathInfo(HashResult)> mkInfo)
|
2016-02-24 13:48:16 +00:00
|
|
|
|
{
|
2020-07-10 18:58:02 +00:00
|
|
|
|
auto [fdTemp, fnTemp] = createTempFile();
|
|
|
|
|
|
2020-08-03 16:33:39 +00:00
|
|
|
|
AutoDelete autoDelete(fnTemp);
|
|
|
|
|
|
2020-07-10 18:58:02 +00:00
|
|
|
|
auto now1 = std::chrono::steady_clock::now();
|
|
|
|
|
|
2020-07-13 15:30:42 +00:00
|
|
|
|
/* Read the NAR simultaneously into a CompressionSink+FileSink (to
|
|
|
|
|
write the compressed NAR to disk), into a HashSink (to get the
|
|
|
|
|
NAR hash), and into a NarAccessor (to get the NAR listing). */
|
2020-08-15 16:41:28 +00:00
|
|
|
|
HashSink fileHashSink { htSHA256 };
|
2020-07-13 15:30:42 +00:00
|
|
|
|
std::shared_ptr<FSAccessor> narAccessor;
|
2020-08-15 16:41:28 +00:00
|
|
|
|
HashSink narHashSink { htSHA256 };
|
2020-07-10 18:58:02 +00:00
|
|
|
|
{
|
|
|
|
|
FdSink fileSink(fdTemp.get());
|
2020-08-15 16:41:28 +00:00
|
|
|
|
TeeSink teeSinkCompressed { fileSink, fileHashSink };
|
|
|
|
|
auto compressionSink = makeCompressionSink(compression, teeSinkCompressed);
|
|
|
|
|
TeeSink teeSinkUncompressed { *compressionSink, narHashSink };
|
|
|
|
|
TeeSource teeSource { narSource, teeSinkUncompressed };
|
2020-07-13 15:30:42 +00:00
|
|
|
|
narAccessor = makeNarAccessor(teeSource);
|
2020-07-10 18:58:02 +00:00
|
|
|
|
compressionSink->finish();
|
2020-08-04 12:50:43 +00:00
|
|
|
|
fileSink.flush();
|
2020-07-10 18:58:02 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
auto now2 = std::chrono::steady_clock::now();
|
|
|
|
|
|
2020-08-15 16:41:28 +00:00
|
|
|
|
auto info = mkInfo(narHashSink.finish());
|
2020-07-10 18:58:02 +00:00
|
|
|
|
auto narInfo = make_ref<NarInfo>(info);
|
|
|
|
|
narInfo->compression = compression;
|
|
|
|
|
auto [fileHash, fileSize] = fileHashSink.finish();
|
|
|
|
|
narInfo->fileHash = fileHash;
|
|
|
|
|
narInfo->fileSize = fileSize;
|
2020-07-15 21:19:56 +00:00
|
|
|
|
narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar"
|
2020-07-10 18:58:02 +00:00
|
|
|
|
+ (compression == "xz" ? ".xz" :
|
|
|
|
|
compression == "bzip2" ? ".bz2" :
|
2021-04-09 21:13:08 +00:00
|
|
|
|
compression == "zstd" ? ".zst" :
|
|
|
|
|
compression == "lzip" ? ".lzip" :
|
|
|
|
|
compression == "lz4" ? ".lz4" :
|
2020-07-10 18:58:02 +00:00
|
|
|
|
compression == "br" ? ".br" :
|
|
|
|
|
"");
|
2020-05-29 20:19:48 +00:00
|
|
|
|
|
2020-07-10 18:58:02 +00:00
|
|
|
|
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
|
|
|
|
|
printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache",
|
|
|
|
|
printStorePath(narInfo->path), info.narSize,
|
|
|
|
|
((1.0 - (double) fileSize / info.narSize) * 100.0),
|
|
|
|
|
duration);
|
2016-05-04 11:36:54 +00:00
|
|
|
|
|
2016-04-22 10:15:06 +00:00
|
|
|
|
/* Verify that all references are valid. This may do some .narinfo
|
|
|
|
|
reads, but typically they'll already be cached. */
|
|
|
|
|
for (auto & ref : info.references)
|
|
|
|
|
try {
|
|
|
|
|
if (ref != info.path)
|
|
|
|
|
queryPathInfo(ref);
|
|
|
|
|
} catch (InvalidPath &) {
|
2019-12-05 18:11:09 +00:00
|
|
|
|
throw Error("cannot add '%s' to the binary cache because the reference '%s' is not valid",
|
|
|
|
|
printStorePath(info.path), printStorePath(ref));
|
2016-04-22 10:15:06 +00:00
|
|
|
|
}
|
|
|
|
|
|
2016-10-21 14:50:28 +00:00
|
|
|
|
/* Optionally write a JSON file containing a listing of the
|
|
|
|
|
contents of the NAR. */
|
|
|
|
|
if (writeNARListing) {
|
|
|
|
|
std::ostringstream jsonOut;
|
|
|
|
|
|
|
|
|
|
{
|
|
|
|
|
JSONObject jsonRoot(jsonOut);
|
|
|
|
|
jsonRoot.attr("version", 1);
|
|
|
|
|
|
|
|
|
|
{
|
|
|
|
|
auto res = jsonRoot.placeholder("root");
|
2020-07-13 15:30:42 +00:00
|
|
|
|
listNar(res, ref<FSAccessor>(narAccessor), "", true);
|
2016-10-21 14:50:28 +00:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2020-08-07 19:18:29 +00:00
|
|
|
|
upsertFile(std::string(info.path.hashPart()) + ".ls", jsonOut.str(), "application/json");
|
2016-10-21 14:50:28 +00:00
|
|
|
|
}
|
|
|
|
|
|
2019-09-04 17:24:35 +00:00
|
|
|
|
/* Optionally maintain an index of DWARF debug info files
|
|
|
|
|
consisting of JSON files named 'debuginfo/<build-id>' that
|
|
|
|
|
specify the NAR file and member containing the debug info. */
|
|
|
|
|
if (writeDebugInfo) {
|
|
|
|
|
|
|
|
|
|
std::string buildIdDir = "/lib/debug/.build-id";
|
|
|
|
|
|
|
|
|
|
if (narAccessor->stat(buildIdDir).type == FSAccessor::tDirectory) {
|
|
|
|
|
|
|
|
|
|
ThreadPool threadPool(25);
|
|
|
|
|
|
|
|
|
|
auto doFile = [&](std::string member, std::string key, std::string target) {
|
|
|
|
|
checkInterrupt();
|
|
|
|
|
|
|
|
|
|
nlohmann::json json;
|
|
|
|
|
json["archive"] = target;
|
|
|
|
|
json["member"] = member;
|
|
|
|
|
|
|
|
|
|
// FIXME: or should we overwrite? The previous link may point
|
|
|
|
|
// to a GC'ed file, so overwriting might be useful...
|
|
|
|
|
if (fileExists(key)) return;
|
|
|
|
|
|
|
|
|
|
printMsg(lvlTalkative, "creating debuginfo link from '%s' to '%s'", key, target);
|
|
|
|
|
|
|
|
|
|
upsertFile(key, json.dump(), "application/json");
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
std::regex regex1("^[0-9a-f]{2}$");
|
|
|
|
|
std::regex regex2("^[0-9a-f]{38}\\.debug$");
|
|
|
|
|
|
|
|
|
|
for (auto & s1 : narAccessor->readDirectory(buildIdDir)) {
|
|
|
|
|
auto dir = buildIdDir + "/" + s1;
|
|
|
|
|
|
|
|
|
|
if (narAccessor->stat(dir).type != FSAccessor::tDirectory
|
|
|
|
|
|| !std::regex_match(s1, regex1))
|
|
|
|
|
continue;
|
|
|
|
|
|
|
|
|
|
for (auto & s2 : narAccessor->readDirectory(dir)) {
|
|
|
|
|
auto debugPath = dir + "/" + s2;
|
|
|
|
|
|
|
|
|
|
if (narAccessor->stat(debugPath).type != FSAccessor::tRegular
|
|
|
|
|
|| !std::regex_match(s2, regex2))
|
|
|
|
|
continue;
|
|
|
|
|
|
|
|
|
|
auto buildId = s1 + s2;
|
|
|
|
|
|
|
|
|
|
std::string key = "debuginfo/" + buildId;
|
|
|
|
|
std::string target = "../" + narInfo->url;
|
|
|
|
|
|
|
|
|
|
threadPool.enqueue(std::bind(doFile, std::string(debugPath, 1), key, target));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
threadPool.process();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Atomically write the NAR file. */
|
2016-05-04 11:36:54 +00:00
|
|
|
|
if (repair || !fileExists(narInfo->url)) {
|
2016-02-24 13:48:16 +00:00
|
|
|
|
stats.narWrite++;
|
2020-07-13 18:07:19 +00:00
|
|
|
|
upsertFile(narInfo->url,
|
2020-08-04 14:00:59 +00:00
|
|
|
|
std::make_shared<std::fstream>(fnTemp, std::ios_base::in | std::ios_base::binary),
|
2020-07-13 18:07:19 +00:00
|
|
|
|
"application/x-nix-nar");
|
2016-02-24 13:48:16 +00:00
|
|
|
|
} else
|
|
|
|
|
stats.narWriteAverted++;
|
|
|
|
|
|
2020-07-10 18:58:02 +00:00
|
|
|
|
stats.narWriteBytes += info.narSize;
|
|
|
|
|
stats.narWriteCompressedBytes += fileSize;
|
2016-02-24 13:48:16 +00:00
|
|
|
|
stats.narWriteCompressionTimeMs += duration;
|
|
|
|
|
|
|
|
|
|
/* Atomically write the NAR info file.*/
|
2019-12-05 18:11:09 +00:00
|
|
|
|
if (secretKey) narInfo->sign(*this, *secretKey);
|
2016-02-24 13:48:16 +00:00
|
|
|
|
|
2017-11-14 17:44:05 +00:00
|
|
|
|
writeNarInfo(narInfo);
|
2016-04-20 13:27:48 +00:00
|
|
|
|
|
2016-02-24 13:48:16 +00:00
|
|
|
|
stats.narInfoWrite++;
|
2020-08-15 16:41:28 +00:00
|
|
|
|
|
2020-09-23 14:40:41 +00:00
|
|
|
|
return narInfo;
|
2020-08-15 16:41:28 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
|
|
|
|
|
RepairFlag repair, CheckSigsFlag checkSigs)
|
|
|
|
|
{
|
|
|
|
|
if (!repair && isValidPath(info.path)) {
|
|
|
|
|
// FIXME: copyNAR -> null sink
|
|
|
|
|
narSource.drain();
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
2020-09-28 15:32:58 +00:00
|
|
|
|
addToStoreCommon(narSource, repair, checkSigs, {[&](HashResult nar) {
|
2020-08-15 16:41:28 +00:00
|
|
|
|
/* FIXME reinstate these, once we can correctly do hash modulo sink as
|
2020-09-23 14:36:55 +00:00
|
|
|
|
needed. We need to throw here in case we uploaded a corrupted store path. */
|
2020-08-15 16:41:28 +00:00
|
|
|
|
// assert(info.narHash == nar.first);
|
|
|
|
|
// assert(info.narSize == nar.second);
|
|
|
|
|
return info;
|
|
|
|
|
}});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & name,
|
|
|
|
|
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair)
|
|
|
|
|
{
|
|
|
|
|
if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256)
|
|
|
|
|
unsupported("addToStoreFromDump");
|
|
|
|
|
return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) {
|
|
|
|
|
ValidPathInfo info {
|
|
|
|
|
makeFixedOutputPath(method, nar.first, name),
|
|
|
|
|
nar.first,
|
|
|
|
|
};
|
|
|
|
|
info.narSize = nar.second;
|
|
|
|
|
return info;
|
2020-09-23 14:40:41 +00:00
|
|
|
|
})->path;
|
2016-02-24 13:48:16 +00:00
|
|
|
|
}
|
|
|
|
|
|
2019-12-05 18:11:09 +00:00
|
|
|
|
bool BinaryCacheStore::isValidPathUncached(const StorePath & storePath)
|
2016-02-24 13:48:16 +00:00
|
|
|
|
{
|
2016-02-25 16:43:19 +00:00
|
|
|
|
// FIXME: this only checks whether a .narinfo with a matching hash
|
2016-11-25 23:37:43 +00:00
|
|
|
|
// part exists. So ‘f4kb...-foo’ matches ‘f4kb...-bar’, even
|
2016-02-25 16:43:19 +00:00
|
|
|
|
// though they shouldn't. Not easily fixed.
|
2016-02-24 13:48:16 +00:00
|
|
|
|
return fileExists(narInfoFileFor(storePath));
|
|
|
|
|
}
|
|
|
|
|
|
2019-12-05 18:11:09 +00:00
|
|
|
|
void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink)
|
2016-02-24 13:48:16 +00:00
|
|
|
|
{
|
2016-04-19 16:50:15 +00:00
|
|
|
|
auto info = queryPathInfo(storePath).cast<const NarInfo>();
|
2016-02-24 13:48:16 +00:00
|
|
|
|
|
2020-08-13 14:47:53 +00:00
|
|
|
|
LengthSink narSize;
|
|
|
|
|
TeeSink tee { sink, narSize };
|
2016-02-24 13:48:16 +00:00
|
|
|
|
|
2020-08-13 14:47:53 +00:00
|
|
|
|
auto decompressor = makeDecompressionSink(info->compression, tee);
|
2016-02-24 13:48:16 +00:00
|
|
|
|
|
2018-08-06 13:40:29 +00:00
|
|
|
|
try {
|
|
|
|
|
getFile(info->url, *decompressor);
|
|
|
|
|
} catch (NoSuchBinaryCacheFile & e) {
|
2020-04-30 00:57:05 +00:00
|
|
|
|
throw SubstituteGone(e.info());
|
2018-08-06 13:40:29 +00:00
|
|
|
|
}
|
|
|
|
|
|
2018-08-21 13:19:20 +00:00
|
|
|
|
decompressor->finish();
|
2018-08-06 13:40:29 +00:00
|
|
|
|
|
|
|
|
|
stats.narRead++;
|
|
|
|
|
//stats.narReadCompressedBytes += nar->size(); // FIXME
|
2020-08-13 14:47:53 +00:00
|
|
|
|
stats.narReadBytes += narSize.length;
|
2016-03-21 16:55:57 +00:00
|
|
|
|
}
|
|
|
|
|
|
2019-12-05 18:11:09 +00:00
|
|
|
|
void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
|
2018-09-25 16:54:16 +00:00
|
|
|
|
Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
|
2016-02-24 13:48:16 +00:00
|
|
|
|
{
|
2017-08-31 13:25:58 +00:00
|
|
|
|
auto uri = getUri();
|
2019-12-05 18:11:09 +00:00
|
|
|
|
auto storePathS = printStorePath(storePath);
|
2017-08-31 13:25:58 +00:00
|
|
|
|
auto act = std::make_shared<Activity>(*logger, lvlTalkative, actQueryPathInfo,
|
2019-12-05 18:11:09 +00:00
|
|
|
|
fmt("querying info about '%s' on '%s'", storePathS, uri), Logger::Fields{storePathS, uri});
|
2017-08-31 13:25:58 +00:00
|
|
|
|
PushActivity pact(act->id);
|
|
|
|
|
|
2016-04-19 16:50:15 +00:00
|
|
|
|
auto narInfoFile = narInfoFileFor(storePath);
|
|
|
|
|
|
2019-09-03 10:51:35 +00:00
|
|
|
|
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
|
|
|
|
|
|
2016-09-16 16:54:14 +00:00
|
|
|
|
getFile(narInfoFile,
|
2018-03-27 20:16:01 +00:00
|
|
|
|
{[=](std::future<std::shared_ptr<std::string>> fut) {
|
|
|
|
|
try {
|
|
|
|
|
auto data = fut.get();
|
|
|
|
|
|
2019-09-03 10:51:35 +00:00
|
|
|
|
if (!data) return (*callbackPtr)(nullptr);
|
2016-04-19 16:50:15 +00:00
|
|
|
|
|
2018-03-27 20:16:01 +00:00
|
|
|
|
stats.narInfoRead++;
|
2016-04-19 16:50:15 +00:00
|
|
|
|
|
2019-09-03 10:51:35 +00:00
|
|
|
|
(*callbackPtr)((std::shared_ptr<ValidPathInfo>)
|
2018-03-27 20:16:01 +00:00
|
|
|
|
std::make_shared<NarInfo>(*this, *data, narInfoFile));
|
2017-08-31 13:25:58 +00:00
|
|
|
|
|
2018-03-27 20:16:01 +00:00
|
|
|
|
(void) act; // force Activity into this lambda to ensure it stays alive
|
|
|
|
|
} catch (...) {
|
2019-09-03 10:51:35 +00:00
|
|
|
|
callbackPtr->rethrow();
|
2018-03-27 20:16:01 +00:00
|
|
|
|
}
|
|
|
|
|
}});
|
2016-02-24 13:48:16 +00:00
|
|
|
|
}
|
|
|
|
|
|
2019-12-05 18:11:09 +00:00
|
|
|
|
StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
|
2020-05-27 18:04:20 +00:00
|
|
|
|
FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
|
2016-02-24 15:52:28 +00:00
|
|
|
|
{
|
2020-08-15 16:41:28 +00:00
|
|
|
|
/* FIXME: Make BinaryCacheStore::addToStoreCommon support
|
|
|
|
|
non-recursive+sha256 so we can just use the default
|
|
|
|
|
implementation of this method in terms of addToStoreFromDump. */
|
2020-09-26 04:56:29 +00:00
|
|
|
|
|
|
|
|
|
HashSink sink { hashAlgo };
|
2020-05-27 18:04:20 +00:00
|
|
|
|
if (method == FileIngestionMethod::Recursive) {
|
2016-03-22 13:21:45 +00:00
|
|
|
|
dumpPath(srcPath, sink, filter);
|
2016-02-24 15:52:28 +00:00
|
|
|
|
} else {
|
2020-09-26 04:56:29 +00:00
|
|
|
|
readFile(srcPath, sink);
|
2016-02-24 15:52:28 +00:00
|
|
|
|
}
|
2020-09-26 04:56:29 +00:00
|
|
|
|
auto h = sink.finish().first;
|
2016-02-24 15:52:28 +00:00
|
|
|
|
|
2020-09-26 04:56:29 +00:00
|
|
|
|
auto source = sinkToSource([&](Sink & sink) {
|
|
|
|
|
dumpPath(srcPath, sink, filter);
|
|
|
|
|
});
|
|
|
|
|
return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) {
|
|
|
|
|
ValidPathInfo info {
|
|
|
|
|
makeFixedOutputPath(method, h, name),
|
|
|
|
|
nar.first,
|
|
|
|
|
};
|
|
|
|
|
info.narSize = nar.second;
|
|
|
|
|
info.ca = FixedOutputHash {
|
|
|
|
|
.method = method,
|
|
|
|
|
.hash = h,
|
|
|
|
|
};
|
|
|
|
|
return info;
|
|
|
|
|
})->path;
|
2016-02-24 15:52:28 +00:00
|
|
|
|
}
|
|
|
|
|
|
2019-12-05 18:11:09 +00:00
|
|
|
|
StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s,
|
|
|
|
|
const StorePathSet & references, RepairFlag repair)
|
2016-02-24 15:52:28 +00:00
|
|
|
|
{
|
2020-09-26 03:21:36 +00:00
|
|
|
|
auto textHash = hashString(htSHA256, s);
|
|
|
|
|
auto path = makeTextPath(name, textHash, references);
|
2020-08-15 16:41:28 +00:00
|
|
|
|
|
|
|
|
|
if (!repair && isValidPath(path))
|
|
|
|
|
return path;
|
|
|
|
|
|
2020-10-20 13:03:54 +00:00
|
|
|
|
StringSink sink;
|
|
|
|
|
dumpString(s, sink);
|
|
|
|
|
auto source = StringSource { *sink.s };
|
2020-08-15 16:41:28 +00:00
|
|
|
|
return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
|
|
|
|
|
ValidPathInfo info { path, nar.first };
|
|
|
|
|
info.narSize = nar.second;
|
2020-09-26 03:21:36 +00:00
|
|
|
|
info.ca = TextHash { textHash };
|
2020-08-15 16:41:28 +00:00
|
|
|
|
info.references = references;
|
|
|
|
|
return info;
|
2020-09-23 14:40:41 +00:00
|
|
|
|
})->path;
|
2016-02-24 15:52:28 +00:00
|
|
|
|
}
|
|
|
|
|
|
2020-10-08 15:36:51 +00:00
|
|
|
|
std::optional<const Realisation> BinaryCacheStore::queryRealisation(const DrvOutput & id)
|
|
|
|
|
{
|
2021-05-06 14:45:09 +00:00
|
|
|
|
if (diskCache) {
|
|
|
|
|
auto [cacheOutcome, maybeCachedRealisation] =
|
|
|
|
|
diskCache->lookupRealisation(getUri(), id);
|
|
|
|
|
switch (cacheOutcome) {
|
2021-05-10 15:36:49 +00:00
|
|
|
|
case NarInfoDiskCache::oValid:
|
2021-05-06 14:45:09 +00:00
|
|
|
|
debug("Returning a cached realisation for %s", id.to_string());
|
|
|
|
|
return *maybeCachedRealisation;
|
2021-05-10 15:36:49 +00:00
|
|
|
|
case NarInfoDiskCache::oInvalid:
|
2021-05-06 14:45:09 +00:00
|
|
|
|
debug("Returning a cached missing realisation for %s", id.to_string());
|
|
|
|
|
return {};
|
2021-05-10 15:36:49 +00:00
|
|
|
|
case NarInfoDiskCache::oUnknown:
|
2021-05-06 14:45:09 +00:00
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2020-10-08 15:36:51 +00:00
|
|
|
|
auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi";
|
|
|
|
|
auto rawOutputInfo = getFile(outputInfoFilePath);
|
|
|
|
|
|
|
|
|
|
if (rawOutputInfo) {
|
2021-05-06 14:45:09 +00:00
|
|
|
|
auto realisation = Realisation::fromJSON(
|
|
|
|
|
nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath);
|
|
|
|
|
|
|
|
|
|
if (diskCache)
|
|
|
|
|
diskCache->upsertRealisation(
|
|
|
|
|
getUri(), realisation);
|
|
|
|
|
|
|
|
|
|
return {realisation};
|
2020-10-08 15:36:51 +00:00
|
|
|
|
} else {
|
2021-05-06 14:45:09 +00:00
|
|
|
|
if (diskCache)
|
|
|
|
|
diskCache->upsertAbsentRealisation(getUri(), id);
|
2020-10-08 15:36:51 +00:00
|
|
|
|
return std::nullopt;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void BinaryCacheStore::registerDrvOutput(const Realisation& info) {
|
2021-05-06 14:45:09 +00:00
|
|
|
|
if (diskCache)
|
|
|
|
|
diskCache->upsertRealisation(getUri(), info);
|
2020-10-08 15:36:51 +00:00
|
|
|
|
auto filePath = realisationsPrefix + "/" + info.id.to_string() + ".doi";
|
2020-12-15 08:34:45 +00:00
|
|
|
|
upsertFile(filePath, info.toJSON().dump(), "application/json");
|
2020-10-08 15:36:51 +00:00
|
|
|
|
}
|
|
|
|
|
|
2016-02-25 16:43:19 +00:00
|
|
|
|
ref<FSAccessor> BinaryCacheStore::getFSAccessor()
|
|
|
|
|
{
|
2017-10-17 19:15:33 +00:00
|
|
|
|
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), localNarCache);
|
2016-02-25 16:43:19 +00:00
|
|
|
|
}
|
|
|
|
|
|
2019-12-05 18:11:09 +00:00
|
|
|
|
void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSet & sigs)
|
2017-11-14 17:44:05 +00:00
|
|
|
|
{
|
|
|
|
|
/* Note: this is inherently racy since there is no locking on
|
|
|
|
|
binary caches. In particular, with S3 this unreliable, even
|
|
|
|
|
when addSignatures() is called sequentially on a path, because
|
|
|
|
|
S3 might return an outdated cached version. */
|
|
|
|
|
|
|
|
|
|
auto narInfo = make_ref<NarInfo>((NarInfo &) *queryPathInfo(storePath));
|
|
|
|
|
|
|
|
|
|
narInfo->sigs.insert(sigs.begin(), sigs.end());
|
|
|
|
|
|
|
|
|
|
writeNarInfo(narInfo);
|
|
|
|
|
}
|
|
|
|
|
|
2019-12-05 18:11:09 +00:00
|
|
|
|
std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const StorePath & path)
|
2017-03-13 13:07:58 +00:00
|
|
|
|
{
|
2020-06-16 20:20:18 +00:00
|
|
|
|
auto drvPath = path;
|
2017-03-13 13:07:58 +00:00
|
|
|
|
|
2019-12-05 18:11:09 +00:00
|
|
|
|
if (!path.isDerivation()) {
|
2017-03-13 13:07:58 +00:00
|
|
|
|
try {
|
|
|
|
|
auto info = queryPathInfo(path);
|
|
|
|
|
// FIXME: add a "Log" field to .narinfo
|
2019-12-05 18:11:09 +00:00
|
|
|
|
if (!info->deriver) return nullptr;
|
2020-06-16 20:20:18 +00:00
|
|
|
|
drvPath = *info->deriver;
|
2017-03-13 13:07:58 +00:00
|
|
|
|
} catch (InvalidPath &) {
|
|
|
|
|
return nullptr;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2019-12-05 18:11:09 +00:00
|
|
|
|
auto logPath = "log/" + std::string(baseNameOf(printStorePath(drvPath)));
|
2017-03-13 13:07:58 +00:00
|
|
|
|
|
2017-07-30 11:27:57 +00:00
|
|
|
|
debug("fetching build log from binary cache '%s/%s'", getUri(), logPath);
|
2017-03-13 13:07:58 +00:00
|
|
|
|
|
|
|
|
|
return getFile(logPath);
|
|
|
|
|
}
|
|
|
|
|
|
2016-02-24 13:48:16 +00:00
|
|
|
|
}
|