BinaryCacheStore::readFile(): Return a shared_ptr to a string

This allows readFile() to indicate that a file doesn't exist, and
might eliminate some large string copying.
This commit is contained in:
Eelco Dolstra 2016-04-15 15:11:34 +02:00
parent 99851c6f06
commit d1b0909894
11 changed files with 52 additions and 28 deletions

View file

@ -119,7 +119,10 @@ NarInfo BinaryCacheStore::readNarInfo(const Path & storePath)
} }
auto narInfoFile = narInfoFileFor(storePath); auto narInfoFile = narInfoFileFor(storePath);
auto narInfo = make_ref<NarInfo>(getFile(narInfoFile), narInfoFile); auto data = getFile(narInfoFile);
if (!data)
throw InvalidPath(format("path %s is not valid") % storePath);
auto narInfo = make_ref<NarInfo>(*data, narInfoFile);
if (narInfo->path != storePath) if (narInfo->path != storePath)
throw Error(format("NAR info file for store path %1% does not match %2%") % narInfo->path % storePath); throw Error(format("NAR info file for store path %1% does not match %2%") % narInfo->path % storePath);
@ -162,25 +165,27 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink)
auto nar = getFile(res.url); auto nar = getFile(res.url);
if (!nar) throw Error(format("file %s missing from binary cache") % res.url);
stats.narRead++; stats.narRead++;
stats.narReadCompressedBytes += nar.size(); stats.narReadCompressedBytes += nar->size();
/* Decompress the NAR. FIXME: would be nice to have the remote /* Decompress the NAR. FIXME: would be nice to have the remote
side do this. */ side do this. */
if (res.compression == "none") if (res.compression == "none")
; ;
else if (res.compression == "xz") else if (res.compression == "xz")
nar = decompressXZ(nar); nar = decompressXZ(*nar);
else else
throw Error(format("unknown NAR compression type %1%") % nar); throw Error(format("unknown NAR compression type %1%") % nar);
stats.narReadBytes += nar.size(); stats.narReadBytes += nar->size();
printMsg(lvlTalkative, format("exporting path %1% (%2% bytes)") % storePath % nar.size()); printMsg(lvlTalkative, format("exporting path %1% (%2% bytes)") % storePath % nar->size());
assert(nar.size() % 8 == 0); assert(nar->size() % 8 == 0);
sink((unsigned char *) nar.c_str(), nar.size()); sink((unsigned char *) nar->c_str(), nar->size());
} }
void BinaryCacheStore::exportPath(const Path & storePath, bool sign, Sink & sink) void BinaryCacheStore::exportPath(const Path & storePath, bool sign, Sink & sink)

View file

@ -39,7 +39,9 @@ protected:
virtual void upsertFile(const std::string & path, const std::string & data) = 0; virtual void upsertFile(const std::string & path, const std::string & data) = 0;
virtual std::string getFile(const std::string & path) = 0; /* Return the contents of the specified file, or null if it
doesn't exist. */
virtual std::shared_ptr<std::string> getFile(const std::string & path) = 0;
public: public:

View file

@ -20,6 +20,7 @@ void builtinFetchurl(const BasicDerivation & drv)
options.showProgress = DownloadOptions::yes; options.showProgress = DownloadOptions::yes;
auto data = makeDownloader()->download(url->second, options); auto data = makeDownloader()->download(url->second, options);
assert(data.data);
auto out = drv.env.find("out"); auto out = drv.env.find("out");
if (out == drv.env.end()) throw Error("attribute url missing"); if (out == drv.env.end()) throw Error("attribute url missing");
@ -29,12 +30,12 @@ void builtinFetchurl(const BasicDerivation & drv)
auto unpack = drv.env.find("unpack"); auto unpack = drv.env.find("unpack");
if (unpack != drv.env.end() && unpack->second == "1") { if (unpack != drv.env.end() && unpack->second == "1") {
if (string(data.data, 0, 6) == string("\xfd" "7zXZ\0", 6)) if (string(*data.data, 0, 6) == string("\xfd" "7zXZ\0", 6))
data.data = decompressXZ(data.data); data.data = decompressXZ(*data.data);
StringSource source(data.data); StringSource source(*data.data);
restorePath(storePath, source); restorePath(storePath, source);
} else } else
writeFile(storePath, data.data); writeFile(storePath, *data.data);
auto executable = drv.env.find("executable"); auto executable = drv.env.find("executable");
if (executable != drv.env.end() && executable->second == "1") { if (executable != drv.env.end() && executable->second == "1") {

View file

@ -29,7 +29,7 @@ std::string resolveUri(const std::string & uri)
struct CurlDownloader : public Downloader struct CurlDownloader : public Downloader
{ {
CURL * curl; CURL * curl;
string data; ref<std::string> data;
string etag, status, expectedETag; string etag, status, expectedETag;
struct curl_slist * requestHeaders; struct curl_slist * requestHeaders;
@ -41,7 +41,7 @@ struct CurlDownloader : public Downloader
size_t writeCallback(void * contents, size_t size, size_t nmemb) size_t writeCallback(void * contents, size_t size, size_t nmemb)
{ {
size_t realSize = size * nmemb; size_t realSize = size * nmemb;
data.append((char *) contents, realSize); data->append((char *) contents, realSize);
return realSize; return realSize;
} }
@ -110,6 +110,7 @@ struct CurlDownloader : public Downloader
} }
CurlDownloader() CurlDownloader()
: data(make_ref<std::string>())
{ {
requestHeaders = 0; requestHeaders = 0;
@ -156,7 +157,7 @@ struct CurlDownloader : public Downloader
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0);
} }
data.clear(); data->clear();
if (requestHeaders) { if (requestHeaders) {
curl_slist_free_all(requestHeaders); curl_slist_free_all(requestHeaders);
@ -269,7 +270,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
auto res = download(url, options); auto res = download(url, options);
if (!res.cached) if (!res.cached)
storePath = store->addTextToStore(name, res.data, PathSet(), false); storePath = store->addTextToStore(name, *res.data, PathSet(), false);
assert(!storePath.empty()); assert(!storePath.empty());
replaceSymlink(storePath, fileLink); replaceSymlink(storePath, fileLink);

View file

@ -17,7 +17,8 @@ struct DownloadOptions
struct DownloadResult struct DownloadResult
{ {
bool cached; bool cached;
string data, etag; string etag;
std::shared_ptr<std::string> data;
}; };
class Store; class Store;

View file

@ -58,12 +58,18 @@ protected:
throw Error("uploading to an HTTP binary cache is not supported"); throw Error("uploading to an HTTP binary cache is not supported");
} }
std::string getFile(const std::string & path) override std::shared_ptr<std::string> getFile(const std::string & path) override
{ {
auto downloader(downloaders.get()); auto downloader(downloaders.get());
DownloadOptions options; DownloadOptions options;
options.showProgress = DownloadOptions::no; options.showProgress = DownloadOptions::no;
return downloader->download(cacheUri + "/" + path, options).data; try {
return downloader->download(cacheUri + "/" + path, options).data;
} catch (DownloadError & e) {
if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
return 0;
throw;
}
} }
}; };

View file

@ -22,7 +22,7 @@ protected:
void upsertFile(const std::string & path, const std::string & data) override; void upsertFile(const std::string & path, const std::string & data) override;
std::string getFile(const std::string & path) override; std::shared_ptr<std::string> getFile(const std::string & path) override;
}; };
@ -59,9 +59,14 @@ void LocalBinaryCacheStore::upsertFile(const std::string & path, const std::stri
atomicWrite(binaryCacheDir + "/" + path, data); atomicWrite(binaryCacheDir + "/" + path, data);
} }
std::string LocalBinaryCacheStore::getFile(const std::string & path) std::shared_ptr<std::string> LocalBinaryCacheStore::getFile(const std::string & path)
{ {
return readFile(binaryCacheDir + "/" + path); try {
return std::make_shared<std::string>(readFile(binaryCacheDir + "/" + path));
} catch (SysError & e) {
if (e.errNo == ENOENT) return 0;
throw;
}
} }
ref<Store> openLocalBinaryCacheStore(std::shared_ptr<Store> localStore, ref<Store> openLocalBinaryCacheStore(std::shared_ptr<Store> localStore,

View file

@ -511,6 +511,7 @@ ValidPathInfo decodeValidPathInfo(std::istream & str,
MakeError(SubstError, Error) MakeError(SubstError, Error)
MakeError(BuildError, Error) /* denotes a permanent build failure */ MakeError(BuildError, Error) /* denotes a permanent build failure */
MakeError(InvalidPath, Error)
} }

View file

@ -55,7 +55,7 @@ std::string compressXZ(const std::string & in)
} }
} }
std::string decompressXZ(const std::string & in) ref<std::string> decompressXZ(const std::string & in)
{ {
LzmaStream strm; LzmaStream strm;
@ -66,7 +66,7 @@ std::string decompressXZ(const std::string & in)
lzma_action action = LZMA_RUN; lzma_action action = LZMA_RUN;
uint8_t outbuf[BUFSIZ]; uint8_t outbuf[BUFSIZ];
string res; ref<std::string> res = make_ref<std::string>();
strm().next_in = (uint8_t *) in.c_str(); strm().next_in = (uint8_t *) in.c_str();
strm().avail_in = in.size(); strm().avail_in = in.size();
strm().next_out = outbuf; strm().next_out = outbuf;
@ -80,7 +80,7 @@ std::string decompressXZ(const std::string & in)
lzma_ret ret = lzma_code(&strm(), action); lzma_ret ret = lzma_code(&strm(), action);
if (strm().avail_out == 0 || ret == LZMA_STREAM_END) { if (strm().avail_out == 0 || ret == LZMA_STREAM_END) {
res.append((char *) outbuf, sizeof(outbuf) - strm().avail_out); res->append((char *) outbuf, sizeof(outbuf) - strm().avail_out);
strm().next_out = outbuf; strm().next_out = outbuf;
strm().avail_out = sizeof(outbuf); strm().avail_out = sizeof(outbuf);
} }

View file

@ -1,11 +1,13 @@
#pragma once #pragma once
#include "ref.hh"
#include <string> #include <string>
namespace nix { namespace nix {
std::string compressXZ(const std::string & in); std::string compressXZ(const std::string & in);
std::string decompressXZ(const std::string & in); ref<std::string> decompressXZ(const std::string & in);
} }

View file

@ -162,7 +162,7 @@ int main(int argc, char * * argv)
AutoDelete tmpDir(createTempDir(), true); AutoDelete tmpDir(createTempDir(), true);
Path tmpFile = (Path) tmpDir + "/tmp"; Path tmpFile = (Path) tmpDir + "/tmp";
writeFile(tmpFile, result.data); writeFile(tmpFile, *result.data);
/* Optionally unpack the file. */ /* Optionally unpack the file. */
if (unpack) { if (unpack) {
@ -186,7 +186,7 @@ int main(int argc, char * * argv)
/* FIXME: inefficient; addToStore() will also hash /* FIXME: inefficient; addToStore() will also hash
this. */ this. */
hash = unpack ? hashPath(ht, tmpFile).first : hashString(ht, result.data); hash = unpack ? hashPath(ht, tmpFile).first : hashString(ht, *result.data);
if (expectedHash != Hash(ht) && expectedHash != hash) if (expectedHash != Hash(ht) && expectedHash != hash)
throw Error(format("hash mismatch for %1%") % uri); throw Error(format("hash mismatch for %1%") % uri);