raito/hydra
0
0
Fork 0
forked from lix-project/hydra
Code Pull requests Activity

hydra-queue-runner: Compress binary cache NARs using xz

Browse source
This commit is contained in:
Eelco Dolstra 2016-02-15 21:56:53 +01:00
parent 2d0dd7fb49
commit 744cee134e
2 changed files with 37 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
src/hydra-queue-runner/local-binary-cache.cc
View file

@ -1,6 +1,7 @@
#include "local-binary-cache.hh" #include "local-binary-cache.hh"
#include "archive.hh" #include "archive.hh"
#include "compression.hh"
#include "derivations.hh" #include "derivations.hh"
#include "globals.hh" #include "globals.hh"
#include "worker-protocol.hh" #include "worker-protocol.hh"
@ -32,6 +33,9 @@ void atomicWrite(const Path & path, const std::string & s)
void LocalBinaryCache::addToCache(const ValidPathInfo & info, void LocalBinaryCache::addToCache(const ValidPathInfo & info,
const string & nar) const string & nar)
{ {
Path narInfoFile = narInfoFileFor(info.path);
if (pathExists(narInfoFile)) return;
size_t narSize = nar.size(); size_t narSize = nar.size();
Hash narHash = hashString(htSHA256, nar); Hash narHash = hashString(htSHA256, nar);
@ -41,16 +45,16 @@ void LocalBinaryCache::addToCache(const ValidPathInfo & info,
printMsg(lvlTalkative, format("copying path ‘%1%’ (%2% bytes) to binary cache") printMsg(lvlTalkative, format("copying path ‘%1%’ (%2% bytes) to binary cache")
% info.path % narSize); % info.path % narSize);
/* Compress the NAR. */
string narXz = compressXZ(nar);
Hash narXzHash = hashString(htSHA256, narXz);
/* Atomically write the NAR file. */ /* Atomically write the NAR file. */
string narFileRel = "nar/" + printHash(narHash) + ".nar"; string narFileRel = "nar/" + printHash32(narXzHash) + ".nar.xz";
Path narFile = binaryCacheDir + "/" + narFileRel; Path narFile = binaryCacheDir + "/" + narFileRel;
if (!pathExists(narFile)) atomicWrite(narFile, nar); if (!pathExists(narFile)) atomicWrite(narFile, narXz);
/* Atomically write the NAR info file.*/ /* Atomically write the NAR info file.*/
Path narInfoFile = narInfoFileFor(info.path);
if (!pathExists(narInfoFile)) {
Strings refs; Strings refs;
for (auto & r : info.references) for (auto & r : info.references)
refs.push_back(baseNameOf(r)); refs.push_back(baseNameOf(r));
@ -58,17 +62,16 @@ void LocalBinaryCache::addToCache(const ValidPathInfo & info,
std::string narInfo; std::string narInfo;
narInfo += "StorePath: " + info.path + "\n"; narInfo += "StorePath: " + info.path + "\n";
narInfo += "URL: " + narFileRel + "\n"; narInfo += "URL: " + narFileRel + "\n";
narInfo += "Compression: none\n"; narInfo += "Compression: xz\n";
narInfo += "FileHash: sha256:" + printHash(narHash) + "\n"; narInfo += "FileHash: sha256:" + printHash32(narXzHash) + "\n";
narInfo += "FileSize: " + std::to_string(narSize) + "\n"; narInfo += "FileSize: " + std::to_string(narXz.size()) + "\n";
narInfo += "NarHash: sha256:" + printHash(narHash) + "\n"; narInfo += "NarHash: sha256:" + printHash32(narHash) + "\n";
narInfo += "NarSize: " + std::to_string(narSize) + "\n"; narInfo += "NarSize: " + std::to_string(narSize) + "\n";
narInfo += "References: " + concatStringsSep(" ", refs) + "\n"; narInfo += "References: " + concatStringsSep(" ", refs) + "\n";
// FIXME: add signature // FIXME: add signature
atomicWrite(narInfoFile, narInfo); atomicWrite(narInfoFile, narInfo);
}
} }
LocalBinaryCache::NarInfo LocalBinaryCache::readNarInfo(const Path & storePath) LocalBinaryCache::NarInfo LocalBinaryCache::readNarInfo(const Path & storePath)
@ -111,6 +114,9 @@ LocalBinaryCache::NarInfo LocalBinaryCache::readNarInfo(const Path & storePath)
else if (name == "URL") { else if (name == "URL") {
res.narUrl = value; res.narUrl = value;
} }
else if (name == "Compression") {
res.compression = value;
}
pos = eol + 1; pos = eol + 1;
} }
@ -137,6 +143,15 @@ void LocalBinaryCache::exportPath(const Path & storePath, bool sign, Sink & sink
auto nar = readFile(binaryCacheDir + "/" + res.narUrl); auto nar = readFile(binaryCacheDir + "/" + res.narUrl);
/* Decompress the NAR. FIXME: would be nice to have the remote
side do this. */
if (res.compression == "none")
;
else if (res.compression == "xz")
nar = decompressXZ(nar);
else
throw Error(format("unknown NAR compression type ‘%1%’") % nar);
printMsg(lvlTalkative, format("exporting path ‘%1%’ (%2% bytes)") % storePath % nar.size()); printMsg(lvlTalkative, format("exporting path ‘%1%’ (%2% bytes)") % storePath % nar.size());
assert(nar.size() % 8 == 0); assert(nar.size() % 8 == 0);

1
src/hydra-queue-runner/local-binary-cache.hh
View file

@ -24,6 +24,7 @@ private:
{ {
ValidPathInfo info; ValidPathInfo info;
std::string narUrl; std::string narUrl;
std::string compression = "none";
}; };
NarInfo readNarInfo(const Path & storePath); NarInfo readNarInfo(const Path & storePath);