BinaryCacheStore: Support "none" compression method

src/libstore/binary-cache-store.cc

@@ -17,6 +17,7 @@ namespace nix {
 BinaryCacheStore::BinaryCacheStore(std::shared_ptr<Store> localStore,
     const StoreParams & params)
     : localStore(localStore)
+    , compression(get(params, "compression", "xz"))
 {
     auto secretKeyFile = get(params, "secret-key", "");
     if (secretKeyFile != "")
@@ -45,8 +46,7 @@ Path BinaryCacheStore::narInfoFileFor(const Path & storePath)
     return storePathToHash(storePath) + ".narinfo";
 }
 
-void BinaryCacheStore::addToCache(const ValidPathInfo & info,
-    const string & nar)
+void BinaryCacheStore::addToCache(const ValidPathInfo & info, ref<std::string> nar)
 {
     /* Verify that all references are valid. This may do some .narinfo
        reads, but typically they'll already be cached. */
@@ -62,40 +62,40 @@ void BinaryCacheStore::addToCache(const ValidPathInfo & info,
     auto narInfoFile = narInfoFileFor(info.path);
     if (fileExists(narInfoFile)) return;
 
-    assert(nar.compare(0, narMagic.size(), narMagic) == 0);
+    assert(nar->compare(0, narMagic.size(), narMagic) == 0);
 
     auto narInfo = make_ref<NarInfo>(info);
 
-    narInfo->narSize = nar.size();
-    narInfo->narHash = hashString(htSHA256, nar);
+    narInfo->narSize = nar->size();
+    narInfo->narHash = hashString(htSHA256, *nar);
 
     if (info.narHash && info.narHash != narInfo->narHash)
         throw Error(format("refusing to copy corrupted path ‘%1%’ to binary cache") % info.path);
 
     /* Compress the NAR. */
-    narInfo->compression = "xz";
+    narInfo->compression = compression;
     auto now1 = std::chrono::steady_clock::now();
-    string narXz = compressXZ(nar);
+    auto narCompressed = compress(compression, nar);
     auto now2 = std::chrono::steady_clock::now();
-    narInfo->fileHash = hashString(htSHA256, narXz);
-    narInfo->fileSize = narXz.size();
+    narInfo->fileHash = hashString(htSHA256, *narCompressed);
+    narInfo->fileSize = narCompressed->size();
 
     auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
     printMsg(lvlTalkative, format("copying path ‘%1%’ (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache")
         % narInfo->path % narInfo->narSize
-        % ((1.0 - (double) narXz.size() / nar.size()) * 100.0)
+        % ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0)
         % duration);
 
     /* Atomically write the NAR file. */
     narInfo->url = "nar/" + printHash32(narInfo->fileHash) + ".nar.xz";
     if (!fileExists(narInfo->url)) {
         stats.narWrite++;
-        upsertFile(narInfo->url, narXz);
+        upsertFile(narInfo->url, *narCompressed);
     } else
         stats.narWriteAverted++;
 
-    stats.narWriteBytes += nar.size();
-    stats.narWriteCompressedBytes += narXz.size();
+    stats.narWriteBytes += nar->size();
+    stats.narWriteCompressedBytes += narCompressed->size();
     stats.narWriteCompressionTimeMs += duration;
 
     /* Atomically write the NAR info file.*/
@@ -137,12 +137,7 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink)
 
     /* Decompress the NAR. FIXME: would be nice to have the remote
        side do this. */
-    if (info->compression == "none")
-        ;
-    else if (info->compression == "xz")
-        nar = decompressXZ(*nar);
-    else
-        throw Error(format("unknown NAR compression type ‘%1%’") % info->compression);
+    nar = decompress(info->compression, ref<std::string>(nar));
 
     stats.narReadBytes += nar->size();
 
@@ -261,7 +256,7 @@ Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
     info.path = makeFixedOutputPath(recursive, hashAlgo, h, name);
 
     if (repair || !isValidPath(info.path))
-        addToCache(info, *sink.s);
+        addToCache(info, sink.s);
 
     return info.path;
 }
@@ -276,7 +271,7 @@ Path BinaryCacheStore::addTextToStore(const string & name, const string & s,
     if (repair || !isValidPath(info.path)) {
         StringSink sink;
         dumpString(s, sink);
-        addToCache(info, *sink.s);
+        addToCache(info, sink.s);
     }
 
     return info.path;
@@ -306,7 +301,7 @@ void BinaryCacheStore::buildPaths(const PathSet & paths, BuildMode buildMode)
         StringSink sink;
         dumpPath(storePath, sink);
 
-        addToCache(*info, *sink.s);
+        addToCache(*info, sink.s);
     }
 }
 
@@ -404,7 +399,7 @@ Path BinaryCacheStore::importPath(Source & source, std::shared_ptr<FSAccessor> a
     bool haveSignature = readInt(source) == 1;
     assert(!haveSignature);
 
-    addToCache(info, *tee.data);
+    addToCache(info, tee.data);
 
     auto accessor_ = std::dynamic_pointer_cast<BinaryCacheStoreAccessor>(accessor);
     if (accessor_)

src/libstore/binary-cache-store.hh

@@ -19,6 +19,8 @@ private:
 
     std::shared_ptr<Store> localStore;
 
+    std::string compression;
+
 protected:
 
     BinaryCacheStore(std::shared_ptr<Store> localStore,
@@ -44,7 +46,7 @@ private:
 
     std::string narInfoFileFor(const Path & storePath);
 
-    void addToCache(const ValidPathInfo & info, const string & nar);
+    void addToCache(const ValidPathInfo & info, ref<std::string> nar);
 
 public:
 

src/libstore/builtins.cc

@@ -31,7 +31,7 @@ void builtinFetchurl(const BasicDerivation & drv)
     auto unpack = drv.env.find("unpack");
     if (unpack != drv.env.end() && unpack->second == "1") {
         if (string(*data.data, 0, 6) == string("\xfd" "7zXZ\0", 6))
-            data.data = decompressXZ(*data.data);
+            data.data = decompress("xz", ref<std::string>(data.data));
         StringSource source(*data.data);
         restorePath(storePath, source);
     } else

src/libutil/compression.cc

@@ -15,7 +15,7 @@ struct LzmaStream
     lzma_stream & operator()() { return strm; }
 };
 
-std::string compressXZ(const std::string & in)
+static ref<std::string> compressXZ(const std::string & in)
 {
     LzmaStream strm;
 
@@ -28,7 +28,7 @@ std::string compressXZ(const std::string & in)
 
     lzma_action action = LZMA_RUN;
     uint8_t outbuf[BUFSIZ];
-    string res;
+    ref<std::string> res = make_ref<std::string>();
     strm().next_in = (uint8_t *) in.c_str();
     strm().avail_in = in.size();
     strm().next_out = outbuf;
@@ -43,7 +43,7 @@ std::string compressXZ(const std::string & in)
         lzma_ret ret = lzma_code(&strm(), action);
 
         if (strm().avail_out == 0 || ret == LZMA_STREAM_END) {
-            res.append((char *) outbuf, sizeof(outbuf) - strm().avail_out);
+            res->append((char *) outbuf, sizeof(outbuf) - strm().avail_out);
             strm().next_out = outbuf;
             strm().avail_out = sizeof(outbuf);
         }
@@ -56,7 +56,7 @@ std::string compressXZ(const std::string & in)
     }
 }
 
-ref<std::string> decompressXZ(const std::string & in)
+static ref<std::string> decompressXZ(const std::string & in)
 {
     LzmaStream strm;
 
@@ -95,4 +95,24 @@ ref<std::string> decompressXZ(const std::string & in)
     }
 }
 
+ref<std::string> compress(const std::string & method, ref<std::string> in)
+{
+    if (method == "none")
+        return in;
+    else if (method == "xz")
+        return compressXZ(*in);
+    else
+        throw UnknownCompressionMethod(format("unknown compression method ‘%s’") % method);
+}
+
+ref<std::string> decompress(const std::string & method, ref<std::string> in)
+{
+    if (method == "none")
+        return in;
+    else if (method == "xz")
+        return decompressXZ(*in);
+    else
+        throw UnknownCompressionMethod(format("unknown compression method ‘%s’") % method);
+}
+
 }

src/libutil/compression.hh

@@ -1,13 +1,16 @@
 #pragma once
 
 #include "ref.hh"
+#include "types.hh"
 
 #include <string>
 
 namespace nix {
 
-std::string compressXZ(const std::string & in);
+ref<std::string> compress(const std::string & method, ref<std::string> in);
 
-ref<std::string> decompressXZ(const std::string & in);
+ref<std::string> decompress(const std::string & method, ref<std::string> in);
+
+MakeError(UnknownCompressionMethod, Error);
 
 }