Allow content-addressable paths to have references

This adds a command 'nix make-content-addressable' that rewrites the
specified store paths into content-addressable paths. The advantage of
such paths is that 1) they can be imported without signatures; 2) they
can enable deduplication in cases where derivation changes do not
cause output changes (apart from store path hashes).

For example,

  $ nix make-content-addressable -r nixpkgs.cowsay
  rewrote '/nix/store/g1g31ah55xdia1jdqabv1imf6mcw0nb1-glibc-2.25-49' to '/nix/store/48jfj7bg78a8n4f2nhg269rgw1936vj4-glibc-2.25-49'
  ...
  rewrote '/nix/store/qbi6rzpk0bxjw8lw6azn2mc7ynnn455q-cowsay-3.03+dfsg1-16' to '/nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16'

We can then copy the resulting closure to another store without
signatures:

  $ nix copy --trusted-public-keys '' ---to ~/my-nix /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16

In order to support self-references in content-addressable paths,
these paths are hashed "modulo" self-references, meaning that
self-references are zeroed out during hashing. Somewhat annoyingly,
this means that the NAR hash stored in the Nix database is no longer
necessarily equal to the output of "nix hash-path"; for
content-addressable paths, you need to pass the --modulo flag:

  $ nix path-info --json /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16  | jq -r .[].narHash
  sha256:0ri611gdilz2c9rsibqhsipbfs9vwcqvs811a52i2bnkhv7w9mgw

  $ nix hash-path --type sha256 --base32 /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16
  1ggznh07khq0hz6id09pqws3a8q9pn03ya3c03nwck1kwq8rclzs

  $ nix hash-path --type sha256 --base32 /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16 --modulo iq6g2x4q62xp7y7493bibx0qn5w7xz67
  0ri611gdilz2c9rsibqhsipbfs9vwcqvs811a52i2bnkhv7w9mgw

src/libstore/build.cc

@@ -727,23 +727,6 @@ HookInstance::~HookInstance()
 //////////////////////////////////////////////////////////////////////
 
 
-typedef map<std::string, std::string> StringRewrites;
-
-
-std::string rewriteStrings(std::string s, const StringRewrites & rewrites)
-{
-    for (auto & i : rewrites) {
-        size_t j = 0;
-        while ((j = s.find(i.first, j)) != string::npos)
-            s.replace(j, i.first.size(), i.second);
-    }
-    return s;
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
 typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
 
 class SubstitutionGoal;
@@ -865,7 +848,7 @@ private:
 #endif
 
     /* Hash rewriting. */
-    StringRewrites inputRewrites, outputRewrites;
+    StringMap inputRewrites, outputRewrites;
     typedef map<Path, Path> RedirectedOutputs;
     RedirectedOutputs redirectedOutputs;
 

src/libstore/local-store.cc

@@ -5,6 +5,7 @@
 #include "worker-protocol.hh"
 #include "derivations.hh"
 #include "nar-info.hh"
+#include "references.hh"
 
 #include <iostream>
 #include <algorithm>
@@ -1009,17 +1010,21 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
             /* While restoring the path from the NAR, compute the hash
                of the NAR. */
-            HashSink hashSink(htSHA256);
+            std::unique_ptr<AbstractHashSink> hashSink;
+            if (info.ca == "")
+                hashSink = std::make_unique<HashSink>(htSHA256);
+            else
+                hashSink = std::make_unique<HashModuloSink>(htSHA256, storePathToHash(info.path));
 
             LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t {
                 size_t n = source.read(data, len);
-                hashSink(data, n);
+                (*hashSink)(data, n);
                 return n;
             });
 
             restorePath(realPath, wrapperSource);
 
-            auto hashResult = hashSink.finish();
+            auto hashResult = hashSink->finish();
 
             if (hashResult.first != info.narHash)
                 throw Error("hash mismatch importing path '%s';\n  wanted: %s\n  got:    %s",
@@ -1241,7 +1246,15 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 
                 /* Check the content hash (optionally - slow). */
                 printMsg(lvlTalkative, format("checking contents of '%1%'") % i);
-                HashResult current = hashPath(info->narHash.type, toRealPath(i));
+
+                std::unique_ptr<AbstractHashSink> hashSink;
+                if (info->ca == "")
+                    hashSink = std::make_unique<HashSink>(info->narHash.type);
+                else
+                    hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(info->path));
+
+                dumpPath(toRealPath(i), *hashSink);
+                auto current = hashSink->finish();
 
                 if (info->narHash != nullHash && info->narHash != current.first) {
                     printError(format("path '%1%' was modified! "

src/libstore/references.cc

@@ -118,4 +118,66 @@ PathSet scanForReferences(const string & path,
 }
 
 
+RewritingSink::RewritingSink(const std::string & from, const std::string & to, Sink & nextSink)
+    : from(from), to(to), nextSink(nextSink)
+{
+    assert(from.size() == to.size());
+}
+
+void RewritingSink::operator () (const unsigned char * data, size_t len)
+{
+    std::string s(prev);
+    s.append((const char *) data, len);
+
+    size_t j = 0;
+    while ((j = s.find(from, j)) != string::npos) {
+        matches.push_back(pos + j);
+        s.replace(j, from.size(), to);
+    }
+
+    prev = s.size() < from.size() ? s : std::string(s, s.size() - from.size() + 1, from.size() - 1);
+
+    auto consumed = s.size() - prev.size();
+
+    pos += consumed;
+
+    if (consumed) nextSink((unsigned char *) s.data(), consumed);
+}
+
+void RewritingSink::flush()
+{
+    if (prev.empty()) return;
+    pos += prev.size();
+    nextSink((unsigned char *) prev.data(), prev.size());
+    prev.clear();
+}
+
+HashModuloSink::HashModuloSink(HashType ht, const std::string & modulus)
+    : hashSink(ht)
+    , rewritingSink(modulus, std::string(modulus.size(), 0), hashSink)
+{
+}
+
+void HashModuloSink::operator () (const unsigned char * data, size_t len)
+{
+    rewritingSink(data, len);
+}
+
+HashResult HashModuloSink::finish()
+{
+    rewritingSink.flush();
+
+    /* Hash the positions of the self-references. This ensures that a
+       NAR with self-references and a NAR with some of the
+       self-references already zeroed out do not produce a hash
+       collision. FIXME: proof. */
+    for (auto & pos : rewritingSink.matches) {
+        auto s = fmt("|%d", pos);
+        hashSink((unsigned char *) s.data(), s.size());
+    }
+
+    auto h = hashSink.finish();
+    return {h.first, rewritingSink.pos};
+}
+
 }

src/libstore/references.hh

@@ -8,4 +8,31 @@ namespace nix {
 PathSet scanForReferences(const Path & path, const PathSet & refs,
     HashResult & hash);
 
+struct RewritingSink : Sink
+{
+    std::string from, to, prev;
+    Sink & nextSink;
+    uint64_t pos = 0;
+
+    std::vector<uint64_t> matches;
+
+    RewritingSink(const std::string & from, const std::string & to, Sink & nextSink);
+
+    void operator () (const unsigned char * data, size_t len) override;
+
+    void flush();
+};
+
+struct HashModuloSink : AbstractHashSink
+{
+    HashSink hashSink;
+    RewritingSink rewritingSink;
+
+    HashModuloSink(HashType ht, const std::string & modulus);
+
+    void operator () (const unsigned char * data, size_t len) override;
+
+    HashResult finish() override;
+};
+
 }

src/libstore/store-api.cc

@@ -205,15 +205,27 @@ Path Store::makeOutputPath(const string & id,
 }
 
 
-Path Store::makeFixedOutputPath(bool recursive,
+static std::string makeType(string && type, const PathSet & references)
-    const Hash & hash, const string & name) const
 {
-    return hash.type == htSHA256 && recursive
+    for (auto & i : references) {
-        ? makeStorePath("source", hash, name)
+        type += ":";
-        : makeStorePath("output:out", hashString(htSHA256,
+        type += i;
+    }
+    return type;
+}
+
+
+Path Store::makeFixedOutputPath(bool recursive,
+    const Hash & hash, const string & name, const PathSet & references) const
+{
+    if (hash.type == htSHA256 && recursive) {
+        return makeStorePath(makeType("source", references), hash, name);
+    } else {
+        assert(references.empty());
+        return makeStorePath("output:out", hashString(htSHA256,
                 "fixed:out:" + (recursive ? (string) "r:" : "") +
-                hash.to_string(Base16) + ":"),
+                hash.to_string(Base16) + ":"), name);
-            name);
+    }
 }
 
 
@@ -224,12 +236,7 @@ Path Store::makeTextPath(const string & name, const Hash & hash,
     /* Stuff the references (if any) into the type.  This is a bit
        hacky, but we can't put them in `s' since that would be
        ambiguous. */
-    string type = "text";
+    return makeStorePath(makeType("text", references), hash, name);
-    for (auto & i : references) {
-        type += ":";
-        type += i;
-    }
-    return makeStorePath(type, hash, name);
 }
 
 
@@ -785,8 +792,9 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const
     else if (hasPrefix(ca, "fixed:")) {
         bool recursive = ca.compare(6, 2, "r:") == 0;
         Hash hash(std::string(ca, recursive ? 8 : 6));
-        if (references.empty() &&
+        auto refs = references;
-            store.makeFixedOutputPath(recursive, hash, storePathToName(path)) == path)
+        replaceInSet(refs, path, std::string("self"));
+        if (store.makeFixedOutputPath(recursive, hash, storePathToName(path), refs) == path)
             return true;
         else
             warn();

src/libstore/store-api.hh

@@ -303,7 +303,8 @@ public:
         const Hash & hash, const string & name) const;
 
     Path makeFixedOutputPath(bool recursive,
-        const Hash & hash, const string & name) const;
+        const Hash & hash, const string & name,
+        const PathSet & references = {}) const;
 
     Path makeTextPath(const string & name, const Hash & hash,
         const PathSet & references) const;

src/libutil/hash.cc

@@ -256,23 +256,9 @@ Hash hashString(HashType ht, const string & s)
 
 Hash hashFile(HashType ht, const Path & path)
 {
-    Ctx ctx;
+    HashSink sink(ht);
-    Hash hash(ht);
+    readFile(path, sink);
-    start(ht, ctx);
+    return sink.finish().first;
-
-    AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
-    if (!fd) throw SysError(format("opening file '%1%'") % path);
-
-    std::vector<unsigned char> buf(8192);
-    ssize_t n;
-    while ((n = read(fd.get(), buf.data(), buf.size()))) {
-        checkInterrupt();
-        if (n == -1) throw SysError(format("reading file '%1%'") % path);
-        update(ht, ctx, buf.data(), n);
-    }
-
-    finish(ht, ctx, hash.hash);
-    return hash;
 }
 
 

src/libutil/hash.hh

@@ -111,7 +111,12 @@ string printHashType(HashType ht);
 
 union Ctx;
 
-class HashSink : public BufferedSink
+struct AbstractHashSink : virtual Sink
+{
+    virtual HashResult finish() = 0;
+};
+
+class HashSink : public BufferedSink, public AbstractHashSink
 {
 private:
     HashType ht;
@@ -122,8 +127,8 @@ public:
     HashSink(HashType ht);
     HashSink(const HashSink & h);
     ~HashSink();
-    void write(const unsigned char * data, size_t len);
+    void write(const unsigned char * data, size_t len) override;
-    HashResult finish();
+    HashResult finish() override;
     HashResult currentHash();
 };
 

src/libutil/serialise.hh

@@ -24,7 +24,7 @@ struct Sink
 
 
 /* A buffered abstract sink. */
-struct BufferedSink : Sink
+struct BufferedSink : virtual Sink
 {
     size_t bufSize, bufPos;
     std::unique_ptr<unsigned char[]> buffer;

src/libutil/util.cc

@@ -1260,6 +1260,19 @@ string replaceStrings(const std::string & s,
 }
 
 
+std::string rewriteStrings(const std::string & _s, const StringMap & rewrites)
+{
+    auto s = _s;
+    for (auto & i : rewrites) {
+        if (i.first == i.second) continue;
+        size_t j = 0;
+        while ((j = s.find(i.first, j)) != string::npos)
+            s.replace(j, i.first.size(), i.second);
+    }
+    return s;
+}
+
+
 string statusToString(int status)
 {
     if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {

src/libutil/util.hh

@@ -362,6 +362,20 @@ string replaceStrings(const std::string & s,
     const std::string & from, const std::string & to);
 
 
+std::string rewriteStrings(const std::string & s, const StringMap & rewrites);
+
+
+/* If a set contains 'from', remove it and insert 'to'. */
+template<typename T>
+void replaceInSet(std::set<T> & set, const T & from, const T & to)
+{
+    auto i = set.find(from);
+    if (i == set.end()) return;
+    set.erase(i);
+    set.insert(to);
+}
+
+
 /* Convert the exit status of a child as returned by wait() into an
    error string. */
 string statusToString(int status);

src/nix/command.cc

@@ -129,7 +129,7 @@ void StorePathsCommand::run(ref<Store> store)
     }
 
     else {
-        for (auto & p : toStorePaths(store, NoBuild, installables))
+        for (auto & p : toStorePaths(store, realiseMode, installables))
             storePaths.push_back(p);
 
         if (recursive) {

src/nix/command.hh

@@ -139,6 +139,10 @@ private:
     bool recursive = false;
     bool all = false;
 
+protected:
+
+    RealiseMode realiseMode = NoBuild;
+
 public:
 
     StorePathsCommand(bool recursive = false);

src/nix/hash.cc

@@ -2,6 +2,8 @@
 #include "hash.hh"
 #include "legacy.hh"
 #include "shared.hh"
+#include "references.hh"
+#include "archive.hh"
 
 using namespace nix;
 
@@ -13,6 +15,7 @@ struct CmdHash : Command
     bool truncate = false;
     HashType ht = htSHA256;
     std::vector<std::string> paths;
+    std::experimental::optional<std::string> modulus;
 
     CmdHash(Mode mode) : mode(mode)
     {
@@ -23,6 +26,11 @@ struct CmdHash : Command
         mkFlag()
             .longName("type")
             .mkHashTypeFlag(&ht);
+        mkFlag()
+            .longName("modulo")
+            .description("compute hash modulo specified string")
+            .labels({"modulus"})
+            .dest(&modulus);
         expectArgs("paths", &paths);
     }
 
@@ -41,7 +49,19 @@ struct CmdHash : Command
     void run() override
     {
         for (auto path : paths) {
-            Hash h = mode == mFile ? hashFile(ht, path) : hashPath(ht, path).first;
+
+            std::unique_ptr<AbstractHashSink> hashSink;
+            if (modulus)
+                hashSink = std::make_unique<HashModuloSink>(ht, *modulus);
+            else
+                hashSink = std::make_unique<HashSink>(ht);
+
+            if (mode == mFile)
+                readFile(path, *hashSink);
+            else
+                dumpPath(path, *hashSink);
+
+            Hash h = hashSink->finish().first;
             if (truncate && h.hashSize > 20) h = compressHash(h, 20);
             std::cout << format("%1%\n") %
                 h.to_string(base, base == SRI);

src/nix/make-content-addressable.cc

@@ -0,0 +1,82 @@
+#include "command.hh"
+#include "store-api.hh"
+#include "references.hh"
+
+using namespace nix;
+
+struct CmdMakeContentAddressable : StorePathsCommand
+{
+    CmdMakeContentAddressable()
+    {
+        realiseMode = Build;
+    }
+
+    std::string name() override
+    {
+        return "make-content-addressable";
+    }
+
+    std::string description() override
+    {
+        return "test";
+    }
+
+    void run(ref<Store> store, Paths storePaths) override
+    {
+        auto paths = store->topoSortPaths(PathSet(storePaths.begin(), storePaths.end()));
+
+        paths.reverse();
+
+        std::map<Path, Path> remappings;
+
+        for (auto & path : paths) {
+            auto oldInfo = store->queryPathInfo(path);
+            auto oldHashPart = storePathToHash(path);
+            auto name = storePathToName(path);
+
+            StringSink sink;
+            store->narFromPath(path, sink);
+
+            StringMap rewrites;
+
+            ValidPathInfo info;
+            for (auto & ref : oldInfo->references) {
+                if (ref == path)
+                    info.references.insert("self");
+                else {
+                    auto replacement = get(remappings, ref, ref);
+                    // FIXME: warn about unremapped paths?
+                    info.references.insert(replacement);
+                    if (replacement != ref)
+                        rewrites[storePathToHash(ref)] = storePathToHash(replacement);
+                }
+            }
+
+            *sink.s = rewriteStrings(*sink.s, rewrites);
+
+            HashModuloSink hashModuloSink(htSHA256, oldHashPart);
+            hashModuloSink((unsigned char *) sink.s->data(), sink.s->size());
+
+            info.narHash = hashModuloSink.finish().first;
+            info.narSize = sink.s->size();
+            replaceInSet(info.references, path, std::string("self"));
+            info.path = store->makeFixedOutputPath(true, info.narHash, name, info.references);
+            replaceInSet(info.references, std::string("self"), info.path);
+            info.ca = makeFixedOutputCA(true, info.narHash);
+
+            printError("rewrote '%s' to '%s'", path, info.path);
+
+            auto source = sinkToSource([&](Sink & nextSink) {
+                RewritingSink rsink2(oldHashPart, storePathToHash(info.path), nextSink);
+                rsink2((unsigned char *) sink.s->data(), sink.s->size());
+                rsink2.flush();
+            });
+
+            store->addToStore(info, *source);
+
+            remappings[path] = info.path;
+        }
+    }
+};
+
+static RegisterCommand r1(make_ref<CmdMakeContentAddressable>());

src/nix/verify.cc

@@ -3,6 +3,7 @@
 #include "store-api.hh"
 #include "sync.hh"
 #include "thread-pool.hh"
+#include "references.hh"
 
 #include <atomic>
 
@@ -88,10 +89,15 @@ struct CmdVerify : StorePathsCommand
 
                 if (!noContents) {
 
-                    HashSink sink(info->narHash.type);
+                    std::unique_ptr<AbstractHashSink> hashSink;
-                    store->narFromPath(info->path, sink);
+                    if (info->ca == "")
+                        hashSink = std::make_unique<HashSink>(info->narHash.type);
+                    else
+                        hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(info->path));
 
-                    auto hash = sink.finish();
+                    store->narFromPath(info->path, *hashSink);
+
+                    auto hash = hashSink->finish();
 
                     if (hash.first != info->narHash) {
                         corrupted++;