Allow content-addressable paths to have references

This adds a command 'nix make-content-addressable' that rewrites the
specified store paths into content-addressable paths. The advantage of
such paths is that 1) they can be imported without signatures; 2) they
can enable deduplication in cases where derivation changes do not
cause output changes (apart from store path hashes).

For example,

  $ nix make-content-addressable -r nixpkgs.cowsay
  rewrote '/nix/store/g1g31ah55xdia1jdqabv1imf6mcw0nb1-glibc-2.25-49' to '/nix/store/48jfj7bg78a8n4f2nhg269rgw1936vj4-glibc-2.25-49'
  ...
  rewrote '/nix/store/qbi6rzpk0bxjw8lw6azn2mc7ynnn455q-cowsay-3.03+dfsg1-16' to '/nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16'

We can then copy the resulting closure to another store without
signatures:

  $ nix copy --trusted-public-keys '' ---to ~/my-nix /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16

In order to support self-references in content-addressable paths,
these paths are hashed "modulo" self-references, meaning that
self-references are zeroed out during hashing. Somewhat annoyingly,
this means that the NAR hash stored in the Nix database is no longer
necessarily equal to the output of "nix hash-path"; for
content-addressable paths, you need to pass the --modulo flag:

  $ nix path-info --json /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16  | jq -r .[].narHash
  sha256:0ri611gdilz2c9rsibqhsipbfs9vwcqvs811a52i2bnkhv7w9mgw

  $ nix hash-path --type sha256 --base32 /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16
  1ggznh07khq0hz6id09pqws3a8q9pn03ya3c03nwck1kwq8rclzs

  $ nix hash-path --type sha256 --base32 /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16 --modulo iq6g2x4q62xp7y7493bibx0qn5w7xz67
  0ri611gdilz2c9rsibqhsipbfs9vwcqvs811a52i2bnkhv7w9mgw
This commit is contained in:
Eelco Dolstra 2018-03-30 00:56:13 +02:00
parent aabf5c86c9
commit 0abb3ad537
16 changed files with 289 additions and 65 deletions

View file

@ -727,23 +727,6 @@ HookInstance::~HookInstance()
////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////
typedef map<std::string, std::string> StringRewrites;
std::string rewriteStrings(std::string s, const StringRewrites & rewrites)
{
for (auto & i : rewrites) {
size_t j = 0;
while ((j = s.find(i.first, j)) != string::npos)
s.replace(j, i.first.size(), i.second);
}
return s;
}
//////////////////////////////////////////////////////////////////////
typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
class SubstitutionGoal; class SubstitutionGoal;
@ -865,7 +848,7 @@ private:
#endif #endif
/* Hash rewriting. */ /* Hash rewriting. */
StringRewrites inputRewrites, outputRewrites; StringMap inputRewrites, outputRewrites;
typedef map<Path, Path> RedirectedOutputs; typedef map<Path, Path> RedirectedOutputs;
RedirectedOutputs redirectedOutputs; RedirectedOutputs redirectedOutputs;

View file

@ -5,6 +5,7 @@
#include "worker-protocol.hh" #include "worker-protocol.hh"
#include "derivations.hh" #include "derivations.hh"
#include "nar-info.hh" #include "nar-info.hh"
#include "references.hh"
#include <iostream> #include <iostream>
#include <algorithm> #include <algorithm>
@ -1009,17 +1010,21 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
/* While restoring the path from the NAR, compute the hash /* While restoring the path from the NAR, compute the hash
of the NAR. */ of the NAR. */
HashSink hashSink(htSHA256); std::unique_ptr<AbstractHashSink> hashSink;
if (info.ca == "")
hashSink = std::make_unique<HashSink>(htSHA256);
else
hashSink = std::make_unique<HashModuloSink>(htSHA256, storePathToHash(info.path));
LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t { LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t {
size_t n = source.read(data, len); size_t n = source.read(data, len);
hashSink(data, n); (*hashSink)(data, n);
return n; return n;
}); });
restorePath(realPath, wrapperSource); restorePath(realPath, wrapperSource);
auto hashResult = hashSink.finish(); auto hashResult = hashSink->finish();
if (hashResult.first != info.narHash) if (hashResult.first != info.narHash)
throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s", throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s",
@ -1241,7 +1246,15 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
/* Check the content hash (optionally - slow). */ /* Check the content hash (optionally - slow). */
printMsg(lvlTalkative, format("checking contents of '%1%'") % i); printMsg(lvlTalkative, format("checking contents of '%1%'") % i);
HashResult current = hashPath(info->narHash.type, toRealPath(i));
std::unique_ptr<AbstractHashSink> hashSink;
if (info->ca == "")
hashSink = std::make_unique<HashSink>(info->narHash.type);
else
hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(info->path));
dumpPath(toRealPath(i), *hashSink);
auto current = hashSink->finish();
if (info->narHash != nullHash && info->narHash != current.first) { if (info->narHash != nullHash && info->narHash != current.first) {
printError(format("path '%1%' was modified! " printError(format("path '%1%' was modified! "

View file

@ -118,4 +118,66 @@ PathSet scanForReferences(const string & path,
} }
RewritingSink::RewritingSink(const std::string & from, const std::string & to, Sink & nextSink)
: from(from), to(to), nextSink(nextSink)
{
assert(from.size() == to.size());
}
void RewritingSink::operator () (const unsigned char * data, size_t len)
{
std::string s(prev);
s.append((const char *) data, len);
size_t j = 0;
while ((j = s.find(from, j)) != string::npos) {
matches.push_back(pos + j);
s.replace(j, from.size(), to);
}
prev = s.size() < from.size() ? s : std::string(s, s.size() - from.size() + 1, from.size() - 1);
auto consumed = s.size() - prev.size();
pos += consumed;
if (consumed) nextSink((unsigned char *) s.data(), consumed);
}
void RewritingSink::flush()
{
if (prev.empty()) return;
pos += prev.size();
nextSink((unsigned char *) prev.data(), prev.size());
prev.clear();
}
HashModuloSink::HashModuloSink(HashType ht, const std::string & modulus)
: hashSink(ht)
, rewritingSink(modulus, std::string(modulus.size(), 0), hashSink)
{
}
void HashModuloSink::operator () (const unsigned char * data, size_t len)
{
rewritingSink(data, len);
}
HashResult HashModuloSink::finish()
{
rewritingSink.flush();
/* Hash the positions of the self-references. This ensures that a
NAR with self-references and a NAR with some of the
self-references already zeroed out do not produce a hash
collision. FIXME: proof. */
for (auto & pos : rewritingSink.matches) {
auto s = fmt("|%d", pos);
hashSink((unsigned char *) s.data(), s.size());
}
auto h = hashSink.finish();
return {h.first, rewritingSink.pos};
}
} }

View file

@ -8,4 +8,31 @@ namespace nix {
PathSet scanForReferences(const Path & path, const PathSet & refs, PathSet scanForReferences(const Path & path, const PathSet & refs,
HashResult & hash); HashResult & hash);
struct RewritingSink : Sink
{
std::string from, to, prev;
Sink & nextSink;
uint64_t pos = 0;
std::vector<uint64_t> matches;
RewritingSink(const std::string & from, const std::string & to, Sink & nextSink);
void operator () (const unsigned char * data, size_t len) override;
void flush();
};
struct HashModuloSink : AbstractHashSink
{
HashSink hashSink;
RewritingSink rewritingSink;
HashModuloSink(HashType ht, const std::string & modulus);
void operator () (const unsigned char * data, size_t len) override;
HashResult finish() override;
};
} }

View file

@ -205,15 +205,27 @@ Path Store::makeOutputPath(const string & id,
} }
Path Store::makeFixedOutputPath(bool recursive, static std::string makeType(string && type, const PathSet & references)
const Hash & hash, const string & name) const
{ {
return hash.type == htSHA256 && recursive for (auto & i : references) {
? makeStorePath("source", hash, name) type += ":";
: makeStorePath("output:out", hashString(htSHA256, type += i;
}
return type;
}
Path Store::makeFixedOutputPath(bool recursive,
const Hash & hash, const string & name, const PathSet & references) const
{
if (hash.type == htSHA256 && recursive) {
return makeStorePath(makeType("source", references), hash, name);
} else {
assert(references.empty());
return makeStorePath("output:out", hashString(htSHA256,
"fixed:out:" + (recursive ? (string) "r:" : "") + "fixed:out:" + (recursive ? (string) "r:" : "") +
hash.to_string(Base16) + ":"), hash.to_string(Base16) + ":"), name);
name); }
} }
@ -224,12 +236,7 @@ Path Store::makeTextPath(const string & name, const Hash & hash,
/* Stuff the references (if any) into the type. This is a bit /* Stuff the references (if any) into the type. This is a bit
hacky, but we can't put them in `s' since that would be hacky, but we can't put them in `s' since that would be
ambiguous. */ ambiguous. */
string type = "text"; return makeStorePath(makeType("text", references), hash, name);
for (auto & i : references) {
type += ":";
type += i;
}
return makeStorePath(type, hash, name);
} }
@ -785,8 +792,9 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const
else if (hasPrefix(ca, "fixed:")) { else if (hasPrefix(ca, "fixed:")) {
bool recursive = ca.compare(6, 2, "r:") == 0; bool recursive = ca.compare(6, 2, "r:") == 0;
Hash hash(std::string(ca, recursive ? 8 : 6)); Hash hash(std::string(ca, recursive ? 8 : 6));
if (references.empty() && auto refs = references;
store.makeFixedOutputPath(recursive, hash, storePathToName(path)) == path) replaceInSet(refs, path, std::string("self"));
if (store.makeFixedOutputPath(recursive, hash, storePathToName(path), refs) == path)
return true; return true;
else else
warn(); warn();

View file

@ -303,7 +303,8 @@ public:
const Hash & hash, const string & name) const; const Hash & hash, const string & name) const;
Path makeFixedOutputPath(bool recursive, Path makeFixedOutputPath(bool recursive,
const Hash & hash, const string & name) const; const Hash & hash, const string & name,
const PathSet & references = {}) const;
Path makeTextPath(const string & name, const Hash & hash, Path makeTextPath(const string & name, const Hash & hash,
const PathSet & references) const; const PathSet & references) const;

View file

@ -256,23 +256,9 @@ Hash hashString(HashType ht, const string & s)
Hash hashFile(HashType ht, const Path & path) Hash hashFile(HashType ht, const Path & path)
{ {
Ctx ctx; HashSink sink(ht);
Hash hash(ht); readFile(path, sink);
start(ht, ctx); return sink.finish().first;
AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
if (!fd) throw SysError(format("opening file '%1%'") % path);
std::vector<unsigned char> buf(8192);
ssize_t n;
while ((n = read(fd.get(), buf.data(), buf.size()))) {
checkInterrupt();
if (n == -1) throw SysError(format("reading file '%1%'") % path);
update(ht, ctx, buf.data(), n);
}
finish(ht, ctx, hash.hash);
return hash;
} }

View file

@ -111,7 +111,12 @@ string printHashType(HashType ht);
union Ctx; union Ctx;
class HashSink : public BufferedSink struct AbstractHashSink : virtual Sink
{
virtual HashResult finish() = 0;
};
class HashSink : public BufferedSink, public AbstractHashSink
{ {
private: private:
HashType ht; HashType ht;
@ -122,8 +127,8 @@ public:
HashSink(HashType ht); HashSink(HashType ht);
HashSink(const HashSink & h); HashSink(const HashSink & h);
~HashSink(); ~HashSink();
void write(const unsigned char * data, size_t len); void write(const unsigned char * data, size_t len) override;
HashResult finish(); HashResult finish() override;
HashResult currentHash(); HashResult currentHash();
}; };

View file

@ -24,7 +24,7 @@ struct Sink
/* A buffered abstract sink. */ /* A buffered abstract sink. */
struct BufferedSink : Sink struct BufferedSink : virtual Sink
{ {
size_t bufSize, bufPos; size_t bufSize, bufPos;
std::unique_ptr<unsigned char[]> buffer; std::unique_ptr<unsigned char[]> buffer;

View file

@ -1260,6 +1260,19 @@ string replaceStrings(const std::string & s,
} }
std::string rewriteStrings(const std::string & _s, const StringMap & rewrites)
{
auto s = _s;
for (auto & i : rewrites) {
if (i.first == i.second) continue;
size_t j = 0;
while ((j = s.find(i.first, j)) != string::npos)
s.replace(j, i.first.size(), i.second);
}
return s;
}
string statusToString(int status) string statusToString(int status)
{ {
if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {

View file

@ -362,6 +362,20 @@ string replaceStrings(const std::string & s,
const std::string & from, const std::string & to); const std::string & from, const std::string & to);
std::string rewriteStrings(const std::string & s, const StringMap & rewrites);
/* If a set contains 'from', remove it and insert 'to'. */
template<typename T>
void replaceInSet(std::set<T> & set, const T & from, const T & to)
{
auto i = set.find(from);
if (i == set.end()) return;
set.erase(i);
set.insert(to);
}
/* Convert the exit status of a child as returned by wait() into an /* Convert the exit status of a child as returned by wait() into an
error string. */ error string. */
string statusToString(int status); string statusToString(int status);

View file

@ -129,7 +129,7 @@ void StorePathsCommand::run(ref<Store> store)
} }
else { else {
for (auto & p : toStorePaths(store, NoBuild, installables)) for (auto & p : toStorePaths(store, realiseMode, installables))
storePaths.push_back(p); storePaths.push_back(p);
if (recursive) { if (recursive) {

View file

@ -139,6 +139,10 @@ private:
bool recursive = false; bool recursive = false;
bool all = false; bool all = false;
protected:
RealiseMode realiseMode = NoBuild;
public: public:
StorePathsCommand(bool recursive = false); StorePathsCommand(bool recursive = false);

View file

@ -2,6 +2,8 @@
#include "hash.hh" #include "hash.hh"
#include "legacy.hh" #include "legacy.hh"
#include "shared.hh" #include "shared.hh"
#include "references.hh"
#include "archive.hh"
using namespace nix; using namespace nix;
@ -13,6 +15,7 @@ struct CmdHash : Command
bool truncate = false; bool truncate = false;
HashType ht = htSHA256; HashType ht = htSHA256;
std::vector<std::string> paths; std::vector<std::string> paths;
std::experimental::optional<std::string> modulus;
CmdHash(Mode mode) : mode(mode) CmdHash(Mode mode) : mode(mode)
{ {
@ -23,6 +26,11 @@ struct CmdHash : Command
mkFlag() mkFlag()
.longName("type") .longName("type")
.mkHashTypeFlag(&ht); .mkHashTypeFlag(&ht);
mkFlag()
.longName("modulo")
.description("compute hash modulo specified string")
.labels({"modulus"})
.dest(&modulus);
expectArgs("paths", &paths); expectArgs("paths", &paths);
} }
@ -41,7 +49,19 @@ struct CmdHash : Command
void run() override void run() override
{ {
for (auto path : paths) { for (auto path : paths) {
Hash h = mode == mFile ? hashFile(ht, path) : hashPath(ht, path).first;
std::unique_ptr<AbstractHashSink> hashSink;
if (modulus)
hashSink = std::make_unique<HashModuloSink>(ht, *modulus);
else
hashSink = std::make_unique<HashSink>(ht);
if (mode == mFile)
readFile(path, *hashSink);
else
dumpPath(path, *hashSink);
Hash h = hashSink->finish().first;
if (truncate && h.hashSize > 20) h = compressHash(h, 20); if (truncate && h.hashSize > 20) h = compressHash(h, 20);
std::cout << format("%1%\n") % std::cout << format("%1%\n") %
h.to_string(base, base == SRI); h.to_string(base, base == SRI);

View file

@ -0,0 +1,82 @@
#include "command.hh"
#include "store-api.hh"
#include "references.hh"
using namespace nix;
struct CmdMakeContentAddressable : StorePathsCommand
{
CmdMakeContentAddressable()
{
realiseMode = Build;
}
std::string name() override
{
return "make-content-addressable";
}
std::string description() override
{
return "test";
}
void run(ref<Store> store, Paths storePaths) override
{
auto paths = store->topoSortPaths(PathSet(storePaths.begin(), storePaths.end()));
paths.reverse();
std::map<Path, Path> remappings;
for (auto & path : paths) {
auto oldInfo = store->queryPathInfo(path);
auto oldHashPart = storePathToHash(path);
auto name = storePathToName(path);
StringSink sink;
store->narFromPath(path, sink);
StringMap rewrites;
ValidPathInfo info;
for (auto & ref : oldInfo->references) {
if (ref == path)
info.references.insert("self");
else {
auto replacement = get(remappings, ref, ref);
// FIXME: warn about unremapped paths?
info.references.insert(replacement);
if (replacement != ref)
rewrites[storePathToHash(ref)] = storePathToHash(replacement);
}
}
*sink.s = rewriteStrings(*sink.s, rewrites);
HashModuloSink hashModuloSink(htSHA256, oldHashPart);
hashModuloSink((unsigned char *) sink.s->data(), sink.s->size());
info.narHash = hashModuloSink.finish().first;
info.narSize = sink.s->size();
replaceInSet(info.references, path, std::string("self"));
info.path = store->makeFixedOutputPath(true, info.narHash, name, info.references);
replaceInSet(info.references, std::string("self"), info.path);
info.ca = makeFixedOutputCA(true, info.narHash);
printError("rewrote '%s' to '%s'", path, info.path);
auto source = sinkToSource([&](Sink & nextSink) {
RewritingSink rsink2(oldHashPart, storePathToHash(info.path), nextSink);
rsink2((unsigned char *) sink.s->data(), sink.s->size());
rsink2.flush();
});
store->addToStore(info, *source);
remappings[path] = info.path;
}
}
};
static RegisterCommand r1(make_ref<CmdMakeContentAddressable>());

View file

@ -3,6 +3,7 @@
#include "store-api.hh" #include "store-api.hh"
#include "sync.hh" #include "sync.hh"
#include "thread-pool.hh" #include "thread-pool.hh"
#include "references.hh"
#include <atomic> #include <atomic>
@ -88,10 +89,15 @@ struct CmdVerify : StorePathsCommand
if (!noContents) { if (!noContents) {
HashSink sink(info->narHash.type); std::unique_ptr<AbstractHashSink> hashSink;
store->narFromPath(info->path, sink); if (info->ca == "")
hashSink = std::make_unique<HashSink>(info->narHash.type);
else
hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(info->path));
auto hash = sink.finish(); store->narFromPath(info->path, *hashSink);
auto hash = hashSink->finish();
if (hash.first != info->narHash) { if (hash.first != info->narHash) {
corrupted++; corrupted++;