Get rid of Hash::dummy from BinaryCacheStore

This commit is contained in:
John Ericson 2020-08-15 16:41:28 +00:00
parent 980edd1f3a
commit 9fbc31a65b
3 changed files with 69 additions and 44 deletions

View file

@ -142,17 +142,10 @@ struct FileSource : FdSource
} }
}; };
void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource, StorePath BinaryCacheStore::addToStoreCommon(
RepairFlag repair, CheckSigsFlag checkSigs) Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs,
std::function<ValidPathInfo(HashResult)> mkInfo)
{ {
assert(info.narSize);
if (!repair && isValidPath(info.path)) {
// FIXME: copyNAR -> null sink
narSource.drain();
return;
}
auto [fdTemp, fnTemp] = createTempFile(); auto [fdTemp, fnTemp] = createTempFile();
AutoDelete autoDelete(fnTemp); AutoDelete autoDelete(fnTemp);
@ -162,13 +155,15 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
/* Read the NAR simultaneously into a CompressionSink+FileSink (to /* Read the NAR simultaneously into a CompressionSink+FileSink (to
write the compressed NAR to disk), into a HashSink (to get the write the compressed NAR to disk), into a HashSink (to get the
NAR hash), and into a NarAccessor (to get the NAR listing). */ NAR hash), and into a NarAccessor (to get the NAR listing). */
HashSink fileHashSink(htSHA256); HashSink fileHashSink { htSHA256 };
std::shared_ptr<FSAccessor> narAccessor; std::shared_ptr<FSAccessor> narAccessor;
HashSink narHashSink { htSHA256 };
{ {
FdSink fileSink(fdTemp.get()); FdSink fileSink(fdTemp.get());
TeeSink teeSink(fileSink, fileHashSink); TeeSink teeSinkCompressed { fileSink, fileHashSink };
auto compressionSink = makeCompressionSink(compression, teeSink); auto compressionSink = makeCompressionSink(compression, teeSinkCompressed);
TeeSource teeSource(narSource, *compressionSink); TeeSink teeSinkUncompressed { *compressionSink, narHashSink };
TeeSource teeSource { narSource, teeSinkUncompressed };
narAccessor = makeNarAccessor(teeSource); narAccessor = makeNarAccessor(teeSource);
compressionSink->finish(); compressionSink->finish();
fileSink.flush(); fileSink.flush();
@ -176,9 +171,10 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
auto now2 = std::chrono::steady_clock::now(); auto now2 = std::chrono::steady_clock::now();
auto info = mkInfo(narHashSink.finish());
auto narInfo = make_ref<NarInfo>(info); auto narInfo = make_ref<NarInfo>(info);
narInfo->narSize = info.narSize; narInfo->narSize = info.narSize; // FIXME needed?
narInfo->narHash = info.narHash; narInfo->narHash = info.narHash; // FIXME needed?
narInfo->compression = compression; narInfo->compression = compression;
auto [fileHash, fileSize] = fileHashSink.finish(); auto [fileHash, fileSize] = fileHashSink.finish();
narInfo->fileHash = fileHash; narInfo->fileHash = fileHash;
@ -300,6 +296,41 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
writeNarInfo(narInfo); writeNarInfo(narInfo);
stats.narInfoWrite++; stats.narInfoWrite++;
return narInfo->path;
}
void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair, CheckSigsFlag checkSigs)
{
if (!repair && isValidPath(info.path)) {
// FIXME: copyNAR -> null sink
narSource.drain();
return;
}
(void) addToStoreCommon(narSource, repair, checkSigs, {[&](HashResult nar) {
/* FIXME reinstate these, once we can correctly do hash modulo sink as
needed. */
// assert(info.narHash == nar.first);
// assert(info.narSize == nar.second);
return info;
}});
}
StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & name,
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair)
{
if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256)
unsupported("addToStoreFromDump");
return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info {
makeFixedOutputPath(method, nar.first, name),
nar.first,
};
info.narSize = nar.second;
return info;
});
} }
bool BinaryCacheStore::isValidPathUncached(const StorePath & storePath) bool BinaryCacheStore::isValidPathUncached(const StorePath & storePath)
@ -367,11 +398,9 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath, StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
{ {
// FIXME: some cut&paste from LocalStore::addToStore(). /* FIXME: Make BinaryCacheStore::addToStoreCommon support
non-recursive+sha256 so we can just use the default
/* Read the whole path into memory. This is not a very scalable implementation of this method in terms of addToStoreFromDump. */
method for very large paths, but `copyPath' is mainly used for
small files. */
StringSink sink; StringSink sink;
std::optional<Hash> h; std::optional<Hash> h;
if (method == FileIngestionMethod::Recursive) { if (method == FileIngestionMethod::Recursive) {
@ -383,34 +412,25 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
h = hashString(hashAlgo, s); h = hashString(hashAlgo, s);
} }
ValidPathInfo info {
makeFixedOutputPath(method, *h, name),
Hash::dummy, // Will be fixed in addToStore, which recomputes nar hash
};
auto source = StringSource { *sink.s }; auto source = StringSource { *sink.s };
addToStore(info, source, repair, CheckSigs); return addToStoreFromDump(source, name, FileIngestionMethod::Recursive, htSHA256, repair);
return std::move(info.path);
} }
StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s, StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) const StorePathSet & references, RepairFlag repair)
{ {
ValidPathInfo info { auto path = computeStorePathForText(name, s, references);
computeStorePathForText(name, s, references),
Hash::dummy, // Will be fixed in addToStore, which recomputes nar hash if (!repair && isValidPath(path))
}; return path;
auto source = StringSource { s };
return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info { path, nar.first };
info.narSize = nar.second;
info.references = references; info.references = references;
return info;
if (repair || !isValidPath(info.path)) { });
StringSink sink;
dumpString(s, sink);
auto source = StringSource { *sink.s };
addToStore(info, source, repair, CheckSigs);
}
return std::move(info.path);
} }
ref<FSAccessor> BinaryCacheStore::getFSAccessor() ref<FSAccessor> BinaryCacheStore::getFSAccessor()

View file

@ -72,6 +72,10 @@ private:
void writeNarInfo(ref<NarInfo> narInfo); void writeNarInfo(ref<NarInfo> narInfo);
StorePath addToStoreCommon(
Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs,
std::function<ValidPathInfo(HashResult)> mkInfo);
public: public:
bool isValidPathUncached(const StorePath & path) override; bool isValidPathUncached(const StorePath & path) override;
@ -85,6 +89,9 @@ public:
void addToStore(const ValidPathInfo & info, Source & narSource, void addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair, CheckSigsFlag checkSigs) override; RepairFlag repair, CheckSigsFlag checkSigs) override;
StorePath addToStoreFromDump(Source & dump, const string & name,
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) override;
StorePath addToStore(const string & name, const Path & srcPath, StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method, HashType hashAlgo, FileIngestionMethod method, HashType hashAlgo,
PathFilter & filter, RepairFlag repair) override; PathFilter & filter, RepairFlag repair) override;

View file

@ -454,9 +454,7 @@ public:
// FIXME: remove? // FIXME: remove?
virtual StorePath addToStoreFromDump(Source & dump, const string & name, virtual StorePath addToStoreFromDump(Source & dump, const string & name,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)
{ { unsupported("addToStoreFromDump"); }
throw Error("addToStoreFromDump() is not supported by this store");
}
/* Like addToStore, but the contents written to the output path is /* Like addToStore, but the contents written to the output path is
a regular file containing the given string. */ a regular file containing the given string. */