Merge pull request #3822 from obsidiansystems/dump-thrice-fixme

Optimize `addToStoreSlow` and remove `TeeParseSink`
This commit is contained in:
Eelco Dolstra 2020-07-20 18:55:05 +02:00 committed by GitHub
commit 1c5f8bbfb5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 69 additions and 54 deletions

View file

@ -173,31 +173,6 @@ struct TunnelSource : BufferedSource
} }
}; };
/* If the NAR archive contains a single file at top-level, then save
the contents of the file to `s'. Otherwise barf. */
struct RetrieveRegularNARSink : ParseSink
{
bool regular;
string s;
RetrieveRegularNARSink() : regular(true) { }
void createDirectory(const Path & path)
{
regular = false;
}
void receiveContents(unsigned char * data, unsigned int len)
{
s.append((const char *) data, len);
}
void createSymlink(const Path & path, const string & target)
{
regular = false;
}
};
struct ClientSettings struct ClientSettings
{ {
bool keepFailed; bool keepFailed;
@ -391,9 +366,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
} }
HashType hashAlgo = parseHashType(s); HashType hashAlgo = parseHashType(s);
StringSink savedNAR; StringSink saved;
TeeSource savedNARSource(from, savedNAR); TeeSource savedNARSource(from, saved);
RetrieveRegularNARSink savedRegular; RetrieveRegularNARSink savedRegular { saved };
if (method == FileIngestionMethod::Recursive) { if (method == FileIngestionMethod::Recursive) {
/* Get the entire NAR dump from the client and save it to /* Get the entire NAR dump from the client and save it to
@ -407,11 +382,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork(); logger->startWork();
if (!savedRegular.regular) throw Error("regular file expected"); if (!savedRegular.regular) throw Error("regular file expected");
auto path = store->addToStoreFromDump( auto path = store->addToStoreFromDump(*saved.s, baseName, method, hashAlgo);
method == FileIngestionMethod::Recursive ? *savedNAR.s : savedRegular.s,
baseName,
method,
hashAlgo);
logger->stopWork(); logger->stopWork();
to << store->printStorePath(path); to << store->printStorePath(path);
@ -727,15 +698,15 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
if (!trusted) if (!trusted)
info.ultimate = false; info.ultimate = false;
std::string saved;
std::unique_ptr<Source> source; std::unique_ptr<Source> source;
if (GET_PROTOCOL_MINOR(clientVersion) >= 21) if (GET_PROTOCOL_MINOR(clientVersion) >= 21)
source = std::make_unique<TunnelSource>(from, to); source = std::make_unique<TunnelSource>(from, to);
else { else {
TeeParseSink tee(from); StringSink saved;
parseDump(tee, tee.source); TeeSource tee { from, saved };
saved = std::move(*tee.saved.s); ParseSink ether;
source = std::make_unique<StringSource>(saved); parseDump(ether, tee);
source = std::make_unique<StringSource>(std::move(*saved.s));
} }
logger->startWork(); logger->startWork();

View file

@ -60,8 +60,10 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
if (n != 1) throw Error("input doesn't look like something created by 'nix-store --export'"); if (n != 1) throw Error("input doesn't look like something created by 'nix-store --export'");
/* Extract the NAR from the source. */ /* Extract the NAR from the source. */
TeeParseSink tee(source); StringSink saved;
parseDump(tee, tee.source); TeeSource tee { source, saved };
ParseSink ether;
parseDump(ether, tee);
uint32_t magic = readInt(source); uint32_t magic = readInt(source);
if (magic != exportMagic) if (magic != exportMagic)
@ -77,15 +79,15 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
if (deriver != "") if (deriver != "")
info.deriver = parseStorePath(deriver); info.deriver = parseStorePath(deriver);
info.narHash = hashString(htSHA256, *tee.saved.s); info.narHash = hashString(htSHA256, *saved.s);
info.narSize = tee.saved.s->size(); info.narSize = saved.s->size();
// Ignore optional legacy signature. // Ignore optional legacy signature.
if (readInt(source) == 1) if (readInt(source) == 1)
readString(source); readString(source);
// Can't use underlying source, which would have been exhausted // Can't use underlying source, which would have been exhausted
auto source = StringSource { *tee.saved.s }; auto source = StringSource { *saved.s };
addToStore(info, source, NoRepair, checkSigs); addToStore(info, source, NoRepair, checkSigs);
res.push_back(info.path); res.push_back(info.path);

View file

@ -226,16 +226,41 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
FileIngestionMethod method, HashType hashAlgo, FileIngestionMethod method, HashType hashAlgo,
std::optional<Hash> expectedCAHash) std::optional<Hash> expectedCAHash)
{ {
/* FIXME: inefficient: we're reading/hashing 'tmpFile' three /* FIXME: inefficient: we're reading/hashing 'tmpFile' two
times. */ times. */
HashSink narHashSink { htSHA256 };
HashSink caHashSink { hashAlgo };
RetrieveRegularNARSink fileSink { caHashSink };
auto [narHash, narSize] = hashPath(htSHA256, srcPath); TeeSink sinkIfNar { narHashSink, caHashSink };
auto hash = method == FileIngestionMethod::Recursive /* We use the tee sink if we need to hash the nar twice */
? hashAlgo == htSHA256 auto & sink = method == FileIngestionMethod::Recursive && hashAlgo != htSHA256
? static_cast<Sink &>(sinkIfNar)
: narHashSink;
auto fileSource = sinkToSource([&](Sink & sink) {
dumpPath(srcPath, sink);
});
TeeSource tapped { *fileSource, sink };
ParseSink blank;
auto & parseSink = method == FileIngestionMethod::Flat
? fileSink
: blank;
parseDump(
parseSink,
method == FileIngestionMethod::Recursive && hashAlgo == htSHA256
? *fileSource // don't need to hash twice if we just can use the `narHash` twice
: tapped);
auto [narHash, narSize] = narHashSink.finish();
auto hash = method == FileIngestionMethod::Recursive && hashAlgo == htSHA256
? narHash ? narHash
: hashPath(hashAlgo, srcPath).first : caHashSink.finish().first;
: hashFile(hashAlgo, srcPath);
if (expectedCAHash && expectedCAHash != hash) if (expectedCAHash && expectedCAHash != hash)
throw Error("hash mismatch for '%s'", srcPath); throw Error("hash mismatch for '%s'", srcPath);

View file

@ -63,12 +63,29 @@ struct ParseSink
virtual void createSymlink(const Path & path, const string & target) { }; virtual void createSymlink(const Path & path, const string & target) { };
}; };
struct TeeParseSink : ParseSink /* If the NAR archive contains a single file at top-level, then save
the contents of the file to `s'. Otherwise barf. */
struct RetrieveRegularNARSink : ParseSink
{ {
StringSink saved; bool regular = true;
TeeSource source; Sink & sink;
TeeParseSink(Source & source) : source(source, saved) { } RetrieveRegularNARSink(Sink & sink) : sink(sink) { }
void createDirectory(const Path & path)
{
regular = false;
}
void receiveContents(unsigned char * data, unsigned int len)
{
sink(data, len);
}
void createSymlink(const Path & path, const string & target)
{
regular = false;
}
}; };
void parseDump(ParseSink & sink, Source & source); void parseDump(ParseSink & sink, Source & source);