lix-project/lix
20
57
Fork 31
Code Issues 347 Code Review (Gerrit) Projects 3 Releases Wiki Activity

Merge pull request #5696 from obsidiansystems/fix-5299

Browse source 
Fix #5299 and remove uncesssary unbounded buffer
This commit is contained in:
Eelco Dolstra 2021-12-01 21:32:26 +01:00 committed by GitHub
parent f3f32f0c30 2f5c913d4a
commit fb662e0acf
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 27 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
src/libstore/daemon.cc
View file

@ -431,25 +431,30 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
hashAlgo = parseHashType(hashAlgoRaw); hashAlgo = parseHashType(hashAlgoRaw);
} }
StringSink saved; auto dumpSource = sinkToSource([&](Sink & saved) {
TeeSource savedNARSource(from, saved);
RetrieveRegularNARSink savedRegular { saved };
if (method == FileIngestionMethod::Recursive) { if (method == FileIngestionMethod::Recursive) {
/* Get the entire NAR dump from the client and save it to /* We parse the NAR dump through into `saved` unmodified,
a string so that we can pass it to so why all this extra work? We still parse the NAR so
addToStoreFromDump(). */ that we aren't sending arbitrary data to `saved`
unwittingly`, and we know when the NAR ends so we don't
consume the rest of `from` and can't parse another
command. (We don't trust `addToStoreFromDump` to not
eagerly consume the entire stream it's given, past the
length of the Nar. */
TeeSource savedNARSource(from, saved);
ParseSink sink; /* null sink; just parse the NAR */ ParseSink sink; /* null sink; just parse the NAR */
parseDump(sink, savedNARSource); parseDump(sink, savedNARSource);
} else } else {
/* Incrementally parse the NAR file, stripping the
metadata, and streaming the sole file we expect into
`saved`. */
RetrieveRegularNARSink savedRegular { saved };
parseDump(savedRegular, from); parseDump(savedRegular, from);
logger->startWork();
if (!savedRegular.regular) throw Error("regular file expected"); if (!savedRegular.regular) throw Error("regular file expected");
}
// FIXME: try to stream directly from `from`. });
StringSource dumpSource { *saved.s }; logger->startWork();
auto path = store->addToStoreFromDump(dumpSource, baseName, method, hashAlgo); auto path = store->addToStoreFromDump(*dumpSource, baseName, method, hashAlgo);
logger->stopWork(); logger->stopWork();
to << store->printStorePath(path); to << store->printStorePath(path);

5
src/libstore/local-store.cc
View file

@ -8,6 +8,7 @@
#include "references.hh" #include "references.hh"
#include "callback.hh" #include "callback.hh"
#include "topo-sort.hh" #include "topo-sort.hh"
#include "finally.hh"
#include <iostream> #include <iostream>
#include <algorithm> #include <algorithm>
@ -1333,13 +1334,15 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
auto want = std::min(chunkSize, settings.narBufferSize - oldSize); auto want = std::min(chunkSize, settings.narBufferSize - oldSize);
dump.resize(oldSize + want); dump.resize(oldSize + want);
auto got = 0; auto got = 0;
Finally cleanup([&]() {
dump.resize(oldSize + got);
});
try { try {
got = source.read(dump.data() + oldSize, want); got = source.read(dump.data() + oldSize, want);
} catch (EndOfFile &) { } catch (EndOfFile &) {
inMemory = true; inMemory = true;
break; break;
} }
dump.resize(oldSize + got);
} }
std::unique_ptr<AutoDelete> delTempDir; std::unique_ptr<AutoDelete> delTempDir;