S3BinaryCacheStore:: Eliminate a string copy while uploading

This cuts hydra-queue-runner's peak memory usage by about a third.
This commit is contained in:
Eelco Dolstra 2016-11-16 16:21:30 +01:00
parent 10ae8fabf1
commit 4be4f6de56
No known key found for this signature in database
GPG key ID: 8170B4726D7198DE
4 changed files with 17 additions and 3 deletions

View file

@ -155,7 +155,7 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
static Derivation parseDerivation(const string & s) static Derivation parseDerivation(const string & s)
{ {
Derivation drv; Derivation drv;
std::istringstream str(s); istringstream_nocopy str(s);
expect(str, "Derive(["); expect(str, "Derive([");
/* Parse the list of outputs. */ /* Parse the list of outputs. */

View file

@ -145,7 +145,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
.WithBucket(bucketName) .WithBucket(bucketName)
.WithKey(path); .WithKey(path);
auto stream = std::make_shared<std::stringstream>(data); auto stream = std::make_shared<istringstream_nocopy>(data);
request.SetBody(stream); request.SetBody(stream);

View file

@ -106,7 +106,7 @@ Hash parseHash(HashType ht, const string & s)
string s2(s, i * 2, 2); string s2(s, i * 2, 2);
if (!isxdigit(s2[0]) || !isxdigit(s2[1])) if (!isxdigit(s2[0]) || !isxdigit(s2[1]))
throw BadHash(format("invalid hash %1%") % s); throw BadHash(format("invalid hash %1%") % s);
std::istringstream str(s2); istringstream_nocopy str(s2);
int n; int n;
str >> std::hex >> n; str >> std::hex >> n;
hash.hash[i] = n; hash.hash[i] = n;

View file

@ -431,4 +431,18 @@ void callSuccess(
} }
/* A variant of std::istringstream that doesn't its string
argument. This is useful for large strings. The caller must ensure
that the string object is not destroyed while it's referenced by
this object. */
struct istringstream_nocopy : public std::stringstream
{
istringstream_nocopy(const std::string & s)
{
rdbuf()->pubsetbuf(
(char *) s.data(), s.size());
}
};
} }