lix-project/lix-releng-staging
0
0
Fork 0
Code Packages Activity

Sink: Use std::string_view

Browse source
This commit is contained in:
Eelco Dolstra 2020-12-02 14:00:43 +01:00
parent aa68486112
commit faa31f4084
21 changed files with 182 additions and 211 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
src/libstore/binary-cache-store.cc
View file

@ -86,8 +86,7 @@ void BinaryCacheStore::getFile(const std::string & path, Sink & sink)
promise.set_exception(std::current_exception());
}
}});
auto data = promise.get_future().get();
sink((unsigned char *) data->data(), data->size());
sink(*promise.get_future().get());
}
std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)

8
src/libstore/build/derivation-goal.cc
View file

@ -916,10 +916,8 @@ void DerivationGoal::buildDone()
LogSink(Activity & act) : act(act) { }
void operator() (const unsigned char * data, size_t len) override {
for (size_t i = 0; i < len; i++) {
auto c = data[i];
void operator() (std::string_view data) override {
for (auto c : data) {
if (c == '\n') {
flushLine();
} else {
@ -3127,7 +3125,7 @@ void DerivationGoal::registerOutputs()
StringSink sink;
dumpPath(actualPath, sink);
RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink);
rsink2((unsigned char *) sink.s->data(), sink.s->size());
rsink2(*sink.s);
rsink2.flush();
});
Path tmpPath = actualPath + ".tmp";

4
src/libstore/daemon.cc
View file

@ -153,10 +153,10 @@ struct TunnelSink : Sink
{
Sink & to;
TunnelSink(Sink & to) : to(to) { }
virtual void operator () (const unsigned char * data, size_t len)
void operator () (std::string_view data)
{
to << STDERR_WRITE;
writeString(data, len, to);
writeString(data, to);
}
};

18
src/libstore/filetransfer.cc
View file

@ -95,18 +95,18 @@ struct curlFileTransfer : public FileTransfer
fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri),
{request.uri}, request.parentAct)
, callback(std::move(callback))
, finalSink([this](const unsigned char * data, size_t len) {
, finalSink([this](std::string_view data) {
if (this->request.dataCallback) {
auto httpStatus = getHTTPStatus();
/* Only write data to the sink if this is a
successful response. */
if (successfulStatuses.count(httpStatus)) {
writtenToSink += len;
this->request.dataCallback((char *) data, len);
writtenToSink += data.size();
this->request.dataCallback(data);
}
} else
this->result.data->append((char *) data, len);
this->result.data->append(data);
})
{
if (!request.expectedETag.empty())
@ -171,8 +171,8 @@ struct curlFileTransfer : public FileTransfer
}
if (errorSink)
(*errorSink)((unsigned char *) contents, realSize);
(*decompressionSink)((unsigned char *) contents, realSize);
(*errorSink)({(char *) contents, realSize});
(*decompressionSink)({(char *) contents, realSize});
return realSize;
} catch (...) {
@ -776,7 +776,7 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink)
state->request.notify_one();
});
request.dataCallback = [_state](char * buf, size_t len) {
request.dataCallback = [_state](std::string_view data) {
auto state(_state->lock());
@ -794,7 +794,7 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink)
/* Append data to the buffer and wake up the calling
thread. */
state->data.append(buf, len);
state->data.append(data);
state->avail.notify_one();
};
@ -840,7 +840,7 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink)
if it's blocked on a full buffer. We don't hold the state
lock while doing this to prevent blocking the download
thread if sink() takes a long time. */
sink((unsigned char *) chunk.data(), chunk.size());
sink(chunk);
}
}

2
src/libstore/filetransfer.hh
View file

@ -61,7 +61,7 @@ struct FileTransferRequest
bool decompress = true;
std::shared_ptr<std::string> data;
std::string mimeType;
std::function<void(char *, size_t)> dataCallback;
std::function<void(std::string_view data)> dataCallback;
FileTransferRequest(const std::string & uri)
: uri(uri), parentAct(getCurActivity()) { }

2
src/libstore/nar-accessor.cc
View file

@ -87,7 +87,7 @@ struct NarAccessor : public FSAccessor
parents.top()->start = pos;
}
void receiveContents(unsigned char * data, size_t len) override
void receiveContents(std::string_view data) override
{ }
void createSymlink(const Path & path, const string & target) override

50
src/libstore/references.cc
View file

@ -55,27 +55,23 @@ struct RefScanSink : Sink
RefScanSink() { }
void operator () (const unsigned char * data, size_t len);
void operator () (std::string_view data) override
{
/* It's possible that a reference spans the previous and current
fragment, so search in the concatenation of the tail of the
previous fragment and the start of the current fragment. */
string s = tail + std::string(data, 0, refLength);
search((const unsigned char *) s.data(), s.size(), hashes, seen);
search((const unsigned char *) data.data(), data.size(), hashes, seen);
size_t tailLen = data.size() <= refLength ? data.size() : refLength;
tail = std::string(tail, tail.size() < refLength - tailLen ? 0 : tail.size() - (refLength - tailLen));
tail.append({data.data() + data.size() - tailLen, tailLen});
}
};
void RefScanSink::operator () (const unsigned char * data, size_t len)
{
/* It's possible that a reference spans the previous and current
fragment, so search in the concatenation of the tail of the
previous fragment and the start of the current fragment. */
string s = tail + string((const char *) data, len > refLength ? refLength : len);
search((const unsigned char *) s.data(), s.size(), hashes, seen);
search(data, len, hashes, seen);
size_t tailLen = len <= refLength ? len : refLength;
tail =
string(tail, tail.size() < refLength - tailLen ? 0 : tail.size() - (refLength - tailLen)) +
string((const char *) data + len - tailLen, tailLen);
}
std::pair<PathSet, HashResult> scanForReferences(const string & path,
const PathSet & refs)
{
@ -129,10 +125,10 @@ RewritingSink::RewritingSink(const std::string & from, const std::string & to, S
assert(from.size() == to.size());
}
void RewritingSink::operator () (const unsigned char * data, size_t len)
void RewritingSink::operator () (std::string_view data)
{
std::string s(prev);
s.append((const char *) data, len);
s.append(data);
size_t j = 0;
while ((j = s.find(from, j)) != string::npos) {
@ -146,14 +142,14 @@ void RewritingSink::operator () (const unsigned char * data, size_t len)
pos += consumed;
if (consumed) nextSink((unsigned char *) s.data(), consumed);
if (consumed) nextSink(s.substr(0, consumed));
}
void RewritingSink::flush()
{
if (prev.empty()) return;
pos += prev.size();
nextSink((unsigned char *) prev.data(), prev.size());
nextSink(prev);
prev.clear();
}
@ -163,9 +159,9 @@ HashModuloSink::HashModuloSink(HashType ht, const std::string & modulus)
{
}
void HashModuloSink::operator () (const unsigned char * data, size_t len)
void HashModuloSink::operator () (std::string_view data)
{
rewritingSink(data, len);
rewritingSink(data);
}
HashResult HashModuloSink::finish()
@ -176,10 +172,8 @@ HashResult HashModuloSink::finish()
NAR with self-references and a NAR with some of the
self-references already zeroed out do not produce a hash
collision. FIXME: proof. */
for (auto & pos : rewritingSink.matches) {
auto s = fmt("|%d", pos);
hashSink((unsigned char *) s.data(), s.size());
}
for (auto & pos : rewritingSink.matches)
hashSink(fmt("|%d", pos));
auto h = hashSink.finish();
return {h.first, rewritingSink.pos};

4
src/libstore/references.hh
View file

@ -19,7 +19,7 @@ struct RewritingSink : Sink
RewritingSink(const std::string & from, const std::string & to, Sink & nextSink);
void operator () (const unsigned char * data, size_t len) override;
void operator () (std::string_view data) override;
void flush();
};
@ -31,7 +31,7 @@ struct HashModuloSink : AbstractHashSink
HashModuloSink(HashType ht, const std::string & modulus);
void operator () (const unsigned char * data, size_t len) override;
void operator () (std::string_view data) override;
HashResult finish() override;
};

2
src/libstore/remote-store.cc
View file

@ -857,7 +857,7 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
if (!source) throw Error("no source");
size_t len = readNum<size_t>(from);
auto buf = std::make_unique<unsigned char[]>(len);
writeString(buf.get(), source->read(buf.get(), len), to);
writeString({(const char *) buf.get(), source->read(buf.get(), len)}, to);
to.flush();
}

2
src/libstore/s3-binary-cache-store.cc
View file

@ -398,7 +398,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCache
printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms",
bucketName, path, res.data->size(), res.durationMs);
sink((unsigned char *) res.data->data(), res.data->size());
sink(*res.data);
} else
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
}

4
src/libstore/store-api.cc
View file

@ -772,8 +772,8 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
}
auto source = sinkToSource([&](Sink & sink) {
LambdaSink progressSink([&](const unsigned char * data, size_t len) {
total += len;
LambdaSink progressSink([&](std::string_view data) {
total += data.size();
act.progress(total, info->narSize);
});
TeeSink tee { sink, progressSink };

18
src/libutil/archive.cc
View file

@ -57,7 +57,7 @@ static void dumpContents(const Path & path, size_t size,
auto n = std::min(left, buf.size());
readFull(fd.get(), buf.data(), n);
left -= n;
sink(buf.data(), n);
sink({(char *) buf.data(), n});
}
writePadding(size, sink);
@ -162,7 +162,7 @@ static void parseContents(ParseSink & sink, Source & source, const Path & path)
auto n = buf.size();
if ((uint64_t)n > left) n = left;
source(buf.data(), n);
sink.receiveContents(buf.data(), n);
sink.receiveContents({(char *) buf.data(), n});
left -= n;
}
@ -300,21 +300,21 @@ struct RestoreSink : ParseSink
Path dstPath;
AutoCloseFD fd;
void createDirectory(const Path & path)
void createDirectory(const Path & path) override
{
Path p = dstPath + path;
if (mkdir(p.c_str(), 0777) == -1)
throw SysError("creating directory '%1%'", p);
};
void createRegularFile(const Path & path)
void createRegularFile(const Path & path) override
{
Path p = dstPath + path;
fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666);
if (!fd) throw SysError("creating file '%1%'", p);
}
void isExecutable()
void isExecutable() override
{
struct stat st;
if (fstat(fd.get(), &st) == -1)
@ -323,7 +323,7 @@ struct RestoreSink : ParseSink
throw SysError("fchmod");
}
void preallocateContents(uint64_t len)
void preallocateContents(uint64_t len) override
{
if (!archiveSettings.preallocateContents)
return;
@ -341,12 +341,12 @@ struct RestoreSink : ParseSink
#endif
}
void receiveContents(unsigned char * data, size_t len)
void receiveContents(std::string_view data) override
{
writeFull(fd.get(), data, len);
writeFull(fd.get(), data);
}
void createSymlink(const Path & path, const string & target)
void createSymlink(const Path & path, const string & target) override
{
Path p = dstPath + path;
nix::createSymlink(target, p);

8
src/libutil/archive.hh
View file

@ -58,7 +58,7 @@ struct ParseSink
virtual void createRegularFile(const Path & path) { };
virtual void isExecutable() { };
virtual void preallocateContents(uint64_t size) { };
virtual void receiveContents(unsigned char * data, size_t len) { };
virtual void receiveContents(std::string_view data) { };
virtual void createSymlink(const Path & path, const string & target) { };
};
@ -72,14 +72,14 @@ struct RetrieveRegularNARSink : ParseSink
RetrieveRegularNARSink(Sink & sink) : sink(sink) { }
void createDirectory(const Path & path)
void createDirectory(const Path & path) override
{
regular = false;
}
void receiveContents(unsigned char * data, size_t len)
void receiveContents(std::string_view data) override
{
sink(data, len);
sink(data);
}
void createSymlink(const Path & path, const string & target)

113
src/libutil/compression.cc
View file

@ -22,18 +22,17 @@ struct ChunkedCompressionSink : CompressionSink
{
uint8_t outbuf[32 * 1024];
void write(const unsigned char * data, size_t len) override
void write(std::string_view data) override
{
const size_t CHUNK_SIZE = sizeof(outbuf) << 2;
while (len) {
size_t n = std::min(CHUNK_SIZE, len);
writeInternal(data, n);
data += n;
len -= n;
while (!data.empty()) {
size_t n = std::min(CHUNK_SIZE, data.size());
writeInternal(data);
data.remove_prefix(n);
}
}
virtual void writeInternal(const unsigned char * data, size_t len) = 0;
virtual void writeInternal(std::string_view data) = 0;
};
struct NoneSink : CompressionSink
@ -41,7 +40,7 @@ struct NoneSink : CompressionSink
Sink & nextSink;
NoneSink(Sink & nextSink) : nextSink(nextSink) { }
void finish() override { flush(); }
void write(const unsigned char * data, size_t len) override { nextSink(data, len); }
void write(std::string_view data) override { nextSink(data); }
};
struct GzipDecompressionSink : CompressionSink
@ -75,28 +74,28 @@ struct GzipDecompressionSink : CompressionSink
void finish() override
{
CompressionSink::flush();
write(nullptr, 0);
write({});
}
void write(const unsigned char * data, size_t len) override
void write(std::string_view data) override
{
assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max());
assert(data.size() <= std::numeric_limits<decltype(strm.avail_in)>::max());
strm.next_in = (Bytef *) data;
strm.avail_in = len;
strm.next_in = (Bytef *) data.data();
strm.avail_in = data.size();
while (!finished && (!data || strm.avail_in)) {
while (!finished && (!data.data() || strm.avail_in)) {
checkInterrupt();
int ret = inflate(&strm,Z_SYNC_FLUSH);
if (ret != Z_OK && ret != Z_STREAM_END)
throw CompressionError("error while decompressing gzip file: %d (%d, %d)",
zError(ret), len, strm.avail_in);
zError(ret), data.size(), strm.avail_in);
finished = ret == Z_STREAM_END;
if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out});
strm.next_out = (Bytef *) outbuf;
strm.avail_out = sizeof(outbuf);
}
@ -130,25 +129,25 @@ struct XzDecompressionSink : CompressionSink
void finish() override
{
CompressionSink::flush();
write(nullptr, 0);
write({});
}
void write(const unsigned char * data, size_t len) override
void write(std::string_view data) override
{
strm.next_in = data;
strm.avail_in = len;
strm.next_in = (const unsigned char *) data.data();
strm.avail_in = data.size();
while (!finished && (!data || strm.avail_in)) {
while (!finished && (!data.data() || strm.avail_in)) {
checkInterrupt();
lzma_ret ret = lzma_code(&strm, data ? LZMA_RUN : LZMA_FINISH);
lzma_ret ret = lzma_code(&strm, data.data() ? LZMA_RUN : LZMA_FINISH);
if (ret != LZMA_OK && ret != LZMA_STREAM_END)
throw CompressionError("error %d while decompressing xz file", ret);
finished = ret == LZMA_STREAM_END;
if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out});
strm.next_out = outbuf;
strm.avail_out = sizeof(outbuf);
}
@ -181,15 +180,15 @@ struct BzipDecompressionSink : ChunkedCompressionSink
void finish() override
{
flush();
write(nullptr, 0);
write({});
}
void writeInternal(const unsigned char * data, size_t len) override
void writeInternal(std::string_view data) override
{
assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max());
assert(data.size() <= std::numeric_limits<decltype(strm.avail_in)>::max());
strm.next_in = (char *) data;
strm.avail_in = len;
strm.next_in = (char *) data.data();
strm.avail_in = data.size();
while (strm.avail_in) {
checkInterrupt();
@ -201,7 +200,7 @@ struct BzipDecompressionSink : ChunkedCompressionSink
finished = ret == BZ_STREAM_END;
if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out});
strm.next_out = (char *) outbuf;
strm.avail_out = sizeof(outbuf);
}
@ -230,17 +229,17 @@ struct BrotliDecompressionSink : ChunkedCompressionSink
void finish() override
{
flush();
writeInternal(nullptr, 0);
writeInternal({});
}
void writeInternal(const unsigned char * data, size_t len) override
void writeInternal(std::string_view data) override
{
const uint8_t * next_in = data;
size_t avail_in = len;
auto next_in = (const uint8_t *) data.data();
size_t avail_in = data.size();
uint8_t * next_out = outbuf;
size_t avail_out = sizeof(outbuf);
while (!finished && (!data || avail_in)) {
while (!finished && (!data.data() || avail_in)) {
checkInterrupt();
if (!BrotliDecoderDecompressStream(state,
@ -250,7 +249,7 @@ struct BrotliDecompressionSink : ChunkedCompressionSink
throw CompressionError("error while decompressing brotli file");
if (avail_out < sizeof(outbuf) || avail_in == 0) {
nextSink(outbuf, sizeof(outbuf) - avail_out);
nextSink({(char *) outbuf, sizeof(outbuf) - avail_out});
next_out = outbuf;
avail_out = sizeof(outbuf);
}
@ -338,25 +337,25 @@ struct XzCompressionSink : CompressionSink
void finish() override
{
CompressionSink::flush();
write(nullptr, 0);
write({});
}
void write(const unsigned char * data, size_t len) override
void write(std::string_view data) override
{
strm.next_in = data;
strm.avail_in = len;
strm.next_in = (const unsigned char *) data.data();
strm.avail_in = data.size();
while (!finished && (!data || strm.avail_in)) {
while (!finished && (!data.data() || strm.avail_in)) {
checkInterrupt();
lzma_ret ret = lzma_code(&strm, data ? LZMA_RUN : LZMA_FINISH);
lzma_ret ret = lzma_code(&strm, data.data() ? LZMA_RUN : LZMA_FINISH);
if (ret != LZMA_OK && ret != LZMA_STREAM_END)
throw CompressionError("error %d while compressing xz file", ret);
finished = ret == LZMA_STREAM_END;
if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
nextSink({(const char *) outbuf, sizeof(outbuf) - strm.avail_out});
strm.next_out = outbuf;
strm.avail_out = sizeof(outbuf);
}
@ -389,27 +388,27 @@ struct BzipCompressionSink : ChunkedCompressionSink
void finish() override
{
flush();
writeInternal(nullptr, 0);
writeInternal({});
}
void writeInternal(const unsigned char * data, size_t len) override
void writeInternal(std::string_view data) override
{
assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max());
assert(data.size() <= std::numeric_limits<decltype(strm.avail_in)>::max());
strm.next_in = (char *) data;
strm.avail_in = len;
strm.next_in = (char *) data.data();
strm.avail_in = data.size();
while (!finished && (!data || strm.avail_in)) {
while (!finished && (!data.data() || strm.avail_in)) {
checkInterrupt();
int ret = BZ2_bzCompress(&strm, data ? BZ_RUN : BZ_FINISH);
int ret = BZ2_bzCompress(&strm, data.data() ? BZ_RUN : BZ_FINISH);
if (ret != BZ_RUN_OK && ret != BZ_FINISH_OK && ret != BZ_STREAM_END)
throw CompressionError("error %d while compressing bzip2 file", ret);
finished = ret == BZ_STREAM_END;
if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
nextSink({(const char *) outbuf, sizeof(outbuf) - strm.avail_out});
strm.next_out = (char *) outbuf;
strm.avail_out = sizeof(outbuf);
}
@ -439,28 +438,28 @@ struct BrotliCompressionSink : ChunkedCompressionSink
void finish() override
{
flush();
writeInternal(nullptr, 0);
writeInternal({});
}
void writeInternal(const unsigned char * data, size_t len) override
void writeInternal(std::string_view data) override
{
const uint8_t * next_in = data;
size_t avail_in = len;
auto next_in = (const uint8_t *) data.data();
size_t avail_in = data.size();
uint8_t * next_out = outbuf;
size_t avail_out = sizeof(outbuf);
while (!finished && (!data || avail_in)) {
while (!finished && (!data.data() || avail_in)) {
checkInterrupt();
if (!BrotliEncoderCompressStream(state,
data ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH,
data.data() ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH,
&avail_in, &next_in,
&avail_out, &next_out,
nullptr))
throw CompressionError("error while compressing brotli compression");
if (avail_out < sizeof(outbuf) || avail_in == 0) {
nextSink(outbuf, sizeof(outbuf) - avail_out);
nextSink({(const char *) outbuf, sizeof(outbuf) - avail_out});
next_out = outbuf;
avail_out = sizeof(outbuf);
}

18
src/libutil/hash.cc
View file

@ -291,12 +291,12 @@ static void start(HashType ht, Ctx & ctx)
static void update(HashType ht, Ctx & ctx,
const unsigned char * bytes, size_t len)
std::string_view data)
{
if (ht == htMD5) MD5_Update(&ctx.md5, bytes, len);
else if (ht == htSHA1) SHA1_Update(&ctx.sha1, bytes, len);
else if (ht == htSHA256) SHA256_Update(&ctx.sha256, bytes, len);
else if (ht == htSHA512) SHA512_Update(&ctx.sha512, bytes, len);
if (ht == htMD5) MD5_Update(&ctx.md5, data.data(), data.size());
else if (ht == htSHA1) SHA1_Update(&ctx.sha1, data.data(), data.size());
else if (ht == htSHA256) SHA256_Update(&ctx.sha256, data.data(), data.size());
else if (ht == htSHA512) SHA512_Update(&ctx.sha512, data.data(), data.size());
}
@ -314,7 +314,7 @@ Hash hashString(HashType ht, std::string_view s)
Ctx ctx;
Hash hash(ht);
start(ht, ctx);
update(ht, ctx, (const unsigned char *) s.data(), s.length());
update(ht, ctx, s);
finish(ht, ctx, hash.hash);
return hash;
}
@ -341,10 +341,10 @@ HashSink::~HashSink()
delete ctx;
}
void HashSink::write(const unsigned char * data, size_t len)
void HashSink::write(std::string_view data)
{
bytes += len;
update(ht, *ctx, data, len);
bytes += data.size();
update(ht, *ctx, data);
}
HashResult HashSink::finish()

2
src/libutil/hash.hh
View file

@ -156,7 +156,7 @@ public:
HashSink(HashType ht);
HashSink(const HashSink & h);
~HashSink();
void write(const unsigned char * data, size_t len) override;
void write(std::string_view data) override;
HashResult finish() override;
HashResult currentHash();
};

48
src/libutil/serialise.cc
View file

@ -11,23 +11,23 @@
namespace nix {
void BufferedSink::operator () (const unsigned char * data, size_t len)
void BufferedSink::operator () (std::string_view data)
{
if (!buffer) buffer = decltype(buffer)(new unsigned char[bufSize]);
if (!buffer) buffer = decltype(buffer)(new char[bufSize]);
while (len) {
while (!data.empty()) {
/* Optimisation: bypass the buffer if the data exceeds the
buffer size. */
if (bufPos + len >= bufSize) {
if (bufPos + data.size() >= bufSize) {
flush();
write(data, len);
write(data);
break;
}
/* Otherwise, copy the bytes to the buffer. Flush the buffer
when it's full. */
size_t n = bufPos + len > bufSize ? bufSize - bufPos : len;
memcpy(buffer.get() + bufPos, data, n);
data += n; bufPos += n; len -= n;
size_t n = bufPos + data.size() > bufSize ? bufSize - bufPos : data.size();
memcpy(buffer.get() + bufPos, data.data(), n);
data.remove_prefix(n); bufPos += n;
if (bufPos == bufSize) flush();
}
}
@ -38,7 +38,7 @@ void BufferedSink::flush()
if (bufPos == 0) return;
size_t n = bufPos;
bufPos = 0; // don't trigger the assert() in ~BufferedSink()
write(buffer.get(), n);
write({buffer.get(), n});
}
@ -59,9 +59,9 @@ static void warnLargeDump()
}
void FdSink::write(const unsigned char * data, size_t len)
void FdSink::write(std::string_view data)
{
written += len;
written += data.size();
static bool warned = false;
if (warn && !warned) {
if (written > threshold) {
@ -70,7 +70,7 @@ void FdSink::write(const unsigned char * data, size_t len)
}
}
try {
writeFull(fd, data, len);
writeFull(fd, data);
} catch (SysError & e) {
_good = false;
throw;
@ -101,7 +101,7 @@ void Source::drainInto(Sink & sink)
size_t n;
try {
n = read(buf.data(), buf.size());
sink(buf.data(), n);
sink({(char *) buf.data(), n});
} catch (EndOfFile &) {
break;
}
@ -229,9 +229,9 @@ std::unique_ptr<Source> sinkToSource(
{
if (!coro)
coro = coro_t::pull_type(VirtualStackAllocator{}, [&](coro_t::push_type & yield) {
LambdaSink sink([&](const unsigned char * data, size_t len) {
if (len) yield(std::string((const char *) data, len));
});
LambdaSink sink([&](std::string_view data) {
if (!data.empty()) yield(std::string(data));
});
fun(sink);
});
@ -260,22 +260,22 @@ void writePadding(size_t len, Sink & sink)
if (len % 8) {
unsigned char zero[8];
memset(zero, 0, sizeof(zero));
sink(zero, 8 - (len % 8));
sink({(char *) zero, 8 - (len % 8)});
}
}
void writeString(const unsigned char * buf, size_t len, Sink & sink)
void writeString(std::string_view data, Sink & sink)
{
sink << len;
sink(buf, len);
writePadding(len, sink);
sink << data.size();
sink(data);
writePadding(data.size(), sink);
}
Sink & operator << (Sink & sink, const string & s)
{
writeString((const unsigned char *) s.data(), s.size(), sink);
writeString(s, sink);
return sink;
}
@ -394,14 +394,14 @@ Error readError(Source & source)
}
void StringSink::operator () (const unsigned char * data, size_t len)
void StringSink::operator () (std::string_view data)
{
static bool warned = false;
if (!warned && s->size() > threshold) {
warnLargeDump();
warned = true;
}
s->append((const char *) data, len);
s->append(data);
}
size_t ChainSource::read(unsigned char * data, size_t len)

52
src/libutil/serialise.hh
View file

@ -14,19 +14,14 @@ namespace nix {
struct Sink
{
virtual ~Sink() { }
virtual void operator () (const unsigned char * data, size_t len) = 0;
virtual void operator () (std::string_view data) = 0;
virtual bool good() { return true; }
void operator () (const std::string & s)
{
(*this)((const unsigned char *) s.data(), s.size());
}
};
/* Just throws away data. */
struct NullSink : Sink
{
void operator () (const unsigned char * data, size_t len) override
void operator () (std::string_view data) override
{ }
};
@ -35,21 +30,16 @@ struct NullSink : Sink
struct BufferedSink : virtual Sink
{
size_t bufSize, bufPos;
std::unique_ptr<unsigned char[]> buffer;
std::unique_ptr<char[]> buffer;
BufferedSink(size_t bufSize = 32 * 1024)
: bufSize(bufSize), bufPos(0), buffer(nullptr) { }
void operator () (const unsigned char * data, size_t len) override;
void operator () (const std::string & s)
{
Sink::operator()(s);
}
void operator () (std::string_view data) override;
void flush();
virtual void write(const unsigned char * data, size_t len) = 0;
virtual void write(std::string_view data) = 0;
};
@ -119,7 +109,7 @@ struct FdSink : BufferedSink
~FdSink();
void write(const unsigned char * data, size_t len) override;
void write(std::string_view data) override;
bool good() override;
@ -163,7 +153,7 @@ struct StringSink : Sink
s->reserve(reservedSize);
};
StringSink(ref<std::string> s) : s(s) { };
void operator () (const unsigned char * data, size_t len) override;
void operator () (std::string_view data) override;
};
@ -182,10 +172,10 @@ struct TeeSink : Sink
{
Sink & sink1, & sink2;
TeeSink(Sink & sink1, Sink & sink2) : sink1(sink1), sink2(sink2) { }
virtual void operator () (const unsigned char * data, size_t len)
virtual void operator () (std::string_view data)
{
sink1(data, len);
sink2(data, len);
sink1(data);
sink2(data);
}
};
@ -200,7 +190,7 @@ struct TeeSource : Source
size_t read(unsigned char * data, size_t len)
{
size_t n = orig.read(data, len);
sink(data, n);
sink({(char *) data, n});
return n;
}
};
@ -241,24 +231,24 @@ struct LengthSink : Sink
{
uint64_t length = 0;
virtual void operator () (const unsigned char * _, size_t len)
void operator () (std::string_view data) override
{
length += len;
length += data.size();
}
};
/* Convert a function into a sink. */
struct LambdaSink : Sink
{
typedef std::function<void(const unsigned char *, size_t)> lambda_t;
typedef std::function<void(std::string_view data)> lambda_t;
lambda_t lambda;
LambdaSink(const lambda_t & lambda) : lambda(lambda) { }
virtual void operator () (const unsigned char * data, size_t len)
void operator () (std::string_view data) override
{
lambda(data, len);
lambda(data);
}
};
@ -302,7 +292,7 @@ std::unique_ptr<Source> sinkToSource(
void writePadding(size_t len, Sink & sink);
void writeString(const unsigned char * buf, size_t len, Sink & sink);
void writeString(std::string_view s, Sink & sink);
inline Sink & operator << (Sink & sink, uint64_t n)
{
@ -315,7 +305,7 @@ inline Sink & operator << (Sink & sink, uint64_t n)
buf[5] = (n >> 40) & 0xff;
buf[6] = (n >> 48) & 0xff;
buf[7] = (unsigned char) (n >> 56) & 0xff;
sink(buf, sizeof(buf));
sink({(char *) buf, sizeof(buf)});
return sink;
}
@ -484,7 +474,7 @@ struct FramedSink : nix::BufferedSink
}
}
void write(const unsigned char * data, size_t len) override
void write(std::string_view data) override
{
/* Don't send more data if the remote has
encountered an error. */
@ -493,8 +483,8 @@ struct FramedSink : nix::BufferedSink
ex = nullptr;
std::rethrow_exception(ex2);
}
to << len;
to(data, len);
to << data.size();
to(data);
};
};

30
src/libutil/util.cc
View file

@ -346,7 +346,7 @@ void writeFile(const Path & path, Source & source, mode_t mode)
while (true) {
try {
auto n = source.read(buf.data(), buf.size());
writeFull(fd.get(), (unsigned char *) buf.data(), n);
writeFull(fd.get(), {(char *) buf.data(), n});
} catch (EndOfFile &) { break; }
}
} catch (Error & e) {
@ -648,24 +648,16 @@ void readFull(int fd, unsigned char * buf, size_t count)
}
void writeFull(int fd, const unsigned char * buf, size_t count, bool allowInterrupts)
{
while (count) {
if (allowInterrupts) checkInterrupt();
ssize_t res = write(fd, (char *) buf, count);
if (res == -1 && errno != EINTR)
throw SysError("writing to file");
if (res > 0) {
count -= res;
buf += res;
}
}
}
void writeFull(int fd, std::string_view s, bool allowInterrupts)
{
writeFull(fd, (const unsigned char *) s.data(), s.size(), allowInterrupts);
while (!s.empty()) {
if (allowInterrupts) checkInterrupt();
ssize_t res = write(fd, s.data(), s.size());
if (res == -1 && errno != EINTR)
throw SysError("writing to file");
if (res > 0)
s.remove_prefix(res);
}
}
@ -705,7 +697,7 @@ void drainFD(int fd, Sink & sink, bool block)
throw SysError("reading from file");
}
else if (rd == 0) break;
else sink(buf.data(), rd);
else sink({(char *) buf.data(), (size_t) rd});
}
}
@ -1153,7 +1145,7 @@ void runProgram2(const RunOptions & options)
} catch (EndOfFile &) {
break;
}
writeFull(in.writeSide.get(), buf.data(), n);
writeFull(in.writeSide.get(), {(char *) buf.data(), n});
}
promise.set_value();
} catch (...) {

1
src/libutil/util.hh
View file

@ -156,7 +156,6 @@ void replaceSymlink(const Path & target, const Path & link,
/* Wrappers arount read()/write() that read/write exactly the
requested number of bytes. */
void readFull(int fd, unsigned char * buf, size_t count);
void writeFull(int fd, const unsigned char * buf, size_t count, bool allowInterrupts = true);
void writeFull(int fd, std::string_view s, bool allowInterrupts = true);
MakeError(EndOfFile, Error);

4
src/nix/make-content-addressable.cc
View file

@ -73,7 +73,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
*sink.s = rewriteStrings(*sink.s, rewrites);
HashModuloSink hashModuloSink(htSHA256, oldHashPart);
hashModuloSink((unsigned char *) sink.s->data(), sink.s->size());
hashModuloSink(*sink.s);
auto narHash = hashModuloSink.finish().first;
@ -94,7 +94,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
auto source = sinkToSource([&](Sink & nextSink) {
RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink);
rsink2((unsigned char *) sink.s->data(), sink.s->size());
rsink2(*sink.s);
rsink2.flush();
});