From 8cd9aa24a8d48caf22225ce32dff3c5aaa111687 Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Tue, 28 May 2024 12:25:49 +0200 Subject: [PATCH] util.{hh,cc}: Split out file-descriptor.{hh,cc} Change-Id: I0dd0f9a9c2003fb887e076127e7f825fd3289c76 --- src/libstore/lock.cc | 1 + src/libstore/pathlocks.hh | 2 +- src/libutil/file-descriptor.cc | 251 ++++++++++++++++++ src/libutil/file-descriptor.hh | 89 +++++++ src/libutil/file-system.cc | 12 +- src/libutil/file-system.hh | 3 +- src/libutil/logging.cc | 1 + src/libutil/meson.build | 2 + src/libutil/serialise.hh | 1 + src/libutil/util.cc | 229 ---------------- src/libutil/util.hh | 77 ------ .../repl_characterization/test-session.hh | 2 +- 12 files changed, 349 insertions(+), 321 deletions(-) create mode 100644 src/libutil/file-descriptor.cc create mode 100644 src/libutil/file-descriptor.hh diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index b52e87ae0..05296757d 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -1,4 +1,5 @@ #include "lock.hh" +#include "logging.hh" #include "file-system.hh" #include "globals.hh" #include "pathlocks.hh" diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index 4921df352..7fcfa2e40 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "util.hh" +#include "file-descriptor.hh" namespace nix { diff --git a/src/libutil/file-descriptor.cc b/src/libutil/file-descriptor.cc new file mode 100644 index 000000000..ec22f17ab --- /dev/null +++ b/src/libutil/file-descriptor.cc @@ -0,0 +1,251 @@ +#include "file-system.hh" +#include "finally.hh" +#include "serialise.hh" +#include "signals.hh" + +#include +#include + +namespace nix { + +std::string readFile(int fd) +{ + struct stat st; + if (fstat(fd, &st) == -1) + throw SysError("statting file"); + + return drainFD(fd, true, st.st_size); +} + + +std::string readLine(int fd) +{ + std::string s; + while (1) { + checkInterrupt(); + char ch; + // FIXME: inefficient + ssize_t rd = read(fd, &ch, 1); + if (rd == -1) { + if (errno != EINTR) + throw SysError("reading a line"); + } else if (rd == 0) + throw EndOfFile("unexpected EOF reading a line"); + else { + if (ch == '\n') return s; + s += ch; + } + } +} + + +void writeLine(int fd, std::string s) +{ + s += '\n'; + writeFull(fd, s); +} + + +void readFull(int fd, char * buf, size_t count) +{ + while (count) { + checkInterrupt(); + ssize_t res = read(fd, buf, count); + if (res == -1) { + if (errno == EINTR) continue; + throw SysError("reading from file"); + } + if (res == 0) throw EndOfFile("unexpected end-of-file"); + count -= res; + buf += res; + } +} + + +void writeFull(int fd, std::string_view s, bool allowInterrupts) +{ + while (!s.empty()) { + if (allowInterrupts) checkInterrupt(); + ssize_t res = write(fd, s.data(), s.size()); + if (res == -1 && errno != EINTR) + throw SysError("writing to file"); + if (res > 0) + s.remove_prefix(res); + } +} + + +std::string drainFD(int fd, bool block, const size_t reserveSize) +{ + // the parser needs two extra bytes to append terminating characters, other users will + // not care very much about the extra memory. + StringSink sink(reserveSize + 2); + drainFD(fd, sink, block); + return std::move(sink.s); +} + + +void drainFD(int fd, Sink & sink, bool block) +{ + // silence GCC maybe-uninitialized warning in finally + int saved = 0; + + if (!block) { + saved = fcntl(fd, F_GETFL); + if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1) + throw SysError("making file descriptor non-blocking"); + } + + Finally finally([&]() { + if (!block) { + if (fcntl(fd, F_SETFL, saved) == -1) + throw SysError("making file descriptor blocking"); + } + }); + + std::array buf; + while (1) { + checkInterrupt(); + ssize_t rd = read(fd, buf.data(), buf.size()); + if (rd == -1) { + if (!block && (errno == EAGAIN || errno == EWOULDBLOCK)) + break; + if (errno != EINTR) + throw SysError("reading from file"); + } + else if (rd == 0) break; + else sink({(char *) buf.data(), (size_t) rd}); + } +} + +AutoCloseFD::AutoCloseFD() : fd{-1} {} + + +AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {} + + +AutoCloseFD::AutoCloseFD(AutoCloseFD && that) : fd{that.fd} +{ + that.fd = -1; +} + + +AutoCloseFD & AutoCloseFD::operator =(AutoCloseFD && that) +{ + close(); + fd = that.fd; + that.fd = -1; + return *this; +} + + +AutoCloseFD::~AutoCloseFD() +{ + try { + close(); + } catch (...) { + ignoreException(); + } +} + + +int AutoCloseFD::get() const +{ + return fd; +} + + +void AutoCloseFD::close() +{ + if (fd != -1) { + if (::close(fd) == -1) + /* This should never happen. */ + throw SysError("closing file descriptor %1%", fd); + fd = -1; + } +} + +void AutoCloseFD::fsync() +{ + if (fd != -1) { + int result; +#if __APPLE__ + result = ::fcntl(fd, F_FULLFSYNC); +#else + result = ::fsync(fd); +#endif + if (result == -1) + throw SysError("fsync file descriptor %1%", fd); + } +} + + +AutoCloseFD::operator bool() const +{ + return fd != -1; +} + + +int AutoCloseFD::release() +{ + int oldFD = fd; + fd = -1; + return oldFD; +} + + +void Pipe::create() +{ + int fds[2]; +#if HAVE_PIPE2 + if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe"); +#else + if (pipe(fds) != 0) throw SysError("creating pipe"); + closeOnExec(fds[0]); + closeOnExec(fds[1]); +#endif + readSide = AutoCloseFD{fds[0]}; + writeSide = AutoCloseFD{fds[1]}; +} + + +void Pipe::close() +{ + readSide.close(); + writeSide.close(); +} + + +void closeMostFDs(const std::set & exceptions) +{ +#if __linux__ + try { + for (auto & s : readDirectory("/proc/self/fd")) { + auto fd = std::stoi(s.name); + if (!exceptions.count(fd)) { + debug("closing leaked FD %d", fd); + close(fd); + } + } + return; + } catch (SysError &) { + } +#endif + + int maxFD = 0; + maxFD = sysconf(_SC_OPEN_MAX); + for (int fd = 0; fd < maxFD; ++fd) + if (!exceptions.count(fd)) + close(fd); /* ignore result */ +} + + +void closeOnExec(int fd) +{ + int prev; + if ((prev = fcntl(fd, F_GETFD, 0)) == -1 || + fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1) + throw SysError("setting close-on-exec flag"); +} + +} diff --git a/src/libutil/file-descriptor.hh b/src/libutil/file-descriptor.hh new file mode 100644 index 000000000..68324b9d9 --- /dev/null +++ b/src/libutil/file-descriptor.hh @@ -0,0 +1,89 @@ +#pragma once +///@file + +#include "error.hh" + +namespace nix { + +struct Sink; +struct Source; + +/** + * Read a line from a file descriptor. + */ +std::string readLine(int fd); + +/** + * Write a line to a file descriptor. + */ +void writeLine(int fd, std::string s); + +/** + * Read the contents of a file into a string. + */ +std::string readFile(int fd); + +/** + * Wrappers arount read()/write() that read/write exactly the + * requested number of bytes. + */ +void readFull(int fd, char * buf, size_t count); +void writeFull(int fd, std::string_view s, bool allowInterrupts = true); + +/** + * Read a file descriptor until EOF occurs. + */ +std::string drainFD(int fd, bool block = true, const size_t reserveSize=0); + +void drainFD(int fd, Sink & sink, bool block = true); + +class AutoCloseFD +{ + int fd; +public: + AutoCloseFD(); + explicit AutoCloseFD(int fd); + AutoCloseFD(const AutoCloseFD & fd) = delete; + AutoCloseFD(AutoCloseFD&& fd); + ~AutoCloseFD(); + AutoCloseFD& operator =(const AutoCloseFD & fd) = delete; + AutoCloseFD& operator =(AutoCloseFD&& fd) noexcept(false); + int get() const; + explicit operator bool() const; + int release(); + void close(); + void fsync(); + void reset() { *this = {}; } +}; + +class Pipe +{ +public: + AutoCloseFD readSide, writeSide; + void create(); + void close(); +}; + +/** + * Close all file descriptors except those listed in the given set. + * Good practice in child processes. + */ +void closeMostFDs(const std::set & exceptions); + +/** + * Set the close-on-exec flag for the given file descriptor. + */ +void closeOnExec(int fd); + +MakeError(EndOfFile, Error); + +/** + * Create a Unix domain socket. + */ +AutoCloseFD createUnixDomainSocket(); + +/** + * Create a Unix domain socket in listen mode. + */ +AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); +} diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index e5ba42eb6..9804c449c 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -3,10 +3,10 @@ #include #include "environment-variables.hh" +#include "file-descriptor.hh" #include "file-system.hh" #include "finally.hh" #include "serialise.hh" -#include "util.hh" #include "signals.hh" #include "types.hh" @@ -278,16 +278,6 @@ unsigned char getFileType(const Path & path) } -std::string readFile(int fd) -{ - struct stat st; - if (fstat(fd, &st) == -1) - throw SysError("statting file"); - - return drainFD(fd, true, st.st_size); -} - - std::string readFile(const Path & path) { AutoCloseFD fd{open(path.c_str(), O_RDONLY | O_CLOEXEC)}; diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh index 1d91ef334..b9b753980 100644 --- a/src/libutil/file-system.hh +++ b/src/libutil/file-system.hh @@ -6,7 +6,7 @@ */ #include "types.hh" -#include "util.hh" +#include "file-descriptor.hh" #include #include @@ -143,7 +143,6 @@ unsigned char getFileType(const Path & path); /** * Read the contents of a file into a string. */ -std::string readFile(int fd); std::string readFile(const Path & path); void readFile(const Path & path, Sink & sink); diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 731d1034d..8d9e18d09 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -1,4 +1,5 @@ #include "environment-variables.hh" +#include "file-descriptor.hh" #include "logging.hh" #include "util.hh" #include "config.hh" diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 8af251d1b..a4813d9a3 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -13,6 +13,7 @@ libutil_sources = files( 'escape-string.cc', 'exit.cc', 'experimental-features.cc', + 'file-descriptor.cc', 'file-system.cc', 'git.cc', 'hash.cc', @@ -62,6 +63,7 @@ libutil_headers = files( 'exit.hh', 'experimental-features.hh', 'experimental-features-json.hh', + 'file-descriptor.hh', 'file-system.hh', 'finally.hh', 'fmt.hh', diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index d1c791823..d4a14bbf0 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -5,6 +5,7 @@ #include "types.hh" #include "util.hh" +#include "file-descriptor.hh" namespace boost::context { struct stack_context; } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index ac3071ba8..55f3154c9 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -52,34 +52,6 @@ namespace nix { -std::string readLine(int fd) -{ - std::string s; - while (1) { - checkInterrupt(); - char ch; - // FIXME: inefficient - ssize_t rd = read(fd, &ch, 1); - if (rd == -1) { - if (errno != EINTR) - throw SysError("reading a line"); - } else if (rd == 0) - throw EndOfFile("unexpected EOF reading a line"); - else { - if (ch == '\n') return s; - s += ch; - } - } -} - - -void writeLine(int fd, std::string s) -{ - s += '\n'; - writeFull(fd, s); -} - - std::string getUserName() { auto pw = getpwuid(geteuid()); @@ -196,78 +168,6 @@ std::optional getSelfExe() -void readFull(int fd, char * buf, size_t count) -{ - while (count) { - checkInterrupt(); - ssize_t res = read(fd, buf, count); - if (res == -1) { - if (errno == EINTR) continue; - throw SysError("reading from file"); - } - if (res == 0) throw EndOfFile("unexpected end-of-file"); - count -= res; - buf += res; - } -} - - -void writeFull(int fd, std::string_view s, bool allowInterrupts) -{ - while (!s.empty()) { - if (allowInterrupts) checkInterrupt(); - ssize_t res = write(fd, s.data(), s.size()); - if (res == -1 && errno != EINTR) - throw SysError("writing to file"); - if (res > 0) - s.remove_prefix(res); - } -} - - -std::string drainFD(int fd, bool block, const size_t reserveSize) -{ - // the parser needs two extra bytes to append terminating characters, other users will - // not care very much about the extra memory. - StringSink sink(reserveSize + 2); - drainFD(fd, sink, block); - return std::move(sink.s); -} - - -void drainFD(int fd, Sink & sink, bool block) -{ - // silence GCC maybe-uninitialized warning in finally - int saved = 0; - - if (!block) { - saved = fcntl(fd, F_GETFL); - if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1) - throw SysError("making file descriptor non-blocking"); - } - - Finally finally([&]() { - if (!block) { - if (fcntl(fd, F_SETFL, saved) == -1) - throw SysError("making file descriptor blocking"); - } - }); - - std::array buf; - while (1) { - checkInterrupt(); - ssize_t rd = read(fd, buf.data(), buf.size()); - if (rd == -1) { - if (!block && (errno == EAGAIN || errno == EWOULDBLOCK)) - break; - if (errno != EINTR) - throw SysError("reading from file"); - } - else if (rd == 0) break; - else sink({(char *) buf.data(), (size_t) rd}); - } -} - ////////////////////////////////////////////////////////////////////// unsigned int getMaxCPU() @@ -303,102 +203,6 @@ unsigned int getMaxCPU() ////////////////////////////////////////////////////////////////////// -AutoCloseFD::AutoCloseFD() : fd{-1} {} - - -AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {} - - -AutoCloseFD::AutoCloseFD(AutoCloseFD && that) : fd{that.fd} -{ - that.fd = -1; -} - - -AutoCloseFD & AutoCloseFD::operator =(AutoCloseFD && that) -{ - close(); - fd = that.fd; - that.fd = -1; - return *this; -} - - -AutoCloseFD::~AutoCloseFD() -{ - try { - close(); - } catch (...) { - ignoreException(); - } -} - - -int AutoCloseFD::get() const -{ - return fd; -} - - -void AutoCloseFD::close() -{ - if (fd != -1) { - if (::close(fd) == -1) - /* This should never happen. */ - throw SysError("closing file descriptor %1%", fd); - fd = -1; - } -} - -void AutoCloseFD::fsync() -{ - if (fd != -1) { - int result; -#if __APPLE__ - result = ::fcntl(fd, F_FULLFSYNC); -#else - result = ::fsync(fd); -#endif - if (result == -1) - throw SysError("fsync file descriptor %1%", fd); - } -} - - -AutoCloseFD::operator bool() const -{ - return fd != -1; -} - - -int AutoCloseFD::release() -{ - int oldFD = fd; - fd = -1; - return oldFD; -} - - -void Pipe::create() -{ - int fds[2]; -#if HAVE_PIPE2 - if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe"); -#else - if (pipe(fds) != 0) throw SysError("creating pipe"); - closeOnExec(fds[0]); - closeOnExec(fds[1]); -#endif - readSide = AutoCloseFD{fds[0]}; - writeSide = AutoCloseFD{fds[1]}; -} - - -void Pipe::close() -{ - readSide.close(); - writeSide.close(); -} ////////////////////////////////////////////////////////////////////// @@ -762,39 +566,6 @@ void runProgram2(const RunOptions & options) } -void closeMostFDs(const std::set & exceptions) -{ -#if __linux__ - try { - for (auto & s : readDirectory("/proc/self/fd")) { - auto fd = std::stoi(s.name); - if (!exceptions.count(fd)) { - debug("closing leaked FD %d", fd); - close(fd); - } - } - return; - } catch (SysError &) { - } -#endif - - int maxFD = 0; - maxFD = sysconf(_SC_OPEN_MAX); - for (int fd = 0; fd < maxFD; ++fd) - if (!exceptions.count(fd)) - close(fd); /* ignore result */ -} - - -void closeOnExec(int fd) -{ - int prev; - if ((prev = fcntl(fd, F_GETFD, 0)) == -1 || - fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1) - throw SysError("setting close-on-exec flag"); -} - - ////////////////////////////////////////////////////////////////////// diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 63a4fcb6a..d0e207146 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -39,17 +39,6 @@ struct Source; extern const std::string nativeSystem; -/** - * Read a line from a file descriptor. - */ -std::string readLine(int fd); - -/** - * Write a line to a file descriptor. - */ -void writeLine(int fd, std::string s); - - std::string getUserName(); /** @@ -102,57 +91,12 @@ Path getStateDir(); Path createNixStateDir(); -/** - * Wrappers arount read()/write() that read/write exactly the - * requested number of bytes. - */ -void readFull(int fd, char * buf, size_t count); -void writeFull(int fd, std::string_view s, bool allowInterrupts = true); - -MakeError(EndOfFile, Error); - - -/** - * Read a file descriptor until EOF occurs. - */ -std::string drainFD(int fd, bool block = true, const size_t reserveSize=0); - -void drainFD(int fd, Sink & sink, bool block = true); - /** * If cgroups are active, attempt to calculate the number of CPUs available. * If cgroups are unavailable or if cpu.max is set to "max", return 0. */ unsigned int getMaxCPU(); -class AutoCloseFD -{ - int fd; -public: - AutoCloseFD(); - explicit AutoCloseFD(int fd); - AutoCloseFD(const AutoCloseFD & fd) = delete; - AutoCloseFD(AutoCloseFD&& fd); - ~AutoCloseFD(); - AutoCloseFD& operator =(const AutoCloseFD & fd) = delete; - AutoCloseFD& operator =(AutoCloseFD&& fd) noexcept(false); - int get() const; - explicit operator bool() const; - int release(); - void close(); - void fsync(); - void reset() { *this = {}; } -}; - -class Pipe -{ -public: - AutoCloseFD readSide, writeSide; - void create(); - void close(); -}; - - class Pid { pid_t pid = -1; @@ -172,7 +116,6 @@ public: pid_t release(); }; - /** * Kill all processes running under the specified uid by sending them * a SIGKILL. @@ -279,17 +222,6 @@ public: */ std::vector stringsToCharPtrs(const Strings & ss); -/** - * Close all file descriptors except those listed in the given set. - * Good practice in child processes. - */ -void closeMostFDs(const std::set & exceptions); - -/** - * Set the close-on-exec flag for the given file descriptor. - */ -void closeOnExec(int fd); - MakeError(FormatError, Error); @@ -596,15 +528,6 @@ struct MaintainCount */ void commonChildInit(); -/** - * Create a Unix domain socket. - */ -AutoCloseFD createUnixDomainSocket(); - -/** - * Create a Unix domain socket in listen mode. - */ -AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); /** * Bind a Unix domain socket to a path. diff --git a/tests/functional/repl_characterization/test-session.hh b/tests/functional/repl_characterization/test-session.hh index 9961849fc..c77cce6d5 100644 --- a/tests/functional/repl_characterization/test-session.hh +++ b/tests/functional/repl_characterization/test-session.hh @@ -6,7 +6,7 @@ #include #include -#include "util.hh" +#include "file-descriptor.hh" #include "tests/terminal-code-eater.hh" namespace nix {