util.{hh,cc}: Split out file-descriptor.{hh,cc}

Change-Id: I0dd0f9a9c2003fb887e076127e7f825fd3289c76
This commit is contained in:
Tom Hubrecht 2024-05-28 12:25:49 +02:00
parent 6b5078c815
commit 8cd9aa24a8
12 changed files with 349 additions and 321 deletions

View file

@ -1,4 +1,5 @@
#include "lock.hh" #include "lock.hh"
#include "logging.hh"
#include "file-system.hh" #include "file-system.hh"
#include "globals.hh" #include "globals.hh"
#include "pathlocks.hh" #include "pathlocks.hh"

View file

@ -1,7 +1,7 @@
#pragma once #pragma once
///@file ///@file
#include "util.hh" #include "file-descriptor.hh"
namespace nix { namespace nix {

View file

@ -0,0 +1,251 @@
#include "file-system.hh"
#include "finally.hh"
#include "serialise.hh"
#include "signals.hh"
#include <fcntl.h>
#include <unistd.h>
namespace nix {
std::string readFile(int fd)
{
struct stat st;
if (fstat(fd, &st) == -1)
throw SysError("statting file");
return drainFD(fd, true, st.st_size);
}
std::string readLine(int fd)
{
std::string s;
while (1) {
checkInterrupt();
char ch;
// FIXME: inefficient
ssize_t rd = read(fd, &ch, 1);
if (rd == -1) {
if (errno != EINTR)
throw SysError("reading a line");
} else if (rd == 0)
throw EndOfFile("unexpected EOF reading a line");
else {
if (ch == '\n') return s;
s += ch;
}
}
}
void writeLine(int fd, std::string s)
{
s += '\n';
writeFull(fd, s);
}
void readFull(int fd, char * buf, size_t count)
{
while (count) {
checkInterrupt();
ssize_t res = read(fd, buf, count);
if (res == -1) {
if (errno == EINTR) continue;
throw SysError("reading from file");
}
if (res == 0) throw EndOfFile("unexpected end-of-file");
count -= res;
buf += res;
}
}
void writeFull(int fd, std::string_view s, bool allowInterrupts)
{
while (!s.empty()) {
if (allowInterrupts) checkInterrupt();
ssize_t res = write(fd, s.data(), s.size());
if (res == -1 && errno != EINTR)
throw SysError("writing to file");
if (res > 0)
s.remove_prefix(res);
}
}
std::string drainFD(int fd, bool block, const size_t reserveSize)
{
// the parser needs two extra bytes to append terminating characters, other users will
// not care very much about the extra memory.
StringSink sink(reserveSize + 2);
drainFD(fd, sink, block);
return std::move(sink.s);
}
void drainFD(int fd, Sink & sink, bool block)
{
// silence GCC maybe-uninitialized warning in finally
int saved = 0;
if (!block) {
saved = fcntl(fd, F_GETFL);
if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1)
throw SysError("making file descriptor non-blocking");
}
Finally finally([&]() {
if (!block) {
if (fcntl(fd, F_SETFL, saved) == -1)
throw SysError("making file descriptor blocking");
}
});
std::array<unsigned char, 64 * 1024> buf;
while (1) {
checkInterrupt();
ssize_t rd = read(fd, buf.data(), buf.size());
if (rd == -1) {
if (!block && (errno == EAGAIN || errno == EWOULDBLOCK))
break;
if (errno != EINTR)
throw SysError("reading from file");
}
else if (rd == 0) break;
else sink({(char *) buf.data(), (size_t) rd});
}
}
AutoCloseFD::AutoCloseFD() : fd{-1} {}
AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {}
AutoCloseFD::AutoCloseFD(AutoCloseFD && that) : fd{that.fd}
{
that.fd = -1;
}
AutoCloseFD & AutoCloseFD::operator =(AutoCloseFD && that)
{
close();
fd = that.fd;
that.fd = -1;
return *this;
}
AutoCloseFD::~AutoCloseFD()
{
try {
close();
} catch (...) {
ignoreException();
}
}
int AutoCloseFD::get() const
{
return fd;
}
void AutoCloseFD::close()
{
if (fd != -1) {
if (::close(fd) == -1)
/* This should never happen. */
throw SysError("closing file descriptor %1%", fd);
fd = -1;
}
}
void AutoCloseFD::fsync()
{
if (fd != -1) {
int result;
#if __APPLE__
result = ::fcntl(fd, F_FULLFSYNC);
#else
result = ::fsync(fd);
#endif
if (result == -1)
throw SysError("fsync file descriptor %1%", fd);
}
}
AutoCloseFD::operator bool() const
{
return fd != -1;
}
int AutoCloseFD::release()
{
int oldFD = fd;
fd = -1;
return oldFD;
}
void Pipe::create()
{
int fds[2];
#if HAVE_PIPE2
if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe");
#else
if (pipe(fds) != 0) throw SysError("creating pipe");
closeOnExec(fds[0]);
closeOnExec(fds[1]);
#endif
readSide = AutoCloseFD{fds[0]};
writeSide = AutoCloseFD{fds[1]};
}
void Pipe::close()
{
readSide.close();
writeSide.close();
}
void closeMostFDs(const std::set<int> & exceptions)
{
#if __linux__
try {
for (auto & s : readDirectory("/proc/self/fd")) {
auto fd = std::stoi(s.name);
if (!exceptions.count(fd)) {
debug("closing leaked FD %d", fd);
close(fd);
}
}
return;
} catch (SysError &) {
}
#endif
int maxFD = 0;
maxFD = sysconf(_SC_OPEN_MAX);
for (int fd = 0; fd < maxFD; ++fd)
if (!exceptions.count(fd))
close(fd); /* ignore result */
}
void closeOnExec(int fd)
{
int prev;
if ((prev = fcntl(fd, F_GETFD, 0)) == -1 ||
fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1)
throw SysError("setting close-on-exec flag");
}
}

View file

@ -0,0 +1,89 @@
#pragma once
///@file
#include "error.hh"
namespace nix {
struct Sink;
struct Source;
/**
* Read a line from a file descriptor.
*/
std::string readLine(int fd);
/**
* Write a line to a file descriptor.
*/
void writeLine(int fd, std::string s);
/**
* Read the contents of a file into a string.
*/
std::string readFile(int fd);
/**
* Wrappers arount read()/write() that read/write exactly the
* requested number of bytes.
*/
void readFull(int fd, char * buf, size_t count);
void writeFull(int fd, std::string_view s, bool allowInterrupts = true);
/**
* Read a file descriptor until EOF occurs.
*/
std::string drainFD(int fd, bool block = true, const size_t reserveSize=0);
void drainFD(int fd, Sink & sink, bool block = true);
class AutoCloseFD
{
int fd;
public:
AutoCloseFD();
explicit AutoCloseFD(int fd);
AutoCloseFD(const AutoCloseFD & fd) = delete;
AutoCloseFD(AutoCloseFD&& fd);
~AutoCloseFD();
AutoCloseFD& operator =(const AutoCloseFD & fd) = delete;
AutoCloseFD& operator =(AutoCloseFD&& fd) noexcept(false);
int get() const;
explicit operator bool() const;
int release();
void close();
void fsync();
void reset() { *this = {}; }
};
class Pipe
{
public:
AutoCloseFD readSide, writeSide;
void create();
void close();
};
/**
* Close all file descriptors except those listed in the given set.
* Good practice in child processes.
*/
void closeMostFDs(const std::set<int> & exceptions);
/**
* Set the close-on-exec flag for the given file descriptor.
*/
void closeOnExec(int fd);
MakeError(EndOfFile, Error);
/**
* Create a Unix domain socket.
*/
AutoCloseFD createUnixDomainSocket();
/**
* Create a Unix domain socket in listen mode.
*/
AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode);
}

View file

@ -3,10 +3,10 @@
#include <atomic> #include <atomic>
#include "environment-variables.hh" #include "environment-variables.hh"
#include "file-descriptor.hh"
#include "file-system.hh" #include "file-system.hh"
#include "finally.hh" #include "finally.hh"
#include "serialise.hh" #include "serialise.hh"
#include "util.hh"
#include "signals.hh" #include "signals.hh"
#include "types.hh" #include "types.hh"
@ -278,16 +278,6 @@ unsigned char getFileType(const Path & path)
} }
std::string readFile(int fd)
{
struct stat st;
if (fstat(fd, &st) == -1)
throw SysError("statting file");
return drainFD(fd, true, st.st_size);
}
std::string readFile(const Path & path) std::string readFile(const Path & path)
{ {
AutoCloseFD fd{open(path.c_str(), O_RDONLY | O_CLOEXEC)}; AutoCloseFD fd{open(path.c_str(), O_RDONLY | O_CLOEXEC)};

View file

@ -6,7 +6,7 @@
*/ */
#include "types.hh" #include "types.hh"
#include "util.hh" #include "file-descriptor.hh"
#include <sys/types.h> #include <sys/types.h>
#include <sys/stat.h> #include <sys/stat.h>
@ -143,7 +143,6 @@ unsigned char getFileType(const Path & path);
/** /**
* Read the contents of a file into a string. * Read the contents of a file into a string.
*/ */
std::string readFile(int fd);
std::string readFile(const Path & path); std::string readFile(const Path & path);
void readFile(const Path & path, Sink & sink); void readFile(const Path & path, Sink & sink);

View file

@ -1,4 +1,5 @@
#include "environment-variables.hh" #include "environment-variables.hh"
#include "file-descriptor.hh"
#include "logging.hh" #include "logging.hh"
#include "util.hh" #include "util.hh"
#include "config.hh" #include "config.hh"

View file

@ -13,6 +13,7 @@ libutil_sources = files(
'escape-string.cc', 'escape-string.cc',
'exit.cc', 'exit.cc',
'experimental-features.cc', 'experimental-features.cc',
'file-descriptor.cc',
'file-system.cc', 'file-system.cc',
'git.cc', 'git.cc',
'hash.cc', 'hash.cc',
@ -62,6 +63,7 @@ libutil_headers = files(
'exit.hh', 'exit.hh',
'experimental-features.hh', 'experimental-features.hh',
'experimental-features-json.hh', 'experimental-features-json.hh',
'file-descriptor.hh',
'file-system.hh', 'file-system.hh',
'finally.hh', 'finally.hh',
'fmt.hh', 'fmt.hh',

View file

@ -5,6 +5,7 @@
#include "types.hh" #include "types.hh"
#include "util.hh" #include "util.hh"
#include "file-descriptor.hh"
namespace boost::context { struct stack_context; } namespace boost::context { struct stack_context; }

View file

@ -52,34 +52,6 @@
namespace nix { namespace nix {
std::string readLine(int fd)
{
std::string s;
while (1) {
checkInterrupt();
char ch;
// FIXME: inefficient
ssize_t rd = read(fd, &ch, 1);
if (rd == -1) {
if (errno != EINTR)
throw SysError("reading a line");
} else if (rd == 0)
throw EndOfFile("unexpected EOF reading a line");
else {
if (ch == '\n') return s;
s += ch;
}
}
}
void writeLine(int fd, std::string s)
{
s += '\n';
writeFull(fd, s);
}
std::string getUserName() std::string getUserName()
{ {
auto pw = getpwuid(geteuid()); auto pw = getpwuid(geteuid());
@ -196,78 +168,6 @@ std::optional<Path> getSelfExe()
void readFull(int fd, char * buf, size_t count)
{
while (count) {
checkInterrupt();
ssize_t res = read(fd, buf, count);
if (res == -1) {
if (errno == EINTR) continue;
throw SysError("reading from file");
}
if (res == 0) throw EndOfFile("unexpected end-of-file");
count -= res;
buf += res;
}
}
void writeFull(int fd, std::string_view s, bool allowInterrupts)
{
while (!s.empty()) {
if (allowInterrupts) checkInterrupt();
ssize_t res = write(fd, s.data(), s.size());
if (res == -1 && errno != EINTR)
throw SysError("writing to file");
if (res > 0)
s.remove_prefix(res);
}
}
std::string drainFD(int fd, bool block, const size_t reserveSize)
{
// the parser needs two extra bytes to append terminating characters, other users will
// not care very much about the extra memory.
StringSink sink(reserveSize + 2);
drainFD(fd, sink, block);
return std::move(sink.s);
}
void drainFD(int fd, Sink & sink, bool block)
{
// silence GCC maybe-uninitialized warning in finally
int saved = 0;
if (!block) {
saved = fcntl(fd, F_GETFL);
if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1)
throw SysError("making file descriptor non-blocking");
}
Finally finally([&]() {
if (!block) {
if (fcntl(fd, F_SETFL, saved) == -1)
throw SysError("making file descriptor blocking");
}
});
std::array<unsigned char, 64 * 1024> buf;
while (1) {
checkInterrupt();
ssize_t rd = read(fd, buf.data(), buf.size());
if (rd == -1) {
if (!block && (errno == EAGAIN || errno == EWOULDBLOCK))
break;
if (errno != EINTR)
throw SysError("reading from file");
}
else if (rd == 0) break;
else sink({(char *) buf.data(), (size_t) rd});
}
}
////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////
unsigned int getMaxCPU() unsigned int getMaxCPU()
@ -303,102 +203,6 @@ unsigned int getMaxCPU()
////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////
AutoCloseFD::AutoCloseFD() : fd{-1} {}
AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {}
AutoCloseFD::AutoCloseFD(AutoCloseFD && that) : fd{that.fd}
{
that.fd = -1;
}
AutoCloseFD & AutoCloseFD::operator =(AutoCloseFD && that)
{
close();
fd = that.fd;
that.fd = -1;
return *this;
}
AutoCloseFD::~AutoCloseFD()
{
try {
close();
} catch (...) {
ignoreException();
}
}
int AutoCloseFD::get() const
{
return fd;
}
void AutoCloseFD::close()
{
if (fd != -1) {
if (::close(fd) == -1)
/* This should never happen. */
throw SysError("closing file descriptor %1%", fd);
fd = -1;
}
}
void AutoCloseFD::fsync()
{
if (fd != -1) {
int result;
#if __APPLE__
result = ::fcntl(fd, F_FULLFSYNC);
#else
result = ::fsync(fd);
#endif
if (result == -1)
throw SysError("fsync file descriptor %1%", fd);
}
}
AutoCloseFD::operator bool() const
{
return fd != -1;
}
int AutoCloseFD::release()
{
int oldFD = fd;
fd = -1;
return oldFD;
}
void Pipe::create()
{
int fds[2];
#if HAVE_PIPE2
if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe");
#else
if (pipe(fds) != 0) throw SysError("creating pipe");
closeOnExec(fds[0]);
closeOnExec(fds[1]);
#endif
readSide = AutoCloseFD{fds[0]};
writeSide = AutoCloseFD{fds[1]};
}
void Pipe::close()
{
readSide.close();
writeSide.close();
}
////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////
@ -762,39 +566,6 @@ void runProgram2(const RunOptions & options)
} }
void closeMostFDs(const std::set<int> & exceptions)
{
#if __linux__
try {
for (auto & s : readDirectory("/proc/self/fd")) {
auto fd = std::stoi(s.name);
if (!exceptions.count(fd)) {
debug("closing leaked FD %d", fd);
close(fd);
}
}
return;
} catch (SysError &) {
}
#endif
int maxFD = 0;
maxFD = sysconf(_SC_OPEN_MAX);
for (int fd = 0; fd < maxFD; ++fd)
if (!exceptions.count(fd))
close(fd); /* ignore result */
}
void closeOnExec(int fd)
{
int prev;
if ((prev = fcntl(fd, F_GETFD, 0)) == -1 ||
fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1)
throw SysError("setting close-on-exec flag");
}
////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////

View file

@ -39,17 +39,6 @@ struct Source;
extern const std::string nativeSystem; extern const std::string nativeSystem;
/**
* Read a line from a file descriptor.
*/
std::string readLine(int fd);
/**
* Write a line to a file descriptor.
*/
void writeLine(int fd, std::string s);
std::string getUserName(); std::string getUserName();
/** /**
@ -102,57 +91,12 @@ Path getStateDir();
Path createNixStateDir(); Path createNixStateDir();
/**
* Wrappers arount read()/write() that read/write exactly the
* requested number of bytes.
*/
void readFull(int fd, char * buf, size_t count);
void writeFull(int fd, std::string_view s, bool allowInterrupts = true);
MakeError(EndOfFile, Error);
/**
* Read a file descriptor until EOF occurs.
*/
std::string drainFD(int fd, bool block = true, const size_t reserveSize=0);
void drainFD(int fd, Sink & sink, bool block = true);
/** /**
* If cgroups are active, attempt to calculate the number of CPUs available. * If cgroups are active, attempt to calculate the number of CPUs available.
* If cgroups are unavailable or if cpu.max is set to "max", return 0. * If cgroups are unavailable or if cpu.max is set to "max", return 0.
*/ */
unsigned int getMaxCPU(); unsigned int getMaxCPU();
class AutoCloseFD
{
int fd;
public:
AutoCloseFD();
explicit AutoCloseFD(int fd);
AutoCloseFD(const AutoCloseFD & fd) = delete;
AutoCloseFD(AutoCloseFD&& fd);
~AutoCloseFD();
AutoCloseFD& operator =(const AutoCloseFD & fd) = delete;
AutoCloseFD& operator =(AutoCloseFD&& fd) noexcept(false);
int get() const;
explicit operator bool() const;
int release();
void close();
void fsync();
void reset() { *this = {}; }
};
class Pipe
{
public:
AutoCloseFD readSide, writeSide;
void create();
void close();
};
class Pid class Pid
{ {
pid_t pid = -1; pid_t pid = -1;
@ -172,7 +116,6 @@ public:
pid_t release(); pid_t release();
}; };
/** /**
* Kill all processes running under the specified uid by sending them * Kill all processes running under the specified uid by sending them
* a SIGKILL. * a SIGKILL.
@ -279,17 +222,6 @@ public:
*/ */
std::vector<char *> stringsToCharPtrs(const Strings & ss); std::vector<char *> stringsToCharPtrs(const Strings & ss);
/**
* Close all file descriptors except those listed in the given set.
* Good practice in child processes.
*/
void closeMostFDs(const std::set<int> & exceptions);
/**
* Set the close-on-exec flag for the given file descriptor.
*/
void closeOnExec(int fd);
MakeError(FormatError, Error); MakeError(FormatError, Error);
@ -596,15 +528,6 @@ struct MaintainCount
*/ */
void commonChildInit(); void commonChildInit();
/**
* Create a Unix domain socket.
*/
AutoCloseFD createUnixDomainSocket();
/**
* Create a Unix domain socket in listen mode.
*/
AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode);
/** /**
* Bind a Unix domain socket to a path. * Bind a Unix domain socket to a path.

View file

@ -6,7 +6,7 @@
#include <span> #include <span>
#include <string> #include <string>
#include "util.hh" #include "file-descriptor.hh"
#include "tests/terminal-code-eater.hh" #include "tests/terminal-code-eater.hh"
namespace nix { namespace nix {