lix-project/lix
18
37
Fork 14
Code Issues 286 Code Review (Gerrit) Projects 1 Releases Wiki Activity

Merge "libstore: Create platform LocalStore subclasses" into main

Browse source
This commit is contained in:
Artemis Tosini 2024-04-24 15:35:32 +00:00 committed by Gerrit Code Review
parent f24223931d b247ef72dc
commit 7114b0465a
12 changed files with 269 additions and 120 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
src/libstore/gc-store.hh
View file

@ -7,7 +7,14 @@
namespace nix {
/**
* Garbage-collector roots, referring to a store path
*/
typedef std::unordered_map<StorePath, std::unordered_set<std::string>> Roots;
/**
* Possible garbage collector roots, referring to any path
*/
typedef std::unordered_map<Path, std::unordered_set<std::string>> UncheckedRoots;
struct GCOptions

111
src/libstore/gc.cc
View file

@ -321,105 +321,8 @@ Roots LocalStore::findRoots(bool censor)
return roots;
}
typedef std::unordered_map<Path, std::unordered_set<std::string>> UncheckedRoots;
static void readProcLink(const std::string & file, UncheckedRoots & roots)
void LocalStore::findPlatformRoots(UncheckedRoots & unchecked)
{
constexpr auto bufsiz = PATH_MAX;
char buf[bufsiz];
auto res = readlink(file.c_str(), buf, bufsiz);
if (res == -1) {
if (errno == ENOENT || errno == EACCES || errno == ESRCH)
return;
throw SysError("reading symlink");
}
if (res == bufsiz) {
throw Error("overly long symlink starting with '%1%'", std::string_view(buf, bufsiz));
}
if (res > 0 && buf[0] == '/')
roots[std::string(static_cast<char *>(buf), res)]
.emplace(file);
}
static std::string quoteRegexChars(const std::string & raw)
{
static auto specialRegex = std::regex(R"([.^$\\*+?()\[\]{}|])");
return std::regex_replace(raw, specialRegex, R"(\$&)");
}
#if __linux__
static void readFileRoots(const char * path, UncheckedRoots & roots)
{
try {
roots[readFile(path)].emplace(path);
} catch (SysError & e) {
if (e.errNo != ENOENT && e.errNo != EACCES)
throw;
}
}
#endif
void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
{
UncheckedRoots unchecked;
auto procDir = AutoCloseDir{opendir("/proc")};
if (procDir) {
struct dirent * ent;
auto digitsRegex = std::regex(R"(^\d+$)");
auto mapRegex = std::regex(R"(^\s*\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(/\S+)\s*$)");
auto storePathRegex = std::regex(quoteRegexChars(storeDir) + R"(/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*)");
while (errno = 0, ent = readdir(procDir.get())) {
checkInterrupt();
if (std::regex_match(ent->d_name, digitsRegex)) {
try {
readProcLink(fmt("/proc/%s/exe" ,ent->d_name), unchecked);
readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked);
auto fdStr = fmt("/proc/%s/fd", ent->d_name);
auto fdDir = AutoCloseDir(opendir(fdStr.c_str()));
if (!fdDir) {
if (errno == ENOENT || errno == EACCES)
continue;
throw SysError("opening %1%", fdStr);
}
struct dirent * fd_ent;
while (errno = 0, fd_ent = readdir(fdDir.get())) {
if (fd_ent->d_name[0] != '.')
readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked);
}
if (errno) {
if (errno == ESRCH)
continue;
throw SysError("iterating /proc/%1%/fd", ent->d_name);
}
fdDir.reset();
auto mapFile = fmt("/proc/%s/maps", ent->d_name);
auto mapLines = tokenizeString<std::vector<std::string>>(readFile(mapFile), "\n");
for (const auto & line : mapLines) {
auto match = std::smatch{};
if (std::regex_match(line, match, mapRegex))
unchecked[match[1]].emplace(mapFile);
}
auto envFile = fmt("/proc/%s/environ", ent->d_name);
auto envString = readFile(envFile);
auto env_end = std::sregex_iterator{};
for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i)
unchecked[i->str()].emplace(envFile);
} catch (SysError & e) {
if (errno == ENOENT || errno == EACCES || errno == ESRCH)
continue;
throw;
}
}
}
if (errno)
throw SysError("iterating /proc");
}
#if !defined(__linux__)
// lsof is really slow on OS X. This actually causes the gc-concurrent.sh test to fail.
// See: https://github.com/NixOS/nix/issues/3011
// Because of this we disable lsof when running the tests.
@ -437,13 +340,13 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
/* lsof not installed, lsof failed */
}
}
#endif
}
#if __linux__
readFileRoots("/proc/sys/kernel/modprobe", unchecked);
readFileRoots("/proc/sys/kernel/fbsplash", unchecked);
readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked);
#endif
void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
{
UncheckedRoots unchecked;
findPlatformRoots(unchecked);
for (auto & [target, links] : unchecked) {
if (!isInStore(target)) continue;

2
src/libstore/local-store.cc
View file

@ -1940,6 +1940,4 @@ std::optional<std::string> LocalStore::getVersion()
return nixVersion;
}
static RegisterStoreImplementation<LocalStore, LocalStoreConfig> regLocalStore;
} // namespace nix

31
src/libstore/local-store.hh
View file

@ -127,6 +127,17 @@ private:
const PublicKeys & getPublicKeys();
protected:
/**
* Initialise the local store, upgrading the schema if
* necessary.
* Protected so that users don't accidentally create a LocalStore
* instead of a platform's subclass.
*/
LocalStore(const Params & params);
LocalStore(std::string scheme, std::string path, const Params & params);
public:
/**
@ -134,18 +145,16 @@ public:
*/
PathSet locksHeld;
/**
* Initialise the local store, upgrading the schema if
* necessary.
*/
LocalStore(const Params & params);
LocalStore(std::string scheme, std::string path, const Params & params);
~LocalStore();
virtual ~LocalStore();
static std::set<std::string> uriSchemes()
{ return {}; }
/**
* Create a LocalStore, possibly a platform-specific subclass
*/
static std::shared_ptr<LocalStore> makeLocalStore(const Params & params);
/**
* Implementations of abstract store API methods.
*/
@ -330,6 +339,12 @@ private:
void findRootsNoTemp(Roots & roots, bool censor);
/**
* Find possible garbage collector roots in a platform-specific manner,
* e.g. by looking in `/proc` or using `lsof`
*/
virtual void findPlatformRoots(UncheckedRoots & unchecked);
void findRuntimeRoots(Roots & roots, bool censor);
std::pair<Path, AutoCloseFD> createTempDirInStore();

5
src/libstore/local.mk
View file

@ -5,6 +5,11 @@ libstore_NAME = libnixstore
libstore_DIR := $(d)
libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/build/*.cc)
ifdef HOST_LINUX
libstore_SOURCES += $(d)/platform/linux.cc
else
libstore_SOURCES += $(d)/platform/fallback.cc
endif
libstore_LIBS = libutil

9
src/libstore/meson.build
View file

@ -66,6 +66,7 @@ libstore_sources = files(
'path-with-outputs.cc',
'path.cc',
'pathlocks.cc',
'platform.cc',
'profiles.cc',
'realisation.cc',
'remote-fs-accessor.cc',
@ -158,6 +159,14 @@ libstore_headers = files(
'worker-protocol.hh',
)
if host_machine.system() == 'linux'
libstore_sources += files('platform/linux.cc')
libstore_headers += files('platform/linux.hh')
else
libstore_sources += files('platform/fallback.cc')
libstore_headers += files('platform/fallback.hh')
endif
# These variables (aside from LSOF) are created pseudo-dynamically, near the beginning of
# the top-level meson.build. Aside from prefix itself, each of these was
# made into an absolute path by joining it with prefix, unless it was already

18
src/libstore/platform.cc Normal file
View file

@ -0,0 +1,18 @@
#include "local-store.hh"
#if __linux__
#include "platform/linux.hh"
#else
#include "platform/fallback.hh"
#endif
namespace nix {
std::shared_ptr<LocalStore> LocalStore::makeLocalStore(const Params & params)
{
#if __linux__
return std::shared_ptr<LocalStore>(new LinuxLocalStore(params));
#else
return std::shared_ptr<LocalStore>(new FallbackLocalStore(params));
#endif
}
}

5
src/libstore/platform/fallback.cc Normal file
View file

@ -0,0 +1,5 @@
#include "platform/fallback.hh"
namespace nix {
static RegisterStoreImplementation<FallbackLocalStore, LocalStoreConfig> regLocalStore;
}

31
src/libstore/platform/fallback.hh Normal file
View file

@ -0,0 +1,31 @@
#pragma once
///@file
#include "local-store.hh"
namespace nix {
/**
* Fallback platform implementation of LocalStore
* Exists so we can make LocalStore constructor protected
*/
class FallbackLocalStore : public LocalStore
{
public:
FallbackLocalStore(const Params & params)
: StoreConfig(params)
, LocalFSStoreConfig(params)
, LocalStoreConfig(params)
, Store(params)
, LocalFSStore(params)
, LocalStore(params)
{
}
FallbackLocalStore(const std::string scheme, std::string path, const Params & params)
: FallbackLocalStore(params)
{
throw UnimplementedError("FallbackLocalStore");
}
};
}

123
src/libstore/platform/linux.cc Normal file
View file

@ -0,0 +1,123 @@
#include "gc-store.hh"
#include "signals.hh"
#include "platform/linux.hh"
#include <regex>
namespace nix {
static RegisterStoreImplementation<LinuxLocalStore, LocalStoreConfig> regLocalStore;
static void readProcLink(const std::string & file, UncheckedRoots & roots)
{
constexpr auto bufsiz = PATH_MAX;
char buf[bufsiz];
auto res = readlink(file.c_str(), buf, bufsiz);
if (res == -1) {
if (errno == ENOENT || errno == EACCES || errno == ESRCH) {
return;
}
throw SysError("reading symlink");
}
if (res == bufsiz) {
throw Error("overly long symlink starting with '%1%'", std::string_view(buf, bufsiz));
}
if (res > 0 && buf[0] == '/') {
roots[std::string(static_cast<char *>(buf), res)].emplace(file);
}
}
static std::string quoteRegexChars(const std::string & raw)
{
static auto specialRegex = std::regex(R"([.^$\\*+?()\[\]{}|])");
return std::regex_replace(raw, specialRegex, R"(\$&)");
}
static void readFileRoots(const char * path, UncheckedRoots & roots)
{
try {
roots[readFile(path)].emplace(path);
} catch (SysError & e) {
if (e.errNo != ENOENT && e.errNo != EACCES) {
throw;
}
}
}
void LinuxLocalStore::findPlatformRoots(UncheckedRoots & unchecked)
{
auto procDir = AutoCloseDir{opendir("/proc")};
if (procDir) {
struct dirent * ent;
auto digitsRegex = std::regex(R"(^\d+$)");
auto mapRegex = std::regex(R"(^\s*\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(/\S+)\s*$)");
auto storePathRegex =
std::regex(quoteRegexChars(storeDir) + R"(/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*)");
while (errno = 0, ent = readdir(procDir.get())) {
checkInterrupt();
if (std::regex_match(ent->d_name, digitsRegex)) {
try {
readProcLink(fmt("/proc/%s/exe", ent->d_name), unchecked);
readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked);
auto fdStr = fmt("/proc/%s/fd", ent->d_name);
auto fdDir = AutoCloseDir(opendir(fdStr.c_str()));
if (!fdDir) {
if (errno == ENOENT || errno == EACCES) {
continue;
}
throw SysError("opening %1%", fdStr);
}
struct dirent * fd_ent;
while (errno = 0, fd_ent = readdir(fdDir.get())) {
if (fd_ent->d_name[0] != '.') {
readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked);
}
}
if (errno) {
if (errno == ESRCH) {
continue;
}
throw SysError("iterating /proc/%1%/fd", ent->d_name);
}
fdDir.reset();
auto mapFile = fmt("/proc/%s/maps", ent->d_name);
auto mapLines =
tokenizeString<std::vector<std::string>>(readFile(mapFile), "\n");
for (const auto & line : mapLines) {
auto match = std::smatch{};
if (std::regex_match(line, match, mapRegex)) {
unchecked[match[1]].emplace(mapFile);
}
}
auto envFile = fmt("/proc/%s/environ", ent->d_name);
auto envString = readFile(envFile);
auto env_end = std::sregex_iterator{};
for (auto i =
std::sregex_iterator{
envString.begin(), envString.end(), storePathRegex
};
i != env_end;
++i)
{
unchecked[i->str()].emplace(envFile);
}
} catch (SysError & e) {
if (errno == ENOENT || errno == EACCES || errno == ESRCH) {
continue;
}
throw;
}
}
}
if (errno) {
throw SysError("iterating /proc");
}
}
readFileRoots("/proc/sys/kernel/modprobe", unchecked);
readFileRoots("/proc/sys/kernel/fbsplash", unchecked);
readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked);
}
}

35
src/libstore/platform/linux.hh Normal file
View file

@ -0,0 +1,35 @@
#pragma once
///@file
#include "gc-store.hh"
#include "local-store.hh"
namespace nix {
/**
* Linux-specific implementation of LocalStore
*/
class LinuxLocalStore : public LocalStore
{
public:
LinuxLocalStore(const Params & params)
: StoreConfig(params)
, LocalFSStoreConfig(params)
, LocalStoreConfig(params)
, Store(params)
, LocalFSStore(params)
, LocalStore(params)
{
}
LinuxLocalStore(const std::string scheme, std::string path, const Params & params)
: LinuxLocalStore(params)
{
throw UnimplementedError("LinuxLocalStore");
}
private:
void findPlatformRoots(UncheckedRoots & unchecked) override;
};
}

12
src/libstore/store-api.cc
View file

@ -1426,7 +1426,7 @@ std::shared_ptr<Store> openFromNonUri(const std::string & uri, const Store::Para
if (uri == "" || uri == "auto") {
auto stateDir = getOr(params, "state", settings.nixStateDir);
if (access(stateDir.c_str(), R_OK | W_OK) == 0)
return std::make_shared<LocalStore>(params);
return LocalStore::makeLocalStore(params);
else if (pathExists(settings.nixDaemonSocketFile))
return std::make_shared<UDSRemoteStore>(params);
#if __linux__
@ -1444,26 +1444,26 @@ std::shared_ptr<Store> openFromNonUri(const std::string & uri, const Store::Para
try {
createDirs(chrootStore);
} catch (Error & e) {
return std::make_shared<LocalStore>(params);
return LocalStore::makeLocalStore(params);
}
warn("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore);
} else
debug("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore);
Store::Params params2;
params2["root"] = chrootStore;
return std::make_shared<LocalStore>(params2);
return LocalStore::makeLocalStore(params);
}
#endif
else
return std::make_shared<LocalStore>(params);
return LocalStore::makeLocalStore(params);
} else if (uri == "daemon") {
return std::make_shared<UDSRemoteStore>(params);
} else if (uri == "local") {
return std::make_shared<LocalStore>(params);
return LocalStore::makeLocalStore(params);
} else if (isNonUriPath(uri)) {
Store::Params params2 = params;
params2["root"] = absPath(uri);
return std::make_shared<LocalStore>(params2);
return LocalStore::makeLocalStore(params2);
} else {
return nullptr;
}