lix-project/lix
20
58
Fork 31
Code Issues 359 Code Review (Gerrit) Projects 3 Releases Wiki Activity

Merge pull request #5149 from edolstra/non-blocking-gc

Browse source 
Non-blocking garbage collector
This commit is contained in:
Eelco Dolstra 2021-10-28 23:55:16 +02:00 committed by GitHub
parent bc4b7521f4 33d04e8a8d
commit 0d00dd6262
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
23 changed files with 553 additions and 507 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
flake.nix
View file

@ -489,12 +489,7 @@
'';
*/
};
checks = forAllSystems (system: {
binaryTarball = self.hydraJobs.binaryTarball.${system};
perlBindings = self.hydraJobs.perlBindings.${system};
installTests =
installTests = forAllSystems (system:
let pkgs = nixpkgsFor.${system}; in
pkgs.runCommand "install-tests" {
againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix;
@ -506,7 +501,14 @@
# Disabled because the latest stable version doesn't handle
# `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work
# againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable;
} "touch $out";
} "touch $out");
};
checks = forAllSystems (system: {
binaryTarball = self.hydraJobs.binaryTarball.${system};
perlBindings = self.hydraJobs.perlBindings.${system};
installTests = self.hydraJobs.installTests.${system};
});
packages = forAllSystems (system: {

8
src/libstore/binary-cache-store.cc
View file

@ -111,15 +111,15 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo");
std::string hashPart(narInfo->path.hashPart());
{
auto state_(state.lock());
state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = std::shared_ptr<NarInfo>(narInfo) });
state_->pathInfoCache.upsert(
std::string(narInfo->path.to_string()),
PathInfoCacheValue { .value = std::shared_ptr<NarInfo>(narInfo) });
}
if (diskCache)
diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo));
diskCache->upsertNarInfo(getUri(), std::string(narInfo->path.hashPart()), std::shared_ptr<NarInfo>(narInfo));
}
AutoCloseFD openFile(const Path & path)

2
src/libstore/build/local-derivation-goal.cc
View file

@ -1353,7 +1353,7 @@ void LocalDerivationGoal::startDaemon()
AutoCloseFD remote = accept(daemonSocket.get(),
(struct sockaddr *) &remoteAddr, &remoteAddrLen);
if (!remote) {
if (errno == EINTR) continue;
if (errno == EINTR || errno == EAGAIN) continue;
if (errno == EINVAL) break;
throw SysError("accepting connection");
}

2
src/libstore/daemon.cc
View file

@ -625,9 +625,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
break;
}
// Obsolete.
case wopSyncWithGC: {
logger->startWork();
store->syncWithGC();
logger->stopWork();
to << 1;
break;

785
src/libstore/gc.cc
View file

@ -10,48 +10,22 @@
#include <regex>
#include <random>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/statvfs.h>
#include <climits>
#include <errno.h>
#include <fcntl.h>
#include <poll.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/statvfs.h>
#include <sys/types.h>
#include <sys/un.h>
#include <unistd.h>
#include <climits>
namespace nix {
static string gcLockName = "gc.lock";
static string gcRootsDir = "gcroots";
/* Acquire the global GC lock. This is used to prevent new Nix
processes from starting after the temporary root files have been
read. To be precise: when they try to create a new temporary root
file, they will block until the garbage collector has finished /
yielded the GC lock. */
AutoCloseFD LocalStore::openGCLock(LockType lockType)
{
Path fnGCLock = (format("%1%/%2%")
% stateDir % gcLockName).str();
debug(format("acquiring global GC lock '%1%'") % fnGCLock);
AutoCloseFD fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
if (!fdGCLock)
throw SysError("opening global GC lock '%1%'", fnGCLock);
if (!lockFile(fdGCLock.get(), lockType, false)) {
printInfo("waiting for the big garbage collector lock...");
lockFile(fdGCLock.get(), lockType, true);
}
/* !!! Restrict read permission on the GC root. Otherwise any
process that can open the file for reading can DoS the
collector. */
return fdGCLock;
}
static std::string gcSocketPath = "/gc-socket/socket";
static std::string gcRootsDir = "gcroots";
static void makeSymlink(const Path & link, const Path & target)
@ -71,12 +45,6 @@ static void makeSymlink(const Path & link, const Path & target)
}
void LocalStore::syncWithGC()
{
AutoCloseFD fdGCLock = openGCLock(ltRead);
}
void LocalStore::addIndirectRoot(const Path & path)
{
string hash = hashString(htSHA1, path).to_string(Base32, false);
@ -95,6 +63,12 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot
"creating a garbage collector root (%1%) in the Nix store is forbidden "
"(are you running nix-build inside the store?)", gcRoot);
/* Register this root with the garbage collector, if it's
running. This should be superfluous since the caller should
have registered this root yet, but let's be on the safe
side. */
addTempRoot(storePath);
/* Don't clobber the link if it already exists and doesn't
point to the Nix store. */
if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot))))
@ -102,11 +76,6 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot
makeSymlink(gcRoot, printStorePath(storePath));
addIndirectRoot(gcRoot);
/* Grab the global GC root, causing us to block while a GC is in
progress. This prevents the set of permanent roots from
increasing while a GC is in progress. */
syncWithGC();
return gcRoot;
}
@ -119,8 +88,6 @@ void LocalStore::addTempRoot(const StorePath & path)
if (!state->fdTempRoots) {
while (1) {
AutoCloseFD fdGCLock = openGCLock(ltRead);
if (pathExists(fnTempRoots))
/* It *must* be stale, since there can be no two
processes with the same pid. */
@ -128,10 +95,8 @@ void LocalStore::addTempRoot(const StorePath & path)
state->fdTempRoots = openLockFile(fnTempRoots, true);
fdGCLock = -1;
debug(format("acquiring read lock on '%1%'") % fnTempRoots);
lockFile(state->fdTempRoots.get(), ltRead, true);
debug("acquiring write lock on '%s'", fnTempRoots);
lockFile(state->fdTempRoots.get(), ltWrite, true);
/* Check whether the garbage collector didn't get in our
way. */
@ -147,24 +112,55 @@ void LocalStore::addTempRoot(const StorePath & path)
}
/* Upgrade the lock to a write lock. This will cause us to block
if the garbage collector is holding our lock. */
debug(format("acquiring write lock on '%1%'") % fnTempRoots);
lockFile(state->fdTempRoots.get(), ltWrite, true);
if (!state->fdGCLock)
state->fdGCLock = openGCLock();
restart:
FdLock gcLock(state->fdGCLock.get(), ltRead, false, "");
if (!gcLock.acquired) {
/* We couldn't get a shared global GC lock, so the garbage
collector is running. So we have to connect to the garbage
collector and inform it about our root. */
if (!state->fdRootsSocket) {
auto socketPath = stateDir.get() + gcSocketPath;
debug("connecting to '%s'", socketPath);
state->fdRootsSocket = createUnixDomainSocket();
nix::connect(state->fdRootsSocket.get(), socketPath);
}
try {
debug("sending GC root '%s'", printStorePath(path));
writeFull(state->fdRootsSocket.get(), printStorePath(path) + "\n", false);
char c;
readFull(state->fdRootsSocket.get(), &c, 1);
assert(c == '1');
debug("got ack for GC root '%s'", printStorePath(path));
} catch (SysError & e) {
/* The garbage collector may have exited, so we need to
restart. */
if (e.errNo == EPIPE) {
debug("GC socket disconnected");
state->fdRootsSocket.close();
goto restart;
}
} catch (EndOfFile & e) {
debug("GC socket disconnected");
state->fdRootsSocket.close();
goto restart;
}
}
/* Append the store path to the temporary roots file. */
string s = printStorePath(path) + '\0';
writeFull(state->fdTempRoots.get(), s);
/* Downgrade to a read lock. */
debug(format("downgrading to read lock on '%1%'") % fnTempRoots);
lockFile(state->fdTempRoots.get(), ltRead, true);
}
static std::string censored = "{censored}";
void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
void LocalStore::findTempRoots(Roots & tempRoots, bool censor)
{
/* Read the `temproots' directory for per-process temporary root
files. */
@ -179,35 +175,25 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
pid_t pid = std::stoi(i.name);
debug(format("reading temporary root file '%1%'") % path);
FDPtr fd(new AutoCloseFD(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666)));
if (!*fd) {
AutoCloseFD fd(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666));
if (!fd) {
/* It's okay if the file has disappeared. */
if (errno == ENOENT) continue;
throw SysError("opening temporary roots file '%1%'", path);
}
/* This should work, but doesn't, for some reason. */
//FDPtr fd(new AutoCloseFD(openLockFile(path, false)));
//if (*fd == -1) continue;
/* Try to acquire a write lock without blocking. This can
only succeed if the owning process has died. In that case
we don't care about its temporary roots. */
if (lockFile(fd->get(), ltWrite, false)) {
if (lockFile(fd.get(), ltWrite, false)) {
printInfo("removing stale temporary roots file '%1%'", path);
unlink(path.c_str());
writeFull(fd->get(), "d");
writeFull(fd.get(), "d");
continue;
}
/* Acquire a read lock. This will prevent the owning process
from upgrading to a write lock, therefore it will block in
addTempRoot(). */
debug(format("waiting for read lock on '%1%'") % path);
lockFile(fd->get(), ltRead, true);
/* Read the entire file. */
string contents = readFile(fd->get());
string contents = readFile(fd.get());
/* Extract the roots. */
string::size_type pos = 0, end;
@ -218,8 +204,6 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
tempRoots[parseStorePath(root)].emplace(censor ? censored : fmt("{temp:%d}", pid));
pos = end + 1;
}
fds.push_back(fd); /* keep open */
}
}
@ -304,8 +288,7 @@ Roots LocalStore::findRoots(bool censor)
Roots roots;
findRootsNoTemp(roots, censor);
FDs fds;
findTempRoots(fds, roots, censor);
findTempRoots(roots, censor);
return roots;
}
@ -455,265 +438,139 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
struct GCLimitReached { };
struct LocalStore::GCState
{
const GCOptions & options;
GCResults & results;
StorePathSet roots;
StorePathSet tempRoots;
StorePathSet dead;
StorePathSet alive;
bool gcKeepOutputs;
bool gcKeepDerivations;
uint64_t bytesInvalidated;
bool moveToTrash = true;
bool shouldDelete;
GCState(const GCOptions & options, GCResults & results)
: options(options), results(results), bytesInvalidated(0) { }
};
bool LocalStore::isActiveTempFile(const GCState & state,
const Path & path, const string & suffix)
{
return hasSuffix(path, suffix)
&& state.tempRoots.count(parseStorePath(string(path, 0, path.size() - suffix.size())));
}
void LocalStore::deleteGarbage(GCState & state, const Path & path)
{
uint64_t bytesFreed;
deletePath(path, bytesFreed);
state.results.bytesFreed += bytesFreed;
}
void LocalStore::deletePathRecursive(GCState & state, const Path & path)
{
checkInterrupt();
uint64_t size = 0;
auto storePath = maybeParseStorePath(path);
if (storePath && isValidPath(*storePath)) {
StorePathSet referrers;
queryReferrers(*storePath, referrers);
for (auto & i : referrers)
if (printStorePath(i) != path) deletePathRecursive(state, printStorePath(i));
size = queryPathInfo(*storePath)->narSize;
invalidatePathChecked(*storePath);
}
Path realPath = realStoreDir + "/" + std::string(baseNameOf(path));
struct stat st;
if (lstat(realPath.c_str(), &st)) {
if (errno == ENOENT) return;
throw SysError("getting status of %1%", realPath);
}
printInfo(format("deleting '%1%'") % path);
state.results.paths.insert(path);
/* If the path is not a regular file or symlink, move it to the
trash directory. The move is to ensure that later (when we're
not holding the global GC lock) we can delete the path without
being afraid that the path has become alive again. Otherwise
delete it right away. */
if (state.moveToTrash && S_ISDIR(st.st_mode)) {
// Estimate the amount freed using the narSize field. FIXME:
// if the path was not valid, need to determine the actual
// size.
try {
if (chmod(realPath.c_str(), st.st_mode | S_IWUSR) == -1)
throw SysError("making '%1%' writable", realPath);
Path tmp = trashDir + "/" + std::string(baseNameOf(path));
if (rename(realPath.c_str(), tmp.c_str()))
throw SysError("unable to rename '%1%' to '%2%'", realPath, tmp);
state.bytesInvalidated += size;
} catch (SysError & e) {
if (e.errNo == ENOSPC) {
printInfo(format("note: can't create move '%1%': %2%") % realPath % e.msg());
deleteGarbage(state, realPath);
}
}
} else
deleteGarbage(state, realPath);
if (state.results.bytesFreed + state.bytesInvalidated > state.options.maxFreed) {
printInfo(format("deleted or invalidated more than %1% bytes; stopping") % state.options.maxFreed);
throw GCLimitReached();
}
}
bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const StorePath & path)
{
if (visited.count(path)) return false;
if (state.alive.count(path)) return true;
if (state.dead.count(path)) return false;
if (state.roots.count(path)) {
debug("cannot delete '%1%' because it's a root", printStorePath(path));
state.alive.insert(path);
return true;
}
visited.insert(path);
if (!isValidPath(path)) return false;
StorePathSet incoming;
/* Don't delete this path if any of its referrers are alive. */
queryReferrers(path, incoming);
/* If keep-derivations is set and this is a derivation, then
don't delete the derivation if any of the outputs are alive. */
if (state.gcKeepDerivations && path.isDerivation()) {
for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(path))
if (maybeOutPath &&
isValidPath(*maybeOutPath) &&
queryPathInfo(*maybeOutPath)->deriver == path
)
incoming.insert(*maybeOutPath);
}
/* If keep-outputs is set, then don't delete this path if there
are derivers of this path that are not garbage. */
if (state.gcKeepOutputs) {
auto derivers = queryValidDerivers(path);
for (auto & i : derivers)
incoming.insert(i);
}
for (auto & i : incoming)
if (i != path)
if (canReachRoot(state, visited, i)) {
state.alive.insert(path);
return true;
}
return false;
}
void LocalStore::tryToDelete(GCState & state, const Path & path)
{
checkInterrupt();
auto realPath = realStoreDir + "/" + std::string(baseNameOf(path));
if (realPath == linksDir || realPath == trashDir) return;
//Activity act(*logger, lvlDebug, format("considering whether to delete '%1%'") % path);
auto storePath = maybeParseStorePath(path);
if (!storePath || !isValidPath(*storePath)) {
/* A lock file belonging to a path that we're building right
now isn't garbage. */
if (isActiveTempFile(state, path, ".lock")) return;
/* Don't delete .chroot directories for derivations that are
currently being built. */
if (isActiveTempFile(state, path, ".chroot")) return;
/* Don't delete .check directories for derivations that are
currently being built, because we may need to run
diff-hook. */
if (isActiveTempFile(state, path, ".check")) return;
}
StorePathSet visited;
if (storePath && canReachRoot(state, visited, *storePath)) {
debug("cannot delete '%s' because it's still reachable", path);
} else {
/* No path we visited was a root, so everything is garbage.
But we only delete ‘path’ and its referrers here so that
‘nix-store --delete’ doesn't have the unexpected effect of
recursing into derivations and outputs. */
for (auto & i : visited)
state.dead.insert(i);
if (state.shouldDelete)
deletePathRecursive(state, path);
}
}
/* Unlink all files in /nix/store/.links that have a link count of 1,
which indicates that there are no other links and so they can be
safely deleted. FIXME: race condition with optimisePath(): we
might see a link count of 1 just before optimisePath() increases
the link count. */
void LocalStore::removeUnusedLinks(const GCState & state)
{
AutoCloseDir dir(opendir(linksDir.c_str()));
if (!dir) throw SysError("opening directory '%1%'", linksDir);
int64_t actualSize = 0, unsharedSize = 0;
struct dirent * dirent;
while (errno = 0, dirent = readdir(dir.get())) {
checkInterrupt();
string name = dirent->d_name;
if (name == "." || name == "..") continue;
Path path = linksDir + "/" + name;
auto st = lstat(path);
if (st.st_nlink != 1) {
actualSize += st.st_size;
unsharedSize += (st.st_nlink - 1) * st.st_size;
continue;
}
printMsg(lvlTalkative, format("deleting unused link '%1%'") % path);
if (unlink(path.c_str()) == -1)
throw SysError("deleting '%1%'", path);
state.results.bytesFreed += st.st_size;
}
struct stat st;
if (stat(linksDir.c_str(), &st) == -1)
throw SysError("statting '%1%'", linksDir);
int64_t overhead = st.st_blocks * 512ULL;
printInfo("note: currently hard linking saves %.2f MiB",
((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
}
void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
{
GCState state(options, results);
state.gcKeepOutputs = settings.gcKeepOutputs;
state.gcKeepDerivations = settings.gcKeepDerivations;
bool shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific;
bool gcKeepOutputs = settings.gcKeepOutputs;
bool gcKeepDerivations = settings.gcKeepDerivations;
StorePathSet roots, dead, alive;
struct Shared
{
// The temp roots only store the hash part to make it easier to
// ignore suffixes like '.lock', '.chroot' and '.check'.
std::unordered_set<std::string> tempRoots;
// Hash part of the store path currently being deleted, if
// any.
std::optional<std::string> pending;
};
Sync<Shared> _shared;
std::condition_variable wakeup;
/* Using `--ignore-liveness' with `--delete' can have unintended
consequences if `keep-outputs' or `keep-derivations' are true
(the garbage collector will recurse into deleting the outputs
or derivers, respectively). So disable them. */
if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) {
state.gcKeepOutputs = false;
state.gcKeepDerivations = false;
gcKeepOutputs = false;
gcKeepDerivations = false;
}
state.shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific;
if (state.shouldDelete)
if (shouldDelete)
deletePath(reservedPath);
/* Acquire the global GC root. This prevents
a) New roots from being added.
b) Processes from creating new temporary root files. */
AutoCloseFD fdGCLock = openGCLock(ltWrite);
/* Acquire the global GC root. Note: we don't use fdGCLock
here because then in auto-gc mode, another thread could
downgrade our exclusive lock. */
auto fdGCLock = openGCLock();
FdLock gcLock(fdGCLock.get(), ltWrite, true, "waiting for the big garbage collector lock...");
/* Start the server for receiving new roots. */
auto socketPath = stateDir.get() + gcSocketPath;
createDirs(dirOf(socketPath));
auto fdServer = createUnixDomainSocket(socketPath, 0666);
if (fcntl(fdServer.get(), F_SETFL, fcntl(fdServer.get(), F_GETFL) | O_NONBLOCK) == -1)
throw SysError("making socket '%1%' non-blocking", socketPath);
Pipe shutdownPipe;
shutdownPipe.create();
std::thread serverThread([&]() {
Sync<std::map<int, std::thread>> connections;
Finally cleanup([&]() {
debug("GC roots server shutting down");
while (true) {
auto item = remove_begin(*connections.lock());
if (!item) break;
auto & [fd, thread] = *item;
shutdown(fd, SHUT_RDWR);
thread.join();
}
});
while (true) {
std::vector<struct pollfd> fds;
fds.push_back({.fd = shutdownPipe.readSide.get(), .events = POLLIN});
fds.push_back({.fd = fdServer.get(), .events = POLLIN});
auto count = poll(fds.data(), fds.size(), -1);
assert(count != -1);
if (fds[0].revents)
/* Parent is asking us to quit. */
break;
if (fds[1].revents) {
/* Accept a new connection. */
assert(fds[1].revents & POLLIN);
AutoCloseFD fdClient = accept(fdServer.get(), nullptr, nullptr);
if (!fdClient) continue;
/* Process the connection in a separate thread. */
auto fdClient_ = fdClient.get();
std::thread clientThread([&, fdClient = std::move(fdClient)]() {
Finally cleanup([&]() {
auto conn(connections.lock());
auto i = conn->find(fdClient.get());
if (i != conn->end()) {
i->second.detach();
conn->erase(i);
}
});
while (true) {
try {
auto path = readLine(fdClient.get());
auto storePath = maybeParseStorePath(path);
if (storePath) {
debug("got new GC root '%s'", path);
auto hashPart = std::string(storePath->hashPart());
auto shared(_shared.lock());
shared->tempRoots.insert(hashPart);
/* If this path is currently being
deleted, then we have to wait until
deletion is finished to ensure that
the client doesn't start
re-creating it before we're
done. FIXME: ideally we would use a
FD for this so we don't block the
poll loop. */
while (shared->pending == hashPart) {
debug("synchronising with deletion of path '%s'", path);
shared.wait(wakeup);
}
} else
printError("received garbage instead of a root from client");
writeFull(fdClient.get(), "1", false);
} catch (Error &) { break; }
}
});
connections.lock()->insert({fdClient_, std::move(clientThread)});
}
}
});
Finally stopServer([&]() {
writeFull(shutdownPipe.writeSide.get(), "x", false);
wakeup.notify_all();
if (serverThread.joinable()) serverThread.join();
});
/* Find the roots. Since we've grabbed the GC lock, the set of
permanent roots cannot increase now. */
@ -722,124 +579,256 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
if (!options.ignoreLiveness)
findRootsNoTemp(rootMap, true);
for (auto & i : rootMap) state.roots.insert(i.first);
for (auto & i : rootMap) roots.insert(i.first);
/* Read the temporary roots. This acquires read locks on all
per-process temporary root files. So after this point no paths
can be added to the set of temporary roots. */
FDs fds;
/* Read the temporary roots created before we acquired the global
GC root. Any new roots will be sent to our socket. */
Roots tempRoots;
findTempRoots(fds, tempRoots, true);
findTempRoots(tempRoots, true);
for (auto & root : tempRoots) {
state.tempRoots.insert(root.first);
state.roots.insert(root.first);
_shared.lock()->tempRoots.insert(std::string(root.first.hashPart()));
roots.insert(root.first);
}
/* After this point the set of roots or temporary roots cannot
increase, since we hold locks on everything. So everything
that is not reachable from `roots' is garbage. */
/* Helper function that deletes a path from the store and throws
GCLimitReached if we've deleted enough garbage. */
auto deleteFromStore = [&](std::string_view baseName)
{
Path path = storeDir + "/" + std::string(baseName);
Path realPath = realStoreDir + "/" + std::string(baseName);
if (state.shouldDelete) {
if (pathExists(trashDir)) deleteGarbage(state, trashDir);
try {
createDirs(trashDir);
} catch (SysError & e) {
if (e.errNo == ENOSPC) {
printInfo("note: can't create trash directory: %s", e.msg());
state.moveToTrash = false;
printInfo("deleting '%1%'", path);
results.paths.insert(path);
uint64_t bytesFreed;
deletePath(realPath, bytesFreed);
results.bytesFreed += bytesFreed;
if (results.bytesFreed > options.maxFreed) {
printInfo("deleted more than %d bytes; stopping", options.maxFreed);
throw GCLimitReached();
}
};
std::map<StorePath, StorePathSet> referrersCache;
/* Helper function that visits all paths reachable from `start`
via the referrers edges and optionally derivers and derivation
output edges. If none of those paths are roots, then all
visited paths are garbage and are deleted. */
auto deleteReferrersClosure = [&](const StorePath & start) {
StorePathSet visited;
std::queue<StorePath> todo;
/* Wake up any GC client waiting for deletion of the paths in
'visited' to finish. */
Finally releasePending([&]() {
auto shared(_shared.lock());
shared->pending.reset();
wakeup.notify_all();
});
auto enqueue = [&](const StorePath & path) {
if (visited.insert(path).second)
todo.push(path);
};
enqueue(start);
while (auto path = pop(todo)) {
checkInterrupt();
/* Bail out if we've previously discovered that this path
is alive. */
if (alive.count(*path)) {
alive.insert(start);
return;
}
/* If we've previously deleted this path, we don't have to
handle it again. */
if (dead.count(*path)) continue;
auto markAlive = [&]()
{
alive.insert(*path);
alive.insert(start);
try {
StorePathSet closure;
computeFSClosure(*path, closure);
for (auto & p : closure)
alive.insert(p);
} catch (InvalidPath &) { }
};
/* If this is a root, bail out. */
if (roots.count(*path)) {
debug("cannot delete '%s' because it's a root", printStorePath(*path));
return markAlive();
}
if (options.action == GCOptions::gcDeleteSpecific
&& !options.pathsToDelete.count(*path))
return;
{
auto hashPart = std::string(path->hashPart());
auto shared(_shared.lock());
if (shared->tempRoots.count(hashPart)) {
debug("cannot delete '%s' because it's a temporary root", printStorePath(*path));
return markAlive();
}
shared->pending = hashPart;
}
if (isValidPath(*path)) {
/* Visit the referrers of this path. */
auto i = referrersCache.find(*path);
if (i == referrersCache.end()) {
StorePathSet referrers;
queryReferrers(*path, referrers);
referrersCache.emplace(*path, std::move(referrers));
i = referrersCache.find(*path);
}
for (auto & p : i->second)
enqueue(p);
/* If keep-derivations is set and this is a
derivation, then visit the derivation outputs. */
if (gcKeepDerivations && path->isDerivation()) {
for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(*path))
if (maybeOutPath &&
isValidPath(*maybeOutPath) &&
queryPathInfo(*maybeOutPath)->deriver == *path)
enqueue(*maybeOutPath);
}
/* If keep-outputs is set, then visit the derivers. */
if (gcKeepOutputs) {
auto derivers = queryValidDerivers(*path);
for (auto & i : derivers)
enqueue(i);
}
}
}
}
/* Now either delete all garbage paths, or just the specified
for (auto & path : topoSortPaths(visited)) {
if (!dead.insert(path).second) continue;
if (shouldDelete) {
invalidatePathChecked(path);
deleteFromStore(path.to_string());
referrersCache.erase(path);
}
}
};
/* Synchronisation point for testing, see tests/gc-concurrent.sh. */
if (auto p = getEnv("_NIX_TEST_GC_SYNC"))
readFile(*p);
/* Either delete all garbage paths, or just the specified
paths (for gcDeleteSpecific). */
if (options.action == GCOptions::gcDeleteSpecific) {
for (auto & i : options.pathsToDelete) {
tryToDelete(state, printStorePath(i));
if (state.dead.find(i) == state.dead.end())
deleteReferrersClosure(i);
if (!dead.count(i))
throw Error(
"cannot delete path '%1%' since it is still alive. "
"To find out why use: "
"Cannot delete path '%1%' since it is still alive. "
"To find out why, use: "
"nix-store --query --roots",
printStorePath(i));
}
} else if (options.maxFreed > 0) {
if (state.shouldDelete)
if (shouldDelete)
printInfo("deleting garbage...");
else
printInfo("determining live/dead paths...");
try {
AutoCloseDir dir(opendir(realStoreDir.get().c_str()));
if (!dir) throw SysError("opening directory '%1%'", realStoreDir);
/* Read the store and immediately delete all paths that
aren't valid. When using --max-freed etc., deleting
invalid paths is preferred over deleting unreachable
paths, since unreachable paths could become reachable
again. We don't use readDirectory() here so that GCing
can start faster. */
/* Read the store and delete all paths that are invalid or
unreachable. We don't use readDirectory() here so that
GCing can start faster. */
auto linksName = baseNameOf(linksDir);
Paths entries;
struct dirent * dirent;
while (errno = 0, dirent = readdir(dir.get())) {
checkInterrupt();
string name = dirent->d_name;
if (name == "." || name == "..") continue;
Path path = storeDir + "/" + name;
auto storePath = maybeParseStorePath(path);
if (storePath && isValidPath(*storePath))
entries.push_back(path);
if (name == "." || name == ".." || name == linksName) continue;
if (auto storePath = maybeParseStorePath(storeDir + "/" + name))
deleteReferrersClosure(*storePath);
else
tryToDelete(state, path);
deleteFromStore(name);
}
dir.reset();
/* Now delete the unreachable valid paths. Randomise the
order in which we delete entries to make the collector
less biased towards deleting paths that come
alphabetically first (e.g. /nix/store/000...). This
matters when using --max-freed etc. */
vector<Path> entries_(entries.begin(), entries.end());
std::mt19937 gen(1);
std::shuffle(entries_.begin(), entries_.end(), gen);
for (auto & i : entries_)
tryToDelete(state, i);
} catch (GCLimitReached & e) {
}
}
if (state.options.action == GCOptions::gcReturnLive) {
for (auto & i : state.alive)
state.results.paths.insert(printStorePath(i));
if (options.action == GCOptions::gcReturnLive) {
for (auto & i : alive)
results.paths.insert(printStorePath(i));
return;
}
if (state.options.action == GCOptions::gcReturnDead) {
for (auto & i : state.dead)
state.results.paths.insert(printStorePath(i));
if (options.action == GCOptions::gcReturnDead) {
for (auto & i : dead)
results.paths.insert(printStorePath(i));
return;
}
/* Allow other processes to add to the store from here on. */
fdGCLock = -1;
fds.clear();
/* Delete the trash directory. */
printInfo(format("deleting '%1%'") % trashDir);
deleteGarbage(state, trashDir);
/* Clean up the links directory. */
/* Unlink all files in /nix/store/.links that have a link count of 1,
which indicates that there are no other links and so they can be
safely deleted. FIXME: race condition with optimisePath(): we
might see a link count of 1 just before optimisePath() increases
the link count. */
if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) {
printInfo("deleting unused links...");
removeUnusedLinks(state);
AutoCloseDir dir(opendir(linksDir.c_str()));
if (!dir) throw SysError("opening directory '%1%'", linksDir);
int64_t actualSize = 0, unsharedSize = 0;
struct dirent * dirent;
while (errno = 0, dirent = readdir(dir.get())) {
checkInterrupt();
string name = dirent->d_name;
if (name == "." || name == "..") continue;
Path path = linksDir + "/" + name;
auto st = lstat(path);
if (st.st_nlink != 1) {
actualSize += st.st_size;
unsharedSize += (st.st_nlink - 1) * st.st_size;
continue;
}
printMsg(lvlTalkative, format("deleting unused link '%1%'") % path);
if (unlink(path.c_str()) == -1)
throw SysError("deleting '%1%'", path);
results.bytesFreed += st.st_size;
}
struct stat st;
if (stat(linksDir.c_str(), &st) == -1)
throw SysError("statting '%1%'", linksDir);
int64_t overhead = st.st_blocks * 512ULL;
printInfo("note: currently hard linking saves %.2f MiB",
((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
}
/* While we're at it, vacuum the database. */

20
src/libstore/local-store.cc
View file

@ -145,7 +145,6 @@ LocalStore::LocalStore(const Params & params)
, linksDir(realStoreDir + "/.links")
, reservedPath(dbDir + "/reserved")
, schemaPath(dbDir + "/schema")
, trashDir(realStoreDir + "/trash")
, tempRootsDir(stateDir + "/temproots")
, fnTempRoots(fmt("%s/%d", tempRootsDir, getpid()))
, locksHeld(tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS").value_or("")))
@ -386,6 +385,16 @@ LocalStore::LocalStore(const Params & params)
}
AutoCloseFD LocalStore::openGCLock()
{
Path fnGCLock = stateDir + "/gc.lock";
auto fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
if (!fdGCLock)
throw SysError("opening global GC lock '%1%'", fnGCLock);
return fdGCLock;
}
LocalStore::~LocalStore()
{
std::shared_future<void> future;
@ -825,7 +834,7 @@ uint64_t LocalStore::addValidPath(State & state,
{
auto state_(Store::state.lock());
state_->pathInfoCache.upsert(std::string(info.path.hashPart()),
state_->pathInfoCache.upsert(std::string(info.path.to_string()),
PathInfoCacheValue{ .value = std::make_shared<const ValidPathInfo>(info) });
}
@ -1198,7 +1207,7 @@ void LocalStore::invalidatePath(State & state, const StorePath & path)
{
auto state_(Store::state.lock());
state_->pathInfoCache.erase(std::string(path.hashPart()));
state_->pathInfoCache.erase(std::string(path.to_string()));
}
}
@ -1505,7 +1514,8 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
/* Acquire the global GC lock to get a consistent snapshot of
existing and valid paths. */
AutoCloseFD fdGCLock = openGCLock(ltWrite);
auto fdGCLock = openGCLock();
FdLock gcLock(fdGCLock.get(), ltRead, true, "waiting for the big garbage collector lock...");
StringSet store;
for (auto & i : readDirectory(realStoreDir)) store.insert(i.name);
@ -1516,8 +1526,6 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
StorePathSet validPaths;
PathSet done;
fdGCLock = -1;
for (auto & i : queryAllValidPaths())
verifyPath(printStorePath(i), store, done, validPaths, repair, errors);

31
src/libstore/local-store.hh
View file

@ -58,9 +58,15 @@ private:
struct Stmts;
std::unique_ptr<Stmts> stmts;
/* The global GC lock */
AutoCloseFD fdGCLock;
/* The file to which we write our temporary roots. */
AutoCloseFD fdTempRoots;
/* Connection to the garbage collector. */
AutoCloseFD fdRootsSocket;
/* The last time we checked whether to do an auto-GC, or an
auto-GC finished. */
std::chrono::time_point<std::chrono::steady_clock> lastGCCheck;
@ -87,7 +93,6 @@ public:
const Path linksDir;
const Path reservedPath;
const Path schemaPath;
const Path trashDir;
const Path tempRootsDir;
const Path fnTempRoots;
@ -149,14 +154,11 @@ public:
void addIndirectRoot(const Path & path) override;
void syncWithGC() override;
private:
typedef std::shared_ptr<AutoCloseFD> FDPtr;
typedef list<FDPtr> FDs;
void findTempRoots(Roots & roots, bool censor);
void findTempRoots(FDs & fds, Roots & roots, bool censor);
AutoCloseFD openGCLock();
public:
@ -236,29 +238,12 @@ private:
PathSet queryValidPathsOld();
ValidPathInfo queryPathInfoOld(const Path & path);
struct GCState;
void deleteGarbage(GCState & state, const Path & path);
void tryToDelete(GCState & state, const Path & path);
bool canReachRoot(GCState & state, StorePathSet & visited, const StorePath & path);
void deletePathRecursive(GCState & state, const Path & path);
bool isActiveTempFile(const GCState & state,
const Path & path, const string & suffix);
AutoCloseFD openGCLock(LockType lockType);
void findRoots(const Path & path, unsigned char type, Roots & roots);
void findRootsNoTemp(Roots & roots, bool censor);
void findRuntimeRoots(Roots & roots, bool censor);
void removeUnusedLinks(const GCState & state);
Path createTempDirInStore();
void checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv);

5
src/libstore/misc.cc
View file

@ -239,12 +239,11 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths)
{
return topoSort(paths,
{[&](const StorePath & path) {
StorePathSet references;
try {
references = queryPathInfo(path)->references;
return queryPathInfo(path)->references;
} catch (InvalidPath &) {
return StorePathSet();
}
return references;
}},
{[&](const StorePath & path, const StorePath & parent) {
return BuildError(

13
src/libstore/pathlocks.cc
View file

@ -176,4 +176,17 @@ void PathLocks::setDeletion(bool deletePaths)
}
FdLock::FdLock(int fd, LockType lockType, bool wait, std::string_view waitMsg)
: fd(fd)
{
if (wait) {
if (!lockFile(fd, lockType, false)) {
printInfo("%s", waitMsg);
acquired = lockFile(fd, lockType, true);
}
} else
acquired = lockFile(fd, lockType, false);
}
}

14
src/libstore/pathlocks.hh
View file

@ -35,4 +35,18 @@ public:
void setDeletion(bool deletePaths);
};
struct FdLock
{
int fd;
bool acquired = false;
FdLock(int fd, LockType lockType, bool wait, std::string_view waitMsg);
~FdLock()
{
if (acquired)
lockFile(fd, ltNone, false);
}
};
}

9
src/libstore/remote-store.cc
View file

@ -797,15 +797,6 @@ void RemoteStore::addIndirectRoot(const Path & path)
}
void RemoteStore::syncWithGC()
{
auto conn(getConnection());
conn->to << wopSyncWithGC;
conn.processStderr();
readInt(conn->from);
}
Roots RemoteStore::findRoots(bool censor)
{
auto conn(getConnection());

2
src/libstore/remote-store.hh
View file

@ -101,8 +101,6 @@ public:
void addIndirectRoot(const Path & path) override;
void syncWithGC() override;
Roots findRoots(bool censor) override;
void collectGarbage(const GCOptions & options, GCResults & results) override;

26
src/libstore/store-api.cc
View file

@ -414,11 +414,9 @@ StorePathSet Store::queryDerivationOutputs(const StorePath & path)
bool Store::isValidPath(const StorePath & storePath)
{
std::string hashPart(storePath.hashPart());
{
auto state_(state.lock());
auto res = state_->pathInfoCache.get(hashPart);
auto res = state_->pathInfoCache.get(std::string(storePath.to_string()));
if (res && res->isKnownNow()) {
stats.narInfoReadAverted++;
return res->didExist();
@ -426,11 +424,11 @@ bool Store::isValidPath(const StorePath & storePath)
}
if (diskCache) {
auto res = diskCache->lookupNarInfo(getUri(), hashPart);
auto res = diskCache->lookupNarInfo(getUri(), std::string(storePath.hashPart()));
if (res.first != NarInfoDiskCache::oUnknown) {
stats.narInfoReadAverted++;
auto state_(state.lock());
state_->pathInfoCache.upsert(hashPart,
state_->pathInfoCache.upsert(std::string(storePath.to_string()),
res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue { .value = res.second });
return res.first == NarInfoDiskCache::oValid;
}
@ -440,7 +438,7 @@ bool Store::isValidPath(const StorePath & storePath)
if (diskCache && !valid)
// FIXME: handle valid = true case.
diskCache->upsertNarInfo(getUri(), hashPart, 0);
diskCache->upsertNarInfo(getUri(), std::string(storePath.hashPart()), 0);
return valid;
}
@ -487,13 +485,11 @@ static bool goodStorePath(const StorePath & expected, const StorePath & actual)
void Store::queryPathInfo(const StorePath & storePath,
Callback<ref<const ValidPathInfo>> callback) noexcept
{
std::string hashPart;
auto hashPart = std::string(storePath.hashPart());
try {
hashPart = storePath.hashPart();
{
auto res = state.lock()->pathInfoCache.get(hashPart);
auto res = state.lock()->pathInfoCache.get(std::string(storePath.to_string()));
if (res && res->isKnownNow()) {
stats.narInfoReadAverted++;
if (!res->didExist())
@ -508,7 +504,7 @@ void Store::queryPathInfo(const StorePath & storePath,
stats.narInfoReadAverted++;
{
auto state_(state.lock());
state_->pathInfoCache.upsert(hashPart,
state_->pathInfoCache.upsert(std::string(storePath.to_string()),
res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second });
if (res.first == NarInfoDiskCache::oInvalid ||
!goodStorePath(storePath, res.second->path))
@ -523,7 +519,7 @@ void Store::queryPathInfo(const StorePath & storePath,
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
queryPathInfoUncached(storePath,
{[this, storePathS{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) {
{[this, storePath, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) {
try {
auto info = fut.get();
@ -533,14 +529,12 @@ void Store::queryPathInfo(const StorePath & storePath,
{
auto state_(state.lock());
state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info });
state_->pathInfoCache.upsert(std::string(storePath.to_string()), PathInfoCacheValue { .value = info });
}
auto storePath = parseStorePath(storePathS);
if (!info || !goodStorePath(storePath, info->path)) {
stats.narInfoMissing++;
throw InvalidPath("path '%s' is not valid", storePathS);
throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
}
(*callbackPtr)(ref<const ValidPathInfo>(info));

21
src/libstore/store-api.hh
View file

@ -232,7 +232,6 @@ protected:
struct State
{
// FIXME: fix key
LRUCache<std::string, PathInfoCacheValue> pathInfoCache;
};
@ -561,26 +560,6 @@ public:
virtual void addIndirectRoot(const Path & path)
{ unsupported("addIndirectRoot"); }
/* Acquire the global GC lock, then immediately release it. This
function must be called after registering a new permanent root,
but before exiting. Otherwise, it is possible that a running
garbage collector doesn't see the new root and deletes the
stuff we've just built. By acquiring the lock briefly, we
ensure that either:
- The collector is already running, and so we block until the
collector is finished. The collector will know about our
*temporary* locks, which should include whatever it is we
want to register as a permanent lock.
- The collector isn't running, or it's just started but hasn't
acquired the GC lock yet. In that case we get and release
the lock right away, then exit. The collector scans the
permanent root and sees ours.
In either case the permanent root is seen by the collector. */
virtual void syncWithGC() { };
/* Find the roots of the garbage collector. Each root is a pair
(link, storepath) where `link' is the path of the symlink
outside of the Nix store that point to `storePath'. If

9
src/libstore/uds-remote-store.cc
View file

@ -56,14 +56,7 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
auto conn = make_ref<Connection>();
/* Connect to a daemon that does the privileged work for us. */
conn->fd = socket(PF_UNIX, SOCK_STREAM
#ifdef SOCK_CLOEXEC
| SOCK_CLOEXEC
#endif
, 0);
if (!conn->fd)
throw SysError("cannot create Unix domain socket");
closeOnExec(conn->fd.get());
conn->fd = createUnixDomainSocket();
nix::connect(conn->fd.get(), path ? *path : settings.nixDaemonSocketFile);

18
src/libutil/util.cc
View file

@ -1670,7 +1670,7 @@ std::unique_ptr<InterruptCallback> createInterruptCallback(std::function<void()>
}
AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode)
AutoCloseFD createUnixDomainSocket()
{
AutoCloseFD fdSocket = socket(PF_UNIX, SOCK_STREAM
#ifdef SOCK_CLOEXEC
@ -1679,8 +1679,14 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode)
, 0);
if (!fdSocket)
throw SysError("cannot create Unix domain socket");
closeOnExec(fdSocket.get());
return fdSocket;
}
AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode)
{
auto fdSocket = nix::createUnixDomainSocket();
bind(fdSocket.get(), path);
@ -1709,7 +1715,7 @@ void bind(int fd, const std::string & path)
std::string base(baseNameOf(path));
if (base.size() + 1 >= sizeof(addr.sun_path))
throw Error("socket path '%s' is too long", base);
strcpy(addr.sun_path, base.c_str());
memcpy(addr.sun_path, base.c_str(), base.size() + 1);
if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
throw SysError("cannot bind to socket '%s'", path);
_exit(0);
@ -1718,7 +1724,7 @@ void bind(int fd, const std::string & path)
if (status != 0)
throw Error("cannot bind to socket '%s'", path);
} else {
strcpy(addr.sun_path, path.c_str());
memcpy(addr.sun_path, path.c_str(), path.size() + 1);
if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
throw SysError("cannot bind to socket '%s'", path);
}
@ -1738,7 +1744,7 @@ void connect(int fd, const std::string & path)
std::string base(baseNameOf(path));
if (base.size() + 1 >= sizeof(addr.sun_path))
throw Error("socket path '%s' is too long", base);
strcpy(addr.sun_path, base.c_str());
memcpy(addr.sun_path, base.c_str(), base.size() + 1);
if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
throw SysError("cannot connect to socket at '%s'", path);
_exit(0);
@ -1747,7 +1753,7 @@ void connect(int fd, const std::string & path)
if (status != 0)
throw Error("cannot connect to socket at '%s'", path);
} else {
strcpy(addr.sun_path, path.c_str());
memcpy(addr.sun_path, path.c_str(), path.size() + 1);
if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
throw SysError("cannot connect to socket at '%s'", path);
}

26
src/libutil/util.hh
View file

@ -511,6 +511,29 @@ std::optional<typename T::mapped_type> get(const T & map, const typename T::key_
}
/* Remove and return the first item from a container. */
template <class T>
std::optional<typename T::value_type> remove_begin(T & c)
{
auto i = c.begin();
if (i == c.end()) return {};
auto v = std::move(*i);
c.erase(i);
return v;
}
/* Remove and return the first item from a container. */
template <class T>
std::optional<typename T::value_type> pop(T & c)
{
if (c.empty()) return {};
auto v = std::move(c.front());
c.pop();
return v;
}
template<typename T>
class Callback;
@ -571,6 +594,9 @@ extern PathFilter defaultPathFilter;
/* Common initialisation performed in child processes. */
void commonChildInit(Pipe & logPipe);
/* Create a Unix domain socket. */
AutoCloseFD createUnixDomainSocket();
/* Create a Unix domain socket in listen mode. */
AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode);

2
tests/ca/build-with-garbage-path.sh
View file

@ -8,7 +8,7 @@ requireDaemonNewerThan "2.4pre20210621"
# Get the output path of `rootCA`, and put some garbage instead
outPath="$(nix-build ./content-addressed.nix -A rootCA --no-out-link)"
nix-store --delete "$outPath"
nix-store --delete $(nix-store -q --referrers-closure "$outPath")
touch "$outPath"
# The build should correctly remove the garbage and put the expected path instead

33
tests/gc-non-blocking.sh Normal file
View file

@ -0,0 +1,33 @@
# Test whether the collector is non-blocking, i.e. a build can run in
# parallel with it.
source common.sh
needLocalStore "the GC test needs a synchronisation point"
clearStore
fifo=$TEST_ROOT/test.fifo
mkfifo "$fifo"
dummy=$(nix store add-path ./simple.nix)
running=$TEST_ROOT/running
touch $running
(_NIX_TEST_GC_SYNC=$fifo nix-store --gc -vvvvv; rm $running) &
pid=$!
sleep 2
outPath=$(nix-build -o "$TEST_ROOT/result" -E "
with import ./config.nix;
mkDerivation {
name = \"non-blocking\";
buildCommand = \"set -x; test -e $running; mkdir \$out; echo > $fifo\";
}")
wait $pid
(! test -e $running)
(! test -e $dummy)
test -e $outPath

12
tests/gc.sh
View file

@ -1,5 +1,7 @@
source common.sh
clearStore
drvPath=$(nix-instantiate dependencies.nix)
outPath=$(nix-store -rvv "$drvPath")
@ -23,6 +25,12 @@ test -e $inUse
if nix-store --delete $outPath; then false; fi
test -e $outPath
for i in $NIX_STORE_DIR/*; do
if [[ $i =~ /trash ]]; then continue; fi # compat with old daemon
touch $i.lock
touch $i.chroot
done
nix-collect-garbage
# Check that the root and its dependencies haven't been deleted.
@ -38,3 +46,7 @@ nix-collect-garbage
# Check that the output has been GC'd.
if test -e $outPath/foobar; then false; fi
# Check that the store is empty.
rmdir $NIX_STORE_DIR/.links
rmdir $NIX_STORE_DIR

1
tests/local.mk
View file

@ -4,6 +4,7 @@ nix_tests = \
gc.sh \
ca/gc.sh \
gc-concurrent.sh \
gc-non-blocking.sh \
gc-auto.sh \
referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \
gc-runtime.sh check-refs.sh filter-source.sh \

3
tests/multiple-outputs.sh
View file

@ -76,7 +76,10 @@ if nix-build multiple-outputs.nix -A cyclic --no-out-link; then
exit 1
fi
# Do a GC. This should leave an empty store.
echo "collecting garbage..."
rm $TEST_ROOT/result*
nix-store --gc --keep-derivations --keep-outputs
nix-store --gc --print-roots
rm -rf $NIX_STORE_DIR/.links
rmdir $NIX_STORE_DIR

2
tests/repair.sh
View file

@ -30,7 +30,7 @@ nix-store --verify-path $path2
chmod u+w $path2
touch $path2/bad
nix-store --delete $(nix-store -qd $path2)
nix-store --delete $(nix-store -q --referrers-closure $(nix-store -qd $path2))
(! nix-store --verify --check-contents --repair)