* Fix and simplify the garbage collector (it's still not concurrent,

though).  In particular it's now much easier to register a GC root.
  Just place a symlink to whatever store path it is that you want to
  keep in /nix/var/nix/gcroots.
This commit is contained in:
Eelco Dolstra 2005-01-27 15:21:29 +00:00
parent 59682e6188
commit c505702265
10 changed files with 124 additions and 109 deletions

View file

@ -23,7 +23,7 @@ for i in "$@"; do
for j in $storeExprs; do for j in $storeExprs; do
echo "store expression is $j" >&2 echo "store expression is $j" >&2
done done
outPaths=$(@bindir@/nix-store -qnfv $extraArgs $storeExprs) outPaths=$(@bindir@/nix-store -rv $extraArgs $storeExprs)
for j in $outPaths; do for j in $outPaths; do
echo "$j" echo "$j"
if test -z "$noLink"; then if test -z "$noLink"; then

View file

@ -9,7 +9,6 @@ my $storeDir = "@storedir@";
my %alive; my %alive;
my $gcOper = "--delete"; my $gcOper = "--delete";
my $minAge = 0;
my @roots = (); my @roots = ();
@ -20,33 +19,11 @@ for (my $i = 0; $i < scalar @ARGV; $i++) {
if ($arg eq "--delete" || $arg eq "--print-live" || $arg eq "--print-dead") { if ($arg eq "--delete" || $arg eq "--print-live" || $arg eq "--print-dead") {
$gcOper = $arg; $gcOper = $arg;
} }
elsif ($arg eq "--min-age") {
$i++;
$minAge = undef;
$minAge = $ARGV[$i];
die "invalid minimum age" unless defined $minAge && $minAge =~ /^\d*$/;
}
else { die "unknown argument `$arg'" }; else { die "unknown argument `$arg'" };
} }
# Read all GC roots from the given file. # Recursively finds all symlinks to the store in the given directory.
sub readRoots {
my $fileName = shift;
open ROOT, "<$fileName" or die "cannot open `$fileName': $!";
while (<ROOT>) {
chomp;
foreach my $root (split ' ') {
die "bad root `$root' in file `$fileName'"
unless $root =~ /^\S+$/;
push @roots, $root;
}
}
close ROOT;
}
# Recursively finds all *.gcroot files in the given directory.
sub findRoots; sub findRoots;
sub findRoots { sub findRoots {
my $followSymlinks = shift; my $followSymlinks = shift;
@ -58,14 +35,26 @@ sub findRoots {
foreach my $name (@names) { foreach my $name (@names) {
next if $name eq "." || $name eq ".."; next if $name eq "." || $name eq "..";
$name = $dir . "/" . $name; my $path = $dir . "/" . $name;
if ($name =~ /.gcroot$/ && -f $name) {
readRoots $name; if (-l $path) {
} my $target = readlink $path
elsif (-d $name) { or die "cannot read symlink `$path': $!";
if ($followSymlinks || !-l $name) {
findRoots 0, $name; if (substr($target, 0, length $storeDir) eq $storeDir) {
# We're only interested in the store-level part.
$target = substr($target, length $storeDir);
$target = "$storeDir/$target";
push @roots, $target;
} }
elsif ($followSymlinks && -d $path) {
findRoots 0, $path;
}
}
elsif (-d $path) {
findRoots $followSymlinks, $path;
} }
} }
@ -77,7 +66,7 @@ findRoots 1, $rootsDir;
# Run the collector with the roots we found. # Run the collector with the roots we found.
my $pid = open2(">&1", \*WRITE, "@bindir@/nix-store --gc $gcOper --min-age $minAge") my $pid = open2(">&1", \*WRITE, "@bindir@/nix-store --gc $gcOper")
or die "cannot run `nix-store --gc'"; or die "cannot run `nix-store --gc'";
foreach my $root (@roots) { foreach my $root (@roots) {

View file

@ -458,7 +458,7 @@ void DerivationGoal::haveStoreExpr()
i != invalidOutputs.end(); ++i) i != invalidOutputs.end(); ++i)
/* Don't bother creating a substitution goal if there are no /* Don't bother creating a substitution goal if there are no
substitutes. */ substitutes. */
if (querySubstitutes(*i).size() > 0) if (querySubstitutes(noTxn, *i).size() > 0)
addWaitee(worker.makeSubstitutionGoal(*i)); addWaitee(worker.makeSubstitutionGoal(*i));
if (waitees.empty()) /* to prevent hang (no wake-up event) */ if (waitees.empty()) /* to prevent hang (no wake-up event) */
@ -1315,7 +1315,7 @@ void SubstitutionGoal::init()
} }
/* Read the substitutes. */ /* Read the substitutes. */
subs = querySubstitutes(storePath); subs = querySubstitutes(noTxn, storePath);
/* To maintain the closure invairant, we first have to realise the /* To maintain the closure invairant, we first have to realise the
paths referenced by this one. */ paths referenced by this one. */

View file

@ -1,12 +1,68 @@
#include "globals.hh" #include "globals.hh"
#include "gc.hh" #include "gc.hh"
#include "build.hh"
#include <sys/types.h> #include <sys/types.h>
#include <sys/stat.h> #include <sys/stat.h>
#include <unistd.h> #include <unistd.h>
void collectGarbage(const PathSet & roots, GCAction action,
PathSet & result)
{
result.clear();
/* !!! TODO: Acquire an exclusive lock on the gcroots directory.
This prevents the set of live paths from increasing after this
point. */
/* Determine the live paths which is just the closure of the
roots under the `references' relation. */
PathSet livePaths;
for (PathSet::const_iterator i = roots.begin(); i != roots.end(); ++i)
computeFSClosure(canonPath(*i), livePaths);
if (action == gcReturnLive) {
result = livePaths;
return;
}
/* !!! TODO: Try to acquire (without blocking) exclusive locks on
the files in the `pending' directory. Delete all files for
which we managed to acquire such a lock (since if we could get
such a lock, that means that the process that owned the file
has died). */
/* !!! TODO: Acquire shared locks on all files in the pending
directories. This prevents the set of pending paths from
increasing while we are garbage-collecting. Read the set of
pending paths from those files. */
/* Read the Nix store directory to find all currently existing
paths. */
Strings storeNames = readDirectory(nixStore);
for (Strings::iterator i = storeNames.begin(); i != storeNames.end(); ++i) {
Path path = canonPath(nixStore + "/" + *i);
if (livePaths.find(path) != livePaths.end()) {
debug(format("live path `%1%'") % path);
continue;
}
debug(format("dead path `%1%'") % path);
result.insert(path);
if (action == gcDeleteDead) {
printMsg(lvlInfo, format("deleting `%1%'") % path);
deleteFromStore(path);
}
}
}
#if 0 #if 0
void followLivePaths(Path nePath, PathSet & live) void followLivePaths(Path nePath, PathSet & live)
{ {

View file

@ -3,24 +3,15 @@
#include "util.hh" #include "util.hh"
/* Garbage collector operation. */
typedef enum { gcReturnLive, gcReturnDead, gcDeleteDead } GCAction;
/* Determine the set of "live" store paths, given a set of root store /* If `action' is set to `soReturnLive', return the set of paths
expressions. The live store paths are those that are reachable reachable from (i.e. in the closure of) the specified roots. If
from the roots. The roots are reachable by definition. Any path `action' is `soReturnDead', return the set of paths not reachable
mentioned in a reachable store expression is also reachable. If a from the roots. If `action' is `soDeleteDead', actually delete the
derivation store expression is reachable, then its successor (if it latter set. */
exists) if also reachable. It is not an error for store void collectGarbage(const PathSet & roots, GCAction action,
expressions not to exist (since this can happen on derivation store PathSet & result);
expressions, for instance, due to the substitute mechanism), but
successor links are followed even for non-existant derivations. */
PathSet findLivePaths(const Paths & roots);
/* Given a set of "live" store paths, determine the set of "dead"
store paths (which are simply all store paths that are not in the
live set). The value `minAge' specifies the minimum age in seconds
for an unreachable file to be considered dead (0 meaning that any
unreachable file is dead). */
PathSet findDeadPaths(const PathSet & live, time_t minAge);
#endif /* !__GC_H */ #endif /* !__GC_H */

View file

@ -363,9 +363,9 @@ void registerSubstitute(const Transaction & txn,
} }
Substitutes querySubstitutes(const Path & srcPath) Substitutes querySubstitutes(const Transaction & txn, const Path & srcPath)
{ {
return readSubstitutes(noTxn, srcPath); return readSubstitutes(txn, srcPath);
} }
@ -411,6 +411,13 @@ static void invalidatePath(const Path & path, Transaction & txn)
debug(format("unregistering path `%1%'") % path); debug(format("unregistering path `%1%'") % path);
nixDB.delPair(txn, dbValidPaths, path); nixDB.delPair(txn, dbValidPaths, path);
/* Clear the `references' entry for this path, as well as the
inverse `referers' entries; but only if there are no
substitutes for this path. This maintains the cleanup
invariant. */
if (querySubstitutes(txn, path).size() == 0)
setReferences(txn, path, PathSet());
} }

View file

@ -45,7 +45,7 @@ void registerSubstitute(const Transaction & txn,
const Path & srcPath, const Substitute & sub); const Path & srcPath, const Substitute & sub);
/* Return the substitutes for the given path. */ /* Return the substitutes for the given path. */
Substitutes querySubstitutes(const Path & srcPath); Substitutes querySubstitutes(const Transaction & txn, const Path & srcPath);
/* Deregister all substitutes. */ /* Deregister all substitutes. */
void clearSubstitutes(); void clearSubstitutes();

View file

@ -573,7 +573,7 @@ static void opQuery(Globals & globals,
Strings columns; Strings columns;
if (printStatus) { if (printStatus) {
Substitutes subs = querySubstitutes(i->drvPath); Substitutes subs = querySubstitutes(noTxn, i->drvPath);
columns.push_back( columns.push_back(
(string) (installedPaths.find(i->outPath) (string) (installedPaths.find(i->outPath)
!= installedPaths.end() ? "I" : "-") != installedPaths.end() ? "I" : "-")

View file

@ -62,11 +62,11 @@ Generations findGenerations(Path profile, int & curGen)
static void makeNames(const Path & profile, unsigned int num, static void makeNames(const Path & profile, unsigned int num,
Path & generation, Path & gcrootDrv) Path & outLink, Path & drvLink)
{ {
Path prefix = (format("%1%-%2%") % profile % num).str(); Path prefix = (format("%1%-%2%") % profile % num).str();
generation = prefix + "-link"; outLink = prefix + "-output";
gcrootDrv = prefix + "-drv.gcroot"; drvLink = prefix + "-drv";
} }
@ -79,20 +79,21 @@ Path createGeneration(Path profile, Path outPath, Path drvPath)
unsigned int num = gens.size() > 0 ? gens.front().number : 0; unsigned int num = gens.size() > 0 ? gens.front().number : 0;
/* Create the new generation. */ /* Create the new generation. */
Path generation, gcrootDrv; Path outLink, drvLink;
while (1) { while (1) {
makeNames(profile, num, generation, gcrootDrv); makeNames(profile, num, outLink, drvLink);
if (symlink(outPath.c_str(), generation.c_str()) == 0) break; if (symlink(outPath.c_str(), outLink.c_str()) == 0) break;
if (errno != EEXIST) if (errno != EEXIST)
throw SysError(format("creating symlink `%1%'") % generation); throw SysError(format("creating symlink `%1%'") % outLink);
/* Somebody beat us to it, retry with a higher number. */ /* Somebody beat us to it, retry with a higher number. */
num++; num++;
} }
writeStringToFile(gcrootDrv, drvPath); if (symlink(drvPath.c_str(), drvLink.c_str()) != 0)
throw SysError(format("creating symlink `%1%'") % drvLink);
return generation; return outLink;
} }

View file

@ -274,61 +274,32 @@ static void opIsValid(Strings opFlags, Strings opArgs)
static void opGC(Strings opFlags, Strings opArgs) static void opGC(Strings opFlags, Strings opArgs)
{ {
#if 0 GCAction action;
/* Do what? */ /* Do what? */
enum { soPrintLive, soPrintDead, soDelete } subOp;
time_t minAge = 0;
for (Strings::iterator i = opFlags.begin(); for (Strings::iterator i = opFlags.begin();
i != opFlags.end(); ++i) i != opFlags.end(); ++i)
if (*i == "--print-live") subOp = soPrintLive; if (*i == "--print-live") action = gcReturnLive;
else if (*i == "--print-dead") subOp = soPrintDead; else if (*i == "--print-dead") action = gcReturnDead;
else if (*i == "--delete") subOp = soDelete; else if (*i == "--delete") action = gcDeleteDead;
else if (*i == "--min-age") {
int n;
if (opArgs.size() == 0 || !string2Int(opArgs.front(), n))
throw UsageError("`--min-age' requires an integer argument");
minAge = n;
}
else throw UsageError(format("bad sub-operation `%1%' in GC") % *i); else throw UsageError(format("bad sub-operation `%1%' in GC") % *i);
Paths roots; /* Read the roots. */
PathSet roots;
while (1) { while (1) {
Path root; Path root;
getline(cin, root); getline(cin, root);
if (cin.eof()) break; if (cin.eof()) break;
roots.push_back(root); roots.insert(root);
} }
PathSet live = findLivePaths(roots); PathSet result;
collectGarbage(roots, action, result);
if (subOp == soPrintLive) { if (action != gcDeleteDead) {
for (PathSet::iterator i = live.begin(); i != live.end(); ++i) for (PathSet::iterator i = result.begin(); i != result.end(); ++i)
cout << *i << endl; cout << *i << endl;
return;
} }
PathSet dead = findDeadPaths(live, minAge * 3600);
if (subOp == soPrintDead) {
for (PathSet::iterator i = dead.begin(); i != dead.end(); ++i)
cout << *i << endl;
return;
}
if (subOp == soDelete) {
/* !!! What happens if the garbage collector run is aborted
halfway through? In particular, dead paths can always
become live again (through re-instantiation), and might
then refer to deleted paths. => check instantiation
invariants */
for (PathSet::iterator i = dead.begin(); i != dead.end(); ++i) {
printMsg(lvlInfo, format("deleting `%1%'") % *i);
deleteFromStore(*i);
}
}
#endif
} }