Merge pull request #6188 from obsidiansystems/store-gc-subclass

Factor out a `GcStore` interface
This commit is contained in:
Eelco Dolstra 2022-03-03 20:53:10 +01:00 committed by GitHub
commit b09baf690b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 143 additions and 93 deletions

View file

@ -1,6 +1,7 @@
#include "globals.hh" #include "globals.hh"
#include "shared.hh" #include "shared.hh"
#include "store-api.hh" #include "store-api.hh"
#include "gc-store.hh"
#include "util.hh" #include "util.hh"
#include "loggers.hh" #include "loggers.hh"

View file

@ -1,4 +1,5 @@
#include "local-derivation-goal.hh" #include "local-derivation-goal.hh"
#include "gc-store.hh"
#include "hook-instance.hh" #include "hook-instance.hh"
#include "worker.hh" #include "worker.hh"
#include "builtins.hh" #include "builtins.hh"
@ -1127,7 +1128,7 @@ struct RestrictedStoreConfig : virtual LocalFSStoreConfig
/* A wrapper around LocalStore that only allows building/querying of /* A wrapper around LocalStore that only allows building/querying of
paths that are in the input closures of the build or were added via paths that are in the input closures of the build or were added via
recursive Nix calls. */ recursive Nix calls. */
struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore, public virtual GcStore
{ {
ref<LocalStore> next; ref<LocalStore> next;

View file

@ -3,6 +3,7 @@
#include "worker-protocol.hh" #include "worker-protocol.hh"
#include "build-result.hh" #include "build-result.hh"
#include "store-api.hh" #include "store-api.hh"
#include "gc-store.hh"
#include "path-with-outputs.hh" #include "path-with-outputs.hh"
#include "finally.hh" #include "finally.hh"
#include "archive.hh" #include "archive.hh"
@ -623,9 +624,12 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopAddIndirectRoot: { case wopAddIndirectRoot: {
Path path = absPath(readString(from)); Path path = absPath(readString(from));
logger->startWork(); logger->startWork();
store->addIndirectRoot(path); auto & gcStore = requireGcStore(*store);
gcStore.addIndirectRoot(path);
logger->stopWork(); logger->stopWork();
to << 1; to << 1;
break; break;
} }
@ -640,7 +644,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopFindRoots: { case wopFindRoots: {
logger->startWork(); logger->startWork();
Roots roots = store->findRoots(!trusted); auto & gcStore = requireGcStore(*store);
Roots roots = gcStore.findRoots(!trusted);
logger->stopWork(); logger->stopWork();
size_t size = 0; size_t size = 0;
@ -671,7 +676,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork(); logger->startWork();
if (options.ignoreLiveness) if (options.ignoreLiveness)
throw Error("you are not allowed to ignore liveness"); throw Error("you are not allowed to ignore liveness");
store->collectGarbage(options, results); auto & gcStore = requireGcStore(*store);
gcStore.collectGarbage(options, results);
logger->stopWork(); logger->stopWork();
to << results.paths << results.bytesFreed << 0 /* obsolete */; to << results.paths << results.bytesFreed << 0 /* obsolete */;

13
src/libstore/gc-store.cc Normal file
View file

@ -0,0 +1,13 @@
#include "gc-store.hh"
namespace nix {
GcStore & requireGcStore(Store & store)
{
auto * gcStore = dynamic_cast<GcStore *>(&store);
if (!gcStore)
throw UsageError("Garbage collection not supported by this store");
return *gcStore;
}
}

84
src/libstore/gc-store.hh Normal file
View file

@ -0,0 +1,84 @@
#pragma once
#include "store-api.hh"
namespace nix {
typedef std::unordered_map<StorePath, std::unordered_set<std::string>> Roots;
struct GCOptions
{
/* Garbage collector operation:
- `gcReturnLive': return the set of paths reachable from
(i.e. in the closure of) the roots.
- `gcReturnDead': return the set of paths not reachable from
the roots.
- `gcDeleteDead': actually delete the latter set.
- `gcDeleteSpecific': delete the paths listed in
`pathsToDelete', insofar as they are not reachable.
*/
typedef enum {
gcReturnLive,
gcReturnDead,
gcDeleteDead,
gcDeleteSpecific,
} GCAction;
GCAction action{gcDeleteDead};
/* If `ignoreLiveness' is set, then reachability from the roots is
ignored (dangerous!). However, the paths must still be
unreferenced *within* the store (i.e., there can be no other
store paths that depend on them). */
bool ignoreLiveness{false};
/* For `gcDeleteSpecific', the paths to delete. */
StorePathSet pathsToDelete;
/* Stop after at least `maxFreed' bytes have been freed. */
uint64_t maxFreed{std::numeric_limits<uint64_t>::max()};
};
struct GCResults
{
/* Depending on the action, the GC roots, or the paths that would
be or have been deleted. */
PathSet paths;
/* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the
number of bytes that would be or was freed. */
uint64_t bytesFreed = 0;
};
struct GcStore : public virtual Store
{
/* Add an indirect root, which is merely a symlink to `path' from
/nix/var/nix/gcroots/auto/<hash of `path'>. `path' is supposed
to be a symlink to a store path. The garbage collector will
automatically remove the indirect root when it finds that
`path' has disappeared. */
virtual void addIndirectRoot(const Path & path) = 0;
/* Find the roots of the garbage collector. Each root is a pair
(link, storepath) where `link' is the path of the symlink
outside of the Nix store that point to `storePath'. If
'censor' is true, privacy-sensitive information about roots
found in /proc is censored. */
virtual Roots findRoots(bool censor) = 0;
/* Perform a garbage collection. */
virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0;
};
GcStore & requireGcStore(Store & store);
}

View file

@ -1,6 +1,7 @@
#pragma once #pragma once
#include "store-api.hh" #include "store-api.hh"
#include "gc-store.hh"
namespace nix { namespace nix {
@ -23,7 +24,7 @@ struct LocalFSStoreConfig : virtual StoreConfig
"physical path to the Nix store"}; "physical path to the Nix store"};
}; };
class LocalFSStore : public virtual LocalFSStoreConfig, public virtual Store class LocalFSStore : public virtual LocalFSStoreConfig, public virtual Store, virtual GcStore
{ {
public: public:

View file

@ -5,6 +5,7 @@
#include "pathlocks.hh" #include "pathlocks.hh"
#include "store-api.hh" #include "store-api.hh"
#include "local-fs-store.hh" #include "local-fs-store.hh"
#include "gc-store.hh"
#include "sync.hh" #include "sync.hh"
#include "util.hh" #include "util.hh"
@ -43,7 +44,7 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig
}; };
class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore, public virtual GcStore
{ {
private: private:

View file

@ -22,9 +22,9 @@ DerivedPath StorePathWithOutputs::toDerivedPath() const
std::vector<DerivedPath> toDerivedPaths(const std::vector<StorePathWithOutputs> ss) std::vector<DerivedPath> toDerivedPaths(const std::vector<StorePathWithOutputs> ss)
{ {
std::vector<DerivedPath> reqs; std::vector<DerivedPath> reqs;
for (auto & s : ss) reqs.push_back(s.toDerivedPath()); for (auto & s : ss) reqs.push_back(s.toDerivedPath());
return reqs; return reqs;
} }

View file

@ -1,6 +1,7 @@
#include "serialise.hh" #include "serialise.hh"
#include "util.hh" #include "util.hh"
#include "path-with-outputs.hh" #include "path-with-outputs.hh"
#include "gc-store.hh"
#include "remote-fs-accessor.hh" #include "remote-fs-accessor.hh"
#include "build-result.hh" #include "build-result.hh"
#include "remote-store.hh" #include "remote-store.hh"

View file

@ -4,6 +4,7 @@
#include <string> #include <string>
#include "store-api.hh" #include "store-api.hh"
#include "gc-store.hh"
namespace nix { namespace nix {
@ -29,7 +30,7 @@ struct RemoteStoreConfig : virtual StoreConfig
/* FIXME: RemoteStore is a misnomer - should be something like /* FIXME: RemoteStore is a misnomer - should be something like
DaemonStore. */ DaemonStore. */
class RemoteStore : public virtual RemoteStoreConfig, public virtual Store class RemoteStore : public virtual RemoteStoreConfig, public virtual Store, public virtual GcStore
{ {
public: public:

View file

@ -76,59 +76,6 @@ enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true };
const uint32_t exportMagic = 0x4558494e; const uint32_t exportMagic = 0x4558494e;
typedef std::unordered_map<StorePath, std::unordered_set<std::string>> Roots;
struct GCOptions
{
/* Garbage collector operation:
- `gcReturnLive': return the set of paths reachable from
(i.e. in the closure of) the roots.
- `gcReturnDead': return the set of paths not reachable from
the roots.
- `gcDeleteDead': actually delete the latter set.
- `gcDeleteSpecific': delete the paths listed in
`pathsToDelete', insofar as they are not reachable.
*/
typedef enum {
gcReturnLive,
gcReturnDead,
gcDeleteDead,
gcDeleteSpecific,
} GCAction;
GCAction action{gcDeleteDead};
/* If `ignoreLiveness' is set, then reachability from the roots is
ignored (dangerous!). However, the paths must still be
unreferenced *within* the store (i.e., there can be no other
store paths that depend on them). */
bool ignoreLiveness{false};
/* For `gcDeleteSpecific', the paths to delete. */
StorePathSet pathsToDelete;
/* Stop after at least `maxFreed' bytes have been freed. */
uint64_t maxFreed{std::numeric_limits<uint64_t>::max()};
};
struct GCResults
{
/* Depending on the action, the GC roots, or the paths that would
be or have been deleted. */
PathSet paths;
/* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the
number of bytes that would be or was freed. */
uint64_t bytesFreed = 0;
};
enum BuildMode { bmNormal, bmRepair, bmCheck }; enum BuildMode { bmNormal, bmRepair, bmCheck };
struct BuildResult; struct BuildResult;
@ -531,26 +478,6 @@ public:
virtual void addTempRoot(const StorePath & path) virtual void addTempRoot(const StorePath & path)
{ debug("not creating temporary root, store doesn't support GC"); } { debug("not creating temporary root, store doesn't support GC"); }
/* Add an indirect root, which is merely a symlink to `path' from
/nix/var/nix/gcroots/auto/<hash of `path'>. `path' is supposed
to be a symlink to a store path. The garbage collector will
automatically remove the indirect root when it finds that
`path' has disappeared. */
virtual void addIndirectRoot(const Path & path)
{ unsupported("addIndirectRoot"); }
/* Find the roots of the garbage collector. Each root is a pair
(link, storepath) where `link' is the path of the symlink
outside of the Nix store that point to `storePath'. If
'censor' is true, privacy-sensitive information about roots
found in /proc is censored. */
virtual Roots findRoots(bool censor)
{ unsupported("findRoots"); }
/* Perform a garbage collection. */
virtual void collectGarbage(const GCOptions & options, GCResults & results)
{ unsupported("collectGarbage"); }
/* Return a string representing information about the path that /* Return a string representing information about the path that
can be loaded into the database using `nix-store --load-db' or can be loaded into the database using `nix-store --load-db' or
`nix-store --register-validity'. */ `nix-store --register-validity'. */

View file

@ -359,7 +359,7 @@ namespace nix {
// constructing without access violation. // constructing without access violation.
ErrPos ep(invalid); ErrPos ep(invalid);
// assignment without access violation. // assignment without access violation.
ep = invalid; ep = invalid;

View file

@ -1,4 +1,5 @@
#include "store-api.hh" #include "store-api.hh"
#include "gc-store.hh"
#include "profiles.hh" #include "profiles.hh"
#include "shared.hh" #include "shared.hh"
#include "globals.hh" #include "globals.hh"
@ -80,10 +81,11 @@ static int main_nix_collect_garbage(int argc, char * * argv)
// Run the actual garbage collector. // Run the actual garbage collector.
if (!dryRun) { if (!dryRun) {
auto store = openStore(); auto store = openStore();
auto & gcStore = requireGcStore(*store);
options.action = GCOptions::gcDeleteDead; options.action = GCOptions::gcDeleteDead;
GCResults results; GCResults results;
PrintFreed freed(true, results); PrintFreed freed(true, results);
store->collectGarbage(options, results); gcStore.collectGarbage(options, results);
} }
return 0; return 0;

View file

@ -3,6 +3,7 @@
#include "dotgraph.hh" #include "dotgraph.hh"
#include "globals.hh" #include "globals.hh"
#include "build-result.hh" #include "build-result.hh"
#include "gc-store.hh"
#include "local-store.hh" #include "local-store.hh"
#include "monitor-fd.hh" #include "monitor-fd.hh"
#include "serve-protocol.hh" #include "serve-protocol.hh"
@ -428,11 +429,12 @@ static void opQuery(Strings opFlags, Strings opArgs)
store->computeFSClosure( store->computeFSClosure(
args, referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations); args, referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations);
Roots roots = store->findRoots(false); auto & gcStore = requireGcStore(*store);
Roots roots = gcStore.findRoots(false);
for (auto & [target, links] : roots) for (auto & [target, links] : roots)
if (referrers.find(target) != referrers.end()) if (referrers.find(target) != referrers.end())
for (auto & link : links) for (auto & link : links)
cout << fmt("%1% -> %2%\n", link, store->printStorePath(target)); cout << fmt("%1% -> %2%\n", link, gcStore.printStorePath(target));
break; break;
} }
@ -588,20 +590,22 @@ static void opGC(Strings opFlags, Strings opArgs)
if (!opArgs.empty()) throw UsageError("no arguments expected"); if (!opArgs.empty()) throw UsageError("no arguments expected");
auto & gcStore = requireGcStore(*store);
if (printRoots) { if (printRoots) {
Roots roots = store->findRoots(false); Roots roots = gcStore.findRoots(false);
std::set<std::pair<Path, StorePath>> roots2; std::set<std::pair<Path, StorePath>> roots2;
// Transpose and sort the roots. // Transpose and sort the roots.
for (auto & [target, links] : roots) for (auto & [target, links] : roots)
for (auto & link : links) for (auto & link : links)
roots2.emplace(link, target); roots2.emplace(link, target);
for (auto & [link, target] : roots2) for (auto & [link, target] : roots2)
std::cout << link << " -> " << store->printStorePath(target) << "\n"; std::cout << link << " -> " << gcStore.printStorePath(target) << "\n";
} }
else { else {
PrintFreed freed(options.action == GCOptions::gcDeleteDead, results); PrintFreed freed(options.action == GCOptions::gcDeleteDead, results);
store->collectGarbage(options, results); gcStore.collectGarbage(options, results);
if (options.action != GCOptions::gcDeleteDead) if (options.action != GCOptions::gcDeleteDead)
for (auto & i : results.paths) for (auto & i : results.paths)
@ -625,9 +629,11 @@ static void opDelete(Strings opFlags, Strings opArgs)
for (auto & i : opArgs) for (auto & i : opArgs)
options.pathsToDelete.insert(store->followLinksToStorePath(i)); options.pathsToDelete.insert(store->followLinksToStorePath(i));
auto & gcStore = requireGcStore(*store);
GCResults results; GCResults results;
PrintFreed freed(true, results); PrintFreed freed(true, results);
store->collectGarbage(options, results); gcStore.collectGarbage(options, results);
} }

View file

@ -2,6 +2,7 @@
#include "common-args.hh" #include "common-args.hh"
#include "shared.hh" #include "shared.hh"
#include "store-api.hh" #include "store-api.hh"
#include "gc-store.hh"
using namespace nix; using namespace nix;
@ -32,12 +33,14 @@ struct CmdStoreDelete : StorePathsCommand
void run(ref<Store> store, std::vector<StorePath> && storePaths) override void run(ref<Store> store, std::vector<StorePath> && storePaths) override
{ {
auto & gcStore = requireGcStore(*store);
for (auto & path : storePaths) for (auto & path : storePaths)
options.pathsToDelete.insert(path); options.pathsToDelete.insert(path);
GCResults results; GCResults results;
PrintFreed freed(true, results); PrintFreed freed(true, results);
store->collectGarbage(options, results); gcStore.collectGarbage(options, results);
} }
}; };

View file

@ -2,6 +2,7 @@
#include "common-args.hh" #include "common-args.hh"
#include "shared.hh" #include "shared.hh"
#include "store-api.hh" #include "store-api.hh"
#include "gc-store.hh"
using namespace nix; using namespace nix;
@ -33,10 +34,12 @@ struct CmdStoreGC : StoreCommand, MixDryRun
void run(ref<Store> store) override void run(ref<Store> store) override
{ {
auto & gcStore = requireGcStore(*store);
options.action = dryRun ? GCOptions::gcReturnDead : GCOptions::gcDeleteDead; options.action = dryRun ? GCOptions::gcReturnDead : GCOptions::gcDeleteDead;
GCResults results; GCResults results;
PrintFreed freed(options.action == GCOptions::gcDeleteDead, results); PrintFreed freed(options.action == GCOptions::gcDeleteDead, results);
store->collectGarbage(options, results); gcStore.collectGarbage(options, results);
} }
}; };