Merge pull request #4834 from NixOS/generic-closure-function

Extract a generic `computeClosure` function
This commit is contained in:
Eelco Dolstra 2021-05-19 13:31:30 +02:00 committed by GitHub
commit af4ff644d5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 193 additions and 79 deletions

View file

@ -6,98 +6,73 @@
#include "thread-pool.hh" #include "thread-pool.hh"
#include "topo-sort.hh" #include "topo-sort.hh"
#include "callback.hh" #include "callback.hh"
#include "closure.hh"
namespace nix { namespace nix {
void Store::computeFSClosure(const StorePathSet & startPaths, void Store::computeFSClosure(const StorePathSet & startPaths,
StorePathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers) StorePathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers)
{ {
struct State std::function<std::set<StorePath>(const StorePath & path, std::future<ref<const ValidPathInfo>> &)> queryDeps;
{ if (flipDirection)
size_t pending; queryDeps = [&](const StorePath& path,
StorePathSet & paths; std::future<ref<const ValidPathInfo>> & fut) {
std::exception_ptr exc; StorePathSet res;
}; StorePathSet referrers;
queryReferrers(path, referrers);
for (auto& ref : referrers)
if (ref != path)
res.insert(ref);
Sync<State> state_(State{0, paths_, 0}); if (includeOutputs)
for (auto& i : queryValidDerivers(path))
res.insert(i);
std::function<void(const StorePath &)> enqueue; if (includeDerivers && path.isDerivation())
for (auto& i : queryDerivationOutputs(path))
if (isValidPath(i) && queryPathInfo(i)->deriver == path)
res.insert(i);
return res;
};
else
queryDeps = [&](const StorePath& path,
std::future<ref<const ValidPathInfo>> & fut) {
StorePathSet res;
auto info = fut.get();
for (auto& ref : info->references)
if (ref != path)
res.insert(ref);
std::condition_variable done; if (includeOutputs && path.isDerivation())
for (auto& i : queryDerivationOutputs(path))
if (isValidPath(i))
res.insert(i);
enqueue = [&](const StorePath & path) -> void { if (includeDerivers && info->deriver && isValidPath(*info->deriver))
{ res.insert(*info->deriver);
auto state(state_.lock()); return res;
if (state->exc) return; };
if (!state->paths.insert(path).second) return;
state->pending++;
}
queryPathInfo(path, {[&](std::future<ref<const ValidPathInfo>> fut) { computeClosure<StorePath>(
// FIXME: calls to isValidPath() should be async startPaths, paths_,
[&](const StorePath& path,
try { std::function<void(std::promise<std::set<StorePath>>&)>
auto info = fut.get(); processEdges) {
std::promise<std::set<StorePath>> promise;
if (flipDirection) { std::function<void(std::future<ref<const ValidPathInfo>>)>
getDependencies =
StorePathSet referrers; [&](std::future<ref<const ValidPathInfo>> fut) {
queryReferrers(path, referrers); try {
for (auto & ref : referrers) promise.set_value(queryDeps(path, fut));
if (ref != path) } catch (...) {
enqueue(ref); promise.set_exception(std::current_exception());
}
if (includeOutputs) };
for (auto & i : queryValidDerivers(path)) queryPathInfo(path, getDependencies);
enqueue(i); processEdges(promise);
});
if (includeDerivers && path.isDerivation())
for (auto & i : queryDerivationOutputs(path))
if (isValidPath(i) && queryPathInfo(i)->deriver == path)
enqueue(i);
} else {
for (auto & ref : info->references)
if (ref != path)
enqueue(ref);
if (includeOutputs && path.isDerivation())
for (auto & i : queryDerivationOutputs(path))
if (isValidPath(i)) enqueue(i);
if (includeDerivers && info->deriver && isValidPath(*info->deriver))
enqueue(*info->deriver);
}
{
auto state(state_.lock());
assert(state->pending);
if (!--state->pending) done.notify_one();
}
} catch (...) {
auto state(state_.lock());
if (!state->exc) state->exc = std::current_exception();
assert(state->pending);
if (!--state->pending) done.notify_one();
};
}});
};
for (auto & startPath : startPaths)
enqueue(startPath);
{
auto state(state_.lock());
while (state->pending) state.wait(done);
if (state->exc) std::rethrow_exception(state->exc);
}
} }
void Store::computeFSClosure(const StorePath & startPath, void Store::computeFSClosure(const StorePath & startPath,
StorePathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers) StorePathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers)
{ {

69
src/libutil/closure.hh Normal file
View file

@ -0,0 +1,69 @@
#include <set>
#include <future>
#include "sync.hh"
using std::set;
namespace nix {
template<typename T>
using GetEdgesAsync = std::function<void(const T &, std::function<void(std::promise<set<T>> &)>)>;
template<typename T>
void computeClosure(
const set<T> startElts,
set<T> & res,
GetEdgesAsync<T> getEdgesAsync
)
{
struct State
{
size_t pending;
set<T> & res;
std::exception_ptr exc;
};
Sync<State> state_(State{0, res, 0});
std::function<void(const T &)> enqueue;
std::condition_variable done;
enqueue = [&](const T & current) -> void {
{
auto state(state_.lock());
if (state->exc) return;
if (!state->res.insert(current).second) return;
state->pending++;
}
getEdgesAsync(current, [&](std::promise<set<T>> & prom) {
try {
auto children = prom.get_future().get();
for (auto & child : children)
enqueue(child);
{
auto state(state_.lock());
assert(state->pending);
if (!--state->pending) done.notify_one();
}
} catch (...) {
auto state(state_.lock());
if (!state->exc) state->exc = std::current_exception();
assert(state->pending);
if (!--state->pending) done.notify_one();
};
});
};
for (auto & startElt : startElts)
enqueue(startElt);
{
auto state(state_.lock());
while (state->pending) state.wait(done);
if (state->exc) std::rethrow_exception(state->exc);
}
}
}

View file

@ -0,0 +1,70 @@
#include "closure.hh"
#include <gtest/gtest.h>
namespace nix {
using namespace std;
map<string, set<string>> testGraph = {
{ "A", { "B", "C", "G" } },
{ "B", { "A" } }, // Loops back to A
{ "C", { "F" } }, // Indirect reference
{ "D", { "A" } }, // Not reachable, but has backreferences
{ "E", {} }, // Just not reachable
{ "F", {} },
{ "G", { "G" } }, // Self reference
};
TEST(closure, correctClosure) {
set<string> aClosure;
set<string> expectedClosure = {"A", "B", "C", "F", "G"};
computeClosure<string>(
{"A"},
aClosure,
[&](const string currentNode, function<void(promise<set<string>> &)> processEdges) {
promise<set<string>> promisedNodes;
promisedNodes.set_value(testGraph[currentNode]);
processEdges(promisedNodes);
}
);
ASSERT_EQ(aClosure, expectedClosure);
}
TEST(closure, properlyHandlesDirectExceptions) {
struct TestExn {};
set<string> aClosure;
EXPECT_THROW(
computeClosure<string>(
{"A"},
aClosure,
[&](const string currentNode, function<void(promise<set<string>> &)> processEdges) {
throw TestExn();
}
),
TestExn
);
}
TEST(closure, properlyHandlesExceptionsInPromise) {
struct TestExn {};
set<string> aClosure;
EXPECT_THROW(
computeClosure<string>(
{"A"},
aClosure,
[&](const string currentNode, function<void(promise<set<string>> &)> processEdges) {
promise<set<string>> promise;
try {
throw TestExn();
} catch (...) {
promise.set_exception(std::current_exception());
}
processEdges(promise);
}
),
TestExn
);
}
}