forked from lix-project/nix-eval-jobs
Move collecting handler to separate function.
s/master/collector/g
This commit is contained in:
John Soo
parent
a27faabd0a
commit
b9a87464a0
|
|
@ -182,7 +182,7 @@ static void worker(
|
||||||
auto vRoot = topLevelValue(state, autoArgs);
|
auto vRoot = topLevelValue(state, autoArgs);
|
||||||
|
|
||||||
while (true) {
|
while (true) {
|
||||||
/* Wait for the master to send us a job name. */
|
/* Wait for the collector to send us a job name. */
|
||||||
writeLine(to.get(), "next");
|
writeLine(to.get(), "next");
|
||||||
|
|
||||||
auto s = readLine(from.get());
|
auto s = readLine(from.get());
|
||||||
|
|
@ -192,7 +192,7 @@ static void worker(
|
||||||
|
|
||||||
debug("worker process %d at '%s'", getpid(), attrPath);
|
debug("worker process %d at '%s'", getpid(), attrPath);
|
||||||
|
|
||||||
/* Evaluate it and send info back to the master. */
|
/* Evaluate it and send info back to the collector. */
|
||||||
nlohmann::json reply;
|
nlohmann::json reply;
|
||||||
reply["attr"] = attrPath;
|
reply["attr"] = attrPath;
|
||||||
|
|
||||||
|
|
@ -287,7 +287,7 @@ static void worker(
|
||||||
|
|
||||||
writeLine(to.get(), reply.dump());
|
writeLine(to.get(), reply.dump());
|
||||||
|
|
||||||
/* If our RSS exceeds the maximum, exit. The master will
|
/* If our RSS exceeds the maximum, exit. The collector will
|
||||||
start a new process. */
|
start a new process. */
|
||||||
struct rusage r;
|
struct rusage r;
|
||||||
getrusage(RUSAGE_SELF, &r);
|
getrusage(RUSAGE_SELF, &r);
|
||||||
|
|
@ -341,6 +341,91 @@ struct Proc {
|
||||||
~Proc() { }
|
~Proc() { }
|
||||||
};
|
};
|
||||||
|
|
||||||
|
struct State
|
||||||
|
{
|
||||||
|
std::set<std::string> todo{""};
|
||||||
|
std::set<std::string> active;
|
||||||
|
std::exception_ptr exc;
|
||||||
|
};
|
||||||
|
|
||||||
|
std::function<void()> collector(Sync<State> & state_, std::condition_variable & wakeup) {
|
||||||
|
return [&]() {
|
||||||
|
try {
|
||||||
|
std::optional<std::unique_ptr<Proc>> proc_;
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
|
||||||
|
auto proc = proc_.has_value()
|
||||||
|
? std::move(proc_.value())
|
||||||
|
: std::make_unique<Proc>(worker);
|
||||||
|
|
||||||
|
/* Check whether the existing worker process is still there. */
|
||||||
|
auto s = readLine(proc->from.get());
|
||||||
|
if (s == "restart") {
|
||||||
|
proc_ = std::nullopt;
|
||||||
|
continue;
|
||||||
|
} else if (s != "next") {
|
||||||
|
auto json = nlohmann::json::parse(s);
|
||||||
|
throw Error("worker error: %s", (std::string) json["error"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Wait for a job name to become available. */
|
||||||
|
std::string attrPath;
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
checkInterrupt();
|
||||||
|
auto state(state_.lock());
|
||||||
|
if ((state->todo.empty() && state->active.empty()) || state->exc) {
|
||||||
|
writeLine(proc->to.get(), "exit");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!state->todo.empty()) {
|
||||||
|
attrPath = *state->todo.begin();
|
||||||
|
state->todo.erase(state->todo.begin());
|
||||||
|
state->active.insert(attrPath);
|
||||||
|
break;
|
||||||
|
} else
|
||||||
|
state.wait(wakeup);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Tell the worker to evaluate it. */
|
||||||
|
writeLine(proc->to.get(), "do " + attrPath);
|
||||||
|
|
||||||
|
/* Wait for the response. */
|
||||||
|
auto respString = readLine(proc->from.get());
|
||||||
|
auto response = nlohmann::json::parse(respString);
|
||||||
|
|
||||||
|
/* Handle the response. */
|
||||||
|
StringSet newAttrs;
|
||||||
|
if (response.find("attrs") != response.end()) {
|
||||||
|
for (auto & i : response["attrs"]) {
|
||||||
|
auto s = (attrPath.empty() ? "" : attrPath + ".") + (std::string) i;
|
||||||
|
newAttrs.insert(s);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
auto state(state_.lock());
|
||||||
|
std::cout << respString << "\n" << std::flush;
|
||||||
|
}
|
||||||
|
|
||||||
|
proc_ = std::move(proc);
|
||||||
|
|
||||||
|
/* Add newly discovered job names to the queue. */
|
||||||
|
{
|
||||||
|
auto state(state_.lock());
|
||||||
|
state->active.erase(attrPath);
|
||||||
|
for (auto & s : newAttrs)
|
||||||
|
state->todo.insert(s);
|
||||||
|
wakeup.notify_all();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (...) {
|
||||||
|
auto state(state_.lock());
|
||||||
|
state->exc = std::current_exception();
|
||||||
|
wakeup.notify_all();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
int main(int argc, char * * argv)
|
int main(int argc, char * * argv)
|
||||||
{
|
{
|
||||||
/* Prevent undeclared dependencies in the evaluation via
|
/* Prevent undeclared dependencies in the evaluation via
|
||||||
|
|
@ -375,98 +460,13 @@ int main(int argc, char * * argv)
|
||||||
loggerSettings.showTrace.assign(true);
|
loggerSettings.showTrace.assign(true);
|
||||||
}
|
}
|
||||||
|
|
||||||
struct State
|
|
||||||
{
|
|
||||||
std::set<std::string> todo{""};
|
|
||||||
std::set<std::string> active;
|
|
||||||
std::exception_ptr exc;
|
|
||||||
};
|
|
||||||
|
|
||||||
std::condition_variable wakeup;
|
|
||||||
|
|
||||||
Sync<State> state_;
|
Sync<State> state_;
|
||||||
|
|
||||||
/* Start a handler thread per worker process. */
|
/* Start a collector thread per worker process. */
|
||||||
auto handler = [&]()
|
|
||||||
{
|
|
||||||
try {
|
|
||||||
std::optional<std::unique_ptr<Proc>> proc_;
|
|
||||||
|
|
||||||
while (true) {
|
|
||||||
|
|
||||||
auto proc = proc_.has_value()
|
|
||||||
? std::move(proc_.value())
|
|
||||||
: std::make_unique<Proc>(worker);
|
|
||||||
|
|
||||||
/* Check whether the existing worker process is still there. */
|
|
||||||
auto s = readLine(proc->from.get());
|
|
||||||
if (s == "restart") {
|
|
||||||
proc_ = std::nullopt;
|
|
||||||
continue;
|
|
||||||
} else if (s != "next") {
|
|
||||||
auto json = nlohmann::json::parse(s);
|
|
||||||
throw Error("worker error: %s", (std::string) json["error"]);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Wait for a job name to become available. */
|
|
||||||
std::string attrPath;
|
|
||||||
|
|
||||||
while (true) {
|
|
||||||
checkInterrupt();
|
|
||||||
auto state(state_.lock());
|
|
||||||
if ((state->todo.empty() && state->active.empty()) || state->exc) {
|
|
||||||
writeLine(proc->to.get(), "exit");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (!state->todo.empty()) {
|
|
||||||
attrPath = *state->todo.begin();
|
|
||||||
state->todo.erase(state->todo.begin());
|
|
||||||
state->active.insert(attrPath);
|
|
||||||
break;
|
|
||||||
} else
|
|
||||||
state.wait(wakeup);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Tell the worker to evaluate it. */
|
|
||||||
writeLine(proc->to.get(), "do " + attrPath);
|
|
||||||
|
|
||||||
/* Wait for the response. */
|
|
||||||
auto respString = readLine(proc->from.get());
|
|
||||||
auto response = nlohmann::json::parse(respString);
|
|
||||||
|
|
||||||
/* Handle the response. */
|
|
||||||
StringSet newAttrs;
|
|
||||||
if (response.find("attrs") != response.end()) {
|
|
||||||
for (auto & i : response["attrs"]) {
|
|
||||||
auto s = (attrPath.empty() ? "" : attrPath + ".") + (std::string) i;
|
|
||||||
newAttrs.insert(s);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
auto state(state_.lock());
|
|
||||||
std::cout << respString << "\n" << std::flush;
|
|
||||||
}
|
|
||||||
|
|
||||||
proc_ = std::move(proc);
|
|
||||||
|
|
||||||
/* Add newly discovered job names to the queue. */
|
|
||||||
{
|
|
||||||
auto state(state_.lock());
|
|
||||||
state->active.erase(attrPath);
|
|
||||||
for (auto & s : newAttrs)
|
|
||||||
state->todo.insert(s);
|
|
||||||
wakeup.notify_all();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (...) {
|
|
||||||
auto state(state_.lock());
|
|
||||||
state->exc = std::current_exception();
|
|
||||||
wakeup.notify_all();
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
std::vector<std::thread> threads;
|
std::vector<std::thread> threads;
|
||||||
|
std::condition_variable wakeup;
|
||||||
for (size_t i = 0; i < myArgs.nrWorkers; i++)
|
for (size_t i = 0; i < myArgs.nrWorkers; i++)
|
||||||
threads.emplace_back(std::thread(handler));
|
threads.emplace_back(std::thread(collector(state_, wakeup)));
|
||||||
|
|
||||||
for (auto & thread : threads)
|
for (auto & thread : threads)
|
||||||
thread.join();
|
thread.join();
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue