snaakey/hydra
0
0
Fork 0
forked from lix-project/hydra
Code Pull requests Activity

Kill active build steps when builds are cancelled

Browse source 
We now kill active build steps when there are no more referring
builds. This is useful e.g. for preventing cancelled multi-hour TPC-H
benchmark runs from hogging build machines.
This commit is contained in:
Eelco Dolstra 2016-10-31 14:58:29 +01:00
parent a816ef873d
commit b3169ce438
6 changed files with 111 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

70
src/hydra-queue-runner/builder.cc
View file

@ -13,7 +13,14 @@ void State::builder(MachineReservation::ptr reservation)
nrStepsStarted++; nrStepsStarted++;
MaintainCount mc(nrActiveSteps); reservation->threadId = pthread_self();
activeSteps_.lock()->insert(reservation);
Finally removeActiveStep([&]() {
reservation->threadId = -1;
activeSteps_.lock()->erase(reservation);
});
auto step = reservation->step; auto step = reservation->step;
@ -63,8 +70,15 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore, Step::ptr step,
purpose of creating build steps. We could create a build step purpose of creating build steps. We could create a build step
record for every build, but that could be very expensive record for every build, but that could be very expensive
(e.g. a stdenv derivation can be a dependency of tens of (e.g. a stdenv derivation can be a dependency of tens of
thousands of builds), so we don't. */ thousands of builds), so we don't.
Build::ptr build;
We don't keep a Build::ptr here to allow
State::processQueueChange() to detect whether a step can be
cancelled (namely if there are no more Builds referring to
it). */
BuildID buildId;
Path buildDrvPath;
unsigned int maxSilentTime, buildTimeout;
{ {
std::set<Build::ptr> dependents; std::set<Build::ptr> dependents;
@ -85,6 +99,8 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore, Step::ptr step,
return sMaybeCancelled; return sMaybeCancelled;
} }
Build::ptr build;
for (auto build2 : dependents) { for (auto build2 : dependents) {
if (build2->drvPath == step->drvPath) { if (build2->drvPath == step->drvPath) {
build = build2; build = build2;
@ -93,11 +109,16 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore, Step::ptr step,
} }
if (!build) build = *dependents.begin(); if (!build) build = *dependents.begin();
buildId = build->id;
buildDrvPath = build->drvPath;
maxSilentTime = build->maxSilentTime;
buildTimeout = build->buildTimeout;
printMsg(lvlInfo, format("performing step ‘%1%’ on ‘%2%’ (needed by build %3% and %4% others)") printMsg(lvlInfo, format("performing step ‘%1%’ on ‘%2%’ (needed by build %3% and %4% others)")
% step->drvPath % machine->sshName % build->id % (dependents.size() - 1)); % step->drvPath % machine->sshName % buildId % (dependents.size() - 1));
} }
bool quit = build->id == buildOne && step->drvPath == build->drvPath; bool quit = buildId == buildOne && step->drvPath == buildDrvPath;
auto conn(dbPool.get()); auto conn(dbPool.get());
@ -108,9 +129,9 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore, Step::ptr step,
Finally clearStep([&]() { Finally clearStep([&]() {
if (stepNr && !stepFinished) { if (stepNr && !stepFinished) {
printError("marking step %d of build %d as orphaned", stepNr, build->id); printError("marking step %d of build %d as orphaned", stepNr, buildId);
auto orphanedSteps_(orphanedSteps.lock()); auto orphanedSteps_(orphanedSteps.lock());
orphanedSteps_->emplace(build->id, stepNr); orphanedSteps_->emplace(buildId, stepNr);
} }
}); });
@ -127,20 +148,33 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore, Step::ptr step,
{ {
auto mc = startDbUpdate(); auto mc = startDbUpdate();
pqxx::work txn(*conn); pqxx::work txn(*conn);
stepNr = createBuildStep(txn, result.startTime, build, step, machine->sshName, bsBusy); stepNr = createBuildStep(txn, result.startTime, buildId, step, machine->sshName, bsBusy);
txn.commit(); txn.commit();
} }
/* Do the build. */ /* Do the build. */
try { try {
/* FIXME: referring builds may have conflicting timeouts. */ /* FIXME: referring builds may have conflicting timeouts. */
buildRemote(destStore, machine, step, build->maxSilentTime, build->buildTimeout, result); buildRemote(destStore, machine, step, maxSilentTime, buildTimeout, result);
} catch (NoTokens & e) { } catch (NoTokens & e) {
result.stepStatus = bsNarSizeLimitExceeded; result.stepStatus = bsNarSizeLimitExceeded;
} catch (Error & e) { } catch (Error & e) {
result.stepStatus = bsAborted; result.stepStatus = bsAborted;
result.errorMsg = e.msg(); result.errorMsg = e.msg();
result.canRetry = true; result.canRetry = true;
} catch (__cxxabiv1::__forced_unwind & e) {
/* The queue monitor thread cancelled this step. */
try {
printInfo("marking step %d of build %d as succeeded", stepNr, buildId);
pqxx::work txn(*conn);
finishBuildStep(txn, result.startTime, time(0), result.overhead, buildId,
stepNr, machine->sshName, bsCancelled, "");
txn.commit();
stepFinished = true;
} catch (...) {
ignoreException();
}
throw;
} }
if (result.stepStatus == bsSuccess) if (result.stepStatus == bsSuccess)
@ -167,7 +201,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore, Step::ptr step,
{ {
auto logCompressorQueue_(logCompressorQueue.lock()); auto logCompressorQueue_(logCompressorQueue.lock());
assert(stepNr); assert(stepNr);
logCompressorQueue_->push({build->id, stepNr, result.logFile}); logCompressorQueue_->push({buildId, stepNr, result.logFile});
} }
logCompressorWakeup.notify_one(); logCompressorWakeup.notify_one();
} }
@ -187,7 +221,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore, Step::ptr step,
auto mc = startDbUpdate(); auto mc = startDbUpdate();
{ {
pqxx::work txn(*conn); pqxx::work txn(*conn);
finishBuildStep(txn, result.startTime, result.stopTime, result.overhead, build->id, finishBuildStep(txn, result.startTime, result.stopTime, result.overhead, buildId,
stepNr, machine->sshName, result.stepStatus, result.errorMsg); stepNr, machine->sshName, result.stepStatus, result.errorMsg);
txn.commit(); txn.commit();
} }
@ -243,11 +277,11 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore, Step::ptr step,
pqxx::work txn(*conn); pqxx::work txn(*conn);
finishBuildStep(txn, result.startTime, result.stopTime, result.overhead, finishBuildStep(txn, result.startTime, result.stopTime, result.overhead,
build->id, stepNr, machine->sshName, bsSuccess); buildId, stepNr, machine->sshName, bsSuccess);
for (auto & b : direct) { for (auto & b : direct) {
printMsg(lvlInfo, format("marking build %1% as succeeded") % b->id); printMsg(lvlInfo, format("marking build %1% as succeeded") % b->id);
markSucceededBuild(txn, b, res, build != b || result.isCached, markSucceededBuild(txn, b, res, buildId != b->id || result.isCached,
result.startTime, result.stopTime); result.startTime, result.stopTime);
} }
@ -340,17 +374,17 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore, Step::ptr step,
redundant with the build's isCachedBuild field). */ redundant with the build's isCachedBuild field). */
for (auto & build2 : indirect) { for (auto & build2 : indirect) {
if ((result.stepStatus == bsCachedFailure && build2->drvPath == step->drvPath) || if ((result.stepStatus == bsCachedFailure && build2->drvPath == step->drvPath) ||
(result.stepStatus != bsCachedFailure && build == build2) || (result.stepStatus != bsCachedFailure && buildId == build2->id) ||
build2->finishedInDB) build2->finishedInDB)
continue; continue;
createBuildStep(txn, 0, build2, step, machine->sshName, createBuildStep(txn, 0, build2->id, step, machine->sshName,
result.stepStatus, result.errorMsg, build == build2 ? 0 : build->id); result.stepStatus, result.errorMsg, buildId == build2->id ? 0 : buildId);
} }
if (result.stepStatus != bsCachedFailure && !stepFinished) { if (result.stepStatus != bsCachedFailure && !stepFinished) {
assert(stepNr); assert(stepNr);
finishBuildStep(txn, result.startTime, result.stopTime, result.overhead, finishBuildStep(txn, result.startTime, result.stopTime, result.overhead,
build->id, stepNr, machine->sshName, result.stepStatus, result.errorMsg); buildId, stepNr, machine->sshName, result.stepStatus, result.errorMsg);
} }
/* Mark all builds that depend on this derivation as failed. */ /* Mark all builds that depend on this derivation as failed. */
@ -392,7 +426,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore, Step::ptr step,
/* Send notification about this build and its dependents. */ /* Send notification about this build and its dependents. */
{ {
auto notificationSenderQueue_(notificationSenderQueue.lock()); auto notificationSenderQueue_(notificationSenderQueue.lock());
notificationSenderQueue_->push(NotificationItem{NotificationItem::Type::BuildFinished, build->id, dependentIDs}); notificationSenderQueue_->push(NotificationItem{NotificationItem::Type::BuildFinished, buildId, dependentIDs});
} }
notificationSenderWakeup.notify_one(); notificationSenderWakeup.notify_one();

16
src/hydra-queue-runner/hydra-queue-runner.cc
View file

@ -224,25 +224,25 @@ void State::clearBusy(Connection & conn, time_t stopTime)
} }
unsigned int State::allocBuildStep(pqxx::work & txn, Build::ptr build) unsigned int State::allocBuildStep(pqxx::work & txn, BuildID buildId)
{ {
/* Acquire an exclusive lock on BuildSteps to ensure that we don't /* Acquire an exclusive lock on BuildSteps to ensure that we don't
race with other threads creating a step of the same build. */ race with other threads creating a step of the same build. */
txn.exec("lock table BuildSteps in exclusive mode"); txn.exec("lock table BuildSteps in exclusive mode");
auto res = txn.parameterized("select max(stepnr) from BuildSteps where build = $1")(build->id).exec(); auto res = txn.parameterized("select max(stepnr) from BuildSteps where build = $1")(buildId).exec();
return res[0][0].is_null() ? 1 : res[0][0].as<int>() + 1; return res[0][0].is_null() ? 1 : res[0][0].as<int>() + 1;
} }
unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, Build::ptr build, Step::ptr step, unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID buildId, Step::ptr step,
const std::string & machine, BuildStatus status, const std::string & errorMsg, BuildID propagatedFrom) const std::string & machine, BuildStatus status, const std::string & errorMsg, BuildID propagatedFrom)
{ {
unsigned int stepNr = allocBuildStep(txn, build); auto stepNr = allocBuildStep(txn, buildId);
txn.parameterized txn.parameterized
("insert into BuildSteps (build, stepnr, type, drvPath, busy, startTime, system, status, propagatedFrom, errorMsg, stopTime, machine) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)") ("insert into BuildSteps (build, stepnr, type, drvPath, busy, startTime, system, status, propagatedFrom, errorMsg, stopTime, machine) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)")
(build->id) (buildId)
(stepNr) (stepNr)
(0) // == build (0) // == build
(step->drvPath) (step->drvPath)
@ -258,7 +258,7 @@ unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, Build::p
for (auto & output : step->drv.outputs) for (auto & output : step->drv.outputs)
txn.parameterized txn.parameterized
("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)") ("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)")
(build->id)(stepNr)(output.first)(output.second.path).exec(); (buildId)(stepNr)(output.first)(output.second.path).exec();
return stepNr; return stepNr;
} }
@ -284,7 +284,7 @@ void State::finishBuildStep(pqxx::work & txn, time_t startTime, time_t stopTime,
int State::createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime, int State::createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime,
Build::ptr build, const Path & drvPath, const string & outputName, const Path & storePath) Build::ptr build, const Path & drvPath, const string & outputName, const Path & storePath)
{ {
int stepNr = allocBuildStep(txn, build); auto stepNr = allocBuildStep(txn, build->id);
txn.parameterized txn.parameterized
("insert into BuildSteps (build, stepnr, type, drvPath, busy, status, startTime, stopTime) values ($1, $2, $3, $4, $5, $6, $7, $8)") ("insert into BuildSteps (build, stepnr, type, drvPath, busy, status, startTime, stopTime) values ($1, $2, $3, $4, $5, $6, $7, $8)")
@ -574,7 +574,7 @@ void State::dumpStatus(Connection & conn, bool log)
if (i->lock()) ++i; else i = runnable_->erase(i); if (i->lock()) ++i; else i = runnable_->erase(i);
root.attr("nrRunnableSteps", runnable_->size()); root.attr("nrRunnableSteps", runnable_->size());
} }
root.attr("nrActiveSteps", nrActiveSteps); root.attr("nrActiveSteps", activeSteps_.lock()->size());
root.attr("nrStepsBuilding", nrStepsBuilding); root.attr("nrStepsBuilding", nrStepsBuilding);
root.attr("nrStepsCopyingTo", nrStepsCopyingTo); root.attr("nrStepsCopyingTo", nrStepsCopyingTo);
root.attr("nrStepsCopyingFrom", nrStepsCopyingFrom); root.attr("nrStepsCopyingFrom", nrStepsCopyingFrom);

26
src/hydra-queue-runner/queue-monitor.cc
View file

@ -2,6 +2,8 @@
#include "build-result.hh" #include "build-result.hh"
#include "globals.hh" #include "globals.hh"
#include <cstring>
using namespace nix; using namespace nix;
@ -180,7 +182,7 @@ bool State::getQueuedBuilds(Connection & conn,
} }
} }
createBuildStep(txn, 0, build, ex.step, "", bsCachedFailure, "", propagatedFrom); createBuildStep(txn, 0, build->id, ex.step, "", bsCachedFailure, "", propagatedFrom);
txn.parameterized txn.parameterized
("update Builds set finished = 1, buildStatus = $2, startTime = $3, stopTime = $3, isCachedBuild = 1 where id = $1 and finished = 0") ("update Builds set finished = 1, buildStatus = $2, startTime = $3, stopTime = $3, isCachedBuild = 1 where id = $1 and finished = 0")
(build->id) (build->id)
@ -312,6 +314,7 @@ void State::processQueueChange(Connection & conn)
currentIds[row["id"].as<BuildID>()] = row["globalPriority"].as<BuildID>(); currentIds[row["id"].as<BuildID>()] = row["globalPriority"].as<BuildID>();
} }
{
auto builds_(builds.lock()); auto builds_(builds.lock());
for (auto i = builds_->begin(); i != builds_->end(); ) { for (auto i = builds_->begin(); i != builds_->end(); ) {
@ -331,6 +334,27 @@ void State::processQueueChange(Connection & conn)
} }
} }
{
auto activeSteps(activeSteps_.lock());
for (auto & activeStep : *activeSteps) {
auto threadId = activeStep->threadId; // FIXME: use Sync or atomic?
if (threadId == 0) continue;
std::set<Build::ptr> dependents;
std::set<Step::ptr> steps;
getDependents(activeStep->step, dependents, steps);
if (!dependents.empty()) continue;
printInfo("cancelling thread for build step ‘%s’", activeStep->step->drvPath);
int err = pthread_cancel(threadId);
if (err)
printError("error cancelling thread for build step ‘%s’: %s",
activeStep->step->drvPath, strerror(err));
}
}
}
Step::ptr State::createStep(ref<Store> destStore, Step::ptr State::createStep(ref<Store> destStore,
Connection & conn, Build::ptr build, const Path & drvPath, Connection & conn, Build::ptr build, const Path & drvPath,

10
src/hydra-queue-runner/state.hh
View file

@ -28,6 +28,7 @@ typedef enum {
bsFailed = 1, bsFailed = 1,
bsDepFailed = 2, // builds only bsDepFailed = 2, // builds only
bsAborted = 3, bsAborted = 3,
bsCancelled = 4,
bsFailedWithOutput = 6, // builds only bsFailedWithOutput = 6, // builds only
bsTimedOut = 7, bsTimedOut = 7,
bsCachedFailure = 8, // steps only bsCachedFailure = 8, // steps only
@ -296,7 +297,6 @@ private:
counter nrBuildsDone{0}; counter nrBuildsDone{0};
counter nrStepsStarted{0}; counter nrStepsStarted{0};
counter nrStepsDone{0}; counter nrStepsDone{0};
counter nrActiveSteps{0};
counter nrStepsBuilding{0}; counter nrStepsBuilding{0};
counter nrStepsCopyingTo{0}; counter nrStepsCopyingTo{0};
counter nrStepsCopyingFrom{0}; counter nrStepsCopyingFrom{0};
@ -370,10 +370,14 @@ private:
State & state; State & state;
Step::ptr step; Step::ptr step;
Machine::ptr machine; Machine::ptr machine;
pthread_t threadId = 0;
bool cancelled = false;
MachineReservation(State & state, Step::ptr step, Machine::ptr machine); MachineReservation(State & state, Step::ptr step, Machine::ptr machine);
~MachineReservation(); ~MachineReservation();
}; };
nix::Sync<std::set<std::shared_ptr<MachineReservation>>> activeSteps_;
std::atomic<time_t> lastDispatcherCheck{0}; std::atomic<time_t> lastDispatcherCheck{0};
std::shared_ptr<nix::Store> localStore; std::shared_ptr<nix::Store> localStore;
@ -413,9 +417,9 @@ private:
/* Thread to reload /etc/nix/machines periodically. */ /* Thread to reload /etc/nix/machines periodically. */
void monitorMachinesFile(); void monitorMachinesFile();
unsigned int allocBuildStep(pqxx::work & txn, Build::ptr build); unsigned int allocBuildStep(pqxx::work & txn, BuildID buildId);
unsigned int createBuildStep(pqxx::work & txn, time_t startTime, Build::ptr build, Step::ptr step, unsigned int createBuildStep(pqxx::work & txn, time_t startTime, BuildID buildId, Step::ptr step,
const std::string & machine, BuildStatus status, const std::string & errorMsg = "", const std::string & machine, BuildStatus status, const std::string & errorMsg = "",
BuildID propagatedFrom = 0); BuildID propagatedFrom = 0);

8
src/root/build.tt
View file

@ -23,7 +23,7 @@ FOR step IN steps; IF step.busy; busy = 1; END; END;
</thead> </thead>
<tbody> <tbody>
[% FOREACH step IN steps %] [% FOREACH step IN steps %]
[% IF ( type == "All" ) || ( type == "Failed" && step.status != 0 ) || ( type == "Running" && step.busy == 1 ) %] [% IF ( type == "All" ) || ( type == "Failed" && step.busy == 0 && step.status != 0 ) || ( type == "Running" && step.busy == 1 ) %]
[% has_log = seen.${step.drvpath} ? 0 : buildStepLogExists(step); [% has_log = seen.${step.drvpath} ? 0 : buildStepLogExists(step);
seen.${step.drvpath} = 1; seen.${step.drvpath} = 1;
log = c.uri_for('/build' build.id 'nixlog' step.stepnr); %] log = c.uri_for('/build' build.id 'nixlog' step.stepnr); %]
@ -49,7 +49,7 @@ FOR step IN steps; IF step.busy; busy = 1; END; END;
INCLUDE renderDuration duration = curTime - step.starttime; INCLUDE renderDuration duration = curTime - step.starttime;
END %] END %]
</td> </td>
<td>[% IF step.busy == 1 || ((step.machine || step.starttime) && (step.status == 0 || step.status == 1 || step.status == 3 || step.status == 7)); INCLUDE renderMachineName machine=step.machine; ELSE; "<em>n/a</em>"; END %]</td> <td>[% IF step.busy == 1 || ((step.machine || step.starttime) && (step.status == 0 || step.status == 1 || step.status == 3 || step.status == 4 || step.status == 7)); INCLUDE renderMachineName machine=step.machine; ELSE; "<em>n/a</em>"; END %]</td>
<td> <td>
[% IF step.busy == 1 %] [% IF step.busy == 1 %]
<strong>Building</strong> <strong>Building</strong>
@ -57,6 +57,8 @@ FOR step IN steps; IF step.busy; busy = 1; END; END;
Succeeded Succeeded
[% ELSIF step.status == 3 %] [% ELSIF step.status == 3 %]
<span class="error"><strong>Aborted</strong>[% IF step.errormsg %]: [% HTML.escape(step.errormsg); END %]</span> <span class="error"><strong>Aborted</strong>[% IF step.errormsg %]: [% HTML.escape(step.errormsg); END %]</span>
[% ELSIF step.status == 4 %]
<span class="error">Cancelled</span>
[% ELSIF step.status == 7 %] [% ELSIF step.status == 7 %]
<span class="error">Timed out</span> <span class="error">Timed out</span>
[% ELSIF step.status == 8 %] [% ELSIF step.status == 8 %]
@ -242,7 +244,7 @@ FOR step IN steps; IF step.busy; busy = 1; END; END;
[% IF build.finished %] [% IF build.finished %]
[% IF steps && build.buildstatus != 0 && build.buildstatus != 6 %] [% IF steps && build.buildstatus != 0 && build.buildstatus != 4 && build.buildstatus != 6 %]
<h3>Failed build steps</h3> <h3>Failed build steps</h3>
[% INCLUDE renderBuildSteps type="Failed" %] [% INCLUDE renderBuildSteps type="Failed" %]
[% END %] [% END %]

2
src/sql/hydra.sql
View file

@ -203,7 +203,7 @@ create table Builds (
-- 1 = regular Nix failure (derivation returned non-zero exit code) -- 1 = regular Nix failure (derivation returned non-zero exit code)
-- 2 = build of a dependency failed [builds only] -- 2 = build of a dependency failed [builds only]
-- 3 = build or step aborted due to misc failure -- 3 = build or step aborted due to misc failure
-- 4 = build cancelled (removed from queue; never built) [builds only] -- 4 = build or step cancelled
-- 5 = [obsolete] -- 5 = [obsolete]
-- 6 = failure with output (i.e. $out/nix-support/failed exists) [builds only] -- 6 = failure with output (i.e. $out/nix-support/failed exists) [builds only]
-- 7 = build timed out -- 7 = build timed out