lix-project/hydra
3
0
Fork 6
Code Issues 2 Pull requests Packages Projects Releases Wiki Activity

Simplify retry handling

Browse source
This commit is contained in:
Eelco Dolstra 2015-06-18 14:51:08 +02:00
parent e039f5f840
commit 9c03b11ca8
1 changed files with 45 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

88
src/hydra-queue-runner/hydra-queue-runner.cc
View file

@ -1100,10 +1100,21 @@ bool State::doBuildStep(std::shared_ptr<StoreAPI> store, Step::ptr step,
if (!result.stopTime) result.stopTime = time(0); if (!result.stopTime) result.stopTime = time(0);
bool retry = false; /* The step had a hopefully temporary failure (e.g. network
issue). Retry a number of times. */
if (result.status == RemoteResult::rrMiscFailure) { if (result.status == RemoteResult::rrMiscFailure) {
auto step_(step->state.lock()); bool retry;
retry = step_->tries + 1 < maxTries; {
auto step_(step->state.lock());
retry = step_->tries + 1 < maxTries;
}
if (retry) {
pqxx::work txn(*conn);
finishBuildStep(txn, result.startTime, result.stopTime, build->id,
stepNr, machine->sshName, bssAborted, result.errorMsg);
txn.commit();
return true;
}
} }
/* Remove this step. After this, incoming builds that depend on /* Remove this step. After this, incoming builds that depend on
@ -1112,10 +1123,8 @@ bool State::doBuildStep(std::shared_ptr<StoreAPI> store, Step::ptr step,
won't conflict with this one, because we're removing it. In any won't conflict with this one, because we're removing it. In any
case, the set of dependent builds for ‘step’ can't increase case, the set of dependent builds for ‘step’ can't increase
anymore because ‘step’ is no longer visible to createStep(). */ anymore because ‘step’ is no longer visible to createStep(). */
if (!retry) { auto steps_(steps.lock());
auto steps_(steps.lock()); steps_->erase(step->drvPath);
steps_->erase(step->drvPath);
}
/* Get the final set of dependent builds. */ /* Get the final set of dependent builds. */
auto dependents = getDependentBuilds(step); auto dependents = getDependentBuilds(step);
@ -1159,20 +1168,16 @@ bool State::doBuildStep(std::shared_ptr<StoreAPI> store, Step::ptr step,
message. */ message. */
if (buildStatus != bsAborted) result.errorMsg = ""; if (buildStatus != bsAborted) result.errorMsg = "";
if (!retry) { /* Create failed build steps for every build that depends
on this. For cached failures, only create a step for
/* Create failed build steps for every build that builds that don't have this step as top-level
depends on this. For cached failures, only create a (otherwise the user won't be able to see what caused
step for builds that don't have this step as the build to fail). */
top-level (otherwise the user won't be able to see for (auto build2 : dependents) {
what caused the build to fail). */ if (build == build2) continue;
for (auto build2 : dependents) { if (cachedFailure && build2->drvPath == step->drvPath) continue;
if (build == build2) continue; createBuildStep(txn, 0, build2, step, machine->sshName,
if (cachedFailure && build2->drvPath == step->drvPath) continue; buildStepStatus, result.errorMsg, build->id);
createBuildStep(txn, 0, build2, step, machine->sshName,
buildStepStatus, result.errorMsg, build->id);
}
} }
if (!cachedFailure) if (!cachedFailure)
@ -1180,19 +1185,18 @@ bool State::doBuildStep(std::shared_ptr<StoreAPI> store, Step::ptr step,
stepNr, machine->sshName, buildStepStatus, result.errorMsg); stepNr, machine->sshName, buildStepStatus, result.errorMsg);
/* Mark all builds that depend on this derivation as failed. */ /* Mark all builds that depend on this derivation as failed. */
if (!retry) for (auto build2 : dependents) {
for (auto build2 : dependents) { printMsg(lvlError, format("marking build %1% as failed") % build2->id);
printMsg(lvlError, format("marking build %1% as failed") % build2->id); txn.parameterized
txn.parameterized ("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $4, isCachedBuild = $5 where id = $1")
("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $4, isCachedBuild = $5 where id = $1") (build2->id)
(build2->id) ((int) (build2->drvPath != step->drvPath && buildStatus == bsFailed ? bsDepFailed : buildStatus))
((int) (build2->drvPath != step->drvPath && buildStatus == bsFailed ? bsDepFailed : buildStatus)) (result.startTime)
(result.startTime) (result.stopTime)
(result.stopTime) (cachedFailure ? 1 : 0).exec();
(cachedFailure ? 1 : 0).exec(); build2->finishedInDB = true; // FIXME: txn might fail
build2->finishedInDB = true; // FIXME: txn might fail nrBuildsDone++;
nrBuildsDone++; }
}
/* Remember failed paths in the database so that they /* Remember failed paths in the database so that they
won't be built again. */ won't be built again. */
@ -1208,20 +1212,18 @@ bool State::doBuildStep(std::shared_ptr<StoreAPI> store, Step::ptr step,
is the top-level derivation. In case of failure, destroy all is the top-level derivation. In case of failure, destroy all
dependent Build objects. Any Steps not referenced by other dependent Build objects. Any Steps not referenced by other
Builds will be destroyed as well. */ Builds will be destroyed as well. */
if (!retry) for (auto build2 : dependents)
for (auto build2 : dependents) if (build2->toplevel == step || result.status != RemoteResult::rrSuccess) {
if (build2->toplevel == step || result.status != RemoteResult::rrSuccess) { auto builds_(builds.lock());
auto builds_(builds.lock()); builds_->erase(build2->id);
builds_->erase(build2->id); }
}
/* Remove the step from the graph. In case of success, make /* Remove the step from the graph. In case of success, make
dependent build steps runnable if they have no other dependent build steps runnable if they have no other
dependencies. */ dependencies. */
if (!retry) destroyStep(step, result.status == RemoteResult::rrSuccess);
destroyStep(step, result.status == RemoteResult::rrSuccess);
return retry; return false;
} }