snaakey/hydra
0
0
Fork 0
forked from lix-project/hydra
Code Pull requests Activity

Handle concurrent finishing of the same build

Browse source 
There is a slight possibility that the queue monitor and a builder
thread simultaneously decide to mark a build as finished. That's fine,
as long as we ensure the DB update is idempotent (as ensured by doing
"update Builds set finished = 1 ... where finished = 0").
This commit is contained in:
Eelco Dolstra 2015-06-18 17:12:51 +02:00
parent 948473c909
commit 9cdbff2fdf
2 changed files with 35 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

61
src/hydra-queue-runner/hydra-queue-runner.cc
View file

@ -163,7 +163,7 @@ struct Step
~Step() ~Step()
{ {
printMsg(lvlError, format("destroying step %1%") % drvPath); //printMsg(lvlError, format("destroying step %1%") % drvPath);
} }
}; };
@ -520,17 +520,18 @@ void State::getQueuedBuilds(Connection & conn, std::shared_ptr<StoreAPI> store,
if (!store->isValidPath(build->drvPath)) { if (!store->isValidPath(build->drvPath)) {
/* Derivation has been GC'ed prematurely. */ /* Derivation has been GC'ed prematurely. */
printMsg(lvlError, format("aborting GC'ed build %1%") % build->id); printMsg(lvlError, format("aborting GC'ed build %1%") % build->id);
pqxx::work txn(conn); if (!build->finishedInDB) {
assert(!build->finishedInDB); pqxx::work txn(conn);
txn.parameterized txn.parameterized
("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $3, errorMsg = $4 where id = $1") ("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $3, errorMsg = $4 where id = $1 and finished = 0")
(build->id) (build->id)
((int) bsAborted) ((int) bsAborted)
(time(0)) (time(0))
("derivation was garbage-collected prior to build").exec(); ("derivation was garbage-collected prior to build").exec();
txn.commit(); txn.commit();
build->finishedInDB = true; build->finishedInDB = true;
nrBuildsDone++; nrBuildsDone++;
}
return; return;
} }
@ -598,18 +599,19 @@ void State::getQueuedBuilds(Connection & conn, std::shared_ptr<StoreAPI> store,
if (buildStatus != bsSuccess) { if (buildStatus != bsSuccess) {
time_t now = time(0); time_t now = time(0);
pqxx::work txn(conn); if (!build->finishedInDB) {
createBuildStep(txn, 0, build, r, "", buildStepStatus); pqxx::work txn(conn);
assert(!build->finishedInDB); createBuildStep(txn, 0, build, r, "", buildStepStatus);
txn.parameterized txn.parameterized
("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $3, isCachedBuild = $4 where id = $1") ("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $3, isCachedBuild = $4 where id = $1 and finished = 0")
(build->id) (build->id)
((int) buildStatus) ((int) buildStatus)
(now) (now)
(buildStatus != bsUnsupported ? 1 : 0).exec(); (buildStatus != bsUnsupported ? 1 : 0).exec();
txn.commit(); txn.commit();
build->finishedInDB = true; build->finishedInDB = true;
nrBuildsDone++; nrBuildsDone++;
}
badStep = true; badStep = true;
break; break;
} }
@ -622,7 +624,8 @@ void State::getQueuedBuilds(Connection & conn, std::shared_ptr<StoreAPI> store,
{ {
auto builds_(builds.lock()); auto builds_(builds.lock());
(*builds_)[build->id] = build; if (!build->finishedInDB) // FIXME: can this happen?
(*builds_)[build->id] = build;
build->toplevel = step; build->toplevel = step;
} }
@ -1232,9 +1235,9 @@ bool State::doBuildStep(std::shared_ptr<StoreAPI> store, Step::ptr step,
/* Mark all builds that depend on this derivation as failed. */ /* Mark all builds that depend on this derivation as failed. */
for (auto & build2 : indirect) { for (auto & build2 : indirect) {
printMsg(lvlError, format("marking build %1% as failed") % build2->id); printMsg(lvlError, format("marking build %1% as failed") % build2->id);
assert(!build->finishedInDB); if (build->finishedInDB) continue;
txn.parameterized txn.parameterized
("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $4, isCachedBuild = $5 where id = $1") ("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $4, isCachedBuild = $5 where id = $1 and finished = 0")
(build2->id) (build2->id)
((int) (build2->drvPath != step->drvPath && buildStatus == bsFailed ? bsDepFailed : buildStatus)) ((int) (build2->drvPath != step->drvPath && buildStatus == bsFailed ? bsDepFailed : buildStatus))
(result.startTime) (result.startTime)
@ -1272,10 +1275,10 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build,
{ {
printMsg(lvlInfo, format("marking build %1% as succeeded") % build->id); printMsg(lvlInfo, format("marking build %1% as succeeded") % build->id);
assert(!build->finishedInDB); if (build->finishedInDB) return;
txn.parameterized txn.parameterized
("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $4, size = $5, closureSize = $6, releaseName = $7, isCachedBuild = $8 where id = $1") ("update Builds set finished = 1, busy = 0, buildStatus = $2, startTime = $3, stopTime = $4, size = $5, closureSize = $6, releaseName = $7, isCachedBuild = $8 where id = $1 and finished = 0")
(build->id) (build->id)
((int) (res.failed ? bsFailedWithOutput : bsSuccess)) ((int) (res.failed ? bsFailedWithOutput : bsSuccess))
(startTime) (startTime)

3
src/hydra-queue-runner/sync.hh
View file

@ -28,6 +28,9 @@ private:
public: public:
Sync() { }
Sync(const T & data) : data(data) { }
class Lock class Lock
{ {
private: private: