Allow build to be bumped to the front of the queue via the web interface

Builds now have a "Bump up" action. This will cause the queue runner
to prioritise the steps of the build above all other steps.
This commit is contained in:
Eelco Dolstra 2015-08-10 16:18:06 +02:00
parent 27182c7c1d
commit eb13007fe6
9 changed files with 92 additions and 25 deletions

View file

@ -144,7 +144,9 @@ system_time State::doDispatch()
{ {
auto a_(a->state.lock()); auto a_(a->state.lock());
auto b_(b->state.lock()); // FIXME: deadlock? auto b_(b->state.lock()); // FIXME: deadlock?
return a_->lowestBuildID < b_->lowestBuildID; return
a_->highestGlobalPriority != b_->highestGlobalPriority ? a_->highestGlobalPriority > b_->highestGlobalPriority :
a_->lowestBuildID < b_->lowestBuildID;
}); });
/* Find a machine with a free slot and find a step to run /* Find a machine with a free slot and find a step to run

View file

@ -25,6 +25,7 @@ void State::queueMonitorLoop()
receiver buildsRestarted(*conn, "builds_restarted"); receiver buildsRestarted(*conn, "builds_restarted");
receiver buildsCancelled(*conn, "builds_cancelled"); receiver buildsCancelled(*conn, "builds_cancelled");
receiver buildsDeleted(*conn, "builds_deleted"); receiver buildsDeleted(*conn, "builds_deleted");
receiver buildsBumped(*conn, "builds_bumped");
auto store = openStore(); // FIXME: pool auto store = openStore(); // FIXME: pool
@ -44,9 +45,9 @@ void State::queueMonitorLoop()
printMsg(lvlTalkative, "got notification: builds restarted"); printMsg(lvlTalkative, "got notification: builds restarted");
lastBuildId = 0; // check all builds lastBuildId = 0; // check all builds
} }
if (buildsCancelled.get() || buildsDeleted.get()) { if (buildsCancelled.get() || buildsDeleted.get() || buildsBumped.get()) {
printMsg(lvlTalkative, "got notification: builds cancelled"); printMsg(lvlTalkative, "got notification: builds cancelled or bumped");
removeCancelledBuilds(*conn); processQueueChange(*conn);
} }
} }
@ -64,7 +65,7 @@ void State::getQueuedBuilds(Connection & conn, std::shared_ptr<StoreAPI> store,
{ {
pqxx::work txn(conn); pqxx::work txn(conn);
auto res = txn.parameterized("select id, project, jobset, job, drvPath, maxsilent, timeout, timestamp from Builds where id > $1 and finished = 0 order by id")(lastBuildId).exec(); auto res = txn.parameterized("select id, project, jobset, job, drvPath, maxsilent, timeout, timestamp, globalPriority from Builds where id > $1 and finished = 0 order by id")(lastBuildId).exec();
for (auto const & row : res) { for (auto const & row : res) {
auto builds_(builds.lock()); auto builds_(builds.lock());
@ -82,6 +83,7 @@ void State::getQueuedBuilds(Connection & conn, std::shared_ptr<StoreAPI> store,
build->maxSilentTime = row["maxsilent"].as<int>(); build->maxSilentTime = row["maxsilent"].as<int>();
build->buildTimeout = row["timeout"].as<int>(); build->buildTimeout = row["timeout"].as<int>();
build->timestamp = row["timestamp"].as<time_t>(); build->timestamp = row["timestamp"].as<time_t>();
build->globalPriority = row["globalPriority"].as<int>();
newBuilds.emplace(std::make_pair(build->drvPath, build)); newBuilds.emplace(std::make_pair(build->drvPath, build));
} }
@ -228,13 +230,7 @@ void State::getQueuedBuilds(Connection & conn, std::shared_ptr<StoreAPI> store,
throw; throw;
} }
/* Update the lowest build ID field of each dependency. This build->propagatePriorities();
is used by the dispatcher to start steps in order of build
ID. */
visitDependencies([&](const Step::ptr & step) {
auto step_(step->state.lock());
step_->lowestBuildID = std::min(step_->lowestBuildID, build->id);
}, build->toplevel);
/* Add the new runnable build steps to runnable and wake up /* Add the new runnable build steps to runnable and wake up
the builder threads. */ the builder threads. */
@ -247,25 +243,46 @@ void State::getQueuedBuilds(Connection & conn, std::shared_ptr<StoreAPI> store,
} }
void State::removeCancelledBuilds(Connection & conn) void Build::propagatePriorities()
{
/* Update the highest global priority and lowest build ID fields
of each dependency. This is used by the dispatcher to start
steps in order of descending global priority and ascending
build ID. */
visitDependencies([&](const Step::ptr & step) {
auto step_(step->state.lock());
step_->highestGlobalPriority = std::max(step_->highestGlobalPriority, globalPriority);
step_->lowestBuildID = std::min(step_->lowestBuildID, id);
}, toplevel);
}
void State::processQueueChange(Connection & conn)
{ {
/* Get the current set of queued builds. */ /* Get the current set of queued builds. */
std::set<BuildID> currentIds; std::map<BuildID, int> currentIds;
{ {
pqxx::work txn(conn); pqxx::work txn(conn);
auto res = txn.exec("select id from Builds where finished = 0"); auto res = txn.exec("select id, globalPriority from Builds where finished = 0");
for (auto const & row : res) for (auto const & row : res)
currentIds.insert(row["id"].as<BuildID>()); currentIds[row["id"].as<BuildID>()] = row["globalPriority"].as<BuildID>();
} }
auto builds_(builds.lock()); auto builds_(builds.lock());
for (auto i = builds_->begin(); i != builds_->end(); ) { for (auto i = builds_->begin(); i != builds_->end(); ) {
if (currentIds.find(i->first) == currentIds.end()) { auto b = currentIds.find(i->first);
if (b == currentIds.end()) {
printMsg(lvlInfo, format("discarding cancelled build %1%") % i->first); printMsg(lvlInfo, format("discarding cancelled build %1%") % i->first);
i = builds_->erase(i); i = builds_->erase(i);
// FIXME: ideally we would interrupt active build steps here. // FIXME: ideally we would interrupt active build steps here.
} else continue;
}
if (i->second->globalPriority < b->second) {
printMsg(lvlInfo, format("priority of build %1% increased") % i->first);
i->second->globalPriority = b->second;
i->second->propagatePriorities();
}
++i; ++i;
} }
} }

View file

@ -71,6 +71,7 @@ struct Build
std::string projectName, jobsetName, jobName; std::string projectName, jobsetName, jobName;
time_t timestamp; time_t timestamp;
unsigned int maxSilentTime, buildTimeout; unsigned int maxSilentTime, buildTimeout;
int globalPriority;
std::shared_ptr<Step> toplevel; std::shared_ptr<Step> toplevel;
@ -80,6 +81,8 @@ struct Build
{ {
return projectName + ":" + jobsetName + ":" + jobName; return projectName + ":" + jobsetName + ":" + jobName;
} }
void propagatePriorities();
}; };
@ -113,7 +116,11 @@ struct Step
/* Point in time after which the step can be retried. */ /* Point in time after which the step can be retried. */
system_time after; system_time after;
/* The lowest build ID depending on this step. */ /* The highest global priority of any build depending on this
step. */
int highestGlobalPriority{0};
/* The lowest ID of any build depending on this step. */
BuildID lowestBuildID{std::numeric_limits<BuildID>::max()}; BuildID lowestBuildID{std::numeric_limits<BuildID>::max()};
}; };
@ -282,9 +289,11 @@ private:
void queueMonitorLoop(); void queueMonitorLoop();
/* Check the queue for new builds. */
void getQueuedBuilds(Connection & conn, std::shared_ptr<nix::StoreAPI> store, unsigned int & lastBuildId); void getQueuedBuilds(Connection & conn, std::shared_ptr<nix::StoreAPI> store, unsigned int & lastBuildId);
void removeCancelledBuilds(Connection & conn); /* Handle cancellation, deletion and priority bumps. */
void processQueueChange(Connection & conn);
Step::ptr createStep(std::shared_ptr<nix::StoreAPI> store, const nix::Path & drvPath, Step::ptr createStep(std::shared_ptr<nix::StoreAPI> store, const nix::Path & drvPath,
Build::ptr referringBuild, Step::ptr referringStep, std::set<nix::Path> & finishedDrvs, Build::ptr referringBuild, Step::ptr referringStep, std::set<nix::Path> & finishedDrvs,

View file

@ -475,6 +475,23 @@ sub keep : Chained('buildChain') PathPart Args(1) {
} }
sub bump : Chained('buildChain') PathPart('bump') {
my ($self, $c, $x) = @_;
my $build = $c->stash->{build};
requireProjectOwner($c, $build->project); # FIXME: require admin?
$c->model('DB')->schema->txn_do(sub {
$build->update({globalpriority => time()});
});
$c->flash->{successMsg} = "Build has been bumped to the front of the queue.";
$c->res->redirect($c->uri_for($self->action_for("build"), $c->req->captures));
}
sub add_to_release : Chained('buildChain') PathPart('add-to-release') Args(0) { sub add_to_release : Chained('buildChain') PathPart('add-to-release') Args(0) {
my ($self, $c) = @_; my ($self, $c) = @_;

View file

@ -88,7 +88,7 @@ sub queue_GET {
$c->stash->{flashMsg} //= $c->flash->{buildMsg}; $c->stash->{flashMsg} //= $c->flash->{buildMsg};
$self->status_ok( $self->status_ok(
$c, $c,
entity => [$c->model('DB::Builds')->search({finished => 0}, { order_by => ["id"]})] entity => [$c->model('DB::Builds')->search({finished => 0}, { order_by => ["globalpriority desc", "id"]})]
); );
} }

View file

@ -138,6 +138,12 @@ __PACKAGE__->table("Builds");
default_value: 0 default_value: 0
is_nullable: 0 is_nullable: 0
=head2 globalpriority
data_type: 'integer'
default_value: 0
is_nullable: 0
=head2 busy =head2 busy
data_type: 'integer' data_type: 'integer'
@ -241,6 +247,8 @@ __PACKAGE__->add_columns(
{ data_type => "text", is_nullable => 1 }, { data_type => "text", is_nullable => 1 },
"priority", "priority",
{ data_type => "integer", default_value => 0, is_nullable => 0 }, { data_type => "integer", default_value => 0, is_nullable => 0 },
"globalpriority",
{ data_type => "integer", default_value => 0, is_nullable => 0 },
"busy", "busy",
{ data_type => "integer", default_value => 0, is_nullable => 0 }, { data_type => "integer", default_value => 0, is_nullable => 0 },
"locker", "locker",
@ -550,8 +558,8 @@ __PACKAGE__->many_to_many(
); );
# Created by DBIx::Class::Schema::Loader v0.07043 @ 2015-07-30 16:52:20 # Created by DBIx::Class::Schema::Loader v0.07043 @ 2015-08-10 15:10:41
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Y2lDtgY8EBLOuCHAI8fWRQ # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:rjifgnPtjY96MaQ7eiGzaA
__PACKAGE__->has_many( __PACKAGE__->has_many(
"dependents", "dependents",

View file

@ -96,6 +96,7 @@
<li><a href="[% c.uri_for('/build' build.id 'restart') %]">Restart</a></li> <li><a href="[% c.uri_for('/build' build.id 'restart') %]">Restart</a></li>
[% ELSE %] [% ELSE %]
<li><a href="[% c.uri_for('/build' build.id 'cancel') %]">Cancel</a></li> <li><a href="[% c.uri_for('/build' build.id 'cancel') %]">Cancel</a></li>
<li><a href="[% c.uri_for('/build' build.id 'bump') %]">Bump up</a></li>
[% END %] [% END %]
[% IF available && project.releases %] [% IF available && project.releases %]
[% INCLUDE menuItem [% INCLUDE menuItem

View file

@ -156,9 +156,13 @@ create table Builds (
nixExprInput text, nixExprInput text,
nixExprPath text, nixExprPath text,
-- Information about scheduled builds. -- Priority within a jobset, set via meta.schedulingPriority.
priority integer not null default 0, priority integer not null default 0,
-- Priority among all builds, used by the admin to bump builds to
-- the front of the queue via the web interface.
globalPriority integer not null default 0,
-- FIXME: remove (obsolete with the new queue runner) -- FIXME: remove (obsolete with the new queue runner)
busy integer not null default 0, -- true means someone is building this job now busy integer not null default 0, -- true means someone is building this job now
locker text, -- !!! hostname/pid of the process building this job? locker text, -- !!! hostname/pid of the process building this job?
@ -218,6 +222,10 @@ create function notifyBuildCancelled() returns trigger as 'begin notify builds_c
create trigger BuildCancelled after update on Builds for each row create trigger BuildCancelled after update on Builds for each row
when (old.finished = 0 and new.finished = 1 and new.buildStatus = 4) execute procedure notifyBuildCancelled(); when (old.finished = 0 and new.finished = 1 and new.buildStatus = 4) execute procedure notifyBuildCancelled();
create function notifyBuildBumped() returns trigger as 'begin notify builds_bumped; return null; end;' language plpgsql;
create trigger BuildBumped after update on Builds for each row
when (old.globalPriority != new.globalPriority) execute procedure notifyBuildBumped();
#endif #endif

5
src/sql/upgrade-40.sql Normal file
View file

@ -0,0 +1,5 @@
alter table Builds add column globalPriority integer not null default 0;
create function notifyBuildBumped() returns trigger as 'begin notify builds_bumped; return null; end;' language plpgsql;
create trigger BuildBumped after update on Builds for each row
when (old.globalPriority != new.globalPriority) execute procedure notifyBuildBumped();