hydra-evaluator: add a 'ONE_AT_A_TIME' evaluator style

In the past, jobsets which are automatically evaluated are evaluated
regularly, on a schedule. This schedule means a new evaluation is
created every checkInterval seconds (assuming something changed.)

This model works well for architectures where our build farm can
easily keep up with demand.

This commit adds a new type of evaluation, called ONE_AT_A_TIME, which
only schedules a new evaluation if the previous evaluation of the
jobset has no unfinished builds.

This model of evaluation lets us have 'low-tier' architectures.

For example, we could now have a jobset for ARMv7l builds, where
the buildfarm only has a single, underpowered ARMv7l builder.
Configuring that jobset as ONE_AT_A_TIME will create an evaluation
and then won't schedule another evaluation until every job of
the existing evaluation is complete.

This way, the cache will have a complete collection of pre-built
software for some commits, but the underpowered architecture will
never become backlogged in ancient revisions.
This commit is contained in:
Graham Christensen 2020-03-03 19:53:18 -05:00
parent eaa65f51f4
commit 5fae9d96a2
No known key found for this signature in database
GPG key ID: FE918C3A98C1030F
5 changed files with 86 additions and 8 deletions

View file

@ -15,6 +15,13 @@ using namespace nix;
typedef std::pair<std::string, std::string> JobsetName;
enum class EvaluationStyle
{
SCHEDULE = 1,
ONESHOT = 2,
ONE_AT_A_TIME = 3,
};
struct Evaluator
{
std::unique_ptr<Config> config;
@ -24,6 +31,7 @@ struct Evaluator
struct Jobset
{
JobsetName name;
std::optional<EvaluationStyle> evaluation_style;
time_t lastCheckedTime, triggerTime;
int checkInterval;
Pid pid;
@ -60,7 +68,7 @@ struct Evaluator
pqxx::work txn(*conn);
auto res = txn.parameterized
("select project, j.name, lastCheckedTime, triggerTime, checkInterval from Jobsets j join Projects p on j.project = p.name "
("select project, j.name, lastCheckedTime, triggerTime, checkInterval, j.enabled as jobset_enabled from Jobsets j join Projects p on j.project = p.name "
"where j.enabled != 0 and p.enabled != 0").exec();
auto state(state_.lock());
@ -78,6 +86,17 @@ struct Evaluator
jobset.lastCheckedTime = row["lastCheckedTime"].as<time_t>(0);
jobset.triggerTime = row["triggerTime"].as<time_t>(notTriggered);
jobset.checkInterval = row["checkInterval"].as<time_t>();
switch (row["jobset_enabled"].as<int>(0)) {
case 1:
jobset.evaluation_style = EvaluationStyle::SCHEDULE;
break;
case 2:
jobset.evaluation_style = EvaluationStyle::ONESHOT;
break;
case 3:
jobset.evaluation_style = EvaluationStyle::ONE_AT_A_TIME;
break;
}
seen.insert(name);
}
@ -133,25 +152,83 @@ struct Evaluator
{
if (jobset.pid != -1) {
// Already running.
debug("shouldEvaluate %s:%s? no: already running",
jobset.name.first, jobset.name.second);
return false;
}
if (jobset.triggerTime == std::numeric_limits<time_t>::max()) {
if (jobset.triggerTime != std::numeric_limits<time_t>::max()) {
// An evaluation of this Jobset is requested
debug("shouldEvaluate %s:%s? yes: requested",
jobset.name.first, jobset.name.second);
return true;
}
if (jobset.checkInterval <= 0) {
// Automatic scheduling is disabled. We allow requested
// evaluations, but never schedule start one.
debug("shouldEvaluate %s:%s? no: checkInterval <= 0",
jobset.name.first, jobset.name.second);
return false;
}
if (jobset.lastCheckedTime + jobset.checkInterval <= time(0)) {
// Time to schedule a fresh evaluation. If the jobset
// is a ONE_AT_A_TIME jobset, ensure the previous jobset
// has no remaining, unfinished work.
auto conn(dbPool.get());
pqxx::work txn(*conn);
if (jobset.evaluation_style == EvaluationStyle::ONE_AT_A_TIME) {
auto evaluation_res = txn.parameterized
("select id from JobsetEvals "
"where project = $1 and jobset = $2 "
"order by id desc limit 1")
(jobset.name.first)
(jobset.name.second)
.exec();
if (evaluation_res.empty()) {
// First evaluation, so allow scheduling.
debug("shouldEvaluate(one-at-a-time) %s:%s? yes: no prior eval",
jobset.name.first, jobset.name.second);
return true;
}
auto evaluation_id = evaluation_res[0][0].as<int>();
auto unfinished_build_res = txn.parameterized
("select id from Builds "
"join JobsetEvalMembers "
" on (JobsetEvalMembers.build = Builds.id) "
"where JobsetEvalMembers.eval = $1 "
" and builds.finished = 0 "
" limit 1")
(evaluation_id)
.exec();
// If the previous evaluation has no unfinished builds
// schedule!
if (unfinished_build_res.empty()) {
debug("shouldEvaluate(one-at-a-time) %s:%s? yes: no unfinished builds",
jobset.name.first, jobset.name.second);
return true;
} else {
debug("shouldEvaluate(one-at-a-time) %s:%s? no: at least one unfinished build",
jobset.name.first, jobset.name.second);
return false;
}
if (jobset.lastCheckedTime + jobset.checkInterval <= time(0)) {
// Time to schedule a fresh evaluation
} else {
// EvaluationStyle::ONESHOT, EvaluationStyle::SCHEDULED
debug("shouldEvaluate(oneshot/scheduled) %s:%s? yes: checkInterval elapsed",
jobset.name.first, jobset.name.second);
return true;
}
}
return false;
}

View file

@ -226,7 +226,7 @@ sub updateJobset {
my ($nixExprPath, $nixExprInput) = nixExprPathFromParams $c;
my $enabled = int($c->stash->{params}->{enabled});
die if $enabled < 0 || $enabled > 2;
die if $enabled < 0 || $enabled > 3;
my $shares = int($c->stash->{params}->{schedulingshares} // 1);
error($c, "The number of scheduling shares must be positive.") if $shares <= 0;

View file

@ -68,6 +68,7 @@
<input type="hidden" name="enabled" value="[% jobset.enabled %]" />
<button type="button" class="btn" value="1">Enabled</button>
<button type="button" class="btn" value="2">One-shot</button>
<button type="button" class="btn" value="3">One-at-a-time</button>
<button type="button" class="btn" value="0">Disabled</button>
</div>
</div>

View file

@ -129,7 +129,7 @@
<table class="info-table">
<tr>
<th>State:</th>
<td>[% IF jobset.enabled == 0; "Disabled"; ELSIF jobset.enabled == 1; "Enabled"; ELSIF jobset.enabled == 2; "One-shot"; END %]</td>
<td>[% IF jobset.enabled == 0; "Disabled"; ELSIF jobset.enabled == 1; "Enabled"; ELSIF jobset.enabled == 2; "One-shot"; ELSIF jobset.enabled == 3; "One-at-a-time"; END %]</td>
</tr>
<tr>
<th>Description:</th>

View file

@ -61,7 +61,7 @@ create table Jobsets (
errorTime integer, -- timestamp associated with errorMsg
lastCheckedTime integer, -- last time the evaluator looked at this jobset
triggerTime integer, -- set if we were triggered by a push event
enabled integer not null default 1, -- 0 = disabled, 1 = enabled, 2 = one-shot
enabled integer not null default 1, -- 0 = disabled, 1 = enabled, 2 = one-shot, 3 = one-at-a-time
enableEmail integer not null default 1,
hidden integer not null default 0,
emailOverride text not null,