hydra-evaluator: add a 'ONE_AT_A_TIME' evaluator style

In the past, jobsets which are automatically evaluated are evaluated
regularly, on a schedule. This schedule means a new evaluation is
created every checkInterval seconds (assuming something changed.)

This model works well for architectures where our build farm can
easily keep up with demand.

This commit adds a new type of evaluation, called ONE_AT_A_TIME, which
only schedules a new evaluation if the previous evaluation of the
jobset has no unfinished builds.

This model of evaluation lets us have 'low-tier' architectures.

For example, we could now have a jobset for ARMv7l builds, where
the buildfarm only has a single, underpowered ARMv7l builder.
Configuring that jobset as ONE_AT_A_TIME will create an evaluation
and then won't schedule another evaluation until every job of
the existing evaluation is complete.

This way, the cache will have a complete collection of pre-built
software for some commits, but the underpowered architecture will
never become backlogged in ancient revisions.
This commit is contained in:
Graham Christensen 2020-03-03 19:53:18 -05:00
parent eaa65f51f4
commit 5fae9d96a2
No known key found for this signature in database
GPG key ID: FE918C3A98C1030F
5 changed files with 86 additions and 8 deletions

View file

@ -15,6 +15,13 @@ using namespace nix;
typedef std::pair<std::string, std::string> JobsetName; typedef std::pair<std::string, std::string> JobsetName;
enum class EvaluationStyle
{
SCHEDULE = 1,
ONESHOT = 2,
ONE_AT_A_TIME = 3,
};
struct Evaluator struct Evaluator
{ {
std::unique_ptr<Config> config; std::unique_ptr<Config> config;
@ -24,6 +31,7 @@ struct Evaluator
struct Jobset struct Jobset
{ {
JobsetName name; JobsetName name;
std::optional<EvaluationStyle> evaluation_style;
time_t lastCheckedTime, triggerTime; time_t lastCheckedTime, triggerTime;
int checkInterval; int checkInterval;
Pid pid; Pid pid;
@ -60,7 +68,7 @@ struct Evaluator
pqxx::work txn(*conn); pqxx::work txn(*conn);
auto res = txn.parameterized auto res = txn.parameterized
("select project, j.name, lastCheckedTime, triggerTime, checkInterval from Jobsets j join Projects p on j.project = p.name " ("select project, j.name, lastCheckedTime, triggerTime, checkInterval, j.enabled as jobset_enabled from Jobsets j join Projects p on j.project = p.name "
"where j.enabled != 0 and p.enabled != 0").exec(); "where j.enabled != 0 and p.enabled != 0").exec();
auto state(state_.lock()); auto state(state_.lock());
@ -78,6 +86,17 @@ struct Evaluator
jobset.lastCheckedTime = row["lastCheckedTime"].as<time_t>(0); jobset.lastCheckedTime = row["lastCheckedTime"].as<time_t>(0);
jobset.triggerTime = row["triggerTime"].as<time_t>(notTriggered); jobset.triggerTime = row["triggerTime"].as<time_t>(notTriggered);
jobset.checkInterval = row["checkInterval"].as<time_t>(); jobset.checkInterval = row["checkInterval"].as<time_t>();
switch (row["jobset_enabled"].as<int>(0)) {
case 1:
jobset.evaluation_style = EvaluationStyle::SCHEDULE;
break;
case 2:
jobset.evaluation_style = EvaluationStyle::ONESHOT;
break;
case 3:
jobset.evaluation_style = EvaluationStyle::ONE_AT_A_TIME;
break;
}
seen.insert(name); seen.insert(name);
} }
@ -133,25 +152,83 @@ struct Evaluator
{ {
if (jobset.pid != -1) { if (jobset.pid != -1) {
// Already running. // Already running.
debug("shouldEvaluate %s:%s? no: already running",
jobset.name.first, jobset.name.second);
return false; return false;
} }
if (jobset.triggerTime == std::numeric_limits<time_t>::max()) { if (jobset.triggerTime != std::numeric_limits<time_t>::max()) {
// An evaluation of this Jobset is requested // An evaluation of this Jobset is requested
debug("shouldEvaluate %s:%s? yes: requested",
jobset.name.first, jobset.name.second);
return true; return true;
} }
if (jobset.checkInterval <= 0) { if (jobset.checkInterval <= 0) {
// Automatic scheduling is disabled. We allow requested // Automatic scheduling is disabled. We allow requested
// evaluations, but never schedule start one. // evaluations, but never schedule start one.
debug("shouldEvaluate %s:%s? no: checkInterval <= 0",
jobset.name.first, jobset.name.second);
return false;
}
if (jobset.lastCheckedTime + jobset.checkInterval <= time(0)) {
// Time to schedule a fresh evaluation. If the jobset
// is a ONE_AT_A_TIME jobset, ensure the previous jobset
// has no remaining, unfinished work.
auto conn(dbPool.get());
pqxx::work txn(*conn);
if (jobset.evaluation_style == EvaluationStyle::ONE_AT_A_TIME) {
auto evaluation_res = txn.parameterized
("select id from JobsetEvals "
"where project = $1 and jobset = $2 "
"order by id desc limit 1")
(jobset.name.first)
(jobset.name.second)
.exec();
if (evaluation_res.empty()) {
// First evaluation, so allow scheduling.
debug("shouldEvaluate(one-at-a-time) %s:%s? yes: no prior eval",
jobset.name.first, jobset.name.second);
return true;
}
auto evaluation_id = evaluation_res[0][0].as<int>();
auto unfinished_build_res = txn.parameterized
("select id from Builds "
"join JobsetEvalMembers "
" on (JobsetEvalMembers.build = Builds.id) "
"where JobsetEvalMembers.eval = $1 "
" and builds.finished = 0 "
" limit 1")
(evaluation_id)
.exec();
// If the previous evaluation has no unfinished builds
// schedule!
if (unfinished_build_res.empty()) {
debug("shouldEvaluate(one-at-a-time) %s:%s? yes: no unfinished builds",
jobset.name.first, jobset.name.second);
return true;
} else {
debug("shouldEvaluate(one-at-a-time) %s:%s? no: at least one unfinished build",
jobset.name.first, jobset.name.second);
return false; return false;
} }
if (jobset.lastCheckedTime + jobset.checkInterval <= time(0)) { } else {
// Time to schedule a fresh evaluation // EvaluationStyle::ONESHOT, EvaluationStyle::SCHEDULED
debug("shouldEvaluate(oneshot/scheduled) %s:%s? yes: checkInterval elapsed",
jobset.name.first, jobset.name.second);
return true; return true;
} }
}
return false; return false;
} }

View file

@ -226,7 +226,7 @@ sub updateJobset {
my ($nixExprPath, $nixExprInput) = nixExprPathFromParams $c; my ($nixExprPath, $nixExprInput) = nixExprPathFromParams $c;
my $enabled = int($c->stash->{params}->{enabled}); my $enabled = int($c->stash->{params}->{enabled});
die if $enabled < 0 || $enabled > 2; die if $enabled < 0 || $enabled > 3;
my $shares = int($c->stash->{params}->{schedulingshares} // 1); my $shares = int($c->stash->{params}->{schedulingshares} // 1);
error($c, "The number of scheduling shares must be positive.") if $shares <= 0; error($c, "The number of scheduling shares must be positive.") if $shares <= 0;

View file

@ -68,6 +68,7 @@
<input type="hidden" name="enabled" value="[% jobset.enabled %]" /> <input type="hidden" name="enabled" value="[% jobset.enabled %]" />
<button type="button" class="btn" value="1">Enabled</button> <button type="button" class="btn" value="1">Enabled</button>
<button type="button" class="btn" value="2">One-shot</button> <button type="button" class="btn" value="2">One-shot</button>
<button type="button" class="btn" value="3">One-at-a-time</button>
<button type="button" class="btn" value="0">Disabled</button> <button type="button" class="btn" value="0">Disabled</button>
</div> </div>
</div> </div>

View file

@ -129,7 +129,7 @@
<table class="info-table"> <table class="info-table">
<tr> <tr>
<th>State:</th> <th>State:</th>
<td>[% IF jobset.enabled == 0; "Disabled"; ELSIF jobset.enabled == 1; "Enabled"; ELSIF jobset.enabled == 2; "One-shot"; END %]</td> <td>[% IF jobset.enabled == 0; "Disabled"; ELSIF jobset.enabled == 1; "Enabled"; ELSIF jobset.enabled == 2; "One-shot"; ELSIF jobset.enabled == 3; "One-at-a-time"; END %]</td>
</tr> </tr>
<tr> <tr>
<th>Description:</th> <th>Description:</th>

View file

@ -61,7 +61,7 @@ create table Jobsets (
errorTime integer, -- timestamp associated with errorMsg errorTime integer, -- timestamp associated with errorMsg
lastCheckedTime integer, -- last time the evaluator looked at this jobset lastCheckedTime integer, -- last time the evaluator looked at this jobset
triggerTime integer, -- set if we were triggered by a push event triggerTime integer, -- set if we were triggered by a push event
enabled integer not null default 1, -- 0 = disabled, 1 = enabled, 2 = one-shot enabled integer not null default 1, -- 0 = disabled, 1 = enabled, 2 = one-shot, 3 = one-at-a-time
enableEmail integer not null default 1, enableEmail integer not null default 1,
hidden integer not null default 0, hidden integer not null default 0,
emailOverride text not null, emailOverride text not null,