From 5fae9d96a25c658e4b3ea4f1c121b8d815ba6492 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 3 Mar 2020 19:53:18 -0500 Subject: [PATCH] hydra-evaluator: add a 'ONE_AT_A_TIME' evaluator style In the past, jobsets which are automatically evaluated are evaluated regularly, on a schedule. This schedule means a new evaluation is created every checkInterval seconds (assuming something changed.) This model works well for architectures where our build farm can easily keep up with demand. This commit adds a new type of evaluation, called ONE_AT_A_TIME, which only schedules a new evaluation if the previous evaluation of the jobset has no unfinished builds. This model of evaluation lets us have 'low-tier' architectures. For example, we could now have a jobset for ARMv7l builds, where the buildfarm only has a single, underpowered ARMv7l builder. Configuring that jobset as ONE_AT_A_TIME will create an evaluation and then won't schedule another evaluation until every job of the existing evaluation is complete. This way, the cache will have a complete collection of pre-built software for some commits, but the underpowered architecture will never become backlogged in ancient revisions. --- src/hydra-evaluator/hydra-evaluator.cc | 87 ++++++++++++++++++++++++-- src/lib/Hydra/Controller/Jobset.pm | 2 +- src/root/edit-jobset.tt | 1 + src/root/jobset.tt | 2 +- src/sql/hydra.sql | 2 +- 5 files changed, 86 insertions(+), 8 deletions(-) diff --git a/src/hydra-evaluator/hydra-evaluator.cc b/src/hydra-evaluator/hydra-evaluator.cc index e9103b84..364a5351 100644 --- a/src/hydra-evaluator/hydra-evaluator.cc +++ b/src/hydra-evaluator/hydra-evaluator.cc @@ -15,6 +15,13 @@ using namespace nix; typedef std::pair JobsetName; +enum class EvaluationStyle +{ + SCHEDULE = 1, + ONESHOT = 2, + ONE_AT_A_TIME = 3, +}; + struct Evaluator { std::unique_ptr config; @@ -24,6 +31,7 @@ struct Evaluator struct Jobset { JobsetName name; + std::optional evaluation_style; time_t lastCheckedTime, triggerTime; int checkInterval; Pid pid; @@ -60,7 +68,7 @@ struct Evaluator pqxx::work txn(*conn); auto res = txn.parameterized - ("select project, j.name, lastCheckedTime, triggerTime, checkInterval from Jobsets j join Projects p on j.project = p.name " + ("select project, j.name, lastCheckedTime, triggerTime, checkInterval, j.enabled as jobset_enabled from Jobsets j join Projects p on j.project = p.name " "where j.enabled != 0 and p.enabled != 0").exec(); auto state(state_.lock()); @@ -78,6 +86,17 @@ struct Evaluator jobset.lastCheckedTime = row["lastCheckedTime"].as(0); jobset.triggerTime = row["triggerTime"].as(notTriggered); jobset.checkInterval = row["checkInterval"].as(); + switch (row["jobset_enabled"].as(0)) { + case 1: + jobset.evaluation_style = EvaluationStyle::SCHEDULE; + break; + case 2: + jobset.evaluation_style = EvaluationStyle::ONESHOT; + break; + case 3: + jobset.evaluation_style = EvaluationStyle::ONE_AT_A_TIME; + break; + } seen.insert(name); } @@ -133,24 +152,82 @@ struct Evaluator { if (jobset.pid != -1) { // Already running. + debug("shouldEvaluate %s:%s? no: already running", + jobset.name.first, jobset.name.second); return false; } - if (jobset.triggerTime == std::numeric_limits::max()) { + if (jobset.triggerTime != std::numeric_limits::max()) { // An evaluation of this Jobset is requested + debug("shouldEvaluate %s:%s? yes: requested", + jobset.name.first, jobset.name.second); return true; } if (jobset.checkInterval <= 0) { // Automatic scheduling is disabled. We allow requested // evaluations, but never schedule start one. + debug("shouldEvaluate %s:%s? no: checkInterval <= 0", + jobset.name.first, jobset.name.second); return false; } - if (jobset.lastCheckedTime + jobset.checkInterval <= time(0)) { - // Time to schedule a fresh evaluation - return true; + // Time to schedule a fresh evaluation. If the jobset + // is a ONE_AT_A_TIME jobset, ensure the previous jobset + // has no remaining, unfinished work. + + auto conn(dbPool.get()); + + pqxx::work txn(*conn); + + if (jobset.evaluation_style == EvaluationStyle::ONE_AT_A_TIME) { + auto evaluation_res = txn.parameterized + ("select id from JobsetEvals " + "where project = $1 and jobset = $2 " + "order by id desc limit 1") + (jobset.name.first) + (jobset.name.second) + .exec(); + + if (evaluation_res.empty()) { + // First evaluation, so allow scheduling. + debug("shouldEvaluate(one-at-a-time) %s:%s? yes: no prior eval", + jobset.name.first, jobset.name.second); + return true; + } + + auto evaluation_id = evaluation_res[0][0].as(); + + auto unfinished_build_res = txn.parameterized + ("select id from Builds " + "join JobsetEvalMembers " + " on (JobsetEvalMembers.build = Builds.id) " + "where JobsetEvalMembers.eval = $1 " + " and builds.finished = 0 " + " limit 1") + (evaluation_id) + .exec(); + + // If the previous evaluation has no unfinished builds + // schedule! + if (unfinished_build_res.empty()) { + debug("shouldEvaluate(one-at-a-time) %s:%s? yes: no unfinished builds", + jobset.name.first, jobset.name.second); + return true; + } else { + debug("shouldEvaluate(one-at-a-time) %s:%s? no: at least one unfinished build", + jobset.name.first, jobset.name.second); + return false; + } + + + } else { + // EvaluationStyle::ONESHOT, EvaluationStyle::SCHEDULED + debug("shouldEvaluate(oneshot/scheduled) %s:%s? yes: checkInterval elapsed", + jobset.name.first, jobset.name.second); + return true; + } } return false; diff --git a/src/lib/Hydra/Controller/Jobset.pm b/src/lib/Hydra/Controller/Jobset.pm index 91e21dd4..5ce4aab4 100644 --- a/src/lib/Hydra/Controller/Jobset.pm +++ b/src/lib/Hydra/Controller/Jobset.pm @@ -226,7 +226,7 @@ sub updateJobset { my ($nixExprPath, $nixExprInput) = nixExprPathFromParams $c; my $enabled = int($c->stash->{params}->{enabled}); - die if $enabled < 0 || $enabled > 2; + die if $enabled < 0 || $enabled > 3; my $shares = int($c->stash->{params}->{schedulingshares} // 1); error($c, "The number of scheduling shares must be positive.") if $shares <= 0; diff --git a/src/root/edit-jobset.tt b/src/root/edit-jobset.tt index 6c380a3a..35ac668f 100644 --- a/src/root/edit-jobset.tt +++ b/src/root/edit-jobset.tt @@ -68,6 +68,7 @@ + diff --git a/src/root/jobset.tt b/src/root/jobset.tt index 9cf1202a..50be0f65 100644 --- a/src/root/jobset.tt +++ b/src/root/jobset.tt @@ -129,7 +129,7 @@ - + diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index 8144dd30..a5fdc802 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -61,7 +61,7 @@ create table Jobsets ( errorTime integer, -- timestamp associated with errorMsg lastCheckedTime integer, -- last time the evaluator looked at this jobset triggerTime integer, -- set if we were triggered by a push event - enabled integer not null default 1, -- 0 = disabled, 1 = enabled, 2 = one-shot + enabled integer not null default 1, -- 0 = disabled, 1 = enabled, 2 = one-shot, 3 = one-at-a-time enableEmail integer not null default 1, hidden integer not null default 0, emailOverride text not null,
State:[% IF jobset.enabled == 0; "Disabled"; ELSIF jobset.enabled == 1; "Enabled"; ELSIF jobset.enabled == 2; "One-shot"; END %][% IF jobset.enabled == 0; "Disabled"; ELSIF jobset.enabled == 1; "Enabled"; ELSIF jobset.enabled == 2; "One-shot"; ELSIF jobset.enabled == 3; "One-at-a-time"; END %]
Description: