hydra-evaluator: add a 'ONE_AT_A_TIME' evaluator style

In the past, jobsets which are automatically evaluated are evaluated
regularly, on a schedule. This schedule means a new evaluation is
created every checkInterval seconds (assuming something changed.)

This model works well for architectures where our build farm can
easily keep up with demand.

This commit adds a new type of evaluation, called ONE_AT_A_TIME, which
only schedules a new evaluation if the previous evaluation of the
jobset has no unfinished builds.

This model of evaluation lets us have 'low-tier' architectures.

For example, we could now have a jobset for ARMv7l builds, where
the buildfarm only has a single, underpowered ARMv7l builder.
Configuring that jobset as ONE_AT_A_TIME will create an evaluation
and then won't schedule another evaluation until every job of
the existing evaluation is complete.

This way, the cache will have a complete collection of pre-built
software for some commits, but the underpowered architecture will
never become backlogged in ancient revisions.

src/hydra-evaluator/hydra-evaluator.cc

@@ -15,6 +15,13 @@ using namespace nix;
 
 typedef std::pair<std::string, std::string> JobsetName;
 
+enum class EvaluationStyle
+{
+    SCHEDULE = 1,
+    ONESHOT = 2,
+    ONE_AT_A_TIME = 3,
+};
+
 struct Evaluator
 {
     std::unique_ptr<Config> config;
@@ -24,6 +31,7 @@ struct Evaluator
     struct Jobset
     {
         JobsetName name;
+        std::optional<EvaluationStyle> evaluation_style;
         time_t lastCheckedTime, triggerTime;
         int checkInterval;
         Pid pid;
@@ -60,7 +68,7 @@ struct Evaluator
         pqxx::work txn(*conn);
 
         auto res = txn.parameterized
-            ("select project, j.name, lastCheckedTime, triggerTime, checkInterval from Jobsets j join Projects p on j.project = p.name "
+            ("select project, j.name, lastCheckedTime, triggerTime, checkInterval, j.enabled as jobset_enabled from Jobsets j join Projects p on j.project = p.name "
              "where j.enabled != 0 and p.enabled != 0").exec();
 
         auto state(state_.lock());
@@ -78,6 +86,17 @@ struct Evaluator
             jobset.lastCheckedTime = row["lastCheckedTime"].as<time_t>(0);
             jobset.triggerTime = row["triggerTime"].as<time_t>(notTriggered);
             jobset.checkInterval = row["checkInterval"].as<time_t>();
+            switch (row["jobset_enabled"].as<int>(0)) {
+                case 1:
+                    jobset.evaluation_style = EvaluationStyle::SCHEDULE;
+                    break;
+                case 2:
+                    jobset.evaluation_style = EvaluationStyle::ONESHOT;
+                    break;
+                case 3:
+                    jobset.evaluation_style = EvaluationStyle::ONE_AT_A_TIME;
+                    break;
+            }
 
             seen.insert(name);
         }
@@ -133,24 +152,82 @@ struct Evaluator
     {
         if (jobset.pid != -1) {
             // Already running.
+            debug("shouldEvaluate %s:%s? no: already running",
+                  jobset.name.first, jobset.name.second);
             return false;
         }
 
-        if (jobset.triggerTime == std::numeric_limits<time_t>::max()) {
+        if (jobset.triggerTime != std::numeric_limits<time_t>::max()) {
             // An evaluation of this Jobset is requested
+            debug("shouldEvaluate %s:%s? yes: requested",
+                  jobset.name.first, jobset.name.second);
             return true;
         }
 
         if (jobset.checkInterval <= 0) {
             // Automatic scheduling is disabled. We allow requested
             // evaluations, but never schedule start one.
+            debug("shouldEvaluate %s:%s? no: checkInterval <= 0",
+                  jobset.name.first, jobset.name.second);
             return false;
         }
 
-
         if (jobset.lastCheckedTime + jobset.checkInterval <= time(0)) {
-            // Time to schedule a fresh evaluation
+            // Time to schedule a fresh evaluation. If the jobset
-            return true;
+            // is a ONE_AT_A_TIME jobset, ensure the previous jobset
+            // has no remaining, unfinished work.
+
+            auto conn(dbPool.get());
+
+            pqxx::work txn(*conn);
+
+            if (jobset.evaluation_style == EvaluationStyle::ONE_AT_A_TIME) {
+                auto evaluation_res = txn.parameterized
+                    ("select id from JobsetEvals "
+                     "where project = $1 and jobset = $2 "
+                     "order by id desc limit 1")
+                  (jobset.name.first)
+                  (jobset.name.second)
+                  .exec();
+
+                if (evaluation_res.empty()) {
+                    // First evaluation, so allow scheduling.
+                    debug("shouldEvaluate(one-at-a-time) %s:%s? yes: no prior eval",
+                          jobset.name.first, jobset.name.second);
+                    return true;
+                }
+
+                auto evaluation_id = evaluation_res[0][0].as<int>();
+
+                auto unfinished_build_res = txn.parameterized
+                    ("select id from Builds "
+                     "join JobsetEvalMembers "
+                     "    on (JobsetEvalMembers.build = Builds.id) "
+                     "where JobsetEvalMembers.eval = $1 "
+                     "  and builds.finished = 0 "
+                     " limit 1")
+                  (evaluation_id)
+                  .exec();
+
+                // If the previous evaluation has no unfinished builds
+                // schedule!
+                if (unfinished_build_res.empty()) {
+                    debug("shouldEvaluate(one-at-a-time) %s:%s? yes: no unfinished builds",
+                          jobset.name.first, jobset.name.second);
+                    return true;
+                } else {
+                    debug("shouldEvaluate(one-at-a-time) %s:%s? no: at least one unfinished build",
+                           jobset.name.first, jobset.name.second);
+                    return false;
+                }
+
+
+            } else {
+                // EvaluationStyle::ONESHOT, EvaluationStyle::SCHEDULED
+                debug("shouldEvaluate(oneshot/scheduled) %s:%s? yes: checkInterval elapsed",
+                      jobset.name.first, jobset.name.second);
+                return true;
+            }
         }
 
         return false;

src/lib/Hydra/Controller/Jobset.pm

@@ -226,7 +226,7 @@ sub updateJobset {
     my ($nixExprPath, $nixExprInput) = nixExprPathFromParams $c;
 
     my $enabled = int($c->stash->{params}->{enabled});
-    die if $enabled < 0 || $enabled > 2;
+    die if $enabled < 0 || $enabled > 3;
 
     my $shares = int($c->stash->{params}->{schedulingshares} // 1);
     error($c, "The number of scheduling shares must be positive.") if $shares <= 0;

src/root/edit-jobset.tt

@@ -68,6 +68,7 @@
           <input type="hidden" name="enabled" value="[% jobset.enabled %]" />
           <button type="button" class="btn" value="1">Enabled</button>
           <button type="button" class="btn" value="2">One-shot</button>
+          <button type="button" class="btn" value="3">One-at-a-time</button>
           <button type="button" class="btn" value="0">Disabled</button>
         </div>
       </div>

src/root/jobset.tt

@@ -129,7 +129,7 @@
     <table class="info-table">
       <tr>
         <th>State:</th>
-        <td>[% IF jobset.enabled == 0; "Disabled"; ELSIF jobset.enabled == 1; "Enabled"; ELSIF jobset.enabled == 2; "One-shot"; END %]</td>
+        <td>[% IF jobset.enabled == 0; "Disabled"; ELSIF jobset.enabled == 1; "Enabled"; ELSIF jobset.enabled == 2; "One-shot"; ELSIF jobset.enabled == 3; "One-at-a-time"; END %]</td>
       </tr>
       <tr>
         <th>Description:</th>

src/sql/hydra.sql

@@ -61,7 +61,7 @@ create table Jobsets (
     errorTime     integer, -- timestamp associated with errorMsg
     lastCheckedTime integer, -- last time the evaluator looked at this jobset
     triggerTime   integer, -- set if we were triggered by a push event
-    enabled       integer not null default 1, -- 0 = disabled, 1 = enabled, 2 = one-shot
+    enabled       integer not null default 1, -- 0 = disabled, 1 = enabled, 2 = one-shot, 3 = one-at-a-time
     enableEmail   integer not null default 1,
     hidden        integer not null default 0,
     emailOverride text not null,