Merge pull request #721 from grahamc/one-by-one

hydra-evaluator: add a 'ONE_AT_A_TIME' evaluator style

src/hydra-evaluator/hydra-evaluator.cc

@@ -15,6 +15,13 @@ using namespace nix;
 
 typedef std::pair<std::string, std::string> JobsetName;
 
+enum class EvaluationStyle
+{
+    SCHEDULE = 1,
+    ONESHOT = 2,
+    ONE_AT_A_TIME = 3,
+};
+
 struct Evaluator
 {
     std::unique_ptr<Config> config;
@@ -24,6 +31,7 @@ struct Evaluator
     struct Jobset
     {
         JobsetName name;
+        std::optional<EvaluationStyle> evaluation_style;
         time_t lastCheckedTime, triggerTime;
         int checkInterval;
         Pid pid;
@@ -60,7 +68,7 @@ struct Evaluator
         pqxx::work txn(*conn);
 
         auto res = txn.parameterized
-            ("select project, j.name, lastCheckedTime, triggerTime, checkInterval from Jobsets j join Projects p on j.project = p.name "
+            ("select project, j.name, lastCheckedTime, triggerTime, checkInterval, j.enabled as jobset_enabled from Jobsets j join Projects p on j.project = p.name "
              "where j.enabled != 0 and p.enabled != 0").exec();
 
         auto state(state_.lock());
@@ -78,6 +86,17 @@ struct Evaluator
             jobset.lastCheckedTime = row["lastCheckedTime"].as<time_t>(0);
             jobset.triggerTime = row["triggerTime"].as<time_t>(notTriggered);
             jobset.checkInterval = row["checkInterval"].as<time_t>();
+            switch (row["jobset_enabled"].as<int>(0)) {
+                case 1:
+                    jobset.evaluation_style = EvaluationStyle::SCHEDULE;
+                    break;
+                case 2:
+                    jobset.evaluation_style = EvaluationStyle::ONESHOT;
+                    break;
+                case 3:
+                    jobset.evaluation_style = EvaluationStyle::ONE_AT_A_TIME;
+                    break;
+            }
 
             seen.insert(name);
         }
@@ -129,19 +148,100 @@ struct Evaluator
         childStarted.notify_one();
     }
 
+    bool shouldEvaluate(Jobset & jobset)
+    {
+        if (jobset.pid != -1) {
+            // Already running.
+            debug("shouldEvaluate %s:%s? no: already running",
+                  jobset.name.first, jobset.name.second);
+            return false;
+        }
+
+        if (jobset.triggerTime != std::numeric_limits<time_t>::max()) {
+            // An evaluation of this Jobset is requested
+            debug("shouldEvaluate %s:%s? yes: requested",
+                  jobset.name.first, jobset.name.second);
+            return true;
+        }
+
+        if (jobset.checkInterval <= 0) {
+            // Automatic scheduling is disabled. We allow requested
+            // evaluations, but never schedule start one.
+            debug("shouldEvaluate %s:%s? no: checkInterval <= 0",
+                  jobset.name.first, jobset.name.second);
+            return false;
+        }
+
+        if (jobset.lastCheckedTime + jobset.checkInterval <= time(0)) {
+            // Time to schedule a fresh evaluation. If the jobset
+            // is a ONE_AT_A_TIME jobset, ensure the previous jobset
+            // has no remaining, unfinished work.
+
+            auto conn(dbPool.get());
+
+            pqxx::work txn(*conn);
+
+            if (jobset.evaluation_style == EvaluationStyle::ONE_AT_A_TIME) {
+                auto evaluation_res = txn.parameterized
+                    ("select id from JobsetEvals "
+                     "where project = $1 and jobset = $2 "
+                     "order by id desc limit 1")
+                  (jobset.name.first)
+                  (jobset.name.second)
+                  .exec();
+
+                if (evaluation_res.empty()) {
+                    // First evaluation, so allow scheduling.
+                    debug("shouldEvaluate(one-at-a-time) %s:%s? yes: no prior eval",
+                          jobset.name.first, jobset.name.second);
+                    return true;
+                }
+
+                auto evaluation_id = evaluation_res[0][0].as<int>();
+
+                auto unfinished_build_res = txn.parameterized
+                    ("select id from Builds "
+                     "join JobsetEvalMembers "
+                     "    on (JobsetEvalMembers.build = Builds.id) "
+                     "where JobsetEvalMembers.eval = $1 "
+                     "  and builds.finished = 0 "
+                     " limit 1")
+                  (evaluation_id)
+                  .exec();
+
+                // If the previous evaluation has no unfinished builds
+                // schedule!
+                if (unfinished_build_res.empty()) {
+                    debug("shouldEvaluate(one-at-a-time) %s:%s? yes: no unfinished builds",
+                          jobset.name.first, jobset.name.second);
+                    return true;
+                } else {
+                    debug("shouldEvaluate(one-at-a-time) %s:%s? no: at least one unfinished build",
+                           jobset.name.first, jobset.name.second);
+                    return false;
+                }
+
+
+            } else {
+                // EvaluationStyle::ONESHOT, EvaluationStyle::SCHEDULED
+                debug("shouldEvaluate(oneshot/scheduled) %s:%s? yes: checkInterval elapsed",
+                      jobset.name.first, jobset.name.second);
+                return true;
+            }
+        }
+
+        return false;
+    }
+
     void startEvals(State & state)
     {
         std::vector<Jobsets::iterator> sorted;
 
-        time_t now = time(0);
-
         /* Filter out jobsets that have been evaluated recently and have
            not been triggered. */
         for (auto i = state.jobsets.begin(); i != state.jobsets.end(); ++i)
             if (evalOne ||
-                (i->second.pid == -1 &&
-                 (i->second.triggerTime != std::numeric_limits<time_t>::max() ||
-                     (i->second.checkInterval > 0 && i->second.lastCheckedTime + i->second.checkInterval <= now))))
+                (i->second.evaluation_style && shouldEvaluate(i->second)))
                 sorted.push_back(i);
 
         /* Put jobsets in order of ascending trigger time, last checked

src/lib/Hydra/Controller/Jobset.pm

@@ -226,7 +226,7 @@ sub updateJobset {
     my ($nixExprPath, $nixExprInput) = nixExprPathFromParams $c;
 
     my $enabled = int($c->stash->{params}->{enabled});
-    die if $enabled < 0 || $enabled > 2;
+    die if $enabled < 0 || $enabled > 3;
 
     my $shares = int($c->stash->{params}->{schedulingshares} // 1);
     error($c, "The number of scheduling shares must be positive.") if $shares <= 0;

src/root/edit-jobset.tt

@@ -68,6 +68,7 @@
           <input type="hidden" name="enabled" value="[% jobset.enabled %]" />
           <button type="button" class="btn" value="1">Enabled</button>
           <button type="button" class="btn" value="2">One-shot</button>
+          <button type="button" class="btn" value="3">One-at-a-time</button>
           <button type="button" class="btn" value="0">Disabled</button>
         </div>
       </div>

src/root/jobset.tt

@@ -129,7 +129,7 @@
     <table class="info-table">
       <tr>
         <th>State:</th>
-        <td>[% IF jobset.enabled == 0; "Disabled"; ELSIF jobset.enabled == 1; "Enabled"; ELSIF jobset.enabled == 2; "One-shot"; END %]</td>
+        <td>[% IF jobset.enabled == 0; "Disabled"; ELSIF jobset.enabled == 1; "Enabled"; ELSIF jobset.enabled == 2; "One-shot"; ELSIF jobset.enabled == 3; "One-at-a-time"; END %]</td>
       </tr>
       <tr>
         <th>Description:</th>

src/sql/hydra.sql

@@ -61,7 +61,7 @@ create table Jobsets (
     errorTime     integer, -- timestamp associated with errorMsg
     lastCheckedTime integer, -- last time the evaluator looked at this jobset
     triggerTime   integer, -- set if we were triggered by a push event
-    enabled       integer not null default 1, -- 0 = disabled, 1 = enabled, 2 = one-shot
+    enabled       integer not null default 1, -- 0 = disabled, 1 = enabled, 2 = one-shot, 3 = one-at-a-time
     enableEmail   integer not null default 1,
     hidden        integer not null default 0,
     emailOverride text not null,