diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index cbb6872c..4a4084e0 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -147,9 +147,10 @@ static void worker( nlohmann::json reply; try { - auto v = findAlongAttrPath(state, attrPath, autoArgs, *vRoot).first; + auto vTmp = findAlongAttrPath(state, attrPath, autoArgs, *vRoot).first; - state.forceValue(*v); + auto v = state.allocValue(); + state.autoCallFunction(autoArgs, *vTmp, *v); if (auto drv = getDerivation(state, *v, false)) { @@ -231,6 +232,11 @@ static void worker( reply["attrs"] = std::move(attrs); } + else if (v->type == tNull) + ; + + else throw TypeError("attribute '%s' is %s, which is not supported", attrPath, showType(*v)); + } catch (EvalError & e) { reply["error"] = filterANSIEscapes(e.msg(), true); } diff --git a/src/hydra-evaluator/hydra-evaluator.cc b/src/hydra-evaluator/hydra-evaluator.cc index fd9eb9d7..411d050f 100644 --- a/src/hydra-evaluator/hydra-evaluator.cc +++ b/src/hydra-evaluator/hydra-evaluator.cc @@ -15,6 +15,13 @@ using namespace nix; typedef std::pair JobsetName; +enum class EvaluationStyle +{ + SCHEDULE = 1, + ONESHOT = 2, + ONE_AT_A_TIME = 3, +}; + struct Evaluator { std::unique_ptr config; @@ -24,6 +31,7 @@ struct Evaluator struct Jobset { JobsetName name; + std::optional evaluation_style; time_t lastCheckedTime, triggerTime; int checkInterval; Pid pid; @@ -60,7 +68,7 @@ struct Evaluator pqxx::work txn(*conn); auto res = txn.parameterized - ("select project, j.name, lastCheckedTime, triggerTime, checkInterval from Jobsets j join Projects p on j.project = p.name " + ("select project, j.name, lastCheckedTime, triggerTime, checkInterval, j.enabled as jobset_enabled from Jobsets j join Projects p on j.project = p.name " "where j.enabled != 0 and p.enabled != 0").exec(); auto state(state_.lock()); @@ -78,6 +86,17 @@ struct Evaluator jobset.lastCheckedTime = row["lastCheckedTime"].as(0); jobset.triggerTime = row["triggerTime"].as(notTriggered); jobset.checkInterval = row["checkInterval"].as(); + switch (row["jobset_enabled"].as(0)) { + case 1: + jobset.evaluation_style = EvaluationStyle::SCHEDULE; + break; + case 2: + jobset.evaluation_style = EvaluationStyle::ONESHOT; + break; + case 3: + jobset.evaluation_style = EvaluationStyle::ONE_AT_A_TIME; + break; + } seen.insert(name); } @@ -129,19 +148,100 @@ struct Evaluator childStarted.notify_one(); } + bool shouldEvaluate(Jobset & jobset) + { + if (jobset.pid != -1) { + // Already running. + debug("shouldEvaluate %s:%s? no: already running", + jobset.name.first, jobset.name.second); + return false; + } + + if (jobset.triggerTime != std::numeric_limits::max()) { + // An evaluation of this Jobset is requested + debug("shouldEvaluate %s:%s? yes: requested", + jobset.name.first, jobset.name.second); + return true; + } + + if (jobset.checkInterval <= 0) { + // Automatic scheduling is disabled. We allow requested + // evaluations, but never schedule start one. + debug("shouldEvaluate %s:%s? no: checkInterval <= 0", + jobset.name.first, jobset.name.second); + return false; + } + + if (jobset.lastCheckedTime + jobset.checkInterval <= time(0)) { + // Time to schedule a fresh evaluation. If the jobset + // is a ONE_AT_A_TIME jobset, ensure the previous jobset + // has no remaining, unfinished work. + + auto conn(dbPool.get()); + + pqxx::work txn(*conn); + + if (jobset.evaluation_style == EvaluationStyle::ONE_AT_A_TIME) { + auto evaluation_res = txn.parameterized + ("select id from JobsetEvals " + "where project = $1 and jobset = $2 " + "order by id desc limit 1") + (jobset.name.first) + (jobset.name.second) + .exec(); + + if (evaluation_res.empty()) { + // First evaluation, so allow scheduling. + debug("shouldEvaluate(one-at-a-time) %s:%s? yes: no prior eval", + jobset.name.first, jobset.name.second); + return true; + } + + auto evaluation_id = evaluation_res[0][0].as(); + + auto unfinished_build_res = txn.parameterized + ("select id from Builds " + "join JobsetEvalMembers " + " on (JobsetEvalMembers.build = Builds.id) " + "where JobsetEvalMembers.eval = $1 " + " and builds.finished = 0 " + " limit 1") + (evaluation_id) + .exec(); + + // If the previous evaluation has no unfinished builds + // schedule! + if (unfinished_build_res.empty()) { + debug("shouldEvaluate(one-at-a-time) %s:%s? yes: no unfinished builds", + jobset.name.first, jobset.name.second); + return true; + } else { + debug("shouldEvaluate(one-at-a-time) %s:%s? no: at least one unfinished build", + jobset.name.first, jobset.name.second); + return false; + } + + + } else { + // EvaluationStyle::ONESHOT, EvaluationStyle::SCHEDULED + debug("shouldEvaluate(oneshot/scheduled) %s:%s? yes: checkInterval elapsed", + jobset.name.first, jobset.name.second); + return true; + } + } + + return false; + } + void startEvals(State & state) { std::vector sorted; - time_t now = time(0); - /* Filter out jobsets that have been evaluated recently and have not been triggered. */ for (auto i = state.jobsets.begin(); i != state.jobsets.end(); ++i) if (evalOne || - (i->second.pid == -1 && - (i->second.triggerTime != std::numeric_limits::max() || - (i->second.checkInterval > 0 && i->second.lastCheckedTime + i->second.checkInterval <= now)))) + (i->second.evaluation_style && shouldEvaluate(i->second))) sorted.push_back(i); /* Put jobsets in order of ascending trigger time, last checked diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm index 22bfd98e..52cb71dd 100644 --- a/src/lib/Hydra/Controller/Build.pm +++ b/src/lib/Hydra/Controller/Build.pm @@ -193,7 +193,8 @@ sub checkPath { sub serveFile { my ($c, $path) = @_; - my $res = run(cmd => ["nix", "ls-store", "--store", getStoreUri(), "--json", "$path"]); + my $res = run(cmd => ["nix", "--experimental-features", "nix-command", + "ls-store", "--store", getStoreUri(), "--json", "$path"]); if ($res->{status}) { notFound($c, "File '$path' does not exist.") if $res->{stderr} =~ /does not exist/; @@ -217,7 +218,8 @@ sub serveFile { elsif ($ls->{type} eq "regular") { - $c->stash->{'plain'} = { data => grab(cmd => ["nix", "cat-store", "--store", getStoreUri(), "$path"]) }; + $c->stash->{'plain'} = { data => grab(cmd => ["nix", "--experimental-features", "nix-command", + "cat-store", "--store", getStoreUri(), "$path"]) }; # Detect MIME type. Borrowed from Catalyst::Plugin::Static::Simple. my $type = "text/plain"; diff --git a/src/lib/Hydra/Controller/Jobset.pm b/src/lib/Hydra/Controller/Jobset.pm index 578b9010..b4c20e03 100644 --- a/src/lib/Hydra/Controller/Jobset.pm +++ b/src/lib/Hydra/Controller/Jobset.pm @@ -238,7 +238,7 @@ sub updateJobset { } my $enabled = int($c->stash->{params}->{enabled}); - die if $enabled < 0 || $enabled > 2; + die if $enabled < 0 || $enabled > 3; my $shares = int($c->stash->{params}->{schedulingshares} // 1); error($c, "The number of scheduling shares must be positive.") if $shares <= 0; diff --git a/src/lib/Hydra/Helper/AddBuilds.pm b/src/lib/Hydra/Helper/AddBuilds.pm index 3b8869d6..394cd42a 100644 --- a/src/lib/Hydra/Helper/AddBuilds.pm +++ b/src/lib/Hydra/Helper/AddBuilds.pm @@ -70,8 +70,14 @@ sub handleDeclarativeJobsetBuild { my $id = $build->id; die "Declarative jobset build $id failed" unless $build->buildstatus == 0; my $declPath = ($build->buildoutputs)[0]->path; - my $declText = readNixFile($declPath) - or die "Couldn't read declarative specification file $declPath: $!"; + my $declText = eval { + readNixFile($declPath) + }; + if ($@) { + print STDERR "ERROR: failed to readNixFile $declPath: ", $@, "\n"; + die; + } + my $declSpec = decode_json($declText); txn_do($db, sub { my @kept = keys %$declSpec; diff --git a/src/lib/Hydra/Helper/Nix.pm b/src/lib/Hydra/Helper/Nix.pm index 8ce284ad..5034c81b 100644 --- a/src/lib/Hydra/Helper/Nix.pm +++ b/src/lib/Hydra/Helper/Nix.pm @@ -509,7 +509,8 @@ sub getStoreUri { # Read a file from the (possibly remote) nix store sub readNixFile { my ($path) = @_; - return grab(cmd => ["nix", "cat-store", "--store", getStoreUri(), "$path"]); + return grab(cmd => ["nix", "--experimental-features", "nix-command", + "cat-store", "--store", getStoreUri(), "$path"]); } diff --git a/src/root/edit-jobset.tt b/src/root/edit-jobset.tt index 95b1afaf..02391651 100644 --- a/src/root/edit-jobset.tt +++ b/src/root/edit-jobset.tt @@ -68,6 +68,7 @@ + diff --git a/src/root/jobset.tt b/src/root/jobset.tt index 4b891ff6..b53a9af2 100644 --- a/src/root/jobset.tt +++ b/src/root/jobset.tt @@ -129,7 +129,7 @@ - + diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset index 7ed1ffa9..d76cf306 100755 --- a/src/script/hydra-eval-jobset +++ b/src/script/hydra-eval-jobset @@ -82,7 +82,7 @@ sub getPath { my $substituter = $config->{eval_substituter}; - system("nix", "copy", "--from", $substituter, "--", $path) + system("nix", "--experimental-features", "nix-command", "copy", "--from", $substituter, "--", $path) if defined $substituter; return isValidPath($path); diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index 8144dd30..a5fdc802 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -61,7 +61,7 @@ create table Jobsets ( errorTime integer, -- timestamp associated with errorMsg lastCheckedTime integer, -- last time the evaluator looked at this jobset triggerTime integer, -- set if we were triggered by a push event - enabled integer not null default 1, -- 0 = disabled, 1 = enabled, 2 = one-shot + enabled integer not null default 1, -- 0 = disabled, 1 = enabled, 2 = one-shot, 3 = one-at-a-time enableEmail integer not null default 1, hidden integer not null default 0, emailOverride text not null,
State:[% IF jobset.enabled == 0; "Disabled"; ELSIF jobset.enabled == 1; "Enabled"; ELSIF jobset.enabled == 2; "One-shot"; END %][% IF jobset.enabled == 0; "Disabled"; ELSIF jobset.enabled == 1; "Enabled"; ELSIF jobset.enabled == 2; "One-shot"; ELSIF jobset.enabled == 3; "One-at-a-time"; END %]
Description: