From dc5a0d59c5220bec686484f54730ce061a7cde2a Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 25 Jan 2021 14:37:32 -0500 Subject: [PATCH 01/10] sql: Stop loading SQL if an error occurs Otherwise we may go ahead and create DBIx classes for a half-loaded schema. --- src/sql/update-dbix-harness.sh | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/src/sql/update-dbix-harness.sh b/src/sql/update-dbix-harness.sh index 38407f82..7f381e9f 100755 --- a/src/sql/update-dbix-harness.sh +++ b/src/sql/update-dbix-harness.sh @@ -1,24 +1,26 @@ #!/usr/bin/env bash +set -eux + readonly scratch=$(mktemp -d -t tmp.XXXXXXXXXX) readonly socket=$scratch/socket readonly data=$scratch/data readonly dbname=hydra-update-dbix -function finish { - set +e - pg_ctl -D "$data" \ - -o "-F -h '' -k \"$socket\"" \ - -w stop -m immediate +function finish() { + set +e + pg_ctl -D "$data" \ + -o "-F -h '' -k \"$socket\"" \ + -w stop -m immediate - if [ -f "$data/postmaster.pid" ]; then - pg_ctl -D "$data" \ - -o "-F -h '' -k \"$socket\"" \ - -w kill TERM "$(cat "$data/postmaster.pid")" - fi + if [ -f "$data/postmaster.pid" ]; then + pg_ctl -D "$data" \ + -o "-F -h '' -k \"$socket\"" \ + -w kill TERM "$(cat "$data/postmaster.pid")" + fi - rm -rf "$scratch" + rm -rf "$scratch" } trap finish EXIT @@ -33,8 +35,11 @@ pg_ctl -D "$data" \ createdb -h "$socket" "$dbname" -psql -h "$socket" "$dbname" -f ./hydra.sql +psql --host "$socket" \ + --set ON_ERROR_STOP=1 \ + --file ./hydra.sql \ + "$dbname" perl -I ../lib \ - -MDBIx::Class::Schema::Loader=make_schema_at,dump_to_dir:../lib \ - update-dbix.pl "dbi:Pg:dbname=$dbname;host=$socket" + -MDBIx::Class::Schema::Loader=make_schema_at,dump_to_dir:../lib \ + update-dbix.pl "dbi:Pg:dbname=$dbname;host=$socket" From bf674a96534816819f7d71a91f2274bf4aa59798 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 25 Jan 2021 14:25:18 -0500 Subject: [PATCH 02/10] hydra.sql: embed some in-line docs about schema changes --- src/sql/hydra.sql | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index 07d15ddc..ab2c6159 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -1,3 +1,14 @@ +-- Making a database change: +-- +-- 1. Update this schema document to match what the end result should be. +-- +-- 2. Run `make -C src/sql update-dbix hydra-postgresql.sql` in the root +-- of the project directory, and git add / git commit the changed, +-- generated files. +-- +-- 3. Create a migration in this same directory, named `upgrade-N.sql` +-- + -- Singleton table to keep track of the schema version. create table SchemaVersion ( version integer not null From 99e3c83358719981102ee49e07a1ce73ca209d06 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 25 Jan 2021 14:43:48 -0500 Subject: [PATCH 03/10] JobsetEvals: noop: re-run the generator to update the order of fields --- src/lib/Hydra/Schema/JobsetEvals.pm | 32 ++++++++++++++--------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/lib/Hydra/Schema/JobsetEvals.pm b/src/lib/Hydra/Schema/JobsetEvals.pm index 36bab6c1..3fd5b469 100644 --- a/src/lib/Hydra/Schema/JobsetEvals.pm +++ b/src/lib/Hydra/Schema/JobsetEvals.pm @@ -89,16 +89,6 @@ __PACKAGE__->table("jobsetevals"); data_type: 'text' is_nullable: 0 -=head2 nixexprinput - - data_type: 'text' - is_nullable: 1 - -=head2 nixexprpath - - data_type: 'text' - is_nullable: 1 - =head2 nrbuilds data_type: 'integer' @@ -114,6 +104,16 @@ __PACKAGE__->table("jobsetevals"); data_type: 'text' is_nullable: 1 +=head2 nixexprinput + + data_type: 'text' + is_nullable: 1 + +=head2 nixexprpath + + data_type: 'text' + is_nullable: 1 + =cut __PACKAGE__->add_columns( @@ -142,16 +142,16 @@ __PACKAGE__->add_columns( { data_type => "integer", is_nullable => 0 }, "hash", { data_type => "text", is_nullable => 0 }, - "nixexprinput", - { data_type => "text", is_nullable => 1 }, - "nixexprpath", - { data_type => "text", is_nullable => 1 }, "nrbuilds", { data_type => "integer", is_nullable => 1 }, "nrsucceeded", { data_type => "integer", is_nullable => 1 }, "flake", { data_type => "text", is_nullable => 1 }, + "nixexprinput", + { data_type => "text", is_nullable => 1 }, + "nixexprpath", + { data_type => "text", is_nullable => 1 }, ); =head1 PRIMARY KEY @@ -229,8 +229,8 @@ __PACKAGE__->belongs_to( ); -# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-22 07:11:57 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:hdu+0WWo2363dVvImMKxdA +# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-25 14:43:28 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:VKQNG53wwdbO8p1CTdX+WA __PACKAGE__->has_many( "buildIds", From ac3e8a4a5920797ce04f1fbc0fe8beb086a2472a Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 25 Jan 2021 14:51:33 -0500 Subject: [PATCH 04/10] jobsetevals: refer to jobset by ID --- src/lib/Hydra/Schema/JobsetEvals.pm | 39 ++++++----------------------- src/lib/Hydra/Schema/Jobsets.pm | 9 +++---- src/lib/Hydra/Schema/Projects.pm | 19 ++------------ src/sql/hydra.sql | 10 +++----- src/sql/upgrade-72.sql | 22 ++++++++++++++++ 5 files changed, 39 insertions(+), 60 deletions(-) create mode 100644 src/sql/upgrade-72.sql diff --git a/src/lib/Hydra/Schema/JobsetEvals.pm b/src/lib/Hydra/Schema/JobsetEvals.pm index 3fd5b469..0d4a013f 100644 --- a/src/lib/Hydra/Schema/JobsetEvals.pm +++ b/src/lib/Hydra/Schema/JobsetEvals.pm @@ -42,15 +42,9 @@ __PACKAGE__->table("jobsetevals"); is_nullable: 0 sequence: 'jobsetevals_id_seq' -=head2 project +=head2 jobset_id - data_type: 'text' - is_foreign_key: 1 - is_nullable: 0 - -=head2 jobset - - data_type: 'text' + data_type: 'integer' is_foreign_key: 1 is_nullable: 0 @@ -124,10 +118,8 @@ __PACKAGE__->add_columns( is_nullable => 0, sequence => "jobsetevals_id_seq", }, - "project", - { data_type => "text", is_foreign_key => 1, is_nullable => 0 }, - "jobset", - { data_type => "text", is_foreign_key => 1, is_nullable => 0 }, + "jobset_id", + { data_type => "integer", is_foreign_key => 1, is_nullable => 0 }, "errormsg", { data_type => "text", is_nullable => 1 }, "errortime", @@ -179,8 +171,8 @@ Related object: L __PACKAGE__->belongs_to( "jobset", "Hydra::Schema::Jobsets", - { name => "jobset", project => "project" }, - { is_deferrable => 0, on_delete => "CASCADE", on_update => "CASCADE" }, + { id => "jobset_id" }, + { is_deferrable => 0, on_delete => "CASCADE", on_update => "NO ACTION" }, ); =head2 jobsetevalinputs @@ -213,24 +205,9 @@ __PACKAGE__->has_many( undef, ); -=head2 project -Type: belongs_to - -Related object: L - -=cut - -__PACKAGE__->belongs_to( - "project", - "Hydra::Schema::Projects", - { name => "project" }, - { is_deferrable => 0, on_delete => "CASCADE", on_update => "CASCADE" }, -); - - -# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-25 14:43:28 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:VKQNG53wwdbO8p1CTdX+WA +# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-25 14:44:07 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:OVxeYH+eoZZrAsAJ2/mAAA __PACKAGE__->has_many( "buildIds", diff --git a/src/lib/Hydra/Schema/Jobsets.pm b/src/lib/Hydra/Schema/Jobsets.pm index 6ca83dbb..b2dc0131 100644 --- a/src/lib/Hydra/Schema/Jobsets.pm +++ b/src/lib/Hydra/Schema/Jobsets.pm @@ -301,10 +301,7 @@ Related object: L __PACKAGE__->has_many( "jobsetevals", "Hydra::Schema::JobsetEvals", - { - "foreign.jobset" => "self.name", - "foreign.project" => "self.project", - }, + { "foreign.jobset_id" => "self.id" }, undef, ); @@ -375,8 +372,8 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-22 07:11:57 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:6P1qlC5oVSPRSgRBp6nmrw +# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-25 14:38:14 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:7XtIqrrGAIvReqly1kapog =head2 builds diff --git a/src/lib/Hydra/Schema/Projects.pm b/src/lib/Hydra/Schema/Projects.pm index d3bd1911..09f92af3 100644 --- a/src/lib/Hydra/Schema/Projects.pm +++ b/src/lib/Hydra/Schema/Projects.pm @@ -157,21 +157,6 @@ __PACKAGE__->has_many( undef, ); -=head2 jobsetevals - -Type: has_many - -Related object: L - -=cut - -__PACKAGE__->has_many( - "jobsetevals", - "Hydra::Schema::JobsetEvals", - { "foreign.project" => "self.name" }, - undef, -); - =head2 jobsetrenames Type: has_many @@ -258,8 +243,8 @@ Composing rels: L -> username __PACKAGE__->many_to_many("usernames", "projectmembers", "username"); -# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-22 07:11:57 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Ff5gJejFu+02b0lInobOoQ +# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-25 14:38:14 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:+4yWd9UjCyxxLZYDrVUAxA my %hint = ( columns => [ diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index ab2c6159..6e56960f 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -440,9 +440,7 @@ create table SystemTypes ( create table JobsetEvals ( id serial primary key not null, - - project text not null, - jobset text not null, + jobset_id integer not null, errorMsg text, -- error output from the evaluator errorTime integer, -- timestamp associated with errorMsg @@ -473,8 +471,7 @@ create table JobsetEvals ( nixExprInput text, -- name of the jobsetInput containing the Nix or Guix expression nixExprPath text, -- relative path of the Nix or Guix expression - foreign key (project) references Projects(name) on delete cascade on update cascade, - foreign key (project, jobset) references Jobsets(project, name) on delete cascade on update cascade + foreign key (jobset_id) references Jobsets(id) on delete cascade ); @@ -629,7 +626,8 @@ create index IndexBuildOutputsPath on BuildOutputs using hash(path); create index IndexBuildsOnKeep on Builds(keep) where keep = 1; -- To get the most recent eval for a jobset. -create index IndexJobsetEvalsOnJobsetId on JobsetEvals(project, jobset, id desc) where hasNewBuilds = 1; +create index IndexJobsetEvalsOnJobsetId on JobsetEvals(jobset_id, id desc) where hasNewBuilds = 1; +create index IndexJobsetIdEvals on JobsetEvals(jobset_id) where hasNewBuilds = 1; create index IndexBuildsOnNotificationPendingSince on Builds(notificationPendingSince) where notificationPendingSince is not null; diff --git a/src/sql/upgrade-72.sql b/src/sql/upgrade-72.sql new file mode 100644 index 00000000..89eab500 --- /dev/null +++ b/src/sql/upgrade-72.sql @@ -0,0 +1,22 @@ + +ALTER TABLE JobsetEvals + ADD COLUMN jobset_id integer NULL, + ADD FOREIGN KEY (jobset_id) + REFERENCES Jobsets(id) + ON DELETE CASCADE; + +UPDATE JobsetEvals + SET jobset_id = ( + SELECT jobsets.id + FROM jobsets + WHERE jobsets.name = JobsetEvals.jobset + AND jobsets.project = JobsetEvals.project + ); + + +ALTER TABLE JobsetEvals + ALTER COLUMN jobset_id SET NOT NULL, + DROP COLUMN jobset, + DROP COLUMN project; + +create index IndexJobsetIdEvals on JobsetEvals(jobset_id) where hasNewBuilds = 1; \ No newline at end of file From 705a45df2b3d9cc08321113360efdc5ea6eb0a51 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 25 Jan 2021 15:11:57 -0500 Subject: [PATCH 05/10] hydra-evaluator: reformat readJobsets query --- src/hydra-evaluator/hydra-evaluator.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/hydra-evaluator/hydra-evaluator.cc b/src/hydra-evaluator/hydra-evaluator.cc index 1cb93496..f3397a7c 100644 --- a/src/hydra-evaluator/hydra-evaluator.cc +++ b/src/hydra-evaluator/hydra-evaluator.cc @@ -68,7 +68,9 @@ struct Evaluator pqxx::work txn(*conn); auto res = txn.exec - ("select project, j.name, lastCheckedTime, triggerTime, checkInterval, j.enabled as jobset_enabled from Jobsets j join Projects p on j.project = p.name " + ("select project, j.name, lastCheckedTime, triggerTime, checkInterval, j.enabled as jobset_enabled " + "from Jobsets j " + "join Projects p on j.project = p.name " "where j.enabled != 0 and p.enabled != 0"); From cb0185971865ccfd8fd6c6b332bbd91039fca1dc Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 25 Jan 2021 15:18:52 -0500 Subject: [PATCH 06/10] hydra-evaluator: JobsetName -> JobsetIdentity --- src/hydra-evaluator/hydra-evaluator.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/hydra-evaluator/hydra-evaluator.cc b/src/hydra-evaluator/hydra-evaluator.cc index f3397a7c..b3dec4ac 100644 --- a/src/hydra-evaluator/hydra-evaluator.cc +++ b/src/hydra-evaluator/hydra-evaluator.cc @@ -13,7 +13,7 @@ using namespace nix; -typedef std::pair JobsetName; +typedef std::pair JobsetIdentity; enum class EvaluationStyle { @@ -30,16 +30,16 @@ struct Evaluator struct Jobset { - JobsetName name; + JobsetIdentity name; std::optional evaluation_style; time_t lastCheckedTime, triggerTime; int checkInterval; Pid pid; }; - typedef std::map Jobsets; + typedef std::map Jobsets; - std::optional evalOne; + std::optional evalOne; const size_t maxEvals; @@ -76,10 +76,10 @@ struct Evaluator auto state(state_.lock()); - std::set seen; + std::set seen; for (auto const & row : res) { - auto name = JobsetName{row["project"].as(), row["name"].as()}; + auto name = JobsetIdentity{row["project"].as(), row["name"].as()}; if (evalOne && name != *evalOne) continue; @@ -466,7 +466,7 @@ int main(int argc, char * * argv) else { if (!args.empty()) { if (args.size() != 2) throw UsageError("Syntax: hydra-evaluator [ ]"); - evaluator.evalOne = JobsetName(args[0], args[1]); + evaluator.evalOne = JobsetIdentity(args[0], args[1]); } evaluator.run(); } From 54341cd9f6810258b53503d373352f70d5ddf673 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 25 Jan 2021 16:08:33 -0500 Subject: [PATCH 07/10] hydra-evaluator: deal in jobset IDs --- src/hydra-evaluator/hydra-evaluator.cc | 123 +++++++++++++++++-------- 1 file changed, 84 insertions(+), 39 deletions(-) diff --git a/src/hydra-evaluator/hydra-evaluator.cc b/src/hydra-evaluator/hydra-evaluator.cc index b3dec4ac..2c4d3f96 100644 --- a/src/hydra-evaluator/hydra-evaluator.cc +++ b/src/hydra-evaluator/hydra-evaluator.cc @@ -13,7 +13,57 @@ using namespace nix; -typedef std::pair JobsetIdentity; +typedef std::pair JobsetSymbolicIdentity; + +class JobsetIdentity { + public: + + std::string project; + std::string jobset; + int id; + + + JobsetIdentity(const std::string& project, const std::string& jobset, const int id) + : project{ project }, jobset{ jobset }, id{ id } + { + } + + friend bool operator== (const JobsetIdentity &lhs, const JobsetIdentity &rhs); + friend bool operator!= (const JobsetIdentity &lhs, const JobsetIdentity &rhs); + friend bool operator< (const JobsetIdentity &lhs, const JobsetIdentity &rhs); + + + friend bool operator== (const JobsetIdentity &lhs, const JobsetSymbolicIdentity &rhs); + friend bool operator!= (const JobsetIdentity &lhs, const JobsetSymbolicIdentity &rhs); + + std::string display() const { + return str(format("%1%:%2% (jobset#%3%)") % project % jobset % id); + } +}; +bool operator==(const JobsetIdentity & lhs, const JobsetIdentity & rhs) +{ + return lhs.id == rhs.id; +} + +bool operator!=(const JobsetIdentity & lhs, const JobsetIdentity & rhs) +{ + return lhs.id != rhs.id; +} + +bool operator<(const JobsetIdentity & lhs, const JobsetIdentity & rhs) +{ + return lhs.id < rhs.id; +} + +bool operator==(const JobsetIdentity & lhs, const JobsetSymbolicIdentity & rhs) +{ + return lhs.project == rhs.first && lhs.jobset == rhs.second; +} + +bool operator!=(const JobsetIdentity & lhs, const JobsetSymbolicIdentity & rhs) +{ + return ! (lhs == rhs); +} enum class EvaluationStyle { @@ -39,7 +89,7 @@ struct Evaluator typedef std::map Jobsets; - std::optional evalOne; + std::optional evalOne; const size_t maxEvals; @@ -68,7 +118,7 @@ struct Evaluator pqxx::work txn(*conn); auto res = txn.exec - ("select project, j.name, lastCheckedTime, triggerTime, checkInterval, j.enabled as jobset_enabled " + ("select j.id as id, project, j.name, lastCheckedTime, triggerTime, checkInterval, j.enabled as jobset_enabled " "from Jobsets j " "join Projects p on j.project = p.name " "where j.enabled != 0 and p.enabled != 0"); @@ -79,7 +129,7 @@ struct Evaluator std::set seen; for (auto const & row : res) { - auto name = JobsetIdentity{row["project"].as(), row["name"].as()}; + auto name = JobsetIdentity{row["project"].as(), row["name"].as(), row["id"].as()}; if (evalOne && name != *evalOne) continue; @@ -113,7 +163,7 @@ struct Evaluator if (seen.count(i->first)) ++i; else { - printInfo("forgetting jobset ‘%s:%s’", i->first.first, i->first.second); + printInfo("forgetting jobset ‘%s’", i->first.display()); i = state->jobsets.erase(i); } } @@ -122,25 +172,24 @@ struct Evaluator { time_t now = time(0); - printInfo("starting evaluation of jobset ‘%s:%s’ (last checked %d s ago)", - jobset.name.first, jobset.name.second, + printInfo("starting evaluation of jobset ‘%s’ (last checked %d s ago)", + jobset.name.display(), now - jobset.lastCheckedTime); { auto conn(dbPool.get()); pqxx::work txn(*conn); txn.exec_params0 - ("update Jobsets set startTime = $1 where project = $2 and name = $3", + ("update Jobsets set startTime = $1 where id = $2", now, - jobset.name.first, - jobset.name.second); + jobset.name.id); txn.commit(); } assert(jobset.pid == -1); jobset.pid = startProcess([&]() { - Strings args = { "hydra-eval-jobset", jobset.name.first, jobset.name.second }; + Strings args = { "hydra-eval-jobset", jobset.name.project, jobset.name.jobset }; execvp(args.front().c_str(), stringsToCharPtrs(args).data()); throw SysError("executing ‘%1%’", args.front()); }); @@ -154,23 +203,23 @@ struct Evaluator { if (jobset.pid != -1) { // Already running. - debug("shouldEvaluate %s:%s? no: already running", - jobset.name.first, jobset.name.second); + debug("shouldEvaluate %s? no: already running", + jobset.name.display()); return false; } if (jobset.triggerTime != std::numeric_limits::max()) { // An evaluation of this Jobset is requested - debug("shouldEvaluate %s:%s? yes: requested", - jobset.name.first, jobset.name.second); + debug("shouldEvaluate %s? yes: requested", + jobset.name.display()); return true; } if (jobset.checkInterval <= 0) { // Automatic scheduling is disabled. We allow requested // evaluations, but never schedule start one. - debug("shouldEvaluate %s:%s? no: checkInterval <= 0", - jobset.name.first, jobset.name.second); + debug("shouldEvaluate %s? no: checkInterval <= 0", + jobset.name.display()); return false; } @@ -186,16 +235,15 @@ struct Evaluator if (jobset.evaluation_style == EvaluationStyle::ONE_AT_A_TIME) { auto evaluation_res = txn.parameterized ("select id from JobsetEvals " - "where project = $1 and jobset = $2 " + "where jobset_id = $1 " "order by id desc limit 1") - (jobset.name.first) - (jobset.name.second) + (jobset.name.id) .exec(); if (evaluation_res.empty()) { // First evaluation, so allow scheduling. - debug("shouldEvaluate(one-at-a-time) %s:%s? yes: no prior eval", - jobset.name.first, jobset.name.second); + debug("shouldEvaluate(one-at-a-time) %s? yes: no prior eval", + jobset.name.display()); return true; } @@ -214,20 +262,20 @@ struct Evaluator // If the previous evaluation has no unfinished builds // schedule! if (unfinished_build_res.empty()) { - debug("shouldEvaluate(one-at-a-time) %s:%s? yes: no unfinished builds", - jobset.name.first, jobset.name.second); + debug("shouldEvaluate(one-at-a-time) %s? yes: no unfinished builds", + jobset.name.display()); return true; } else { debug("shouldEvaluate(one-at-a-time) %s:%s? no: at least one unfinished build", - jobset.name.first, jobset.name.second); + jobset.name.display()); return false; } } else { // EvaluationStyle::ONESHOT, EvaluationStyle::SCHEDULED - debug("shouldEvaluate(oneshot/scheduled) %s:%s? yes: checkInterval elapsed", - jobset.name.first, jobset.name.second); + debug("shouldEvaluate(oneshot/scheduled) %s? yes: checkInterval elapsed", + jobset.name.display()); return true; } } @@ -352,8 +400,8 @@ struct Evaluator auto & jobset(i.second); if (jobset.pid == pid) { - printInfo("evaluation of jobset ‘%s:%s’ %s", - jobset.name.first, jobset.name.second, statusToString(status)); + printInfo("evaluation of jobset ‘%s’ %s", + jobset.name.display(), statusToString(status)); auto now = time(0); @@ -369,23 +417,20 @@ struct Evaluator jobset from getting stuck in an endless failing eval loop. */ txn.exec_params0 - ("update Jobsets set triggerTime = null where project = $1 and name = $2 and startTime is not null and triggerTime <= startTime", - jobset.name.first, - jobset.name.second); + ("update Jobsets set triggerTime = null where id = $1 and startTime is not null and triggerTime <= startTime", + jobset.name.id); /* Clear the start time. */ txn.exec_params0 - ("update Jobsets set startTime = null where project = $1 and name = $2", - jobset.name.first, - jobset.name.second); + ("update Jobsets set startTime = null where id = $1", + jobset.name.id); if (!WIFEXITED(status) || WEXITSTATUS(status) > 1) { txn.exec_params0 - ("update Jobsets set errorMsg = $1, lastCheckedTime = $2, errorTime = $2, fetchErrorMsg = null where project = $3 and name = $4", + ("update Jobsets set errorMsg = $1, lastCheckedTime = $2, errorTime = $2, fetchErrorMsg = null where id = $3", fmt("evaluation %s", statusToString(status)), now, - jobset.name.first, - jobset.name.second); + jobset.name.id); } txn.commit(); @@ -466,7 +511,7 @@ int main(int argc, char * * argv) else { if (!args.empty()) { if (args.size() != 2) throw UsageError("Syntax: hydra-evaluator [ ]"); - evaluator.evalOne = JobsetIdentity(args[0], args[1]); + evaluator.evalOne = JobsetSymbolicIdentity(args[0], args[1]); } evaluator.run(); } From 54b8cb188e0c77421b6617cd80cb4e83a4d992dc Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 26 Jan 2021 09:50:59 -0500 Subject: [PATCH 08/10] perl: jobsetevals -> jobset via by jobset_id Frankly, this was suspiciously little work. --- src/lib/Hydra/Controller/Admin.pm | 2 +- src/lib/Hydra/Controller/JobsetEval.pm | 2 +- src/lib/Hydra/Helper/Nix.pm | 2 +- src/root/common.tt | 2 +- src/script/hydra-eval-jobset | 5 ++++- 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/lib/Hydra/Controller/Admin.pm b/src/lib/Hydra/Controller/Admin.pm index 20ef73fa..e2a219ff 100644 --- a/src/lib/Hydra/Controller/Admin.pm +++ b/src/lib/Hydra/Controller/Admin.pm @@ -34,7 +34,7 @@ sub machines : Chained('admin') PathPart('machines') Args(0) { sub clear_queue_non_current : Chained('admin') PathPart('clear-queue-non-current') Args(0) { my ($self, $c) = @_; my $builds = $c->model('DB::Builds')->search( - { id => { -in => \ "select id from Builds where id in ((select id from Builds where finished = 0) except (select build from JobsetEvalMembers where eval in (select max(id) from JobsetEvals where hasNewBuilds = 1 group by project, jobset)))" } + { id => { -in => \ "select id from Builds where id in ((select id from Builds where finished = 0) except (select build from JobsetEvalMembers where eval in (select max(id) from JobsetEvals where hasNewBuilds = 1 group by jobset_id)))" } }); my $n = cancelBuilds($c->model('DB')->schema, $builds); $c->flash->{successMsg} = "$n builds have been cancelled."; diff --git a/src/lib/Hydra/Controller/JobsetEval.pm b/src/lib/Hydra/Controller/JobsetEval.pm index 41c5471d..cc231833 100644 --- a/src/lib/Hydra/Controller/JobsetEval.pm +++ b/src/lib/Hydra/Controller/JobsetEval.pm @@ -16,8 +16,8 @@ sub evalChain : Chained('/') PathPart('eval') CaptureArgs(1) { or notFound($c, "Evaluation $evalId doesn't exist."); $c->stash->{eval} = $eval; - $c->stash->{project} = $eval->project; $c->stash->{jobset} = $eval->jobset; + $c->stash->{project} = $eval->jobset->project; } diff --git a/src/lib/Hydra/Helper/Nix.pm b/src/lib/Hydra/Helper/Nix.pm index fd7a3170..eb6e7dce 100644 --- a/src/lib/Hydra/Helper/Nix.pm +++ b/src/lib/Hydra/Helper/Nix.pm @@ -219,7 +219,7 @@ sub getEvals { foreach my $curEval (@evals) { my ($prevEval) = $c->model('DB::JobsetEvals')->search( - { project => $curEval->get_column('project'), jobset => $curEval->get_column('jobset') + { jobset_id => $curEval->get_column('jobset_id') , hasnewbuilds => 1, id => { '<', $curEval->id } }, { order_by => "id DESC", rows => 1 }); diff --git a/src/root/common.tt b/src/root/common.tt index 8d4f4f70..bfb5982f 100644 --- a/src/root/common.tt +++ b/src/root/common.tt @@ -463,7 +463,7 @@ BLOCK renderEvals %] [% eval.id %] [% IF !jobset && !build %] - [% INCLUDE renderFullJobsetName project=eval.get_column('project') jobset=eval.get_column('jobset') %] + [% INCLUDE renderFullJobsetName project=eval.jobset.project.name jobset=eval.jobset.name %] [% END %] [% INCLUDE renderRelativeDate timestamp = eval.timestamp %] diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset index dbf03499..f128af88 100755 --- a/src/script/hydra-eval-jobset +++ b/src/script/hydra-eval-jobset @@ -191,8 +191,11 @@ sub fetchInputEval { $eval = getLatestFinishedEval($jobset); die "jobset ‘$value’ does not have a finished evaluation\n" unless defined $eval; } elsif ($value =~ /^($projectNameRE):($jobsetNameRE):($jobNameRE)$/) { + my $jobset = $db->resultset('Jobsets')->find({ project => $1, name => $2 }); + die "jobset ‘$1:$2’ does not exist\n" unless defined $jobset; + $eval = $db->resultset('JobsetEvals')->find( - { project => $1, jobset => $2, hasnewbuilds => 1 }, + { jobset_id => $jobset->id, hasnewbuilds => 1 }, { order_by => "id DESC", rows => 1 , where => \ [ # All builds in this jobset should be finished... From 091d58c128e7c1212a0c7265afd529c0762cdcac Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 26 Jan 2021 12:41:56 -0500 Subject: [PATCH 09/10] hydra-dev-server: autoflush stderr/stdout --- src/script/hydra-dev-server | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/script/hydra-dev-server b/src/script/hydra-dev-server index 8d30624f..4f03ee39 100755 --- a/src/script/hydra-dev-server +++ b/src/script/hydra-dev-server @@ -5,6 +5,11 @@ BEGIN { } use Catalyst::ScriptRunner; + +STDOUT->autoflush(); +STDERR->autoflush(1); +binmode STDERR, ":encoding(utf8)"; + Catalyst::ScriptRunner->run('Hydra', 'DevServer'); 1; From 5fcc2018db26be38c7c62456970d1123f83c0449 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Thu, 28 Jan 2021 09:11:51 -0500 Subject: [PATCH 10/10] hydra-evaluator: clean up names, clean up & / * spacing --- src/hydra-evaluator/hydra-evaluator.cc | 38 +++++++++++++------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/src/hydra-evaluator/hydra-evaluator.cc b/src/hydra-evaluator/hydra-evaluator.cc index 2c4d3f96..9a852536 100644 --- a/src/hydra-evaluator/hydra-evaluator.cc +++ b/src/hydra-evaluator/hydra-evaluator.cc @@ -13,9 +13,9 @@ using namespace nix; -typedef std::pair JobsetSymbolicIdentity; +typedef std::pair JobsetName; -class JobsetIdentity { +class JobsetId { public: std::string project; @@ -23,44 +23,44 @@ class JobsetIdentity { int id; - JobsetIdentity(const std::string& project, const std::string& jobset, const int id) + JobsetId(const std::string & project, const std::string & jobset, int id) : project{ project }, jobset{ jobset }, id{ id } { } - friend bool operator== (const JobsetIdentity &lhs, const JobsetIdentity &rhs); - friend bool operator!= (const JobsetIdentity &lhs, const JobsetIdentity &rhs); - friend bool operator< (const JobsetIdentity &lhs, const JobsetIdentity &rhs); + friend bool operator== (const JobsetId & lhs, const JobsetId & rhs); + friend bool operator!= (const JobsetId & lhs, const JobsetId & rhs); + friend bool operator< (const JobsetId & lhs, const JobsetId & rhs); - friend bool operator== (const JobsetIdentity &lhs, const JobsetSymbolicIdentity &rhs); - friend bool operator!= (const JobsetIdentity &lhs, const JobsetSymbolicIdentity &rhs); + friend bool operator== (const JobsetId & lhs, const JobsetName & rhs); + friend bool operator!= (const JobsetId & lhs, const JobsetName & rhs); std::string display() const { return str(format("%1%:%2% (jobset#%3%)") % project % jobset % id); } }; -bool operator==(const JobsetIdentity & lhs, const JobsetIdentity & rhs) +bool operator==(const JobsetId & lhs, const JobsetId & rhs) { return lhs.id == rhs.id; } -bool operator!=(const JobsetIdentity & lhs, const JobsetIdentity & rhs) +bool operator!=(const JobsetId & lhs, const JobsetId & rhs) { return lhs.id != rhs.id; } -bool operator<(const JobsetIdentity & lhs, const JobsetIdentity & rhs) +bool operator<(const JobsetId & lhs, const JobsetId & rhs) { return lhs.id < rhs.id; } -bool operator==(const JobsetIdentity & lhs, const JobsetSymbolicIdentity & rhs) +bool operator==(const JobsetId & lhs, const JobsetName & rhs) { return lhs.project == rhs.first && lhs.jobset == rhs.second; } -bool operator!=(const JobsetIdentity & lhs, const JobsetSymbolicIdentity & rhs) +bool operator!=(const JobsetId & lhs, const JobsetName & rhs) { return ! (lhs == rhs); } @@ -80,16 +80,16 @@ struct Evaluator struct Jobset { - JobsetIdentity name; + JobsetId name; std::optional evaluation_style; time_t lastCheckedTime, triggerTime; int checkInterval; Pid pid; }; - typedef std::map Jobsets; + typedef std::map Jobsets; - std::optional evalOne; + std::optional evalOne; const size_t maxEvals; @@ -126,10 +126,10 @@ struct Evaluator auto state(state_.lock()); - std::set seen; + std::set seen; for (auto const & row : res) { - auto name = JobsetIdentity{row["project"].as(), row["name"].as(), row["id"].as()}; + auto name = JobsetId{row["project"].as(), row["name"].as(), row["id"].as()}; if (evalOne && name != *evalOne) continue; @@ -511,7 +511,7 @@ int main(int argc, char * * argv) else { if (!args.empty()) { if (args.size() != 2) throw UsageError("Syntax: hydra-evaluator [ ]"); - evaluator.evalOne = JobsetSymbolicIdentity(args[0], args[1]); + evaluator.evalOne = JobsetName(args[0], args[1]); } evaluator.run(); }