From f1e75c8bffec3f5f86122ee73e3c0f19e26ddb27 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 1 Feb 2021 18:35:48 -0500 Subject: [PATCH] Move evaluation errors from evaluations to EvaluationErrors, a new table DBIx likes to eagerly select all columns without a way to really tell it so. Therefore, this splits this one large column in to its own table. I'd also like to make "jobsets" use this table too, but that is on hold to stop the bleeding caused by the extreme amount of traffic this is causing. --- src/lib/Hydra/Helper/Nix.pm | 4 +- src/lib/Hydra/Schema/EvaluationErrors.pm | 108 +++++++++++++++++++++++ src/lib/Hydra/Schema/JobsetEvals.pm | 38 +++++--- src/root/common.tt | 2 +- src/root/jobset-eval.tt | 14 +-- src/script/hydra-eval-jobset | 10 ++- src/sql/hydra.sql | 11 ++- src/sql/update-dbix.pl | 1 + src/sql/upgrade-73.sql | 35 ++++++++ 9 files changed, 196 insertions(+), 27 deletions(-) create mode 100644 src/lib/Hydra/Schema/EvaluationErrors.pm create mode 100644 src/sql/upgrade-73.sql diff --git a/src/lib/Hydra/Helper/Nix.pm b/src/lib/Hydra/Helper/Nix.pm index eb6e7dce..e50723bf 100644 --- a/src/lib/Hydra/Helper/Nix.pm +++ b/src/lib/Hydra/Helper/Nix.pm @@ -211,8 +211,8 @@ sub getEvals { my @evals = $evals->search( { hasnewbuilds => 1 }, - { order_by => "id DESC", rows => $rows, offset => $offset }); - + { order_by => "me.id DESC", rows => $rows, offset => $offset + , prefetch => { evaluationerror => [ ] } }); my @res = (); my $cache = {}; diff --git a/src/lib/Hydra/Schema/EvaluationErrors.pm b/src/lib/Hydra/Schema/EvaluationErrors.pm new file mode 100644 index 00000000..e3fafc9a --- /dev/null +++ b/src/lib/Hydra/Schema/EvaluationErrors.pm @@ -0,0 +1,108 @@ +use utf8; +package Hydra::Schema::EvaluationErrors; + +# Created by DBIx::Class::Schema::Loader +# DO NOT MODIFY THE FIRST PART OF THIS FILE + +=head1 NAME + +Hydra::Schema::EvaluationErrors + +=cut + +use strict; +use warnings; + +use base 'DBIx::Class::Core'; + +=head1 COMPONENTS LOADED + +=over 4 + +=item * L + +=back + +=cut + +__PACKAGE__->load_components("+Hydra::Component::ToJSON"); + +=head1 TABLE: C + +=cut + +__PACKAGE__->table("evaluationerrors"); + +=head1 ACCESSORS + +=head2 id + + data_type: 'integer' + is_auto_increment: 1 + is_nullable: 0 + sequence: 'evaluationerrors_id_seq' + +=head2 errormsg + + data_type: 'text' + is_nullable: 1 + +=head2 errortime + + data_type: 'integer' + is_nullable: 1 + +=cut + +__PACKAGE__->add_columns( + "id", + { + data_type => "integer", + is_auto_increment => 1, + is_nullable => 0, + sequence => "evaluationerrors_id_seq", + }, + "errormsg", + { data_type => "text", is_nullable => 1 }, + "errortime", + { data_type => "integer", is_nullable => 1 }, +); + +=head1 PRIMARY KEY + +=over 4 + +=item * L + +=back + +=cut + +__PACKAGE__->set_primary_key("id"); + +=head1 RELATIONS + +=head2 jobsetevals + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "jobsetevals", + "Hydra::Schema::JobsetEvals", + { "foreign.evaluationerror_id" => "self.id" }, + undef, +); + + +# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-02-01 20:17:39 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:sZIg35KWCO8MOsQ5cfN1IA + +__PACKAGE__->add_column( + "+id" => { retrieve_on_insert => 1 } +); + +1; diff --git a/src/lib/Hydra/Schema/JobsetEvals.pm b/src/lib/Hydra/Schema/JobsetEvals.pm index 0d4a013f..af2c69bc 100644 --- a/src/lib/Hydra/Schema/JobsetEvals.pm +++ b/src/lib/Hydra/Schema/JobsetEvals.pm @@ -48,14 +48,10 @@ __PACKAGE__->table("jobsetevals"); is_foreign_key: 1 is_nullable: 0 -=head2 errormsg - - data_type: 'text' - is_nullable: 1 - -=head2 errortime +=head2 evaluationerror_id data_type: 'integer' + is_foreign_key: 1 is_nullable: 1 =head2 timestamp @@ -120,10 +116,8 @@ __PACKAGE__->add_columns( }, "jobset_id", { data_type => "integer", is_foreign_key => 1, is_nullable => 0 }, - "errormsg", - { data_type => "text", is_nullable => 1 }, - "errortime", - { data_type => "integer", is_nullable => 1 }, + "evaluationerror_id", + { data_type => "integer", is_foreign_key => 1, is_nullable => 1 }, "timestamp", { data_type => "integer", is_nullable => 0 }, "checkouttime", @@ -160,6 +154,26 @@ __PACKAGE__->set_primary_key("id"); =head1 RELATIONS +=head2 evaluationerror + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "evaluationerror", + "Hydra::Schema::EvaluationErrors", + { id => "evaluationerror_id" }, + { + is_deferrable => 0, + join_type => "LEFT", + on_delete => "SET NULL", + on_update => "NO ACTION", + }, +); + =head2 jobset Type: belongs_to @@ -206,8 +220,8 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-25 14:44:07 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:OVxeYH+eoZZrAsAJ2/mAAA +# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-02-01 20:17:39 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:SGtK0PwRkbxiMuitQvs4wQ __PACKAGE__->has_many( "buildIds", diff --git a/src/root/common.tt b/src/root/common.tt index bfb5982f..4dfee8c1 100644 --- a/src/root/common.tt +++ b/src/root/common.tt @@ -476,7 +476,7 @@ BLOCK renderEvals %] ELSE %] - [% END %] - [% IF eval.errormsg %] + [% IF eval.evaluationerror.errormsg %] Eval Errors [% END %] diff --git a/src/root/jobset-eval.tt b/src/root/jobset-eval.tt index 139ee2fe..fb3c3190 100644 --- a/src/root/jobset-eval.tt +++ b/src/root/jobset-eval.tt @@ -89,7 +89,7 @@ c.uri_for(c.controller('JobsetEval').action_for('view'), [% END %]
  • Inputs
    • - [% IF eval.errormsg %] + [% IF eval.evaluationerror.errormsg %]
  • Evaluation errors
    • [% END %] @@ -108,10 +108,10 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
    - [% IF eval.errormsg %] + [% IF eval.evaluationerror.errormsg %]
    -

    Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.errortime || eval.timestamp) %].

    - 
    [% HTML.escape(eval.errormsg) %]
    +

    Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.evaluationerror.errortime || eval.timestamp) %].

    + 
    [% HTML.escape(eval.evaluationerror.errormsg) %]
    [% END %] @@ -172,10 +172,10 @@ c.uri_for(c.controller('JobsetEval').action_for('view'), [% END %]
    - [% IF eval.errormsg %] + [% IF eval.evaluationerror.errormsg %]
    -

    Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.errortime || eval.timestamp) %].

    - 
    [% HTML.escape(eval.errormsg) %]
    +

    Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.evaluationerror.errortime || eval.timestamp) %].

    + 
    [% HTML.escape(eval.evaluationerror.errormsg) %]
    [% END %] diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset index f128af88..a608dc91 100755 --- a/src/script/hydra-eval-jobset +++ b/src/script/hydra-eval-jobset @@ -693,6 +693,12 @@ sub checkJobsetWrapped { } setJobsetError($jobset, $evaluationErrorMsg, $evaluationErrorTime); + my $evaluationErrorRecord = $db->resultset('EvaluationErrors')->create( + { errormsg => $evaluationErrorMsg + , errortime => $evaluationErrorTime + } + ); + my %buildMap; $db->txn_do(sub { @@ -715,12 +721,12 @@ sub checkJobsetWrapped { (scalar(grep { $_->{new} } values(%buildMap)) > 0) || (defined $prevEval && $prevEval->jobsetevalmembers->count != scalar(keys %buildMap)); + my $ev = $jobset->jobsetevals->create( { hash => $argsHash + , evaluationerror => $evaluationErrorRecord , timestamp => time , checkouttime => abs(int($checkoutStop - $checkoutStart)) - , errormsg => $evaluationErrorMsg - , errortime => $evaluationErrorTime , evaltime => abs(int($evalStop - $evalStart)) , hasnewbuilds => $jobsetChanged ? 1 : 0 , nrbuilds => $jobsetChanged ? scalar(keys %buildMap) : undef diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index 6e56960f..2736929d 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -437,13 +437,17 @@ create table SystemTypes ( maxConcurrent integer not null default 2 ); +create table EvaluationErrors ( + id serial primary key not null, + errorMsg text, -- error output from the evaluator + errorTime integer -- timestamp associated with errorMsg +); create table JobsetEvals ( id serial primary key not null, jobset_id integer not null, - errorMsg text, -- error output from the evaluator - errorTime integer, -- timestamp associated with errorMsg + evaluationerror_id integer, timestamp integer not null, -- when this entry was added checkoutTime integer not null, -- how long obtaining the inputs took (in seconds) @@ -471,7 +475,8 @@ create table JobsetEvals ( nixExprInput text, -- name of the jobsetInput containing the Nix or Guix expression nixExprPath text, -- relative path of the Nix or Guix expression - foreign key (jobset_id) references Jobsets(id) on delete cascade + foreign key (jobset_id) references Jobsets(id) on delete cascade, + foreign key (evaluationerror_id) references EvaluationErrors(id) on delete set null ); diff --git a/src/sql/update-dbix.pl b/src/sql/update-dbix.pl index df6c0526..0d8404b8 100644 --- a/src/sql/update-dbix.pl +++ b/src/sql/update-dbix.pl @@ -21,6 +21,7 @@ make_schema_at("Hydra::Schema", { "cachedhginputs" => "CachedHgInputs", "cachedpathinputs" => "CachedPathInputs", "cachedsubversioninputs" => "CachedSubversionInputs", + "evaluationerrors" => "EvaluationErrors", "failedpaths" => "FailedPaths", "jobsetevalinputs" => "JobsetEvalInputs", "jobsetevalmembers" => "JobsetEvalMembers", diff --git a/src/sql/upgrade-73.sql b/src/sql/upgrade-73.sql new file mode 100644 index 00000000..1fa59ed3 --- /dev/null +++ b/src/sql/upgrade-73.sql @@ -0,0 +1,35 @@ +create table EvaluationErrors ( + id serial primary key not null, + errorMsg text, -- error output from the evaluator + errorTime integer, -- timestamp associated with errorMsg + jobsetEvalId integer not null, + + FOREIGN KEY (jobsetEvalId) + REFERENCES JobsetEvals(id) + ON DELETE SET NULL +); + +ALTER TABLE JobsetEvals + ADD COLUMN evaluationerror_id integer NULL, + ADD FOREIGN KEY (evaluationerror_id) + REFERENCES EvaluationErrors(id) + ON DELETE SET NULL; + +INSERT INTO EvaluationErrors + (errorMsg, errorTime, jobsetEvalId) +SELECT errorMsg, errorTime, id +FROM JobsetEvals +WHERE JobsetEvals.errorMsg != '' and JobsetEvals.errorMsg is not null; + +UPDATE JobsetEvals +SET evaluationerror_id = EvaluationErrors.id +FROM EvaluationErrors +WHERE JobsetEvals.id = EvaluationErrors.jobsetEvalId +AND JobsetEvals.errorMsg != '' and JobsetEvals.errorMsg is not null; + +ALTER TABLE JobsetEvals + DROP COLUMN errorMsg, + DROP COLUMN errorTime; + +ALTER TABLE EvaluationErrors + DROP COLUMN jobsetEvalId;