Move evaluation errors from evaluations to EvaluationErrors, a new table

DBIx likes to eagerly select all columns without a way to really tell
it so. Therefore, this splits this one large column in to its own
table.

I'd also like to make "jobsets" use this table too, but that is on hold
to stop the bleeding caused by the extreme amount of traffic this is
causing.

src/lib/Hydra/Helper/Nix.pm

@@ -211,8 +211,8 @@ sub getEvals {
 
     my @evals = $evals->search(
         { hasnewbuilds => 1 },
-        { order_by => "id DESC", rows => $rows, offset => $offset });
-
+        { order_by => "me.id DESC", rows => $rows, offset => $offset
+        , prefetch => { evaluationerror => [  ] } });
     my @res = ();
     my $cache = {};
 

src/lib/Hydra/Schema/EvaluationErrors.pm

@@ -0,0 +1,108 @@
+use utf8;
+package Hydra::Schema::EvaluationErrors;
+
+# Created by DBIx::Class::Schema::Loader
+# DO NOT MODIFY THE FIRST PART OF THIS FILE
+
+=head1 NAME
+
+Hydra::Schema::EvaluationErrors
+
+=cut
+
+use strict;
+use warnings;
+
+use base 'DBIx::Class::Core';
+
+=head1 COMPONENTS LOADED
+
+=over 4
+
+=item * L<Hydra::Component::ToJSON>
+
+=back
+
+=cut
+
+__PACKAGE__->load_components("+Hydra::Component::ToJSON");
+
+=head1 TABLE: C<evaluationerrors>
+
+=cut
+
+__PACKAGE__->table("evaluationerrors");
+
+=head1 ACCESSORS
+
+=head2 id
+
+  data_type: 'integer'
+  is_auto_increment: 1
+  is_nullable: 0
+  sequence: 'evaluationerrors_id_seq'
+
+=head2 errormsg
+
+  data_type: 'text'
+  is_nullable: 1
+
+=head2 errortime
+
+  data_type: 'integer'
+  is_nullable: 1
+
+=cut
+
+__PACKAGE__->add_columns(
+  "id",
+  {
+    data_type         => "integer",
+    is_auto_increment => 1,
+    is_nullable       => 0,
+    sequence          => "evaluationerrors_id_seq",
+  },
+  "errormsg",
+  { data_type => "text", is_nullable => 1 },
+  "errortime",
+  { data_type => "integer", is_nullable => 1 },
+);
+
+=head1 PRIMARY KEY
+
+=over 4
+
+=item * L</id>
+
+=back
+
+=cut
+
+__PACKAGE__->set_primary_key("id");
+
+=head1 RELATIONS
+
+=head2 jobsetevals
+
+Type: has_many
+
+Related object: L<Hydra::Schema::JobsetEvals>
+
+=cut
+
+__PACKAGE__->has_many(
+  "jobsetevals",
+  "Hydra::Schema::JobsetEvals",
+  { "foreign.evaluationerror_id" => "self.id" },
+  undef,
+);
+
+
+# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-02-01 20:17:39
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:sZIg35KWCO8MOsQ5cfN1IA
+
+__PACKAGE__->add_column(
+    "+id" => { retrieve_on_insert => 1 }
+);
+
+1;

src/lib/Hydra/Schema/JobsetEvals.pm

@@ -48,14 +48,10 @@ __PACKAGE__->table("jobsetevals");
   is_foreign_key: 1
   is_nullable: 0
 
-=head2 errormsg
-
-  data_type: 'text'
-  is_nullable: 1
-
-=head2 errortime
+=head2 evaluationerror_id
 
   data_type: 'integer'
+  is_foreign_key: 1
   is_nullable: 1
 
 =head2 timestamp
@@ -120,10 +116,8 @@ __PACKAGE__->add_columns(
   },
   "jobset_id",
   { data_type => "integer", is_foreign_key => 1, is_nullable => 0 },
-  "errormsg",
-  { data_type => "text", is_nullable => 1 },
-  "errortime",
-  { data_type => "integer", is_nullable => 1 },
+  "evaluationerror_id",
+  { data_type => "integer", is_foreign_key => 1, is_nullable => 1 },
   "timestamp",
   { data_type => "integer", is_nullable => 0 },
   "checkouttime",
@@ -160,6 +154,26 @@ __PACKAGE__->set_primary_key("id");
 
 =head1 RELATIONS
 
+=head2 evaluationerror
+
+Type: belongs_to
+
+Related object: L<Hydra::Schema::EvaluationErrors>
+
+=cut
+
+__PACKAGE__->belongs_to(
+  "evaluationerror",
+  "Hydra::Schema::EvaluationErrors",
+  { id => "evaluationerror_id" },
+  {
+    is_deferrable => 0,
+    join_type     => "LEFT",
+    on_delete     => "SET NULL",
+    on_update     => "NO ACTION",
+  },
+);
+
 =head2 jobset
 
 Type: belongs_to
@@ -206,8 +220,8 @@ __PACKAGE__->has_many(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-25 14:44:07
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:OVxeYH+eoZZrAsAJ2/mAAA
+# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-02-01 20:17:39
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:SGtK0PwRkbxiMuitQvs4wQ
 
 __PACKAGE__->has_many(
   "buildIds",

src/root/common.tt

@@ -476,7 +476,7 @@ BLOCK renderEvals %]
             ELSE %]
              -
             [% END %]
-            [% IF eval.errormsg %]
+            [% IF eval.evaluationerror.errormsg %]
               <span class="label label-warning">Eval Errors</span>
             [% END %]
           </td>

src/root/jobset-eval.tt

@@ -89,7 +89,7 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
   [% END %]
   <li><a href="#tabs-inputs" data-toggle="tab">Inputs</a></li>
 
-  [% IF eval.errormsg %]
+  [% IF eval.evaluationerror.errormsg %]
     <li><a href="#tabs-errors" data-toggle="tab"><span class="text-warning">Evaluation errors</span></a></li>
   [% END %]
 
@@ -108,10 +108,10 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
 
 <div class="tab-content">
 
-  [% IF eval.errormsg %]
+  [% IF eval.evaluationerror.errormsg %]
     <div id="tabs-errors" class="tab-pane">
-      <p>Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.errortime || eval.timestamp) %].</p>
-      <pre class="alert alert-error">[% HTML.escape(eval.errormsg) %]</pre>
+      <p>Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.evaluationerror.errortime || eval.timestamp) %].</p>
+      <pre class="alert alert-error">[% HTML.escape(eval.evaluationerror.errormsg) %]</pre>
     </div>
   [% END %]
 
@@ -172,10 +172,10 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
     [% END %]
   </div>
 
-  [% IF eval.errormsg %]
+  [% IF eval.evaluationerror.errormsg %]
     <div id="tabs-errors" class="tab-pane">
-      <p>Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.errortime || eval.timestamp) %].</p>
-      <pre class="alert alert-error">[% HTML.escape(eval.errormsg) %]</pre>
+      <p>Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.evaluationerror.errortime || eval.timestamp) %].</p>
+      <pre class="alert alert-error">[% HTML.escape(eval.evaluationerror.errormsg) %]</pre>
     </div>
   [% END %]
 </div>

src/script/hydra-eval-jobset

@@ -693,6 +693,12 @@ sub checkJobsetWrapped {
     }
     setJobsetError($jobset, $evaluationErrorMsg, $evaluationErrorTime);
 
+    my $evaluationErrorRecord = $db->resultset('EvaluationErrors')->create(
+        { errormsg => $evaluationErrorMsg
+        , errortime => $evaluationErrorTime
+        }
+    );
+
     my %buildMap;
     $db->txn_do(sub {
 
@@ -715,12 +721,12 @@ sub checkJobsetWrapped {
             (scalar(grep { $_->{new} } values(%buildMap)) > 0)
             || (defined $prevEval && $prevEval->jobsetevalmembers->count != scalar(keys %buildMap));
 
+
         my $ev = $jobset->jobsetevals->create(
             { hash => $argsHash
+            , evaluationerror => $evaluationErrorRecord
             , timestamp => time
             , checkouttime => abs(int($checkoutStop - $checkoutStart))
-            , errormsg => $evaluationErrorMsg
-            , errortime => $evaluationErrorTime
             , evaltime => abs(int($evalStop - $evalStart))
             , hasnewbuilds => $jobsetChanged ? 1 : 0
             , nrbuilds => $jobsetChanged ? scalar(keys %buildMap) : undef

src/sql/hydra.sql

@@ -437,13 +437,17 @@ create table SystemTypes (
     maxConcurrent integer not null default 2
 );
 
+create table EvaluationErrors (
+    id            serial primary key not null,
+    errorMsg      text,    -- error output from the evaluator
+    errorTime     integer  -- timestamp associated with errorMsg
+);
 
 create table JobsetEvals (
     id            serial primary key not null,
     jobset_id     integer not null,
 
-    errorMsg      text, -- error output from the evaluator
-    errorTime     integer, -- timestamp associated with errorMsg
+    evaluationerror_id integer,
 
     timestamp     integer not null, -- when this entry was added
     checkoutTime  integer not null, -- how long obtaining the inputs took (in seconds)
@@ -471,7 +475,8 @@ create table JobsetEvals (
     nixExprInput  text, -- name of the jobsetInput containing the Nix or Guix expression
     nixExprPath   text, -- relative path of the Nix or Guix expression
 
-    foreign key   (jobset_id) references Jobsets(id) on delete cascade
+    foreign key   (jobset_id) references Jobsets(id) on delete cascade,
+    foreign key   (evaluationerror_id) references EvaluationErrors(id) on delete set null
 );
 
 

src/sql/update-dbix.pl

@@ -21,6 +21,7 @@ make_schema_at("Hydra::Schema", {
             "cachedhginputs" => "CachedHgInputs",
             "cachedpathinputs" => "CachedPathInputs",
             "cachedsubversioninputs" => "CachedSubversionInputs",
+            "evaluationerrors" => "EvaluationErrors",
             "failedpaths" => "FailedPaths",
             "jobsetevalinputs" => "JobsetEvalInputs",
             "jobsetevalmembers" => "JobsetEvalMembers",

src/sql/upgrade-73.sql

@@ -0,0 +1,35 @@
+create table EvaluationErrors (
+    id            serial primary key not null,
+    errorMsg      text,    -- error output from the evaluator
+    errorTime     integer, -- timestamp associated with errorMsg
+    jobsetEvalId  integer not null,
+
+    FOREIGN KEY (jobsetEvalId)
+        REFERENCES JobsetEvals(id)
+        ON DELETE SET NULL
+);
+
+ALTER TABLE JobsetEvals
+    ADD COLUMN evaluationerror_id integer NULL,
+    ADD FOREIGN KEY (evaluationerror_id)
+        REFERENCES EvaluationErrors(id)
+        ON DELETE SET NULL;
+
+INSERT INTO EvaluationErrors
+    (errorMsg, errorTime, jobsetEvalId)
+SELECT errorMsg, errorTime, id
+FROM JobsetEvals
+WHERE JobsetEvals.errorMsg != '' and JobsetEvals.errorMsg is not null;
+
+UPDATE JobsetEvals
+SET evaluationerror_id = EvaluationErrors.id
+FROM EvaluationErrors
+WHERE JobsetEvals.id = EvaluationErrors.jobsetEvalId
+AND JobsetEvals.errorMsg != '' and JobsetEvals.errorMsg is not null;
+
+ALTER TABLE JobsetEvals
+    DROP COLUMN errorMsg,
+    DROP COLUMN errorTime;
+
+ALTER TABLE EvaluationErrors
+    DROP COLUMN jobsetEvalId;