Merge pull request #848 from grahamc/normalize-nixexprinputpath

Normalize nixexpr{input,path} from builds to jobsetevals.

src/lib/Hydra/Schema/Builds.pm

@@ -134,16 +134,6 @@ __PACKAGE__->table("builds");
   default_value: 0
   is_nullable: 1
 
-=head2 nixexprinput
-
-  data_type: 'text'
-  is_nullable: 1
-
-=head2 nixexprpath
-
-  data_type: 'text'
-  is_nullable: 1
-
 =head2 priority
 
   data_type: 'integer'
@@ -246,10 +236,6 @@ __PACKAGE__->add_columns(
   { data_type => "integer", default_value => 0, is_nullable => 0 },
   "iscurrent",
   { data_type => "integer", default_value => 0, is_nullable => 1 },
-  "nixexprinput",
-  { data_type => "text", is_nullable => 1 },
-  "nixexprpath",
-  { data_type => "text", is_nullable => 1 },
   "priority",
   { data_type => "integer", default_value => 0, is_nullable => 0 },
   "globalpriority",
@@ -542,8 +528,8 @@ __PACKAGE__->many_to_many(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.07049 @ 2020-05-27 17:40:41
+# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-22 07:11:57
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:RIKKFfcKXFWIUeM8ma++iw
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Df5N0EByYJqoSUqA0dld/A
 
 __PACKAGE__->has_many(
   "dependents",

src/lib/Hydra/Schema/JobsetEvals.pm

@@ -89,6 +89,16 @@ __PACKAGE__->table("jobsetevals");
   data_type: 'text'
   is_nullable: 0
 
+=head2 nixexprinput
+
+  data_type: 'text'
+  is_nullable: 1
+
+=head2 nixexprpath
+
+  data_type: 'text'
+  is_nullable: 1
+
 =head2 nrbuilds
 
   data_type: 'integer'
@@ -132,6 +142,10 @@ __PACKAGE__->add_columns(
   { data_type => "integer", is_nullable => 0 },
   "hash",
   { data_type => "text", is_nullable => 0 },
+  "nixexprinput",
+  { data_type => "text", is_nullable => 1 },
+  "nixexprpath",
+  { data_type => "text", is_nullable => 1 },
   "nrbuilds",
   { data_type => "integer", is_nullable => 1 },
   "nrsucceeded",
@@ -215,8 +229,8 @@ __PACKAGE__->belongs_to(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-21 11:13:38
+# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-22 07:11:57
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:zDBtAFc4HiFUcL/TpkuCcg
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:hdu+0WWo2363dVvImMKxdA
 
 __PACKAGE__->has_many(
   "buildIds",

src/lib/Hydra/Schema/Jobsets.pm

@@ -375,8 +375,8 @@ __PACKAGE__->has_many(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.07049 @ 2020-05-27 17:40:41
+# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-22 07:11:57
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:aDW78MCelU/ma953aTcHvA
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:6P1qlC5oVSPRSgRBp6nmrw
 
 
 =head2 builds

src/lib/Hydra/Schema/Projects.pm

@@ -258,8 +258,8 @@ Composing rels: L</projectmembers> -> username
 __PACKAGE__->many_to_many("usernames", "projectmembers", "username");
 
 
-# Created by DBIx::Class::Schema::Loader v0.07049 @ 2020-05-27 17:40:41
+# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-22 07:11:57
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:iBGJjFWiI9Wy9zwT7xGOEA
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Ff5gJejFu+02b0lInobOoQ
 
 my %hint = (
     columns => [

src/root/build.tt

@@ -120,7 +120,7 @@ END;
       <b class="caret"></b>
     </a>
     <ul class="dropdown-menu">
-      [% IF build.nixexprinput || eval.flake %]
+      [% IF eval.nixexprinput || eval.flake %]
         <li><a href="#reproduce" data-toggle="modal">Reproduce locally</a></li>
       [% END %]
       [% IF c.user_exists %]
@@ -346,10 +346,10 @@ END;
           <td>[% build.priority %]</td>
         </tr>
       [% END %]
-      [% IF build.nixexprinput %]
+      [% IF eval.nixexprinput %]
         <tr>
           <th>Nix expression:</th>
-          <td>file <tt>[% HTML.escape(build.nixexprpath) %]</tt> in input <tt>[% HTML.escape(build.nixexprinput) %]</tt></td>
+          <td>file <tt>[% HTML.escape(eval.nixexprpath) %]</tt> in input <tt>[% HTML.escape(eval.nixexprinput) %]</tt></td>
         </tr>
       [% END %]
       <tr>

src/root/reproduce.tt

@@ -169,7 +169,7 @@ echo "$0: input ‘[% input.name %]’ has unsupported type ‘[% input.type %]
 exit 1
 [% END %]
 
-[% IF input.name == build.nixexprinput +%]
+[% IF input.name == eval.nixexprinput +%]
 nixExprInputDir="$inputDir"
 [%+ END %]
 
@@ -197,7 +197,7 @@ args+=(--option extra-binary-caches '[% c.uri_for('/') %]')
 # when evaluating jobs that rely on builtins.currentSystem.
 args+=(--option system x86_64-linux)
 
-args+=("$nixExprInputDir/[% build.nixexprpath %]" -A '[% build.job.name %]')
+args+=("$nixExprInputDir/[% eval.nixexprpath %]" -A '[% build.job.name %]')
 
 if [ -n "$printFlags" ]; then
     first=1

src/script/hydra-eval-jobset

@@ -460,8 +460,6 @@ sub checkBuild {
             , nixname => $buildInfo->{nixName}
             , drvpath => $drvPath
             , system => $buildInfo->{system}
-            , nixexprinput => $jobset->nixexprinput
-            , nixexprpath => $jobset->nixexprpath
             , priority => $buildInfo->{schedulingPriority}
             , finished => 0
             , iscurrent => 1
@@ -724,6 +722,8 @@ sub checkJobsetWrapped {
             , hasnewbuilds => $jobsetChanged ? 1 : 0
             , nrbuilds => $jobsetChanged ? scalar(keys %buildMap) : undef
             , flake => $flakeRef
+            , nixexprinput => $jobset->nixexprinput
+            , nixexprpath => $jobset->nixexprpath
             });
 
         $db->storage->dbh->do("notify eval_added, ?", undef,

src/sql/hydra.sql

@@ -162,13 +162,6 @@ create table Builds (
     isChannel     integer not null default 0, -- meta.isHydraChannel
     isCurrent     integer default 0,
 
-    -- Copy of the nixExprInput/nixExprPath fields of the jobset that
-    -- instantiated this build.  Needed if we want to reproduce this
-    -- build.  FIXME: this should be stored in JobsetEvals, storing it
-    -- here is denormal.
-    nixExprInput  text,
-    nixExprPath   text,
-
     -- Priority within a jobset, set via meta.schedulingPriority.
     priority      integer not null default 0,
 
@@ -466,6 +459,8 @@ create table JobsetEvals (
     nrSucceeded   integer, -- set lazily when all builds are finished
 
     flake         text, -- immutable flake reference
+    nixExprInput  text, -- name of the jobsetInput containing the Nix or Guix expression
+    nixExprPath   text, -- relative path of the Nix or Guix expression
 
     foreign key   (project) references Projects(name) on delete cascade on update cascade,
     foreign key   (project, jobset) references Jobsets(project, name) on delete cascade on update cascade

src/sql/upgrade-71.sql

@@ -0,0 +1,31 @@
+ALTER TABLE JobsetEvals
+    ADD COLUMN nixExprInput text,
+    ADD COLUMN nixExprPath text;
+
+
+-- This migration took 4.5 hours on a server
+-- with 5400RPM drives, against a copy of hydra's
+-- production dataset. It might take a significantly
+-- less amount of time there, and not justify a
+-- batched migration.
+UPDATE jobsetevals
+SET (nixexprinput, nixexprpath) = (
+    SELECT builds.nixexprinput, builds.nixexprpath
+    FROM builds
+    LEFT JOIN jobsetevalmembers
+      ON jobsetevalmembers.build = builds.id
+    WHERE jobsetevalmembers.eval = jobsetevals.id
+    LIMIT 1
+)
+WHERE jobsetevals.id in (
+  SELECT jobsetevalsprime.id
+  FROM jobsetevals as jobsetevalsprime
+  WHERE jobsetevalsprime.nixexprinput IS NULL
+  -- AND jobsetevalsprime.id > ? --------- These are in case of a batched migration
+  ORDER BY jobsetevalsprime.id ASC  --  /
+  -- LIMIT ?  --  ----------------------
+);
+
+ALTER TABLE builds
+    DROP COLUMN nixexprinput,
+    DROP COLUMN nixexprpath;