Merge pull request #848 from grahamc/normalize-nixexprinputpath

Normalize nixexpr{input,path} from builds to jobsetevals.
This commit is contained in:
Eelco Dolstra 2021-01-25 15:22:01 +01:00 committed by GitHub
commit d0b3f2dac4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 62 additions and 36 deletions

View file

@ -134,16 +134,6 @@ __PACKAGE__->table("builds");
default_value: 0 default_value: 0
is_nullable: 1 is_nullable: 1
=head2 nixexprinput
data_type: 'text'
is_nullable: 1
=head2 nixexprpath
data_type: 'text'
is_nullable: 1
=head2 priority =head2 priority
data_type: 'integer' data_type: 'integer'
@ -246,10 +236,6 @@ __PACKAGE__->add_columns(
{ data_type => "integer", default_value => 0, is_nullable => 0 }, { data_type => "integer", default_value => 0, is_nullable => 0 },
"iscurrent", "iscurrent",
{ data_type => "integer", default_value => 0, is_nullable => 1 }, { data_type => "integer", default_value => 0, is_nullable => 1 },
"nixexprinput",
{ data_type => "text", is_nullable => 1 },
"nixexprpath",
{ data_type => "text", is_nullable => 1 },
"priority", "priority",
{ data_type => "integer", default_value => 0, is_nullable => 0 }, { data_type => "integer", default_value => 0, is_nullable => 0 },
"globalpriority", "globalpriority",
@ -542,8 +528,8 @@ __PACKAGE__->many_to_many(
); );
# Created by DBIx::Class::Schema::Loader v0.07049 @ 2020-05-27 17:40:41 # Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-22 07:11:57
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:RIKKFfcKXFWIUeM8ma++iw # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Df5N0EByYJqoSUqA0dld/A
__PACKAGE__->has_many( __PACKAGE__->has_many(
"dependents", "dependents",

View file

@ -89,6 +89,16 @@ __PACKAGE__->table("jobsetevals");
data_type: 'text' data_type: 'text'
is_nullable: 0 is_nullable: 0
=head2 nixexprinput
data_type: 'text'
is_nullable: 1
=head2 nixexprpath
data_type: 'text'
is_nullable: 1
=head2 nrbuilds =head2 nrbuilds
data_type: 'integer' data_type: 'integer'
@ -132,6 +142,10 @@ __PACKAGE__->add_columns(
{ data_type => "integer", is_nullable => 0 }, { data_type => "integer", is_nullable => 0 },
"hash", "hash",
{ data_type => "text", is_nullable => 0 }, { data_type => "text", is_nullable => 0 },
"nixexprinput",
{ data_type => "text", is_nullable => 1 },
"nixexprpath",
{ data_type => "text", is_nullable => 1 },
"nrbuilds", "nrbuilds",
{ data_type => "integer", is_nullable => 1 }, { data_type => "integer", is_nullable => 1 },
"nrsucceeded", "nrsucceeded",
@ -215,8 +229,8 @@ __PACKAGE__->belongs_to(
); );
# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-21 11:13:38 # Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-22 07:11:57
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:zDBtAFc4HiFUcL/TpkuCcg # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:hdu+0WWo2363dVvImMKxdA
__PACKAGE__->has_many( __PACKAGE__->has_many(
"buildIds", "buildIds",

View file

@ -375,8 +375,8 @@ __PACKAGE__->has_many(
); );
# Created by DBIx::Class::Schema::Loader v0.07049 @ 2020-05-27 17:40:41 # Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-22 07:11:57
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:aDW78MCelU/ma953aTcHvA # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:6P1qlC5oVSPRSgRBp6nmrw
=head2 builds =head2 builds

View file

@ -258,8 +258,8 @@ Composing rels: L</projectmembers> -> username
__PACKAGE__->many_to_many("usernames", "projectmembers", "username"); __PACKAGE__->many_to_many("usernames", "projectmembers", "username");
# Created by DBIx::Class::Schema::Loader v0.07049 @ 2020-05-27 17:40:41 # Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-22 07:11:57
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:iBGJjFWiI9Wy9zwT7xGOEA # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Ff5gJejFu+02b0lInobOoQ
my %hint = ( my %hint = (
columns => [ columns => [

View file

@ -120,7 +120,7 @@ END;
<b class="caret"></b> <b class="caret"></b>
</a> </a>
<ul class="dropdown-menu"> <ul class="dropdown-menu">
[% IF build.nixexprinput || eval.flake %] [% IF eval.nixexprinput || eval.flake %]
<li><a href="#reproduce" data-toggle="modal">Reproduce locally</a></li> <li><a href="#reproduce" data-toggle="modal">Reproduce locally</a></li>
[% END %] [% END %]
[% IF c.user_exists %] [% IF c.user_exists %]
@ -346,10 +346,10 @@ END;
<td>[% build.priority %]</td> <td>[% build.priority %]</td>
</tr> </tr>
[% END %] [% END %]
[% IF build.nixexprinput %] [% IF eval.nixexprinput %]
<tr> <tr>
<th>Nix expression:</th> <th>Nix expression:</th>
<td>file <tt>[% HTML.escape(build.nixexprpath) %]</tt> in input <tt>[% HTML.escape(build.nixexprinput) %]</tt></td> <td>file <tt>[% HTML.escape(eval.nixexprpath) %]</tt> in input <tt>[% HTML.escape(eval.nixexprinput) %]</tt></td>
</tr> </tr>
[% END %] [% END %]
<tr> <tr>

View file

@ -169,7 +169,7 @@ echo "$0: input [% input.name %] has unsupported type [% input.type %]
exit 1 exit 1
[% END %] [% END %]
[% IF input.name == build.nixexprinput +%] [% IF input.name == eval.nixexprinput +%]
nixExprInputDir="$inputDir" nixExprInputDir="$inputDir"
[%+ END %] [%+ END %]
@ -197,7 +197,7 @@ args+=(--option extra-binary-caches '[% c.uri_for('/') %]')
# when evaluating jobs that rely on builtins.currentSystem. # when evaluating jobs that rely on builtins.currentSystem.
args+=(--option system x86_64-linux) args+=(--option system x86_64-linux)
args+=("$nixExprInputDir/[% build.nixexprpath %]" -A '[% build.job.name %]') args+=("$nixExprInputDir/[% eval.nixexprpath %]" -A '[% build.job.name %]')
if [ -n "$printFlags" ]; then if [ -n "$printFlags" ]; then
first=1 first=1

View file

@ -460,8 +460,6 @@ sub checkBuild {
, nixname => $buildInfo->{nixName} , nixname => $buildInfo->{nixName}
, drvpath => $drvPath , drvpath => $drvPath
, system => $buildInfo->{system} , system => $buildInfo->{system}
, nixexprinput => $jobset->nixexprinput
, nixexprpath => $jobset->nixexprpath
, priority => $buildInfo->{schedulingPriority} , priority => $buildInfo->{schedulingPriority}
, finished => 0 , finished => 0
, iscurrent => 1 , iscurrent => 1
@ -724,6 +722,8 @@ sub checkJobsetWrapped {
, hasnewbuilds => $jobsetChanged ? 1 : 0 , hasnewbuilds => $jobsetChanged ? 1 : 0
, nrbuilds => $jobsetChanged ? scalar(keys %buildMap) : undef , nrbuilds => $jobsetChanged ? scalar(keys %buildMap) : undef
, flake => $flakeRef , flake => $flakeRef
, nixexprinput => $jobset->nixexprinput
, nixexprpath => $jobset->nixexprpath
}); });
$db->storage->dbh->do("notify eval_added, ?", undef, $db->storage->dbh->do("notify eval_added, ?", undef,

View file

@ -162,13 +162,6 @@ create table Builds (
isChannel integer not null default 0, -- meta.isHydraChannel isChannel integer not null default 0, -- meta.isHydraChannel
isCurrent integer default 0, isCurrent integer default 0,
-- Copy of the nixExprInput/nixExprPath fields of the jobset that
-- instantiated this build. Needed if we want to reproduce this
-- build. FIXME: this should be stored in JobsetEvals, storing it
-- here is denormal.
nixExprInput text,
nixExprPath text,
-- Priority within a jobset, set via meta.schedulingPriority. -- Priority within a jobset, set via meta.schedulingPriority.
priority integer not null default 0, priority integer not null default 0,
@ -466,6 +459,8 @@ create table JobsetEvals (
nrSucceeded integer, -- set lazily when all builds are finished nrSucceeded integer, -- set lazily when all builds are finished
flake text, -- immutable flake reference flake text, -- immutable flake reference
nixExprInput text, -- name of the jobsetInput containing the Nix or Guix expression
nixExprPath text, -- relative path of the Nix or Guix expression
foreign key (project) references Projects(name) on delete cascade on update cascade, foreign key (project) references Projects(name) on delete cascade on update cascade,
foreign key (project, jobset) references Jobsets(project, name) on delete cascade on update cascade foreign key (project, jobset) references Jobsets(project, name) on delete cascade on update cascade

31
src/sql/upgrade-71.sql Normal file
View file

@ -0,0 +1,31 @@
ALTER TABLE JobsetEvals
ADD COLUMN nixExprInput text,
ADD COLUMN nixExprPath text;
-- This migration took 4.5 hours on a server
-- with 5400RPM drives, against a copy of hydra's
-- production dataset. It might take a significantly
-- less amount of time there, and not justify a
-- batched migration.
UPDATE jobsetevals
SET (nixexprinput, nixexprpath) = (
SELECT builds.nixexprinput, builds.nixexprpath
FROM builds
LEFT JOIN jobsetevalmembers
ON jobsetevalmembers.build = builds.id
WHERE jobsetevalmembers.eval = jobsetevals.id
LIMIT 1
)
WHERE jobsetevals.id in (
SELECT jobsetevalsprime.id
FROM jobsetevals as jobsetevalsprime
WHERE jobsetevalsprime.nixexprinput IS NULL
-- AND jobsetevalsprime.id > ? --------- These are in case of a batched migration
ORDER BY jobsetevalsprime.id ASC -- /
-- LIMIT ? -- ----------------------
);
ALTER TABLE builds
DROP COLUMN nixexprinput,
DROP COLUMN nixexprpath;