Normalize nixexpr{input,path} from builds to jobsetevals.
Duplicating this data on every record of the builds table cost approximately 4G of duplication. Note that the database migration included took about 4h45m on an untuned server which uses very slow rotational disks in a RAID5 setup, with not a lot of RAM. I imagine in production it might take an hour or two, but not 4. If this should become a chunked migration, I can do that. Note: Because of the question about chunked migrations, I have NOT YET tested this migration thoroughly enough for merge.
This commit is contained in:
Graham Christensen
parent
53c2fc2216
commit
9516b256f1
|
|
@ -134,16 +134,6 @@ __PACKAGE__->table("builds");
|
|||
default_value: 0
|
||||
is_nullable: 1
|
||||
|
||||
=head2 nixexprinput
|
||||
|
||||
data_type: 'text'
|
||||
is_nullable: 1
|
||||
|
||||
=head2 nixexprpath
|
||||
|
||||
data_type: 'text'
|
||||
is_nullable: 1
|
||||
|
||||
=head2 priority
|
||||
|
||||
data_type: 'integer'
|
||||
|
|
@ -246,10 +236,6 @@ __PACKAGE__->add_columns(
|
|||
{ data_type => "integer", default_value => 0, is_nullable => 0 },
|
||||
"iscurrent",
|
||||
{ data_type => "integer", default_value => 0, is_nullable => 1 },
|
||||
"nixexprinput",
|
||||
{ data_type => "text", is_nullable => 1 },
|
||||
"nixexprpath",
|
||||
{ data_type => "text", is_nullable => 1 },
|
||||
"priority",
|
||||
{ data_type => "integer", default_value => 0, is_nullable => 0 },
|
||||
"globalpriority",
|
||||
|
|
@ -542,8 +528,8 @@ __PACKAGE__->many_to_many(
|
|||
);
|
||||
|
||||
|
||||
# Created by DBIx::Class::Schema::Loader v0.07049 @ 2020-05-27 17:40:41
|
||||
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:RIKKFfcKXFWIUeM8ma++iw
|
||||
# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-22 07:11:57
|
||||
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Df5N0EByYJqoSUqA0dld/A
|
||||
|
||||
__PACKAGE__->has_many(
|
||||
"dependents",
|
||||
|
|
|
|||
|
|
@ -89,6 +89,16 @@ __PACKAGE__->table("jobsetevals");
|
|||
data_type: 'text'
|
||||
is_nullable: 0
|
||||
|
||||
=head2 nixexprinput
|
||||
|
||||
data_type: 'text'
|
||||
is_nullable: 1
|
||||
|
||||
=head2 nixexprpath
|
||||
|
||||
data_type: 'text'
|
||||
is_nullable: 1
|
||||
|
||||
=head2 nrbuilds
|
||||
|
||||
data_type: 'integer'
|
||||
|
|
@ -132,6 +142,10 @@ __PACKAGE__->add_columns(
|
|||
{ data_type => "integer", is_nullable => 0 },
|
||||
"hash",
|
||||
{ data_type => "text", is_nullable => 0 },
|
||||
"nixexprinput",
|
||||
{ data_type => "text", is_nullable => 1 },
|
||||
"nixexprpath",
|
||||
{ data_type => "text", is_nullable => 1 },
|
||||
"nrbuilds",
|
||||
{ data_type => "integer", is_nullable => 1 },
|
||||
"nrsucceeded",
|
||||
|
|
@ -215,8 +229,8 @@ __PACKAGE__->belongs_to(
|
|||
);
|
||||
|
||||
|
||||
# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-21 11:13:38
|
||||
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:zDBtAFc4HiFUcL/TpkuCcg
|
||||
# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-22 07:11:57
|
||||
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:hdu+0WWo2363dVvImMKxdA
|
||||
|
||||
__PACKAGE__->has_many(
|
||||
"buildIds",
|
||||
|
|
|
|||
|
|
@ -375,8 +375,8 @@ __PACKAGE__->has_many(
|
|||
);
|
||||
|
||||
|
||||
# Created by DBIx::Class::Schema::Loader v0.07049 @ 2020-05-27 17:40:41
|
||||
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:aDW78MCelU/ma953aTcHvA
|
||||
# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-22 07:11:57
|
||||
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:6P1qlC5oVSPRSgRBp6nmrw
|
||||
|
||||
|
||||
=head2 builds
|
||||
|
|
|
|||
|
|
@ -258,8 +258,8 @@ Composing rels: L</projectmembers> -> username
|
|||
__PACKAGE__->many_to_many("usernames", "projectmembers", "username");
|
||||
|
||||
|
||||
# Created by DBIx::Class::Schema::Loader v0.07049 @ 2020-05-27 17:40:41
|
||||
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:iBGJjFWiI9Wy9zwT7xGOEA
|
||||
# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-22 07:11:57
|
||||
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Ff5gJejFu+02b0lInobOoQ
|
||||
|
||||
my %hint = (
|
||||
columns => [
|
||||
|
|
|
|||
|
|
@ -120,7 +120,7 @@ END;
|
|||
<b class="caret"></b>
|
||||
</a>
|
||||
<ul class="dropdown-menu">
|
||||
[% IF build.nixexprinput || eval.flake %]
|
||||
[% IF eval.nixexprinput || eval.flake %]
|
||||
<li><a href="#reproduce" data-toggle="modal">Reproduce locally</a></li>
|
||||
[% END %]
|
||||
[% IF c.user_exists %]
|
||||
|
|
@ -346,10 +346,10 @@ END;
|
|||
<td>[% build.priority %]</td>
|
||||
</tr>
|
||||
[% END %]
|
||||
[% IF build.nixexprinput %]
|
||||
[% IF eval.nixexprinput %]
|
||||
<tr>
|
||||
<th>Nix expression:</th>
|
||||
<td>file <tt>[% HTML.escape(build.nixexprpath) %]</tt> in input <tt>[% HTML.escape(build.nixexprinput) %]</tt></td>
|
||||
<td>file <tt>[% HTML.escape(eval.nixexprpath) %]</tt> in input <tt>[% HTML.escape(eval.nixexprinput) %]</tt></td>
|
||||
</tr>
|
||||
[% END %]
|
||||
<tr>
|
||||
|
|
|
|||
|
|
@ -169,7 +169,7 @@ echo "$0: input ‘[% input.name %]’ has unsupported type ‘[% input.type %]
|
|||
exit 1
|
||||
[% END %]
|
||||
|
||||
[% IF input.name == build.nixexprinput +%]
|
||||
[% IF input.name == eval.nixexprinput +%]
|
||||
nixExprInputDir="$inputDir"
|
||||
[%+ END %]
|
||||
|
||||
|
|
@ -197,7 +197,7 @@ args+=(--option extra-binary-caches '[% c.uri_for('/') %]')
|
|||
# when evaluating jobs that rely on builtins.currentSystem.
|
||||
args+=(--option system x86_64-linux)
|
||||
|
||||
args+=("$nixExprInputDir/[% build.nixexprpath %]" -A '[% build.job.name %]')
|
||||
args+=("$nixExprInputDir/[% eval.nixexprpath %]" -A '[% build.job.name %]')
|
||||
|
||||
if [ -n "$printFlags" ]; then
|
||||
first=1
|
||||
|
|
|
|||
|
|
@ -460,8 +460,6 @@ sub checkBuild {
|
|||
, nixname => $buildInfo->{nixName}
|
||||
, drvpath => $drvPath
|
||||
, system => $buildInfo->{system}
|
||||
, nixexprinput => $jobset->nixexprinput
|
||||
, nixexprpath => $jobset->nixexprpath
|
||||
, priority => $buildInfo->{schedulingPriority}
|
||||
, finished => 0
|
||||
, iscurrent => 1
|
||||
|
|
@ -724,6 +722,8 @@ sub checkJobsetWrapped {
|
|||
, hasnewbuilds => $jobsetChanged ? 1 : 0
|
||||
, nrbuilds => $jobsetChanged ? scalar(keys %buildMap) : undef
|
||||
, flake => $flakeRef
|
||||
, nixexprinput => $jobset->nixexprinput
|
||||
, nixexprpath => $jobset->nixexprpath
|
||||
});
|
||||
|
||||
$db->storage->dbh->do("notify eval_added, ?", undef,
|
||||
|
|
|
|||
|
|
@ -162,13 +162,6 @@ create table Builds (
|
|||
isChannel integer not null default 0, -- meta.isHydraChannel
|
||||
isCurrent integer default 0,
|
||||
|
||||
-- Copy of the nixExprInput/nixExprPath fields of the jobset that
|
||||
-- instantiated this build. Needed if we want to reproduce this
|
||||
-- build. FIXME: this should be stored in JobsetEvals, storing it
|
||||
-- here is denormal.
|
||||
nixExprInput text,
|
||||
nixExprPath text,
|
||||
|
||||
-- Priority within a jobset, set via meta.schedulingPriority.
|
||||
priority integer not null default 0,
|
||||
|
||||
|
|
@ -466,6 +459,8 @@ create table JobsetEvals (
|
|||
nrSucceeded integer, -- set lazily when all builds are finished
|
||||
|
||||
flake text, -- immutable flake reference
|
||||
nixExprInput text, -- name of the jobsetInput containing the Nix or Guix expression
|
||||
nixExprPath text, -- relative path of the Nix or Guix expression
|
||||
|
||||
foreign key (project) references Projects(name) on delete cascade on update cascade,
|
||||
foreign key (project, jobset) references Jobsets(project, name) on delete cascade on update cascade
|
||||
|
|
|
|||
31
src/sql/upgrade-71.sql
Normal file
31
src/sql/upgrade-71.sql
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
ALTER TABLE JobsetEvals
|
||||
ADD COLUMN nixExprInput text,
|
||||
ADD COLUMN nixExprPath text;
|
||||
|
||||
|
||||
-- This migration took 4.5 hours on a server
|
||||
-- with 5400RPM drives, against a copy of hydra's
|
||||
-- production dataset. It might take a significantly
|
||||
-- less amount of time there, and not justify a
|
||||
-- batched migration.
|
||||
UPDATE jobsetevals
|
||||
SET (nixexprinput, nixexprpath) = (
|
||||
SELECT builds.nixexprinput, builds.nixexprpath
|
||||
FROM builds
|
||||
LEFT JOIN jobsetevalmembers
|
||||
ON jobsetevalmembers.build = builds.id
|
||||
WHERE jobsetevalmembers.eval = jobsetevals.id
|
||||
LIMIT 1
|
||||
)
|
||||
WHERE jobsetevals.id in (
|
||||
SELECT jobsetevalsprime.id
|
||||
FROM jobsetevals as jobsetevalsprime
|
||||
WHERE jobsetevalsprime.nixexprinput IS NULL
|
||||
-- AND jobsetevalsprime.id > ? --------- These are in case of a batched migration
|
||||
ORDER BY jobsetevalsprime.id ASC -- /
|
||||
-- LIMIT ? -- ----------------------
|
||||
);
|
||||
|
||||
ALTER TABLE builds
|
||||
DROP COLUMN nixexprinput,
|
||||
DROP COLUMN nixexprpath;
|
||||
Loading…
Reference in a new issue