From 7daca03e78de7c18c503e3c87877369a01b31b0f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 5 Mar 2010 15:41:10 +0000 Subject: [PATCH] * Store jobset evaluations in the database explicitly. This includes recording the builds that are part of a jobset evaluation. We need this to be able to answer queries such as "return the latest NixOS ISO for which the installation test succeeded". This wasn't previously possible because the database didn't record which builds of (say) the `isoMinimal' job and the `tests.installer.simple' job came from the same evaluation of the nixos:trunk jobset. Keeping a record of evaluations is also useful for logging purposes. --- doc/dev-notes.txt | 15 +- src/lib/Hydra/Helper/AddBuilds.pm | 4 +- src/lib/Hydra/Schema/BuildSchedulingInfo.pm | 8 +- src/lib/Hydra/Schema/Builds.pm | 18 +- src/lib/Hydra/Schema/JobsetEvalMembers.pm | 102 ++++++++++ src/lib/Hydra/Schema/JobsetEvals.pm | 200 ++++++++++++++++++++ src/lib/Hydra/Schema/JobsetInputHashes.pm | 119 ------------ src/lib/Hydra/Schema/Jobsets.pm | 12 +- src/lib/Hydra/Schema/Projects.pm | 12 +- src/script/hydra_scheduler.pl | 43 +++-- src/sql/hydra.sql | 43 ++++- 11 files changed, 415 insertions(+), 161 deletions(-) create mode 100644 src/lib/Hydra/Schema/JobsetEvalMembers.pm create mode 100644 src/lib/Hydra/Schema/JobsetEvals.pm delete mode 100644 src/lib/Hydra/Schema/JobsetInputHashes.pm diff --git a/doc/dev-notes.txt b/doc/dev-notes.txt index 8e95e098..b4373945 100644 --- a/doc/dev-notes.txt +++ b/doc/dev-notes.txt @@ -45,6 +45,10 @@ * Changing the priority of a scheduled build: update buildschedulinginfo set priority = 200 where id = ; + +* Changing the priority of all builds for a jobset: + + update buildschedulinginfo set priority = 20 where id in (select id from builds where finished = 0 and project = 'nixpkgs' and jobset = 'trunk'); * Steps to install: @@ -103,6 +107,10 @@ alter table Builds add column nixExprInput text; alter table Builds add column nixExprPath text; + # Adding JobsetEvals. + drop table JobsetInputHashes; + (add JobsetEvals, JobsetEvalMembers) + * Job selection: @@ -138,7 +146,7 @@ * Installing deps.nix in a profile for testing: - $ nix-env -p /nix/var/nix/profiles/per-user/eelco/hydra-deps -f deps.nix -i \* --arg pkgs 'import /home/eelco/Dev/nixpkgs {}' + $ nix-env -p $NIX_USER_PROFILE_DIR/hydra-deps -f deps.nix -i \* --arg pkgs 'import /etc/nixos/nixpkgs {}' * select x.project, x.jobset, x.job, x.system, x.id, x.timestamp, r.buildstatus, b.id, b.timestamp @@ -154,3 +162,8 @@ * Using PostgreSQL: $ HYDRA_DBI="dbi:Pg:dbname=hydra;" hydra_server.pl + + +* Find the builds with the highest number of build steps: + + select id, (select count(*) from buildsteps where build = x.id) as n from builds x order by n desc; diff --git a/src/lib/Hydra/Helper/AddBuilds.pm b/src/lib/Hydra/Helper/AddBuilds.pm index a9c8fe58..dd9f9283 100644 --- a/src/lib/Hydra/Helper/AddBuilds.pm +++ b/src/lib/Hydra/Helper/AddBuilds.pm @@ -74,7 +74,7 @@ sub fetchInputPath { # Some simple caching: don't check a path more than once every N seconds. (my $cachedInput) = $db->resultset('CachedPathInputs')->search( - {srcpath => $uri, lastseen => {">", $timestamp - 60}}, + {srcpath => $uri, lastseen => {">", $timestamp - 30}}, {rows => 1, order_by => "lastseen DESC"}); if (defined $cachedInput && isValidPath($cachedInput->storepath)) { @@ -505,7 +505,7 @@ sub checkBuild { my @previousBuilds = $job->builds->search({outPath => $outPath, isCurrent => 1}); if (scalar(@previousBuilds) > 0) { print STDERR "already scheduled/built\n"; - $currentBuilds->{$_->id} = 1 foreach @previousBuilds; + $currentBuilds->{$_->id} = 0 foreach @previousBuilds; return; } diff --git a/src/lib/Hydra/Schema/BuildSchedulingInfo.pm b/src/lib/Hydra/Schema/BuildSchedulingInfo.pm index dcecc737..c41be40e 100644 --- a/src/lib/Hydra/Schema/BuildSchedulingInfo.pm +++ b/src/lib/Hydra/Schema/BuildSchedulingInfo.pm @@ -44,7 +44,7 @@ __PACKAGE__->table("BuildSchedulingInfo"); =head2 locker data_type: text - default_value: (empty string) + default_value: '' is_nullable: 0 size: undef @@ -85,7 +85,7 @@ __PACKAGE__->add_columns( "busy", { data_type => "integer", default_value => 0, is_nullable => 0, size => undef }, "locker", - { data_type => "text", default_value => "", is_nullable => 0, size => undef }, + { data_type => "text", default_value => "''", is_nullable => 0, size => undef }, "logfile", { data_type => "text", @@ -118,8 +118,8 @@ Related object: L __PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" }, {}); -# Created by DBIx::Class::Schema::Loader v0.05003 @ 2010-02-25 10:29:41 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:yEhHeANRynKf72dp5URvZA +# Created by DBIx::Class::Schema::Loader v0.05000 @ 2010-03-05 13:07:46 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:qOU/YGv3fgPynBXovV6gfg # You can replace this text with custom content, and it will be preserved on regeneration 1; diff --git a/src/lib/Hydra/Schema/Builds.pm b/src/lib/Hydra/Schema/Builds.pm index de02bd01..6a34322d 100644 --- a/src/lib/Hydra/Schema/Builds.pm +++ b/src/lib/Hydra/Schema/Builds.pm @@ -420,9 +420,23 @@ __PACKAGE__->has_many( { "foreign.build" => "self.id" }, ); +=head2 jobsetevalmembers -# Created by DBIx::Class::Schema::Loader v0.05003 @ 2010-02-25 11:19:24 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:oCkX9bughWPZg6JKaOxDJA +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "jobsetevalmembers", + "Hydra::Schema::JobsetEvalMembers", + { "foreign.build" => "self.id" }, +); + + +# Created by DBIx::Class::Schema::Loader v0.05000 @ 2010-03-05 13:07:46 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:sE2/zTcfETC8Eahh6NQDZA use Hydra::Helper::Nix; diff --git a/src/lib/Hydra/Schema/JobsetEvalMembers.pm b/src/lib/Hydra/Schema/JobsetEvalMembers.pm new file mode 100644 index 00000000..fc37dfa1 --- /dev/null +++ b/src/lib/Hydra/Schema/JobsetEvalMembers.pm @@ -0,0 +1,102 @@ +package Hydra::Schema::JobsetEvalMembers; + +# Created by DBIx::Class::Schema::Loader +# DO NOT MODIFY THE FIRST PART OF THIS FILE + +use strict; +use warnings; + +use base 'DBIx::Class::Core'; + + +=head1 NAME + +Hydra::Schema::JobsetEvalMembers + +=cut + +__PACKAGE__->table("JobsetEvalMembers"); + +=head1 ACCESSORS + +=head2 eval + + data_type: integer + default_value: undef + is_foreign_key: 1 + is_nullable: 0 + size: undef + +=head2 build + + data_type: integer + default_value: undef + is_foreign_key: 1 + is_nullable: 0 + size: undef + +=head2 isnew + + data_type: integer + default_value: undef + is_nullable: 0 + size: undef + +=cut + +__PACKAGE__->add_columns( + "eval", + { + data_type => "integer", + default_value => undef, + is_foreign_key => 1, + is_nullable => 0, + size => undef, + }, + "build", + { + data_type => "integer", + default_value => undef, + is_foreign_key => 1, + is_nullable => 0, + size => undef, + }, + "isnew", + { + data_type => "integer", + default_value => undef, + is_nullable => 0, + size => undef, + }, +); +__PACKAGE__->set_primary_key("eval", "build"); + +=head1 RELATIONS + +=head2 eval + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to("eval", "Hydra::Schema::JobsetEvals", { id => "eval" }, {}); + +=head2 build + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" }, {}); + + +# Created by DBIx::Class::Schema::Loader v0.05000 @ 2010-03-05 13:07:46 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:vwefi8q3HolhFCkB9aEVWw + + +# You can replace this text with custom content, and it will be preserved on regeneration +1; diff --git a/src/lib/Hydra/Schema/JobsetEvals.pm b/src/lib/Hydra/Schema/JobsetEvals.pm new file mode 100644 index 00000000..4c2bf384 --- /dev/null +++ b/src/lib/Hydra/Schema/JobsetEvals.pm @@ -0,0 +1,200 @@ +package Hydra::Schema::JobsetEvals; + +# Created by DBIx::Class::Schema::Loader +# DO NOT MODIFY THE FIRST PART OF THIS FILE + +use strict; +use warnings; + +use base 'DBIx::Class::Core'; + + +=head1 NAME + +Hydra::Schema::JobsetEvals + +=cut + +__PACKAGE__->table("JobsetEvals"); + +=head1 ACCESSORS + +=head2 id + + data_type: integer + default_value: undef + is_auto_increment: 1 + is_nullable: 0 + size: undef + +=head2 project + + data_type: text + default_value: undef + is_foreign_key: 1 + is_nullable: 0 + size: undef + +=head2 jobset + + data_type: text + default_value: undef + is_foreign_key: 1 + is_nullable: 0 + size: undef + +=head2 timestamp + + data_type: integer + default_value: undef + is_nullable: 0 + size: undef + +=head2 checkouttime + + data_type: integer + default_value: undef + is_nullable: 0 + size: undef + +=head2 evaltime + + data_type: integer + default_value: undef + is_nullable: 0 + size: undef + +=head2 hasnewbuilds + + data_type: integer + default_value: undef + is_nullable: 0 + size: undef + +=head2 hash + + data_type: text + default_value: undef + is_nullable: 0 + size: undef + +=cut + +__PACKAGE__->add_columns( + "id", + { + data_type => "integer", + default_value => undef, + is_auto_increment => 1, + is_nullable => 0, + size => undef, + }, + "project", + { + data_type => "text", + default_value => undef, + is_foreign_key => 1, + is_nullable => 0, + size => undef, + }, + "jobset", + { + data_type => "text", + default_value => undef, + is_foreign_key => 1, + is_nullable => 0, + size => undef, + }, + "timestamp", + { + data_type => "integer", + default_value => undef, + is_nullable => 0, + size => undef, + }, + "checkouttime", + { + data_type => "integer", + default_value => undef, + is_nullable => 0, + size => undef, + }, + "evaltime", + { + data_type => "integer", + default_value => undef, + is_nullable => 0, + size => undef, + }, + "hasnewbuilds", + { + data_type => "integer", + default_value => undef, + is_nullable => 0, + size => undef, + }, + "hash", + { + data_type => "text", + default_value => undef, + is_nullable => 0, + size => undef, + }, +); +__PACKAGE__->set_primary_key("id"); + +=head1 RELATIONS + +=head2 project + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to("project", "Hydra::Schema::Projects", { name => "project" }, {}); + +=head2 jobset + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "jobset", + "Hydra::Schema::Jobsets", + { name => "jobset", project => "project" }, + {}, +); + +=head2 jobsetevalmembers + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "jobsetevalmembers", + "Hydra::Schema::JobsetEvalMembers", + { "foreign.eval" => "self.id" }, +); + + +# Created by DBIx::Class::Schema::Loader v0.05000 @ 2010-03-05 13:33:51 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:QD7ZMOLp9HpK0mAYkk0d/Q + +use Hydra::Helper::Nix; + +# !!! Ugly, should be generated. +my $hydradbi = getHydraDBPath; +if ($hydradbi =~ m/^dbi:Pg/) { + __PACKAGE__->sequence('jobsetevals_id_seq'); +} + +# You can replace this text with custom content, and it will be preserved on regeneration +1; diff --git a/src/lib/Hydra/Schema/JobsetInputHashes.pm b/src/lib/Hydra/Schema/JobsetInputHashes.pm deleted file mode 100644 index 318a0fb7..00000000 --- a/src/lib/Hydra/Schema/JobsetInputHashes.pm +++ /dev/null @@ -1,119 +0,0 @@ -package Hydra::Schema::JobsetInputHashes; - -# Created by DBIx::Class::Schema::Loader -# DO NOT MODIFY THE FIRST PART OF THIS FILE - -use strict; -use warnings; - -use base 'DBIx::Class::Core'; - - -=head1 NAME - -Hydra::Schema::JobsetInputHashes - -=cut - -__PACKAGE__->table("JobsetInputHashes"); - -=head1 ACCESSORS - -=head2 project - - data_type: text - default_value: undef - is_foreign_key: 1 - is_nullable: 0 - size: undef - -=head2 jobset - - data_type: text - default_value: undef - is_foreign_key: 1 - is_nullable: 0 - size: undef - -=head2 hash - - data_type: text - default_value: undef - is_nullable: 0 - size: undef - -=head2 timestamp - - data_type: integer - default_value: undef - is_nullable: 0 - size: undef - -=cut - -__PACKAGE__->add_columns( - "project", - { - data_type => "text", - default_value => undef, - is_foreign_key => 1, - is_nullable => 0, - size => undef, - }, - "jobset", - { - data_type => "text", - default_value => undef, - is_foreign_key => 1, - is_nullable => 0, - size => undef, - }, - "hash", - { - data_type => "text", - default_value => undef, - is_nullable => 0, - size => undef, - }, - "timestamp", - { - data_type => "integer", - default_value => undef, - is_nullable => 0, - size => undef, - }, -); -__PACKAGE__->set_primary_key("project", "jobset", "hash"); - -=head1 RELATIONS - -=head2 project - -Type: belongs_to - -Related object: L - -=cut - -__PACKAGE__->belongs_to("project", "Hydra::Schema::Projects", { name => "project" }, {}); - -=head2 jobset - -Type: belongs_to - -Related object: L - -=cut - -__PACKAGE__->belongs_to( - "jobset", - "Hydra::Schema::Jobsets", - { name => "jobset", project => "project" }, - {}, -); - - -# Created by DBIx::Class::Schema::Loader v0.05003 @ 2010-02-25 10:29:41 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:dK9vFHXInejDW/rl1i/kFA - -1; diff --git a/src/lib/Hydra/Schema/Jobsets.pm b/src/lib/Hydra/Schema/Jobsets.pm index 329e7005..65f6448c 100644 --- a/src/lib/Hydra/Schema/Jobsets.pm +++ b/src/lib/Hydra/Schema/Jobsets.pm @@ -253,17 +253,17 @@ __PACKAGE__->has_many( }, ); -=head2 jobsetinputhashes +=head2 jobsetevals Type: has_many -Related object: L +Related object: L =cut __PACKAGE__->has_many( - "jobsetinputhashes", - "Hydra::Schema::JobsetInputHashes", + "jobsetevals", + "Hydra::Schema::JobsetEvals", { "foreign.jobset" => "self.name", "foreign.project" => "self.project", @@ -271,7 +271,7 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.05003 @ 2010-02-25 10:29:41 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ORCZ73BJrscvmyf/4ds0UQ +# Created by DBIx::Class::Schema::Loader v0.05000 @ 2010-03-05 13:07:46 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Z0HutYxnzYVuQc3W51mq5Q 1; diff --git a/src/lib/Hydra/Schema/Projects.pm b/src/lib/Hydra/Schema/Projects.pm index 599c909d..ed8a9448 100644 --- a/src/lib/Hydra/Schema/Projects.pm +++ b/src/lib/Hydra/Schema/Projects.pm @@ -216,22 +216,22 @@ __PACKAGE__->has_many( { "foreign.project" => "self.name" }, ); -=head2 jobsetinputhashes +=head2 jobsetevals Type: has_many -Related object: L +Related object: L =cut __PACKAGE__->has_many( - "jobsetinputhashes", - "Hydra::Schema::JobsetInputHashes", + "jobsetevals", + "Hydra::Schema::JobsetEvals", { "foreign.project" => "self.name" }, ); -# Created by DBIx::Class::Schema::Loader v0.05003 @ 2010-02-25 10:29:41 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:yH/9hz6FH09kgusRNWrqPg +# Created by DBIx::Class::Schema::Loader v0.05000 @ 2010-03-05 13:07:45 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:SXJ+FzgNDad87OKSBH2qrg 1; diff --git a/src/script/hydra_scheduler.pl b/src/script/hydra_scheduler.pl index 77b5cdad..b5278ed4 100755 --- a/src/script/hydra_scheduler.pl +++ b/src/script/hydra_scheduler.pl @@ -20,6 +20,7 @@ STDOUT->autoflush(); my $db = openHydraDB; my %config = new Config::General($ENV{"HYDRA_CONFIG"})->getall; + sub fetchInputs { my ($project, $jobset, $inputInfo) = @_; foreach my $input ($jobset->jobsetinputs->all) { @@ -99,7 +100,9 @@ sub checkJobset { my $inputInfo = {}; # Fetch all values for all inputs. + my $checkoutStart = time; fetchInputs($project, $jobset, $inputInfo); + my $checkoutStop = time; # Hash the arguments to hydra_eval_jobs and check the # JobsetInputHashes to see if we've already evaluated this set of @@ -107,7 +110,7 @@ sub checkJobset { my @args = ($jobset->nixexprinput, $jobset->nixexprpath, inputsToArgs($inputInfo)); my $argsHash = sha256_hex("@args"); - if ($jobset->jobsetinputhashes->find({hash => $argsHash})) { + if ($jobset->jobsetevals->find({hash => $argsHash})) { print " already evaluated, skipping\n"; txn_do($db, sub { $jobset->update({lastcheckedtime => time}); @@ -116,18 +119,20 @@ sub checkJobset { } # Evaluate the job expression. + my $evalStart = time; my ($jobs, $nixExprInput) = evalJobs($inputInfo, $jobset->nixexprinput, $jobset->nixexprpath); - - # Schedule each successfully evaluated job. - my %currentBuilds; - foreach my $job (permute @{$jobs->{job}}) { - next if $job->{jobName} eq ""; - print "considering job " . $job->{jobName} . "\n"; - checkBuild($db, $project, $jobset, $inputInfo, $nixExprInput, $job, \%currentBuilds); - } + my $evalStop = time; txn_do($db, sub { + # Schedule each successfully evaluated job. + my %currentBuilds; + foreach my $job (permute @{$jobs->{job}}) { + next if $job->{jobName} eq ""; + print "considering job " . $job->{jobName} . "\n"; + checkBuild($db, $project, $jobset, $inputInfo, $nixExprInput, $job, \%currentBuilds); + } + # Update the last checked times and error messages for each # job. my %failedJobNames; @@ -149,8 +154,24 @@ sub checkJobset { $build->update({iscurrent => 0}) unless $currentBuilds{$build->id}; } - $jobset->jobsetinputhashes->create({hash => $argsHash, timestamp => time}); - + my $hasNewBuilds = 0; + while (my ($id, $new) = each %currentBuilds) { + $hasNewBuilds = 1 if $new; + } + + my $ev = $jobset->jobsetevals->create( + { hash => $argsHash + , timestamp => time + , checkouttime => abs($checkoutStop - $checkoutStart) + , evaltime => abs($evalStop - $evalStart) + , hasnewbuilds => $hasNewBuilds + }); + + if ($hasNewBuilds) { + while (my ($id, $new) = each %currentBuilds) { + $ev->jobsetevalmembers->create({ build => $id, isnew => $new }); + } + } }); # Store the errors messages for jobs that failed to evaluate. diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index 681291e5..4ebb3934 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -403,24 +403,47 @@ create table ReleaseMembers ( ); --- This table is used to prevent repeated Nix expression evaluation --- for the same set of inputs for a jobset. In the scheduler, after --- obtaining the current inputs for a jobset, we hash the inputs --- together, and if the resulting hash already appears in this table, --- we can skip the jobset. Otherwise it's added to the table, and the --- Nix expression for the jobset is evaluated. The hash is computed --- over the command-line arguments to hydra_eval_jobs. -create table JobsetInputHashes ( +create table JobsetEvals ( +#ifdef POSTGRESQL + id serial primary key not null, +#else + id integer primary key autoincrement not null, +#endif + project text not null, jobset text not null, + + timestamp integer not null, -- when this entry was added + checkoutTime integer not null, -- how long obtaining the inputs took (in seconds) + evalTime integer not null, -- how long evaluation took (in seconds) + + -- If 0, then the evaluation of this jobset did not cause any new + -- builds to be added to the database. Otherwise, *all* the + -- builds resulting from the evaluation of the jobset (including + -- existing ones) can be found in the JobsetEvalMembers table. + hasNewBuilds integer not null, + + -- Used to prevent repeated Nix expression evaluation for the same + -- set of inputs for a jobset. In the scheduler, after obtaining + -- the current inputs for a jobset, we hash the inputs together, + -- and if the resulting hash already appears in this table, we can + -- skip the jobset. Otherwise we proceed. The hash is computed + -- over the command-line arguments to hydra_eval_jobs. hash text not null, - timestamp integer not null, - primary key (project, jobset, hash), + foreign key (project) references Projects(name) on delete cascade on update cascade, foreign key (project, jobset) references Jobsets(project, name) on delete cascade on update cascade ); +create table JobsetEvalMembers ( + eval integer not null references JobsetEvals(id) on delete cascade, + build integer not null references Builds(id) on delete cascade, + isNew integer not null, + primary key (eval, build) +); + + create table UriRevMapper ( baseuri text not null, uri text not null,