diff --git a/doc/dev-notes.txt b/doc/dev-notes.txt index 8e95e098..b4373945 100644 --- a/doc/dev-notes.txt +++ b/doc/dev-notes.txt @@ -45,6 +45,10 @@ * Changing the priority of a scheduled build: update buildschedulinginfo set priority = 200 where id = ; + +* Changing the priority of all builds for a jobset: + + update buildschedulinginfo set priority = 20 where id in (select id from builds where finished = 0 and project = 'nixpkgs' and jobset = 'trunk'); * Steps to install: @@ -103,6 +107,10 @@ alter table Builds add column nixExprInput text; alter table Builds add column nixExprPath text; + # Adding JobsetEvals. + drop table JobsetInputHashes; + (add JobsetEvals, JobsetEvalMembers) + * Job selection: @@ -138,7 +146,7 @@ * Installing deps.nix in a profile for testing: - $ nix-env -p /nix/var/nix/profiles/per-user/eelco/hydra-deps -f deps.nix -i \* --arg pkgs 'import /home/eelco/Dev/nixpkgs {}' + $ nix-env -p $NIX_USER_PROFILE_DIR/hydra-deps -f deps.nix -i \* --arg pkgs 'import /etc/nixos/nixpkgs {}' * select x.project, x.jobset, x.job, x.system, x.id, x.timestamp, r.buildstatus, b.id, b.timestamp @@ -154,3 +162,8 @@ * Using PostgreSQL: $ HYDRA_DBI="dbi:Pg:dbname=hydra;" hydra_server.pl + + +* Find the builds with the highest number of build steps: + + select id, (select count(*) from buildsteps where build = x.id) as n from builds x order by n desc; diff --git a/src/lib/Hydra/Helper/AddBuilds.pm b/src/lib/Hydra/Helper/AddBuilds.pm index a9c8fe58..dd9f9283 100644 --- a/src/lib/Hydra/Helper/AddBuilds.pm +++ b/src/lib/Hydra/Helper/AddBuilds.pm @@ -74,7 +74,7 @@ sub fetchInputPath { # Some simple caching: don't check a path more than once every N seconds. (my $cachedInput) = $db->resultset('CachedPathInputs')->search( - {srcpath => $uri, lastseen => {">", $timestamp - 60}}, + {srcpath => $uri, lastseen => {">", $timestamp - 30}}, {rows => 1, order_by => "lastseen DESC"}); if (defined $cachedInput && isValidPath($cachedInput->storepath)) { @@ -505,7 +505,7 @@ sub checkBuild { my @previousBuilds = $job->builds->search({outPath => $outPath, isCurrent => 1}); if (scalar(@previousBuilds) > 0) { print STDERR "already scheduled/built\n"; - $currentBuilds->{$_->id} = 1 foreach @previousBuilds; + $currentBuilds->{$_->id} = 0 foreach @previousBuilds; return; } diff --git a/src/lib/Hydra/Schema/BuildSchedulingInfo.pm b/src/lib/Hydra/Schema/BuildSchedulingInfo.pm index dcecc737..c41be40e 100644 --- a/src/lib/Hydra/Schema/BuildSchedulingInfo.pm +++ b/src/lib/Hydra/Schema/BuildSchedulingInfo.pm @@ -44,7 +44,7 @@ __PACKAGE__->table("BuildSchedulingInfo"); =head2 locker data_type: text - default_value: (empty string) + default_value: '' is_nullable: 0 size: undef @@ -85,7 +85,7 @@ __PACKAGE__->add_columns( "busy", { data_type => "integer", default_value => 0, is_nullable => 0, size => undef }, "locker", - { data_type => "text", default_value => "", is_nullable => 0, size => undef }, + { data_type => "text", default_value => "''", is_nullable => 0, size => undef }, "logfile", { data_type => "text", @@ -118,8 +118,8 @@ Related object: L __PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" }, {}); -# Created by DBIx::Class::Schema::Loader v0.05003 @ 2010-02-25 10:29:41 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:yEhHeANRynKf72dp5URvZA +# Created by DBIx::Class::Schema::Loader v0.05000 @ 2010-03-05 13:07:46 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:qOU/YGv3fgPynBXovV6gfg # You can replace this text with custom content, and it will be preserved on regeneration 1; diff --git a/src/lib/Hydra/Schema/Builds.pm b/src/lib/Hydra/Schema/Builds.pm index de02bd01..6a34322d 100644 --- a/src/lib/Hydra/Schema/Builds.pm +++ b/src/lib/Hydra/Schema/Builds.pm @@ -420,9 +420,23 @@ __PACKAGE__->has_many( { "foreign.build" => "self.id" }, ); +=head2 jobsetevalmembers -# Created by DBIx::Class::Schema::Loader v0.05003 @ 2010-02-25 11:19:24 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:oCkX9bughWPZg6JKaOxDJA +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "jobsetevalmembers", + "Hydra::Schema::JobsetEvalMembers", + { "foreign.build" => "self.id" }, +); + + +# Created by DBIx::Class::Schema::Loader v0.05000 @ 2010-03-05 13:07:46 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:sE2/zTcfETC8Eahh6NQDZA use Hydra::Helper::Nix; diff --git a/src/lib/Hydra/Schema/JobsetEvalMembers.pm b/src/lib/Hydra/Schema/JobsetEvalMembers.pm new file mode 100644 index 00000000..fc37dfa1 --- /dev/null +++ b/src/lib/Hydra/Schema/JobsetEvalMembers.pm @@ -0,0 +1,102 @@ +package Hydra::Schema::JobsetEvalMembers; + +# Created by DBIx::Class::Schema::Loader +# DO NOT MODIFY THE FIRST PART OF THIS FILE + +use strict; +use warnings; + +use base 'DBIx::Class::Core'; + + +=head1 NAME + +Hydra::Schema::JobsetEvalMembers + +=cut + +__PACKAGE__->table("JobsetEvalMembers"); + +=head1 ACCESSORS + +=head2 eval + + data_type: integer + default_value: undef + is_foreign_key: 1 + is_nullable: 0 + size: undef + +=head2 build + + data_type: integer + default_value: undef + is_foreign_key: 1 + is_nullable: 0 + size: undef + +=head2 isnew + + data_type: integer + default_value: undef + is_nullable: 0 + size: undef + +=cut + +__PACKAGE__->add_columns( + "eval", + { + data_type => "integer", + default_value => undef, + is_foreign_key => 1, + is_nullable => 0, + size => undef, + }, + "build", + { + data_type => "integer", + default_value => undef, + is_foreign_key => 1, + is_nullable => 0, + size => undef, + }, + "isnew", + { + data_type => "integer", + default_value => undef, + is_nullable => 0, + size => undef, + }, +); +__PACKAGE__->set_primary_key("eval", "build"); + +=head1 RELATIONS + +=head2 eval + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to("eval", "Hydra::Schema::JobsetEvals", { id => "eval" }, {}); + +=head2 build + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" }, {}); + + +# Created by DBIx::Class::Schema::Loader v0.05000 @ 2010-03-05 13:07:46 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:vwefi8q3HolhFCkB9aEVWw + + +# You can replace this text with custom content, and it will be preserved on regeneration +1; diff --git a/src/lib/Hydra/Schema/JobsetEvals.pm b/src/lib/Hydra/Schema/JobsetEvals.pm new file mode 100644 index 00000000..4c2bf384 --- /dev/null +++ b/src/lib/Hydra/Schema/JobsetEvals.pm @@ -0,0 +1,200 @@ +package Hydra::Schema::JobsetEvals; + +# Created by DBIx::Class::Schema::Loader +# DO NOT MODIFY THE FIRST PART OF THIS FILE + +use strict; +use warnings; + +use base 'DBIx::Class::Core'; + + +=head1 NAME + +Hydra::Schema::JobsetEvals + +=cut + +__PACKAGE__->table("JobsetEvals"); + +=head1 ACCESSORS + +=head2 id + + data_type: integer + default_value: undef + is_auto_increment: 1 + is_nullable: 0 + size: undef + +=head2 project + + data_type: text + default_value: undef + is_foreign_key: 1 + is_nullable: 0 + size: undef + +=head2 jobset + + data_type: text + default_value: undef + is_foreign_key: 1 + is_nullable: 0 + size: undef + +=head2 timestamp + + data_type: integer + default_value: undef + is_nullable: 0 + size: undef + +=head2 checkouttime + + data_type: integer + default_value: undef + is_nullable: 0 + size: undef + +=head2 evaltime + + data_type: integer + default_value: undef + is_nullable: 0 + size: undef + +=head2 hasnewbuilds + + data_type: integer + default_value: undef + is_nullable: 0 + size: undef + +=head2 hash + + data_type: text + default_value: undef + is_nullable: 0 + size: undef + +=cut + +__PACKAGE__->add_columns( + "id", + { + data_type => "integer", + default_value => undef, + is_auto_increment => 1, + is_nullable => 0, + size => undef, + }, + "project", + { + data_type => "text", + default_value => undef, + is_foreign_key => 1, + is_nullable => 0, + size => undef, + }, + "jobset", + { + data_type => "text", + default_value => undef, + is_foreign_key => 1, + is_nullable => 0, + size => undef, + }, + "timestamp", + { + data_type => "integer", + default_value => undef, + is_nullable => 0, + size => undef, + }, + "checkouttime", + { + data_type => "integer", + default_value => undef, + is_nullable => 0, + size => undef, + }, + "evaltime", + { + data_type => "integer", + default_value => undef, + is_nullable => 0, + size => undef, + }, + "hasnewbuilds", + { + data_type => "integer", + default_value => undef, + is_nullable => 0, + size => undef, + }, + "hash", + { + data_type => "text", + default_value => undef, + is_nullable => 0, + size => undef, + }, +); +__PACKAGE__->set_primary_key("id"); + +=head1 RELATIONS + +=head2 project + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to("project", "Hydra::Schema::Projects", { name => "project" }, {}); + +=head2 jobset + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "jobset", + "Hydra::Schema::Jobsets", + { name => "jobset", project => "project" }, + {}, +); + +=head2 jobsetevalmembers + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "jobsetevalmembers", + "Hydra::Schema::JobsetEvalMembers", + { "foreign.eval" => "self.id" }, +); + + +# Created by DBIx::Class::Schema::Loader v0.05000 @ 2010-03-05 13:33:51 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:QD7ZMOLp9HpK0mAYkk0d/Q + +use Hydra::Helper::Nix; + +# !!! Ugly, should be generated. +my $hydradbi = getHydraDBPath; +if ($hydradbi =~ m/^dbi:Pg/) { + __PACKAGE__->sequence('jobsetevals_id_seq'); +} + +# You can replace this text with custom content, and it will be preserved on regeneration +1; diff --git a/src/lib/Hydra/Schema/JobsetInputHashes.pm b/src/lib/Hydra/Schema/JobsetInputHashes.pm deleted file mode 100644 index 318a0fb7..00000000 --- a/src/lib/Hydra/Schema/JobsetInputHashes.pm +++ /dev/null @@ -1,119 +0,0 @@ -package Hydra::Schema::JobsetInputHashes; - -# Created by DBIx::Class::Schema::Loader -# DO NOT MODIFY THE FIRST PART OF THIS FILE - -use strict; -use warnings; - -use base 'DBIx::Class::Core'; - - -=head1 NAME - -Hydra::Schema::JobsetInputHashes - -=cut - -__PACKAGE__->table("JobsetInputHashes"); - -=head1 ACCESSORS - -=head2 project - - data_type: text - default_value: undef - is_foreign_key: 1 - is_nullable: 0 - size: undef - -=head2 jobset - - data_type: text - default_value: undef - is_foreign_key: 1 - is_nullable: 0 - size: undef - -=head2 hash - - data_type: text - default_value: undef - is_nullable: 0 - size: undef - -=head2 timestamp - - data_type: integer - default_value: undef - is_nullable: 0 - size: undef - -=cut - -__PACKAGE__->add_columns( - "project", - { - data_type => "text", - default_value => undef, - is_foreign_key => 1, - is_nullable => 0, - size => undef, - }, - "jobset", - { - data_type => "text", - default_value => undef, - is_foreign_key => 1, - is_nullable => 0, - size => undef, - }, - "hash", - { - data_type => "text", - default_value => undef, - is_nullable => 0, - size => undef, - }, - "timestamp", - { - data_type => "integer", - default_value => undef, - is_nullable => 0, - size => undef, - }, -); -__PACKAGE__->set_primary_key("project", "jobset", "hash"); - -=head1 RELATIONS - -=head2 project - -Type: belongs_to - -Related object: L - -=cut - -__PACKAGE__->belongs_to("project", "Hydra::Schema::Projects", { name => "project" }, {}); - -=head2 jobset - -Type: belongs_to - -Related object: L - -=cut - -__PACKAGE__->belongs_to( - "jobset", - "Hydra::Schema::Jobsets", - { name => "jobset", project => "project" }, - {}, -); - - -# Created by DBIx::Class::Schema::Loader v0.05003 @ 2010-02-25 10:29:41 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:dK9vFHXInejDW/rl1i/kFA - -1; diff --git a/src/lib/Hydra/Schema/Jobsets.pm b/src/lib/Hydra/Schema/Jobsets.pm index 329e7005..65f6448c 100644 --- a/src/lib/Hydra/Schema/Jobsets.pm +++ b/src/lib/Hydra/Schema/Jobsets.pm @@ -253,17 +253,17 @@ __PACKAGE__->has_many( }, ); -=head2 jobsetinputhashes +=head2 jobsetevals Type: has_many -Related object: L +Related object: L =cut __PACKAGE__->has_many( - "jobsetinputhashes", - "Hydra::Schema::JobsetInputHashes", + "jobsetevals", + "Hydra::Schema::JobsetEvals", { "foreign.jobset" => "self.name", "foreign.project" => "self.project", @@ -271,7 +271,7 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.05003 @ 2010-02-25 10:29:41 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ORCZ73BJrscvmyf/4ds0UQ +# Created by DBIx::Class::Schema::Loader v0.05000 @ 2010-03-05 13:07:46 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Z0HutYxnzYVuQc3W51mq5Q 1; diff --git a/src/lib/Hydra/Schema/Projects.pm b/src/lib/Hydra/Schema/Projects.pm index 599c909d..ed8a9448 100644 --- a/src/lib/Hydra/Schema/Projects.pm +++ b/src/lib/Hydra/Schema/Projects.pm @@ -216,22 +216,22 @@ __PACKAGE__->has_many( { "foreign.project" => "self.name" }, ); -=head2 jobsetinputhashes +=head2 jobsetevals Type: has_many -Related object: L +Related object: L =cut __PACKAGE__->has_many( - "jobsetinputhashes", - "Hydra::Schema::JobsetInputHashes", + "jobsetevals", + "Hydra::Schema::JobsetEvals", { "foreign.project" => "self.name" }, ); -# Created by DBIx::Class::Schema::Loader v0.05003 @ 2010-02-25 10:29:41 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:yH/9hz6FH09kgusRNWrqPg +# Created by DBIx::Class::Schema::Loader v0.05000 @ 2010-03-05 13:07:45 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:SXJ+FzgNDad87OKSBH2qrg 1; diff --git a/src/script/hydra_scheduler.pl b/src/script/hydra_scheduler.pl index 77b5cdad..b5278ed4 100755 --- a/src/script/hydra_scheduler.pl +++ b/src/script/hydra_scheduler.pl @@ -20,6 +20,7 @@ STDOUT->autoflush(); my $db = openHydraDB; my %config = new Config::General($ENV{"HYDRA_CONFIG"})->getall; + sub fetchInputs { my ($project, $jobset, $inputInfo) = @_; foreach my $input ($jobset->jobsetinputs->all) { @@ -99,7 +100,9 @@ sub checkJobset { my $inputInfo = {}; # Fetch all values for all inputs. + my $checkoutStart = time; fetchInputs($project, $jobset, $inputInfo); + my $checkoutStop = time; # Hash the arguments to hydra_eval_jobs and check the # JobsetInputHashes to see if we've already evaluated this set of @@ -107,7 +110,7 @@ sub checkJobset { my @args = ($jobset->nixexprinput, $jobset->nixexprpath, inputsToArgs($inputInfo)); my $argsHash = sha256_hex("@args"); - if ($jobset->jobsetinputhashes->find({hash => $argsHash})) { + if ($jobset->jobsetevals->find({hash => $argsHash})) { print " already evaluated, skipping\n"; txn_do($db, sub { $jobset->update({lastcheckedtime => time}); @@ -116,18 +119,20 @@ sub checkJobset { } # Evaluate the job expression. + my $evalStart = time; my ($jobs, $nixExprInput) = evalJobs($inputInfo, $jobset->nixexprinput, $jobset->nixexprpath); - - # Schedule each successfully evaluated job. - my %currentBuilds; - foreach my $job (permute @{$jobs->{job}}) { - next if $job->{jobName} eq ""; - print "considering job " . $job->{jobName} . "\n"; - checkBuild($db, $project, $jobset, $inputInfo, $nixExprInput, $job, \%currentBuilds); - } + my $evalStop = time; txn_do($db, sub { + # Schedule each successfully evaluated job. + my %currentBuilds; + foreach my $job (permute @{$jobs->{job}}) { + next if $job->{jobName} eq ""; + print "considering job " . $job->{jobName} . "\n"; + checkBuild($db, $project, $jobset, $inputInfo, $nixExprInput, $job, \%currentBuilds); + } + # Update the last checked times and error messages for each # job. my %failedJobNames; @@ -149,8 +154,24 @@ sub checkJobset { $build->update({iscurrent => 0}) unless $currentBuilds{$build->id}; } - $jobset->jobsetinputhashes->create({hash => $argsHash, timestamp => time}); - + my $hasNewBuilds = 0; + while (my ($id, $new) = each %currentBuilds) { + $hasNewBuilds = 1 if $new; + } + + my $ev = $jobset->jobsetevals->create( + { hash => $argsHash + , timestamp => time + , checkouttime => abs($checkoutStop - $checkoutStart) + , evaltime => abs($evalStop - $evalStart) + , hasnewbuilds => $hasNewBuilds + }); + + if ($hasNewBuilds) { + while (my ($id, $new) = each %currentBuilds) { + $ev->jobsetevalmembers->create({ build => $id, isnew => $new }); + } + } }); # Store the errors messages for jobs that failed to evaluate. diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index 681291e5..4ebb3934 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -403,24 +403,47 @@ create table ReleaseMembers ( ); --- This table is used to prevent repeated Nix expression evaluation --- for the same set of inputs for a jobset. In the scheduler, after --- obtaining the current inputs for a jobset, we hash the inputs --- together, and if the resulting hash already appears in this table, --- we can skip the jobset. Otherwise it's added to the table, and the --- Nix expression for the jobset is evaluated. The hash is computed --- over the command-line arguments to hydra_eval_jobs. -create table JobsetInputHashes ( +create table JobsetEvals ( +#ifdef POSTGRESQL + id serial primary key not null, +#else + id integer primary key autoincrement not null, +#endif + project text not null, jobset text not null, + + timestamp integer not null, -- when this entry was added + checkoutTime integer not null, -- how long obtaining the inputs took (in seconds) + evalTime integer not null, -- how long evaluation took (in seconds) + + -- If 0, then the evaluation of this jobset did not cause any new + -- builds to be added to the database. Otherwise, *all* the + -- builds resulting from the evaluation of the jobset (including + -- existing ones) can be found in the JobsetEvalMembers table. + hasNewBuilds integer not null, + + -- Used to prevent repeated Nix expression evaluation for the same + -- set of inputs for a jobset. In the scheduler, after obtaining + -- the current inputs for a jobset, we hash the inputs together, + -- and if the resulting hash already appears in this table, we can + -- skip the jobset. Otherwise we proceed. The hash is computed + -- over the command-line arguments to hydra_eval_jobs. hash text not null, - timestamp integer not null, - primary key (project, jobset, hash), + foreign key (project) references Projects(name) on delete cascade on update cascade, foreign key (project, jobset) references Jobsets(project, name) on delete cascade on update cascade ); +create table JobsetEvalMembers ( + eval integer not null references JobsetEvals(id) on delete cascade, + build integer not null references Builds(id) on delete cascade, + isNew integer not null, + primary key (eval, build) +); + + create table UriRevMapper ( baseuri text not null, uri text not null,