diff --git a/src/c/hydra-eval-jobs.cc b/src/c/hydra-eval-jobs.cc index 4ff01057..ca2fb7fb 100644 --- a/src/c/hydra-eval-jobs.cc +++ b/src/c/hydra-eval-jobs.cc @@ -160,6 +160,23 @@ static void findJobsWrapped(EvalState & state, XMLWriter & doc, } xmlAttrs["maintainers"] = maintainers; + /* If this is an aggregate, then get its members. */ + Bindings::iterator a = v.attrs->find(state.symbols.create("_hydraAggregate")); + if (a != v.attrs->end() && state.forceBool(*a->value)) { + Bindings::iterator a = v.attrs->find(state.symbols.create("members")); + if (a == v.attrs->end()) + throw EvalError("derivation must have a ‘members’ attribute"); + PathSet context; + state.coerceToString(*a->value, context, true, false); + PathSet drvs; + foreach (PathSet::iterator, i, context) + if (i->at(0) == '!') { + size_t index = i->find("!", 1); + drvs.insert(string(*i, index + 1)); + } + xmlAttrs["members"] = concatStringsSep(" ", drvs); + } + /* Register the derivation as a GC root. !!! This registers roots for jobs that we may have already done. */ diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm index cded877f..6fef3ed8 100644 --- a/src/lib/Hydra/Controller/Build.pm +++ b/src/lib/Hydra/Controller/Build.pm @@ -578,7 +578,7 @@ sub clone_submit : Chained('buildChain') PathPart('clone/submit') Args(0) { my %currentBuilds; my $newBuild = checkBuild( - $c->model('DB'), $build->project, $build->jobset, + $c->model('DB'), $build->jobset, $inputInfo, $nixExprInput, $job, \%currentBuilds, undef, {}, $c->hydra_plugins); error($c, "This build has already been performed.") unless $newBuild; diff --git a/src/lib/Hydra/Helper/AddBuilds.pm b/src/lib/Hydra/Helper/AddBuilds.pm index 6f51b98d..f1e21b49 100644 --- a/src/lib/Hydra/Helper/AddBuilds.pm +++ b/src/lib/Hydra/Helper/AddBuilds.pm @@ -288,13 +288,9 @@ sub evalJobs { my $validJob = 1; foreach my $arg (@{$job->{arg}}) { my $input = $inputInfo->{$arg->{name}}->[$arg->{altnr}]; - if ($input->{type} eq "sysbuild" && $input->{system} ne $job->{system}) { - $validJob = 0; - } - } - if ($validJob) { - push(@filteredJobs, $job); + $validJob = 0 if $input->{type} eq "sysbuild" && $input->{system} ne $job->{system}; } + push(@filteredJobs, $job) if $validJob; } $jobs->{job} = \@filteredJobs; @@ -390,7 +386,7 @@ sub getPrevJobsetEval { # Check whether to add the build described by $buildInfo. sub checkBuild { - my ($db, $project, $jobset, $inputInfo, $nixExprInput, $buildInfo, $buildIds, $prevEval, $jobOutPathMap, $plugins) = @_; + my ($db, $jobset, $inputInfo, $nixExprInput, $buildInfo, $buildMap, $prevEval, $jobOutPathMap, $plugins) = @_; my @outputNames = sort keys %{$buildInfo->{output}}; die unless scalar @outputNames; @@ -411,9 +407,7 @@ sub checkBuild { my $build; txn_do($db, sub { - my $job = $jobset->jobs->update_or_create( - { name => $jobName - }); + my $job = $jobset->jobs->update_or_create({ name => $jobName }); # Don't add a build that has already been scheduled for this # job, or has been built but is still a "current" build for @@ -434,19 +428,19 @@ sub checkBuild { # semantically unnecessary (because they're implied by # the eval), but they give a factor 1000 speedup on # the Nixpkgs jobset with PostgreSQL. - { project => $project->name, jobset => $jobset->name, job => $job->name, + { project => $jobset->project->name, jobset => $jobset->name, job => $jobName, name => $firstOutputName, path => $firstOutputPath }, { rows => 1, columns => ['id'], join => ['buildoutputs'] }); if (defined $prevBuild) { print STDERR " already scheduled/built as build ", $prevBuild->id, "\n"; - $buildIds->{$prevBuild->id} = 0; + $buildMap->{$prevBuild->id} = { new => 0, drvPath => $drvPath }; return; } } # Prevent multiple builds with the same (job, outPath) from # being added. - my $prev = $$jobOutPathMap{$job->name . "\t" . $firstOutputPath}; + my $prev = $$jobOutPathMap{$jobName . "\t" . $firstOutputPath}; if (defined $prev) { print STDERR " already scheduled as build ", $prev, "\n"; return; @@ -512,8 +506,8 @@ sub checkBuild { $build->buildoutputs->create({ name => $_, path => $buildInfo->{output}->{$_}->{path} }) foreach @outputNames; - $buildIds->{$build->id} = 1; - $$jobOutPathMap{$job->name . "\t" . $firstOutputPath} = $build->id; + $buildMap->{$build->id} = { new => 1, drvPath => $drvPath }; + $$jobOutPathMap{$jobName . "\t" . $firstOutputPath} = $build->id; if ($build->iscachedbuild) { print STDERR " marked as cached build ", $build->id, "\n"; diff --git a/src/lib/Hydra/Schema/AggregateMembers.pm b/src/lib/Hydra/Schema/AggregateMembers.pm new file mode 100644 index 00000000..4a037663 --- /dev/null +++ b/src/lib/Hydra/Schema/AggregateMembers.pm @@ -0,0 +1,111 @@ +use utf8; +package Hydra::Schema::AggregateMembers; + +# Created by DBIx::Class::Schema::Loader +# DO NOT MODIFY THE FIRST PART OF THIS FILE + +=head1 NAME + +Hydra::Schema::AggregateMembers + +=cut + +use strict; +use warnings; + +use base 'DBIx::Class::Core'; + +=head1 COMPONENTS LOADED + +=over 4 + +=item * L + +=back + +=cut + +__PACKAGE__->load_components("+Hydra::Component::ToJSON"); + +=head1 TABLE: C + +=cut + +__PACKAGE__->table("AggregateMembers"); + +=head1 ACCESSORS + +=head2 aggregate + + data_type: 'integer' + is_foreign_key: 1 + is_nullable: 0 + +=head2 member + + data_type: 'integer' + is_foreign_key: 1 + is_nullable: 0 + +=cut + +__PACKAGE__->add_columns( + "aggregate", + { data_type => "integer", is_foreign_key => 1, is_nullable => 0 }, + "member", + { data_type => "integer", is_foreign_key => 1, is_nullable => 0 }, +); + +=head1 PRIMARY KEY + +=over 4 + +=item * L + +=item * L + +=back + +=cut + +__PACKAGE__->set_primary_key("aggregate", "member"); + +=head1 RELATIONS + +=head2 aggregate + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "aggregate", + "Hydra::Schema::Builds", + { id => "aggregate" }, + { is_deferrable => 0, on_delete => "CASCADE", on_update => "NO ACTION" }, +); + +=head2 member + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "member", + "Hydra::Schema::Builds", + { id => "member" }, + { is_deferrable => 0, on_delete => "CASCADE", on_update => "NO ACTION" }, +); + + +# Created by DBIx::Class::Schema::Loader v0.07033 @ 2013-08-13 22:17:52 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:jHJtO2baXiprv0OcWCLZ+w + + +# You can replace this text with custom code or comments, and it will be preserved on regeneration +1; diff --git a/src/lib/Hydra/Schema/Builds.pm b/src/lib/Hydra/Schema/Builds.pm index ea27d579..6280e08f 100644 --- a/src/lib/Hydra/Schema/Builds.pm +++ b/src/lib/Hydra/Schema/Builds.pm @@ -288,6 +288,36 @@ __PACKAGE__->set_primary_key("id"); =head1 RELATIONS +=head2 aggregatemembers_aggregates + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "aggregatemembers_aggregates", + "Hydra::Schema::AggregateMembers", + { "foreign.aggregate" => "self.id" }, + undef, +); + +=head2 aggregatemembers_members + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "aggregatemembers_members", + "Hydra::Schema::AggregateMembers", + { "foreign.member" => "self.id" }, + undef, +); + =head2 buildinputs_builds Type: has_many @@ -468,9 +498,29 @@ __PACKAGE__->has_many( undef, ); +=head2 aggregates -# Created by DBIx::Class::Schema::Loader v0.07033 @ 2013-06-13 01:54:50 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:isCEXACY/PwkvgKHcXvAIg +Type: many_to_many + +Composing rels: L -> aggregate + +=cut + +__PACKAGE__->many_to_many("aggregates", "aggregatemembers_members", "aggregate"); + +=head2 members + +Type: many_to_many + +Composing rels: L -> member + +=cut + +__PACKAGE__->many_to_many("members", "aggregatemembers_members", "member"); + + +# Created by DBIx::Class::Schema::Loader v0.07033 @ 2013-08-13 22:17:52 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:9jqsol/evbHYjusT09hLtw __PACKAGE__->has_many( "dependents", diff --git a/src/script/hydra-evaluator b/src/script/hydra-evaluator index 5bfed9e3..a6dfde46 100755 --- a/src/script/hydra-evaluator +++ b/src/script/hydra-evaluator @@ -144,11 +144,11 @@ sub checkJobsetWrapped { $jobset->builds->search({iscurrent => 1})->update({iscurrent => 0}); # Schedule each successfully evaluated job. - my %buildIds; + my %buildMap; foreach my $job (permute @{$jobs->{job}}) { next if $job->{jobName} eq ""; print STDERR " considering job " . $project->name, ":", $jobset->name, ":", $job->{jobName} . "\n"; - checkBuild($db, $project, $jobset, $inputInfo, $nixExprInput, $job, \%buildIds, $prevEval, $jobOutPathMap, $plugins); + checkBuild($db, $jobset, $inputInfo, $nixExprInput, $job, \%buildMap, $prevEval, $jobOutPathMap, $plugins); } # Update the last checked times and error messages for each @@ -162,8 +162,8 @@ sub checkJobsetWrapped { foreach $jobset->jobs->all; my $hasNewBuilds = 0; - while (my ($id, $new) = each %buildIds) { - $hasNewBuilds = 1 if $new; + while (my ($id, $x) = each %buildMap) { + $hasNewBuilds = 1 if $x->{new}; } my $ev = $jobset->jobsetevals->create( @@ -172,12 +172,29 @@ sub checkJobsetWrapped { , checkouttime => abs($checkoutStop - $checkoutStart) , evaltime => abs($evalStop - $evalStart) , hasnewbuilds => $hasNewBuilds - , nrbuilds => $hasNewBuilds ? scalar(keys %buildIds) : undef + , nrbuilds => $hasNewBuilds ? scalar(keys %buildMap) : undef }); if ($hasNewBuilds) { - while (my ($id, $new) = each %buildIds) { - $ev->jobsetevalmembers->create({ build => $id, isnew => $new }); + # Create JobsetEvalMembers mappings. + my %drvPathToId; + while (my ($id, $x) = each %buildMap) { + $ev->jobsetevalmembers->create({ build => $id, isnew => $x->{new} }); + $drvPathToId{$x->{drvPath}} = $id; + } + + # Create AggregateMembers mappings. + foreach my $job (@{$jobs->{job}}) { + next unless $job->{members}; + my $id = $drvPathToId{$job->{drvPath}} or die; + foreach my $drvPath (split / /, $job->{members}) { + my $member = $drvPathToId{$drvPath}; + if (defined $member) { + $db->resultset('AggregateMembers')->update_or_create({aggregate => $id, member => $member}); + } else { + warn "aggregate job ‘$job->{jobName}’ has a member ‘$drvPath’ that doesn't correspond to a Hydra build\n"; + } + } } foreach my $name (keys %{$inputInfo}) { diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index 3001ab10..ee601c62 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -514,6 +514,13 @@ create table NewsItems ( ); +create table AggregateMembers ( + aggregate integer not null references Builds(id) on delete cascade, + member integer not null references Builds(id) on delete cascade, + primary key (aggregate, member) +); + + -- Cache of the number of finished builds. create table NrBuilds ( what text primary key not null, diff --git a/src/sql/upgrade-19.sql b/src/sql/upgrade-19.sql new file mode 100644 index 00000000..3d1e849b --- /dev/null +++ b/src/sql/upgrade-19.sql @@ -0,0 +1,5 @@ +create table AggregateMembers ( + aggregate integer not null references Builds(id) on delete cascade, + member integer not null references Builds(id) on delete cascade, + primary key (aggregate, member) +);