raito/hydra
0
0
Fork 0
forked from lix-project/hydra
Code Pull requests Activity

* Represent jobs explicitly in the DB.

Browse source
This commit is contained in:
Eelco Dolstra 2009-03-13 14:49:25 +00:00
parent a39a28dffb
commit ae364b9e5f
25 changed files with 178 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/lib/Hydra/Helper/CatalystUtils.pm
View file

@ -54,7 +54,7 @@ sub getLatestBuilds {
{
my $attrs =
{ project => $build->get_column('project')
, job => $build->job
, job => $build->get_column('job')
, system => $build->system
, finished => 1
};

4
src/lib/Hydra/Helper/Nix.pm
View file

@ -124,7 +124,7 @@ sub attrsToSQL {
sub getPrimaryBuildsForReleaseSet {
my ($project, $primaryJob) = @_;
my @primaryBuilds = $project->builds->search(
{ job => $primaryJob->job, finished => 1 },
{ job => $primaryJob->get_column('job'), finished => 1 },
{ join => 'resultInfo', order_by => "timestamp DESC"
, '+select' => ["resultInfo.releasename"], '+as' => ["releasename"]
, where => \ attrsToSQL($primaryJob->attrs, "me.id")
@ -154,7 +154,7 @@ sub getRelease {
# as input. If there are multiple, prefer successful
# ones, and then oldest. !!! order_by buildstatus is hacky
($thisBuild) = $primaryBuild->dependentBuilds->search(
{ job => $job->job, finished => 1 },
{ job => $job->get_column('job'), finished => 1 },
{ join => 'resultInfo', rows => 1
, order_by => ["buildstatus", "timestamp"]
, where => \ attrsToSQL($job->attrs, "build.id")

4
src/lib/Hydra/Schema.pm
View file

@ -8,8 +8,8 @@ use base 'DBIx::Class::Schema';
__PACKAGE__->load_classes;
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-12 17:44:15
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:oIv4qqdjLKRDgAYXHHTz8A
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-13 13:33:20
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:aJVSTLLx1pgutjETaqTWXA
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/BuildInputs.pm
View file

@ -36,8 +36,8 @@ __PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" });
__PACKAGE__->belongs_to("dependency", "Hydra::Schema::Builds", { id => "dependency" });
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-12 17:44:15
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:9gWr8DPEeH2BjoHT3nt0Lw
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-13 13:33:20
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:uxfS8+GnU06sbx6nvWzTSQ
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/BuildProducts.pm
View file

@ -35,8 +35,8 @@ __PACKAGE__->set_primary_key("build", "productnr");
__PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" });
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-12 17:44:15
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:GsPyt1OUQHWN6z7mSpNW2g
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-13 13:33:20
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:wnnwDGQMGr2YAu++PYRSuA
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/BuildResultInfo.pm
View file

@ -35,8 +35,8 @@ __PACKAGE__->set_primary_key("id");
__PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" });
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-12 17:44:15
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:zIpamqoL0AxNFDpQ/W1TVg
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-13 13:33:20
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:CGlUjhJozOA4VCYaFtyhqw
__PACKAGE__->belongs_to(
"failedDep",

4
src/lib/Hydra/Schema/BuildSchedulingInfo.pm
View file

@ -27,8 +27,8 @@ __PACKAGE__->set_primary_key("id");
__PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" });
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-12 17:44:15
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:McWRZEMJtjFIcR4LplCc9A
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-13 13:33:20
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:X+Pz2XzTBNU3XdEcg49RyQ
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/BuildSteps.pm
View file

@ -35,7 +35,7 @@ __PACKAGE__->set_primary_key("build", "stepnr");
__PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" });
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-12 17:44:15
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:zuyAJq4T5Ynsq920Pd7hLg
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-13 13:33:20
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:TtcOwOIZHO0vLSJ1CXF1bA
1;

9
src/lib/Hydra/Schema/Builds.pm
View file

@ -44,6 +44,11 @@ __PACKAGE__->belongs_to(
"Hydra::Schema::Jobsets",
{ name => "jobset", project => "project" },
);
__PACKAGE__->belongs_to(
"job",
"Hydra::Schema::Jobs",
{ jobset => "jobset", name => "job", project => "project" },
);
__PACKAGE__->has_many(
"buildschedulinginfoes",
"Hydra::Schema::BuildSchedulingInfo",
@ -76,8 +81,8 @@ __PACKAGE__->has_many(
);
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-12 17:44:15
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:6taCYZNB0HaY+fvDjUhrYQ
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-13 13:33:20
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:xqKyjCWVdoTyQJC28K3WXA
__PACKAGE__->has_many(dependents => 'Hydra::Schema::BuildInputs', 'dependency');

4
src/lib/Hydra/Schema/CachedPathInputs.pm
View file

@ -22,8 +22,8 @@ __PACKAGE__->add_columns(
__PACKAGE__->set_primary_key("srcpath", "sha256hash");
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-12 17:44:15
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Jhu1KxMfSF1Y1b4VGUqTzg
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-13 13:33:20
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Eo9F2GRzgzTGGx15JWBv6Q
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/CachedSubversionInputs.pm
View file

@ -20,8 +20,8 @@ __PACKAGE__->add_columns(
__PACKAGE__->set_primary_key("uri", "revision");
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-12 17:44:15
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:9aQa6Zo6z3lj/qFHNF5//w
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-13 13:33:20
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:KeKwcnnPNoVO4eNr9+y+1g
# You can replace this text with custom content, and it will be preserved on regeneration

49
src/lib/Hydra/Schema/Jobs.pm Normal file
View file

@ -0,0 +1,49 @@
package Hydra::Schema::Jobs;
use strict;
use warnings;
use base 'DBIx::Class';
__PACKAGE__->load_components("Core");
__PACKAGE__->table("Jobs");
__PACKAGE__->add_columns(
"project",
{ data_type => "text", is_nullable => 0, size => undef },
"jobset",
{ data_type => "text", is_nullable => 0, size => undef },
"name",
{ data_type => "text", is_nullable => 0, size => undef },
"active",
{ data_type => "integer", is_nullable => 0, size => undef },
"errormsg",
{ data_type => "text", is_nullable => 0, size => undef },
"firstevaltime",
{ data_type => "integer", is_nullable => 0, size => undef },
"lastevaltime",
{ data_type => "integer", is_nullable => 0, size => undef },
);
__PACKAGE__->set_primary_key("project", "jobset", "name");
__PACKAGE__->has_many(
"builds",
"Hydra::Schema::Builds",
{
"foreign.job" => "self.name",
"foreign.jobset" => "self.jobset",
"foreign.project" => "self.project",
},
);
__PACKAGE__->belongs_to("project", "Hydra::Schema::Projects", { name => "project" });
__PACKAGE__->belongs_to(
"jobset",
"Hydra::Schema::Jobsets",
{ name => "jobset", project => "project" },
);
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-13 13:33:20
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:wJFyUvUACQHpaW/ktaYtOQ
# You can replace this text with custom content, and it will be preserved on regeneration
1;

4
src/lib/Hydra/Schema/JobsetInputAlts.pm
View file

@ -31,8 +31,8 @@ __PACKAGE__->belongs_to(
);
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-12 17:44:15
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:fBv361MRwuWvAJQnOLZDIw
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-13 13:33:20
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:V8h/34X4hs4PKhxKsFgy9w
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/JobsetInputs.pm
View file

@ -43,8 +43,8 @@ __PACKAGE__->has_many(
);
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-12 17:44:15
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:S8AQqv8/GkLkXpPutV4N0A
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-13 13:33:20
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:1XdnQh4HnXU/iOyNvv8QWg
# You can replace this text with custom content, and it will be preserved on regeneration

12
src/lib/Hydra/Schema/Jobsets.pm
View file

@ -48,10 +48,18 @@ __PACKAGE__->has_many(
"foreign.project" => "self.project",
},
);
__PACKAGE__->has_many(
"jobs",
"Hydra::Schema::Jobs",
{
"foreign.jobset" => "self.name",
"foreign.project" => "self.project",
},
);
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-12 17:44:15
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:xYDUCEXRPeVdFJBQnG/Wog
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-13 13:33:20
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:oZ81xw7qIjVkQKjRdOFW9A
# You can replace this text with custom content, and it will be preserved on regeneration

9
src/lib/Hydra/Schema/Projects.pm
View file

@ -33,6 +33,11 @@ __PACKAGE__->has_many(
"Hydra::Schema::Jobsets",
{ "foreign.project" => "self.name" },
);
__PACKAGE__->has_many(
"jobs",
"Hydra::Schema::Jobs",
{ "foreign.project" => "self.name" },
);
__PACKAGE__->has_many(
"releasesets",
"Hydra::Schema::ReleaseSets",
@ -45,8 +50,8 @@ __PACKAGE__->has_many(
);
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-12 17:44:15
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:9ZqA/21u40pt4t85YrYbeg
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-13 13:33:20
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:z+RXYaHk0RXJfFirBe175A
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/ReleaseSetJobs.pm
View file

@ -34,8 +34,8 @@ __PACKAGE__->belongs_to(
);
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-12 17:44:15
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Ta71pOzVsej/w0FoHIUMbg
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-13 13:33:20
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:iWBkyucz/pXtzI+s0iP0EA
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/ReleaseSets.pm
View file

@ -29,8 +29,8 @@ __PACKAGE__->has_many(
);
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-12 17:44:15
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:0DtIgm5jznjy1l3809b06Q
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-13 13:33:20
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:eeWkr8kYyCvFVDZ3YzpI1Q
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/SystemTypes.pm
View file

@ -16,8 +16,8 @@ __PACKAGE__->add_columns(
__PACKAGE__->set_primary_key("system");
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-12 17:44:15
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:WUSl5Uivcl6E1KLxiq+89A
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-13 13:33:20
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:z83sYSKRnt5mc2etYvH6Zg
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/UserRoles.pm
View file

@ -17,8 +17,8 @@ __PACKAGE__->set_primary_key("username", "role");
__PACKAGE__->belongs_to("username", "Hydra::Schema::Users", { username => "username" });
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-12 17:44:15
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:x/HXSH7s3bgPJSLK71hV9w
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-13 13:33:20
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:WRiW+nBfh/X+TMqYu0PI6g
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/lib/Hydra/Schema/Users.pm
View file

@ -30,8 +30,8 @@ __PACKAGE__->has_many(
);
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-12 17:44:15
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:uP86PxN/ZWgy5PRVU7Qgow
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2009-03-13 13:33:20
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Gw6X/Et2+whq/S7o63zF8Q
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/root/build.tt
View file

@ -189,7 +189,7 @@
<td><tt>[% type = input.type; inputTypes.$type %]</tt></td>
<td>
[% IF input.type == "build" %]
<a href="[% c.uri_for('/build' input.dependency.id) %]">Job <tt>[% input.dependency.project.name %]:[% input.dependency.job %]</tt> build [% input.dependency.id %]</a>
Job [% PROCESS renderFullJobNameOfBuild build=input.dependency %] <a href="[% c.uri_for('/build' input.dependency.id) %]">build [% input.dependency.id %]</a>
[% ELSIF input.type == "string" || input.type == "boolean" %]
<tt>"[% input.value %]"</tt>
[% ELSE %]
@ -300,7 +300,7 @@
<tbody>
[% FOREACH input IN build.dependents -%]
<tr>
<td><a href="[% c.uri_for('/build' input.build.id) %]">Job <tt>[% input.build.project.name %]:[% input.build.job %]</tt> build [% input.build.id %]</a></td>
<td>Job [% PROCESS renderFullJobNameOfBuild build=input.build %] <a href="[% c.uri_for('/build' input.build.id) %]">build [% input.build.id %]</a></td>
<td><tt>[% input.name %]</tt></td>
<td><tt>[% input.build.system %]</tt></td>
<td>[% PROCESS renderDateTime timestamp = input.build.timestamp %]</td>

9
src/root/common.tt
View file

@ -37,6 +37,13 @@
[% END %]
[% BLOCK renderFullJobNameOfBuild %]
<tt>
[% PROCESS renderFullJobName project=build.get_column("project") jobset = build.get_column("jobset") job = build.get_column("job") %]
</a>
[% END %]
[% BLOCK renderBuildList %]
<table class="buildList tablesorter">
<thead>
@ -79,7 +86,7 @@
<td>[% build.schedulingInfo.priority %]</td>
[% END %]
<td>
[% PROCESS renderFullJobName project=build.get_column("project") jobset = build.get_column("jobset") job = build.job %]
[% PROCESS renderFullJobNameOfBuild %]
</td>
<td>[% build.resultInfo.releasename ? build.resultInfo.releasename : build.nixname %]</td>
<td><tt>[% build.system %]</tt></td>

53
src/script/hydra_scheduler.pl
View file

@ -216,22 +216,28 @@ sub checkJob {
if $job->{schedulingPriority} =~ /^\d+$/;
$db->txn_do(sub {
if (scalar($db->resultset('Builds')->search(
{ project => $project->name, jobset => $jobset->name
, job => $jobName, outPath => $outPath })) > 0)
{
# Mark this job as active in the database.
my $jobInDB = $jobset->jobs->update_or_create(
{ name => $jobName
, active => 1
, lastevaltime => time
});
$jobInDB->update({firstevaltime => time})
unless defined $jobInDB->firstevaltime;
# Have we already done this build (in this job)?
if (scalar($jobInDB->builds->search({outPath => $outPath})) > 0) {
print "already scheduled/done\n";
return;
}
# Nope, so add it.
print "adding to queue\n";
my $build = $db->resultset('Builds')->create(
my $build = $jobInDB->builds->create(
{ finished => 0
, timestamp => time()
, project => $project->name
, jobset => $jobset->name
, job => $jobName
, description => $job->{description}
, longdescription => $job->{longDescription}
, license => $job->{license}
@ -242,18 +248,16 @@ sub checkJob {
, system => $job->{system}
});
$db->resultset('BuildSchedulingInfo')->create(
{ id => $build->id
, priority => $priority
$build->buildschedulinginfoes->create(
{ priority => $priority
, busy => 0
, locker => ""
});
foreach my $arg (@{$job->{arg}}) {
my $input = $inputInfo->{$arg->{name}}->[$arg->{altnr}] or die "invalid input";
$db->resultset('BuildInputs')->create(
{ build => $build->id
, name => $arg->{name}
$build->buildinputs_builds->create(
{ name => $arg->{name}
, type => $input->{type}
, uri => $input->{uri}
, revision => $input->{revision}
@ -337,10 +341,31 @@ sub checkJobSet {
# Schedule each successfully evaluated job.
foreach my $job (@{$jobs->{job}}) {
next if $job->{jobName} eq "";
print "considering job " . $job->{jobName} . "\n";
checkJob($project, $jobset, $inputInfo, $job);
}
# Mark all existing jobs that we haven't seen as inactive.
my %jobNames;
$jobNames{$_->{jobName}}++ foreach @{$jobs->{job}};
my %failedJobNames;
push @{$failedJobNames{$_->{location}}}, $_->{msg} foreach @{$jobs->{error}};
$db->txn_do(sub {
foreach my $jobInDB ($jobset->jobs->all) {
print $jobInDB->name, "\n";
$jobInDB->update({active => $jobNames{$jobInDB->name} || $failedJobNames{$jobInDB->name} ? 1 : 0});
if ($failedJobNames{$jobInDB->name}) {
$jobInDB->update({errormsg => join '\n', @{$failedJobNames{$jobInDB->name}}});
} else {
$jobInDB->update({errormsg => undef});
}
}
});
# Store the errors messages for jobs that failed to evaluate.
my $msg = "";
foreach my $error (@{$jobs->{error}}) {

24
src/sql/hydra.sql
View file

@ -27,7 +27,8 @@ create table Builds (
homepage text, -- meta.homepage
foreign key (project) references Projects(name), -- ignored by sqlite
foreign key (project, jobset) references Jobsets(project, name) -- ignored by sqlite
foreign key (project, jobset) references Jobsets(project, name), -- ignored by sqlite
foreign key (project, jobset, job) references Jobs(project, jobset, name) -- ignored by sqlite
);
@ -253,6 +254,27 @@ create table JobsetInputAlts (
);
create table Jobs (
project text not null,
jobset text not null,
name text not null,
-- `active' means the Nix expression for the jobset currently
-- contains this job. Otherwise it's a job that has been removed
-- from the expression.
active integer not null default 1,
errorMsg text, -- evalution error for this job
firstEvalTime integer, -- first time the scheduler saw this job
lastEvalTime integer, -- last time the scheduler saw this job
primary key (project, jobset, name),
foreign key (project) references Projects(name) on delete cascade, -- ignored by sqlite
foreign key (project, jobset) references Jobsets(project, name) on delete cascade -- ignored by sqlite
);
-- Cache for inputs of type "path" (used for testing Hydra), storing
-- the SHA-256 hash and store path for each source path. Also stores
-- the timestamp when we first saw the path have these contents, which