diff --git a/src/HydraFrontend/lib/HydraFrontend/Schema.pm b/src/HydraFrontend/lib/HydraFrontend/Schema.pm index 88921f2f..675608f3 100644 --- a/src/HydraFrontend/lib/HydraFrontend/Schema.pm +++ b/src/HydraFrontend/lib/HydraFrontend/Schema.pm @@ -8,8 +8,8 @@ use base 'DBIx::Class::Schema'; __PACKAGE__->load_classes; -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-10-28 17:59:29 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Fayli8dtSdcAYhfKSZnJwg +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-04 14:45:23 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:UAjA2VmMoOSjiHk0NUzLfQ # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/HydraFrontend/lib/HydraFrontend/Schema/Buildlogs.pm b/src/HydraFrontend/lib/HydraFrontend/Schema/Buildlogs.pm index 08edcfef..a40934f4 100644 --- a/src/HydraFrontend/lib/HydraFrontend/Schema/Buildlogs.pm +++ b/src/HydraFrontend/lib/HydraFrontend/Schema/Buildlogs.pm @@ -25,8 +25,8 @@ __PACKAGE__->belongs_to( ); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-10-28 17:59:29 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:xXiHLBKW5fHl7ukdYeIsTw +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-04 14:45:23 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:kck2qlNZVLFUnevNPSBVKw # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/HydraFrontend/lib/HydraFrontend/Schema/Buildproducts.pm b/src/HydraFrontend/lib/HydraFrontend/Schema/Buildproducts.pm index e9101f8b..cd945f05 100644 --- a/src/HydraFrontend/lib/HydraFrontend/Schema/Buildproducts.pm +++ b/src/HydraFrontend/lib/HydraFrontend/Schema/Buildproducts.pm @@ -25,8 +25,8 @@ __PACKAGE__->belongs_to( ); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-10-28 17:59:29 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:5SPq4at2/NRvbax49TwfDw +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-04 14:45:23 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:CnCSHdI5+5p+L6+r/YITxQ # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/HydraFrontend/lib/HydraFrontend/Schema/Builds.pm b/src/HydraFrontend/lib/HydraFrontend/Schema/Builds.pm index 562c03de..288e9081 100644 --- a/src/HydraFrontend/lib/HydraFrontend/Schema/Builds.pm +++ b/src/HydraFrontend/lib/HydraFrontend/Schema/Builds.pm @@ -12,7 +12,11 @@ __PACKAGE__->add_columns( { data_type => "integer", is_nullable => 0, size => undef }, "timestamp", { data_type => "integer", is_nullable => 0, size => undef }, - "jobname", + "project", + { data_type => "text", is_nullable => 0, size => undef }, + "jobset", + { data_type => "text", is_nullable => 0, size => undef }, + "attrname", { data_type => "text", is_nullable => 0, size => undef }, "description", { data_type => "text", is_nullable => 0, size => undef }, @@ -44,7 +48,7 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-10-28 17:59:29 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:gp6ZZpDA2VzgnNE9NX99dA +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-04 14:45:23 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Odp6qymLlNXbsD7VOQ7PAQ 1; diff --git a/src/HydraFrontend/lib/HydraFrontend/Schema/Jobsetinputs.pm b/src/HydraFrontend/lib/HydraFrontend/Schema/Jobsetinputs.pm new file mode 100644 index 00000000..66e39603 --- /dev/null +++ b/src/HydraFrontend/lib/HydraFrontend/Schema/Jobsetinputs.pm @@ -0,0 +1,48 @@ +package HydraFrontend::Schema::Jobsetinputs; + +use strict; +use warnings; + +use base 'DBIx::Class'; + +__PACKAGE__->load_components("Core"); +__PACKAGE__->table("jobSetInputs"); +__PACKAGE__->add_columns( + "project", + { data_type => "text", is_nullable => 0, size => undef }, + "job", + { data_type => "text", is_nullable => 0, size => undef }, + "name", + { data_type => "text", is_nullable => 0, size => undef }, + "type", + { data_type => "text", is_nullable => 0, size => undef }, + "uri", + { data_type => "text", is_nullable => 0, size => undef }, + "revision", + { data_type => "integer", is_nullable => 0, size => undef }, + "tag", + { data_type => "text", is_nullable => 0, size => undef }, +); +__PACKAGE__->set_primary_key("project", "job", "name"); +__PACKAGE__->has_many( + "jobsets", + "HydraFrontend::Schema::Jobsets", + { + "foreign.name" => "self.job", + "foreign.nixexprinput" => "self.name", + "foreign.project" => "self.project", + }, +); +__PACKAGE__->belongs_to( + "jobset", + "HydraFrontend::Schema::Jobsets", + { name => "job", project => "project" }, +); + + +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-04 14:45:23 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:pzKFsX3b5wTNZvo8t3WTDg + + +# You can replace this text with custom content, and it will be preserved on regeneration +1; diff --git a/src/HydraFrontend/lib/HydraFrontend/Schema/Jobsets.pm b/src/HydraFrontend/lib/HydraFrontend/Schema/Jobsets.pm new file mode 100644 index 00000000..ac748563 --- /dev/null +++ b/src/HydraFrontend/lib/HydraFrontend/Schema/Jobsets.pm @@ -0,0 +1,45 @@ +package HydraFrontend::Schema::Jobsets; + +use strict; +use warnings; + +use base 'DBIx::Class'; + +__PACKAGE__->load_components("Core"); +__PACKAGE__->table("jobSets"); +__PACKAGE__->add_columns( + "name", + { data_type => "text", is_nullable => 0, size => undef }, + "project", + { data_type => "text", is_nullable => 0, size => undef }, + "description", + { data_type => "text", is_nullable => 0, size => undef }, + "nixexprinput", + { data_type => "text", is_nullable => 0, size => undef }, + "nixexprpath", + { data_type => "text", is_nullable => 0, size => undef }, +); +__PACKAGE__->set_primary_key("project", "name"); +__PACKAGE__->belongs_to( + "project", + "HydraFrontend::Schema::Projects", + { name => "project" }, +); +__PACKAGE__->belongs_to( + "jobsetinput", + "HydraFrontend::Schema::Jobsetinputs", + { job => "name", name => "nixexprinput", project => "project" }, +); +__PACKAGE__->has_many( + "jobsetinputs", + "HydraFrontend::Schema::Jobsetinputs", + { "foreign.job" => "self.name", "foreign.project" => "self.project" }, +); + + +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-04 14:45:23 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:pEIAO9lDM+lMKLCLGWRdXg + + +# You can replace this text with custom content, and it will be preserved on regeneration +1; diff --git a/src/HydraFrontend/lib/HydraFrontend/Schema/Projects.pm b/src/HydraFrontend/lib/HydraFrontend/Schema/Projects.pm new file mode 100644 index 00000000..79b6798f --- /dev/null +++ b/src/HydraFrontend/lib/HydraFrontend/Schema/Projects.pm @@ -0,0 +1,27 @@ +package HydraFrontend::Schema::Projects; + +use strict; +use warnings; + +use base 'DBIx::Class'; + +__PACKAGE__->load_components("Core"); +__PACKAGE__->table("projects"); +__PACKAGE__->add_columns( + "name", + { data_type => "text", is_nullable => 0, size => undef }, +); +__PACKAGE__->set_primary_key("name"); +__PACKAGE__->has_many( + "jobsets", + "HydraFrontend::Schema::Jobsets", + { "foreign.project" => "self.name" }, +); + + +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-04 14:45:23 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:cpO0BGfChpnpm7KBKkSUjw + + +# You can replace this text with custom content, and it will be preserved on regeneration +1; diff --git a/src/HydraFrontend/root/index.tt b/src/HydraFrontend/root/index.tt index 3e70cbfb..2f80a90d 100644 --- a/src/HydraFrontend/root/index.tt +++ b/src/HydraFrontend/root/index.tt @@ -6,7 +6,7 @@ - + [% FOREACH build IN latestBuilds -%] diff --git a/src/hydra.schema b/src/hydra.sql similarity index 51% rename from src/hydra.schema rename to src/hydra.sql index c707b6d7..c206e73c 100644 --- a/src/hydra.schema +++ b/src/hydra.sql @@ -1,7 +1,15 @@ create table builds ( id integer primary key autoincrement not null, timestamp integer not null, -- time this build was added to the db (in Unix time) - jobName text not null, + + -- Info about the inputs. + project text not null, -- !!! foreign key + jobSet text not null, -- !!! foreign key + attrName text not null, + + -- !!! list all the inputs / arguments + + -- Info about the build result. description text, drvPath text not null, outPath text not null, @@ -40,3 +48,36 @@ create trigger cascadeBuildDeletion delete from buildLogs where buildId = old.id; delete from buildProducts where buildId = old.id; end; + + +create table projects ( + name text primary key not null +); + + +-- A jobset consists of a set of inputs (e.g. SVN repositories), one +-- of which contains a Nix expression containing an attribute set +-- describing build jobs. +create table jobSets ( + name text not null, + project text not null, + description text, + nixExprInput text not null, -- name of the jobSetInput containing the Nix expression + nixExprPath text not null, -- relative path of the Nix expression + primary key (project, name), + foreign key (project) references projects(name) on delete cascade, -- ignored by sqlite + foreign key (project, name, nixExprInput) references jobSetInputs(project, job, name) +); + + +create table jobSetInputs ( + project text not null, + job text not null, + name text not null, + type text not null, -- "svn", "cvs", "path", "file" + uri text, + revision integer, -- for svn + tag text, -- for cvs + primary key (project, job, name), + foreign key (project, job) references jobSets(project, name) on delete cascade -- ignored by sqlite +); diff --git a/src/scheduler.pl b/src/scheduler.pl index f69f6ffc..184ee649 100644 --- a/src/scheduler.pl +++ b/src/scheduler.pl @@ -2,37 +2,19 @@ use strict; use XML::Simple; -use DBI; use File::Basename; +use HydraFrontend::Schema; -my $jobsFile = "../test.nix"; +my $db = HydraFrontend::Schema->connect("dbi:SQLite:dbname=hydra.sqlite", "", "", {}); -my $dbh = DBI->connect("dbi:SQLite:dbname=hydra.sqlite", "", ""); +sub buildJob { + my ($project, $jobset, $jobName, $drvPath, $outPath) = @_; - -my $jobsXml = `nix-env -f $jobsFile --query --available "*" --attr-path --out-path --drv-path --meta --xml --system-filter "*"` - or die "cannot evaluate the Nix expression containing the job definitions: $?"; - -print "$jobsXml"; - - -my $jobs = XMLin($jobsXml, KeyAttr => ['attrPath', 'name']) - or die "cannot parse XML output"; - - -foreach my $jobName (keys %{$jobs->{item}}) { - my $job = $jobs->{item}->{$jobName}; - my $description = defined $job->{meta}->{description} ? $job->{meta}->{description}->{value} : ""; - print "JOB: $jobName ($description)\n"; - - my $outPath = $job->{outPath}; - my $drvPath = $job->{drvPath}; - - if (scalar(@{$dbh->selectall_arrayref("select * from builds where jobName = ? and outPath = ?", {}, $jobName, $outPath)}) > 0) { - print " already done\n"; - next; + if (scalar($db->resultset('Builds')->search({project => $project->name, jobset => $jobset->name, attrname => $jobName, outPath => $outPath})) > 0) { + print " already done\n"; + return; } my $isCachedBuild = 1; @@ -45,13 +27,34 @@ foreach my $jobName (keys %{$jobs->{item}}) { $startTime = time(); - my $res = system("nix-build $jobsFile --attr $jobName"); + print " BUILDING\n"; + + my $res = system("nix-store --realise $drvPath"); $stopTime = time(); $buildStatus = $res == 0 ? 0 : 1; } + $db->txn_do(sub { + my $build = $db->resultset('Builds')->create( + { timestamp => time() + , project => $project->name + , jobset => $jobset->name + , attrname => $jobName + , drvpath => $drvPath + , outpath => $outPath + , iscachedbuild => $isCachedBuild + , buildstatus => $buildStatus + , starttime => $startTime + , stoptime => $stopTime + }); + }); + + return 0; + + my $dbh, my $description, my $jobName; + $dbh->begin_work; $dbh->prepare("insert into builds(timestamp, jobName, description, drvPath, outPath, isCachedBuild, buildStatus, errorMsg, startTime, stopTime) values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") @@ -83,3 +86,109 @@ foreach my $jobName (keys %{$jobs->{item}}) { $dbh->commit; } + + +sub fetchInput { + my ($input, $inputInfo) = @_; + my $type = $input->type; + my $uri = $input->uri; + + if ($type eq "path") { + my $storePath = `nix-store --add "$uri"` + or die "cannot copy path $uri to the Nix store"; + chomp $storePath; + print " copied to $storePath\n"; + $$inputInfo{$input->name} = {storePath => $storePath}; + } + + else { + die "input `" . $input->type . "' has unknown type `$type'"; + } +} + + +sub checkJobSet { + my ($project, $jobset) = @_; + + my $inputInfo = {}; + + foreach my $input ($jobset->jobsetinputs) { + print " INPUT ", $input->name, " (", $input->type, " ", $input->uri, ")\n"; + fetchInput($input, $inputInfo); + } + + die unless defined $inputInfo->{$jobset->nixexprinput}; + + my $nixExprPath = $inputInfo->{$jobset->nixexprinput}->{storePath} . "/" . $jobset->nixexprpath; + + print " EVALUATING $nixExprPath\n"; + + my $jobsXml = `nix-instantiate $nixExprPath --eval-only --strict --xml` + or die "cannot evaluate the Nix expression containing the jobs: $?"; + + #print "$jobsXml"; + + my $jobs = XMLin($jobsXml, + ForceArray => [qw(value)], + KeyAttr => ['name'], + SuppressEmpty => '', + ValueAttr => [value => 'value']) + or die "cannot parse XML output"; + + die unless defined $jobs->{attrs}; + + foreach my $jobName (keys(%{$jobs->{attrs}->{attr}})) { + print " JOB $jobName\n"; + + my $jobExpr = $jobs->{attrs}->{attr}->{$jobName}; + + my $extraArgs = ""; + + # If the expression is a function, then look at its formal + # arguments and see if we can supply them. + if (defined $jobExpr->{function}->{attrspat}) { + foreach my $argName (keys(%{$jobExpr->{function}->{attrspat}->{attr}})) { + print " needs input $argName\n"; + die "missing input `$argName'" if !defined $inputInfo->{$argName}; + $extraArgs .= " --arg $argName '{path = " . $inputInfo->{$argName}->{storePath} . ";}'"; + } + } + + # Instantiate the store derivation. + my $drvPath = `nix-instantiate $nixExprPath --attr $jobName $extraArgs` + or die "cannot evaluate the Nix expression containing the job definitions: $?"; + chomp $drvPath; + + # Call nix-env --xml to get info about this job (drvPath, outPath, meta attributes, ...). + my $infoXml = `nix-env -f $nixExprPath --query --available "*" --attr-path --out-path --drv-path --meta --xml --system-filter "*" --attr $jobName $extraArgs` + or die "cannot get information about the job: $?"; + + my $info = XMLin($infoXml, KeyAttr => ['attrPath', 'name']) + or die "cannot parse XML output"; + + my $job = $info->{item}; + die unless !defined $job || $job->{system} ne $jobName; + + my $description = defined $job->{meta}->{description} ? $job->{meta}->{description}->{value} : ""; + die unless $job->{drvPath} eq $drvPath; + my $outPath = $job->{outPath}; + + buildJob($project, $jobset, $jobName, $drvPath, $outPath); + } +} + + +sub checkJobs { + + foreach my $project ($db->resultset('Projects')->all) { + print "PROJECT ", $project->name, "\n"; + foreach my $jobset ($project->jobsets->all) { + print " JOBSET ", $jobset->name, "\n"; + checkJobSet($project, $jobset); + } + } + +} + + +checkJobs; diff --git a/src/test.sql b/src/test.sql new file mode 100644 index 00000000..5633e7a9 --- /dev/null +++ b/src/test.sql @@ -0,0 +1,9 @@ +insert into projects(name) values('patchelf'); +insert into jobSets(project, name, description, nixExprInput, nixExprPath) values('patchelf', 'trunk', 'PatchELF', 'patchelfSrc', 'release.nix'); +insert into jobSetInputs(project, job, name, type, uri) values('patchelf', 'trunk', 'patchelfSrc', 'path', '/home/eelco/Dev/patchelf-wc'); +insert into jobSetInputs(project, job, name, type, uri) values('patchelf', 'trunk', 'nixpkgs', 'path', '/home/eelco/Dev/nixpkgs-wc'); +insert into jobSetInputs(project, job, name, type, uri) values('patchelf', 'trunk', 'release', 'path', '/home/eelco/Dev/release'); + +--insert into projects(name) values('nixpkgs'); +--insert into jobSets(project, name) values('nixpkgs', 'trunk'); +--insert into jobSets(project, name) values('nixpkgs', 'stdenv-branch'); diff --git a/test.nix b/test.nix deleted file mode 100644 index fc258b08..00000000 --- a/test.nix +++ /dev/null @@ -1,17 +0,0 @@ -let - - pkgs = import (builtins.getEnv "NIXPKGS_ALL") {}; - - pkgs64 = import (builtins.getEnv "NIXPKGS_ALL") {system = "x86_64-linux";}; - -in - -{ - - job1 = pkgs.hello; - - job1_64 = pkgs64.hello; - - job2 = pkgs.aterm; - -}
IdJobTimestampDescription
#JobTimestampDescription