* Basic job info in the database.

src/HydraFrontend/lib/HydraFrontend/Schema.pm

@@ -8,8 +8,8 @@ use base 'DBIx::Class::Schema';
 __PACKAGE__->load_classes;
 
 
-# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-10-28 17:59:29
+# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-04 14:45:23
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Fayli8dtSdcAYhfKSZnJwg
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:UAjA2VmMoOSjiHk0NUzLfQ
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/HydraFrontend/lib/HydraFrontend/Schema/Buildlogs.pm

@@ -25,8 +25,8 @@ __PACKAGE__->belongs_to(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-10-28 17:59:29
+# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-04 14:45:23
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:xXiHLBKW5fHl7ukdYeIsTw
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:kck2qlNZVLFUnevNPSBVKw
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/HydraFrontend/lib/HydraFrontend/Schema/Buildproducts.pm

@@ -25,8 +25,8 @@ __PACKAGE__->belongs_to(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-10-28 17:59:29
+# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-04 14:45:23
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:5SPq4at2/NRvbax49TwfDw
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:CnCSHdI5+5p+L6+r/YITxQ
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/HydraFrontend/lib/HydraFrontend/Schema/Builds.pm

@@ -12,7 +12,11 @@ __PACKAGE__->add_columns(
   { data_type => "integer", is_nullable => 0, size => undef },
   "timestamp",
   { data_type => "integer", is_nullable => 0, size => undef },
-  "jobname",
+  "project",
+  { data_type => "text", is_nullable => 0, size => undef },
+  "jobset",
+  { data_type => "text", is_nullable => 0, size => undef },
+  "attrname",
   { data_type => "text", is_nullable => 0, size => undef },
   "description",
   { data_type => "text", is_nullable => 0, size => undef },
@@ -44,7 +48,7 @@ __PACKAGE__->has_many(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-10-28 17:59:29
+# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-04 14:45:23
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:gp6ZZpDA2VzgnNE9NX99dA
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Odp6qymLlNXbsD7VOQ7PAQ
 
 1;

src/HydraFrontend/lib/HydraFrontend/Schema/Jobsetinputs.pm

@@ -0,0 +1,48 @@
+package HydraFrontend::Schema::Jobsetinputs;
+
+use strict;
+use warnings;
+
+use base 'DBIx::Class';
+
+__PACKAGE__->load_components("Core");
+__PACKAGE__->table("jobSetInputs");
+__PACKAGE__->add_columns(
+  "project",
+  { data_type => "text", is_nullable => 0, size => undef },
+  "job",
+  { data_type => "text", is_nullable => 0, size => undef },
+  "name",
+  { data_type => "text", is_nullable => 0, size => undef },
+  "type",
+  { data_type => "text", is_nullable => 0, size => undef },
+  "uri",
+  { data_type => "text", is_nullable => 0, size => undef },
+  "revision",
+  { data_type => "integer", is_nullable => 0, size => undef },
+  "tag",
+  { data_type => "text", is_nullable => 0, size => undef },
+);
+__PACKAGE__->set_primary_key("project", "job", "name");
+__PACKAGE__->has_many(
+  "jobsets",
+  "HydraFrontend::Schema::Jobsets",
+  {
+    "foreign.name"         => "self.job",
+    "foreign.nixexprinput" => "self.name",
+    "foreign.project"      => "self.project",
+  },
+);
+__PACKAGE__->belongs_to(
+  "jobset",
+  "HydraFrontend::Schema::Jobsets",
+  { name => "job", project => "project" },
+);
+
+
+# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-04 14:45:23
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:pzKFsX3b5wTNZvo8t3WTDg
+
+
+# You can replace this text with custom content, and it will be preserved on regeneration
+1;

src/HydraFrontend/lib/HydraFrontend/Schema/Jobsets.pm

@@ -0,0 +1,45 @@
+package HydraFrontend::Schema::Jobsets;
+
+use strict;
+use warnings;
+
+use base 'DBIx::Class';
+
+__PACKAGE__->load_components("Core");
+__PACKAGE__->table("jobSets");
+__PACKAGE__->add_columns(
+  "name",
+  { data_type => "text", is_nullable => 0, size => undef },
+  "project",
+  { data_type => "text", is_nullable => 0, size => undef },
+  "description",
+  { data_type => "text", is_nullable => 0, size => undef },
+  "nixexprinput",
+  { data_type => "text", is_nullable => 0, size => undef },
+  "nixexprpath",
+  { data_type => "text", is_nullable => 0, size => undef },
+);
+__PACKAGE__->set_primary_key("project", "name");
+__PACKAGE__->belongs_to(
+  "project",
+  "HydraFrontend::Schema::Projects",
+  { name => "project" },
+);
+__PACKAGE__->belongs_to(
+  "jobsetinput",
+  "HydraFrontend::Schema::Jobsetinputs",
+  { job => "name", name => "nixexprinput", project => "project" },
+);
+__PACKAGE__->has_many(
+  "jobsetinputs",
+  "HydraFrontend::Schema::Jobsetinputs",
+  { "foreign.job" => "self.name", "foreign.project" => "self.project" },
+);
+
+
+# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-04 14:45:23
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:pEIAO9lDM+lMKLCLGWRdXg
+
+
+# You can replace this text with custom content, and it will be preserved on regeneration
+1;

src/HydraFrontend/lib/HydraFrontend/Schema/Projects.pm

@@ -0,0 +1,27 @@
+package HydraFrontend::Schema::Projects;
+
+use strict;
+use warnings;
+
+use base 'DBIx::Class';
+
+__PACKAGE__->load_components("Core");
+__PACKAGE__->table("projects");
+__PACKAGE__->add_columns(
+  "name",
+  { data_type => "text", is_nullable => 0, size => undef },
+);
+__PACKAGE__->set_primary_key("name");
+__PACKAGE__->has_many(
+  "jobsets",
+  "HydraFrontend::Schema::Jobsets",
+  { "foreign.project" => "self.name" },
+);
+
+
+# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-04 14:45:23
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:cpO0BGfChpnpm7KBKkSUjw
+
+
+# You can replace this text with custom content, and it will be preserved on regeneration
+1;

src/HydraFrontend/root/index.tt

@@ -6,7 +6,7 @@
 
 <table class="tablesorter">
   <thead>
-    <tr><th></th><th>Id</th><th>Job</th><th>Timestamp</th><th>Description</th></tr>
+    <tr><th></th><th>#</th><th>Job</th><th>Timestamp</th><th>Description</th></tr>
   </thead>
   <tbody>
     [% FOREACH build IN latestBuilds -%]

src/hydra.sql

@@ -1,7 +1,15 @@
 create table builds (
     id            integer primary key autoincrement not null,
     timestamp     integer not null, -- time this build was added to the db (in Unix time)
-    jobName       text not null,
+
+    -- Info about the inputs.
+    project       text not null, -- !!! foreign key
+    jobSet        text not null, -- !!! foreign key
+    attrName      text not null,
+
+    -- !!! list all the inputs / arguments
+
+    -- Info about the build result.
     description   text,
     drvPath       text not null,
     outPath       text not null,
@@ -40,3 +48,36 @@ create trigger cascadeBuildDeletion
     delete from buildLogs where buildId = old.id;
     delete from buildProducts where buildId = old.id;
   end;
+
+
+create table projects (
+    name          text primary key not null 
+);
+
+
+-- A jobset consists of a set of inputs (e.g. SVN repositories), one
+-- of which contains a Nix expression containing an attribute set
+-- describing build jobs.
+create table jobSets (
+    name          text not null,
+    project       text not null,
+    description   text,
+    nixExprInput  text not null, -- name of the jobSetInput containing the Nix expression
+    nixExprPath   text not null, -- relative path of the Nix expression
+    primary key   (project, name),
+    foreign key   (project) references projects(name) on delete cascade, -- ignored by sqlite
+    foreign key   (project, name, nixExprInput) references jobSetInputs(project, job, name)
+);
+
+
+create table jobSetInputs (
+    project       text not null,
+    job           text not null,
+    name          text not null,
+    type          text not null, -- "svn", "cvs", "path", "file"
+    uri           text,
+    revision      integer, -- for svn
+    tag           text, -- for cvs
+    primary key   (project, job, name),
+    foreign key   (project, job) references jobSets(project, name) on delete cascade -- ignored by sqlite
+);

src/scheduler.pl

@@ -2,37 +2,19 @@
 
 use strict;
 use XML::Simple;
-use DBI;
 use File::Basename;
+use HydraFrontend::Schema;
 
 
-my $jobsFile = "../test.nix";
+my $db = HydraFrontend::Schema->connect("dbi:SQLite:dbname=hydra.sqlite", "", "", {});
 
 
-my $dbh = DBI->connect("dbi:SQLite:dbname=hydra.sqlite", "", "");
+sub buildJob {
+    my ($project, $jobset, $jobName, $drvPath, $outPath) = @_;
 
-
+    if (scalar($db->resultset('Builds')->search({project => $project->name, jobset => $jobset->name, attrname => $jobName, outPath => $outPath})) > 0) {
-my $jobsXml = `nix-env -f $jobsFile --query --available "*" --attr-path --out-path --drv-path --meta --xml --system-filter "*"`
-    or die "cannot evaluate the Nix expression containing the job definitions: $?";
-
-print "$jobsXml";
-
-
-my $jobs = XMLin($jobsXml, KeyAttr => ['attrPath', 'name'])
-    or die "cannot parse XML output";
-
-
-foreach my $jobName (keys %{$jobs->{item}}) {
-    my $job = $jobs->{item}->{$jobName};
-    my $description = defined $job->{meta}->{description} ? $job->{meta}->{description}->{value} : "";
-    print "JOB: $jobName ($description)\n";
-
-    my $outPath = $job->{outPath};
-    my $drvPath = $job->{drvPath};
-
-    if (scalar(@{$dbh->selectall_arrayref("select * from builds where jobName = ? and outPath = ?", {}, $jobName, $outPath)}) > 0) {
         print "      already done\n";
-        next;
+        return;
     }
 
     my $isCachedBuild = 1;
@@ -45,13 +27,34 @@ foreach my $jobName (keys %{$jobs->{item}}) {
 
         $startTime = time();
 
-        my $res = system("nix-build $jobsFile --attr $jobName");
+        print "      BUILDING\n";
+
+        my $res = system("nix-store --realise $drvPath");
 
         $stopTime = time();
 
         $buildStatus = $res == 0 ? 0 : 1;
     }
 
+    $db->txn_do(sub {
+        my $build = $db->resultset('Builds')->create(
+            { timestamp => time()
+            , project => $project->name
+            , jobset => $jobset->name
+            , attrname => $jobName
+            , drvpath => $drvPath
+            , outpath => $outPath
+            , iscachedbuild => $isCachedBuild
+            , buildstatus => $buildStatus
+            , starttime => $startTime
+            , stoptime => $stopTime
+            });
+    });
+
+    return 0;
+
+    my $dbh, my $description, my $jobName;
+
     $dbh->begin_work;
 
     $dbh->prepare("insert into builds(timestamp, jobName, description, drvPath, outPath, isCachedBuild, buildStatus, errorMsg, startTime, stopTime) values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
@@ -83,3 +86,109 @@ foreach my $jobName (keys %{$jobs->{item}}) {
 
     $dbh->commit;
 }
+
+
+sub fetchInput {
+    my ($input, $inputInfo) = @_;
+    my $type = $input->type;
+    my $uri = $input->uri;
+
+    if ($type eq "path") {
+        my $storePath = `nix-store --add "$uri"`
+            or die "cannot copy path $uri to the Nix store";
+        chomp $storePath;
+        print "      copied to $storePath\n";
+        $$inputInfo{$input->name} = {storePath => $storePath};
+    }
+
+    else {
+        die "input `" . $input->type . "' has unknown type `$type'";
+    }
+}
+
+
+sub checkJobSet {
+    my ($project, $jobset) = @_;
+
+    my $inputInfo = {};
+
+    foreach my $input ($jobset->jobsetinputs) {
+        print "    INPUT ", $input->name, " (", $input->type, " ", $input->uri, ")\n";
+        fetchInput($input, $inputInfo);
+    }
+
+    die unless defined $inputInfo->{$jobset->nixexprinput};
+
+    my $nixExprPath = $inputInfo->{$jobset->nixexprinput}->{storePath} . "/" . $jobset->nixexprpath;
+
+    print "    EVALUATING $nixExprPath\n";
+ 
+    my $jobsXml = `nix-instantiate $nixExprPath --eval-only --strict --xml`
+        or die "cannot evaluate the Nix expression containing the jobs: $?";
+
+    #print "$jobsXml";
+    
+    my $jobs = XMLin($jobsXml,
+                     ForceArray => [qw(value)],
+                     KeyAttr => ['name'],
+                     SuppressEmpty => '',
+                     ValueAttr => [value => 'value'])
+        or die "cannot parse XML output";
+
+    die unless defined $jobs->{attrs};
+
+    foreach my $jobName (keys(%{$jobs->{attrs}->{attr}})) {
+        print "    JOB $jobName\n";
+
+        my $jobExpr = $jobs->{attrs}->{attr}->{$jobName};
+
+        my $extraArgs = "";
+
+        # If the expression is a function, then look at its formal
+        # arguments and see if we can supply them.
+        if (defined $jobExpr->{function}->{attrspat}) {
+            foreach my $argName (keys(%{$jobExpr->{function}->{attrspat}->{attr}})) {
+                print "      needs input $argName\n";
+                die "missing input `$argName'" if !defined $inputInfo->{$argName};
+                $extraArgs .= " --arg $argName '{path = " . $inputInfo->{$argName}->{storePath} . ";}'";
+            }
+        }
+
+        # Instantiate the store derivation.
+        my $drvPath = `nix-instantiate $nixExprPath --attr $jobName $extraArgs`
+            or die "cannot evaluate the Nix expression containing the job definitions: $?";
+        chomp $drvPath;
+
+        # Call nix-env --xml to get info about this job (drvPath, outPath, meta attributes, ...).
+        my $infoXml = `nix-env -f $nixExprPath --query --available "*" --attr-path --out-path --drv-path --meta --xml --system-filter "*" --attr $jobName $extraArgs`
+            or die "cannot get information about the job: $?";
+
+        my $info = XMLin($infoXml, KeyAttr => ['attrPath', 'name'])
+            or die "cannot parse XML output";
+
+        my $job = $info->{item};
+        die unless !defined $job || $job->{system} ne $jobName;
+        
+        my $description = defined $job->{meta}->{description} ? $job->{meta}->{description}->{value} : "";
+        die unless $job->{drvPath} eq $drvPath;
+        my $outPath = $job->{outPath};
+
+        buildJob($project, $jobset, $jobName, $drvPath, $outPath);
+    }
+}
+
+
+sub checkJobs {
+
+    foreach my $project ($db->resultset('Projects')->all) {
+        print "PROJECT ", $project->name, "\n";
+        foreach my $jobset ($project->jobsets->all) {
+            print "  JOBSET ", $jobset->name, "\n";
+            checkJobSet($project, $jobset);
+        }
+    }
+    
+}
+
+
+checkJobs;

src/test.sql

@@ -0,0 +1,9 @@
+insert into projects(name) values('patchelf');
+insert into jobSets(project, name, description, nixExprInput, nixExprPath) values('patchelf', 'trunk', 'PatchELF', 'patchelfSrc', 'release.nix');
+insert into jobSetInputs(project, job, name, type, uri) values('patchelf', 'trunk', 'patchelfSrc', 'path', '/home/eelco/Dev/patchelf-wc');
+insert into jobSetInputs(project, job, name, type, uri) values('patchelf', 'trunk', 'nixpkgs', 'path', '/home/eelco/Dev/nixpkgs-wc');
+insert into jobSetInputs(project, job, name, type, uri) values('patchelf', 'trunk', 'release', 'path', '/home/eelco/Dev/release');
+
+--insert into projects(name) values('nixpkgs');
+--insert into jobSets(project, name) values('nixpkgs', 'trunk');
+--insert into jobSets(project, name) values('nixpkgs', 'stdenv-branch');

test.nix

@@ -1,17 +0,0 @@
-let
-
-  pkgs = import (builtins.getEnv "NIXPKGS_ALL") {};
-  
-  pkgs64 = import (builtins.getEnv "NIXPKGS_ALL") {system = "x86_64-linux";};
-
-in
-
-{
-
-  job1 = pkgs.hello;
-
-  job1_64 = pkgs64.hello;
-
-  job2 = pkgs.aterm;
-
-}