src/HydraFrontend/lib/HydraFrontend/Schema.pm

@@ -8,8 +8,8 @@ use base 'DBIx::Class::Schema';
 __PACKAGE__->load_classes;
 
 
-# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11
+# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:xP97YDrN7Bm2B/BlbQJ7fQ
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:dBO/r6lVlITiJ/HlltKcpQ
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/HydraFrontend/lib/HydraFrontend/Schema/Buildlogs.pm

@@ -21,8 +21,8 @@ __PACKAGE__->set_primary_key("build", "logphase");
 __PACKAGE__->belongs_to("build", "HydraFrontend::Schema::Builds", { id => "build" });
 
 
-# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11
+# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:pt0CJFX1pP9Z2TjqrTjTkw
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:eMNna7u2l0ec+OYuvtGRpg
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/HydraFrontend/lib/HydraFrontend/Schema/Buildproducts.pm

@@ -21,8 +21,8 @@ __PACKAGE__->set_primary_key("build", "path");
 __PACKAGE__->belongs_to("build", "HydraFrontend::Schema::Builds", { id => "build" });
 
 
-# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11
+# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:3NKUaF4u4H6ZmIRCeva8yA
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:LaXQ4zxxvzdKFBRVcjMdMQ
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/HydraFrontend/lib/HydraFrontend/Schema/Builds.pm

@@ -65,8 +65,8 @@ __PACKAGE__->has_many(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11
+# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:8s5Z03ugocOVb021EwGVag
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:C1XPkCXQImyXduKER0Dllg
 
 __PACKAGE__->has_many(dependents => 'HydraFrontend::Schema::Inputs', 'dependency');
 

src/HydraFrontend/lib/HydraFrontend/Schema/Inputs.pm

@@ -35,8 +35,8 @@ __PACKAGE__->set_primary_key("id");
 __PACKAGE__->belongs_to("build", "HydraFrontend::Schema::Builds", { id => "build" });
 
 
-# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11
+# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:AzV6B/6CCrroPlO32n2p3A
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:A3Is4VTFkTl2DzrYjzdrZA
 
 __PACKAGE__->belongs_to("dependency", "HydraFrontend::Schema::Builds", { id => "dependency" });
 

src/HydraFrontend/lib/HydraFrontend/Schema/Jobs.pm

@@ -46,9 +46,13 @@ __PACKAGE__->belongs_to(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11
+# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:GubRofAmJ/sbJbjyV3aKSQ
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ZF8UB1MtbPuOk7wTSFJR5Q
 
+__PACKAGE__->has_many(
+  "inputs",
+  "HydraFrontend::Schema::Inputs",
+  { "foreign.job" => "self.id" },
+);
 
-# You can replace this text with custom content, and it will be preserved on regeneration
 1;

src/HydraFrontend/lib/HydraFrontend/Schema/Jobsetinputalts.pm

@@ -33,8 +33,8 @@ __PACKAGE__->belongs_to(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11
+# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ZjjWLbAWExxOqsDz41A3KA
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ibTncC1AslPWt1eiTtwplA
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/HydraFrontend/lib/HydraFrontend/Schema/Jobsetinputs.pm

@@ -43,8 +43,8 @@ __PACKAGE__->has_many(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11
+# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:6hzbFjPWQ872UxFhhpxjFg
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:D1UzSZwPtwDmOI7q6g8uKQ
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/HydraFrontend/lib/HydraFrontend/Schema/Jobsets.pm

@@ -56,8 +56,8 @@ __PACKAGE__->has_many(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11
+# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:oRV4yw0DWG5PI0agcM7QHA
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:6Pyrgervmq03S5Nx8QfA1Q
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/HydraFrontend/lib/HydraFrontend/Schema/Projects.pm

@@ -29,8 +29,8 @@ __PACKAGE__->has_many(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11
+# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:9SeEXSEOH1ocrdkoa7fx5Q
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:p8LbF31qRl/JfMK5wfkeCg
 
 
 # You can replace this text with custom content, and it will be preserved on regeneration

src/HydraFrontend/root/hydra.css

@@ -157,6 +157,11 @@ ul.productList {
     padding-left: 1em;
 }
 
+tr.runningJob {
+    background-color: #ff3030;
+    color: white;
+}
+
 
 /* Sortable tables */
 

src/HydraFrontend/root/index.tt

@@ -9,7 +9,7 @@
   </thead>
   <tbody>
     [% FOREACH job IN jobs -%]
-      <tr>
+      <tr [% IF job.busy %]class="runningJob"[% END %] >
         <td>[% job.priority %]</td>
         <td><tt>[% job.project.name %]</tt></td>
         <td><tt>[% job.jobset.name %]</tt></td>

src/HydraFrontend/root/project.tt

@@ -0,0 +1,54 @@
+[% WRAPPER layout.tt title="Hydra Overview" %]
+
+<h1>Project <tt>[% project.name %]</tt></h1>
+
+
+<h2>Definition</h2>
+
+[% FOREACH jobset IN project.jobsets -%]
+
+  <h3>Jobset <tt>[% jobset.name %]</tt></h3>
+
+  <p>
+    Description: [% jobset.description %]
+    <br />
+    Nix expression: <tt>[% jobset.nixexprpath %]</tt> in input <tt>[% jobset.nixexprinput %]</tt>
+  </p>
+
+  <table class="tablesorter">
+    <thead>
+      <tr><th>Input name</th><th>Type</th><th>Values</th></tr>
+    </thead>
+    <tbody>
+      [% FOREACH input IN jobset.jobsetinputs -%]
+        <tr>
+          <td><tt>[% input.name %]</tt></td>
+          <td><tt>[% input.type %]</tt></td>
+          <td>
+            [% FOREACH alt IN input.jobsetinputalts -%]
+              [% IF input.type == "string" %]
+                <tt>"[% alt.value %]"</tt>
+              [% ELSE %]
+                <tt>[% alt.uri %]</tt>
+              [% END %]
+            [% END %]
+          </td>
+        </tr>
+      [% END %]
+    </tbody>
+  </table>
+
+[% END -%]
+
+
+<h2>Jobs</h2>
+
+<ul>
+  [% FOREACH jobName IN jobNames -%]
+    <li><a href="[% c.uri_for('/job' project.name jobName.attrname) %]"><tt>[% jobName.attrname %]</tt></a></li>
+  [% END %]
+</ul>
+
+
+
+[% END %]

src/build.pl

@@ -0,0 +1,166 @@
+#! @perl@ -w
+
+use strict;
+use File::Basename;
+use HydraFrontend::Schema;
+
+
+my $db = HydraFrontend::Schema->connect("dbi:SQLite:dbname=hydra.sqlite", "", "", {});
+
+
+sub isValidPath {
+    my $path = shift;
+    return system("nix-store --check-validity $path 2> /dev/null") == 0;
+}
+
+
+sub buildJob {
+    my ($job) = @_;
+
+    my $drvPath = $job->drvpath;
+    my $outPath = $job->outpath;
+
+    my $isCachedBuild = 1;
+    my $outputCreated = 1; # i.e., the Nix build succeeded (but it could be a positive failure)
+    my $startTime = 0;
+    my $stopTime = 0;
+    
+    if (!isValidPath($outPath)) {
+        $isCachedBuild = 0;
+
+        $startTime = time();
+
+        print "      BUILDING\n";
+
+        my $res = system("nix-store --realise $drvPath");
+
+        $stopTime = time();
+
+        $outputCreated = $res == 0;
+    }
+
+    my $buildStatus;
+    
+    if ($outputCreated) {
+        # "Positive" failures, e.g. the builder returned exit code 0
+        # but flagged some error condition.
+        $buildStatus = -e "$outPath/nix-support/failed" ? 2 : 0;
+    } else {
+        $buildStatus = 1; # = Nix failure
+    }
+
+    $db->txn_do(sub {
+        my $build = $db->resultset('Builds')->create(
+            { timestamp => time()
+            , project => $job->project->name
+            , jobset => $job->jobset->name
+            , attrname => $job->attrname
+            , description => $job->description
+            , drvpath => $drvPath
+            , outpath => $outPath
+            , iscachedbuild => $isCachedBuild
+            , buildstatus => $buildStatus
+            , starttime => $startTime
+            , stoptime => $stopTime
+            , system => $job->system
+            });
+        print "      build ID = ", $build->id, "\n";
+
+        foreach my $input ($job->inputs) {
+            $input->job(undef);
+            $input->build($build->id);
+            $input->update;
+        }
+
+        my $logPath = "/nix/var/log/nix/drvs/" . basename $drvPath;
+        if (-e $logPath) {
+            print "      LOG $logPath\n";
+            $db->resultset('Buildlogs')->create(
+                { build => $build->id
+                , logphase => "full"
+                , path => $logPath
+                , type => "raw"
+                });
+        }
+
+        if ($outputCreated) {
+
+            if (-e "$outPath/log") {
+                foreach my $logPath (glob "$outPath/log/*") {
+                    print "      LOG $logPath\n";
+                    $db->resultset('Buildlogs')->create(
+                        { build => $build->id
+                        , logphase => basename($logPath)
+                        , path => $logPath
+                        , type => "raw"
+                        });
+                }
+            }
+
+            if (-e "$outPath/nix-support/hydra-build-products") {
+                open LIST, "$outPath/nix-support/hydra-build-products" or die;
+                while (<LIST>) {
+                    /^(\w+)\s+([\w-]+)\s+(\S+)$/ or die;
+                    my $type = $1;
+                    my $subtype = $2;
+                    my $path = $3;
+                    die unless -e $path;
+                    $db->resultset('Buildproducts')->create(
+                        { build => $build->id
+                        , type => $type
+                        , subtype => $subtype
+                        , path => $path
+                        });
+                }
+                close LIST;
+            } else {
+                $db->resultset('Buildproducts')->create(
+                    { build => $build->id
+                    , type => "nix-build"
+                    , subtype => ""
+                    , path => $outPath
+                    });
+            }
+        }
+
+        $job->delete;
+    });
+}
+
+
+my $jobId = $ARGV[0] or die;
+print "building job $jobId\n";
+
+# Lock the job.  If necessary, steal the lock from the parent process
+# (runner.pl).  This is so that if the runner dies, the children
+# (i.e. the job builders) can continue to run and won't have the lock
+# taken away.
+my $job;
+$db->txn_do(sub {
+    ($job) = $db->resultset('Jobs')->search({ id => $jobId });
+    die "job $jobId doesn't exist" unless defined $job;
+    if ($job->busy != 0 && $job->locker != getppid) {
+        die "job $jobId is already being built";
+    }
+    $job->busy(1);
+    $job->locker($$);
+    $job->update;
+});
+
+die unless $job;
+
+# Build the job.  If it throws an error, unlock the job so that it can
+# be retried.
+eval {
+    print "BUILD\n";
+    buildJob $job;
+    print "DONE\n";
+};
+if ($@) {
+    warn $@;
+    $db->txn_do(sub {
+        $job->busy(0);
+        $job->locker($$);
+        $job->update;
+    });
+}

src/hydra.sql

@@ -129,10 +129,10 @@ create table jobs (
     id            integer primary key autoincrement not null,
     timestamp     integer not null, -- time this build was added to the db (in Unix time)
 
-    priority      integer not null,
+    priority      integer not null default 0,
 
-    busy          integer not null, -- true means someone is building this job now
+    busy          integer not null default 0, -- true means someone is building this job now
-    locker        text not null, -- !!! hostname/pid of the process building this job?
+    locker        text not null default '', -- !!! hostname/pid of the process building this job?
     
     -- Info about the inputs.
     project       text not null, -- !!! foreign key

src/runner.pl

@@ -0,0 +1,65 @@
+#! @perl@ -w
+
+use strict;
+use HydraFrontend::Schema;
+
+
+my $db = HydraFrontend::Schema->connect("dbi:SQLite:dbname=hydra.sqlite", "", "", {});
+
+
+# Unlock jobs whose building process has died.
+$db->txn_do(sub {
+    my @jobs = $db->resultset('Jobs')->search({ busy => 1 });
+    foreach my $job (@jobs) {
+        my $pid = $job->locker;
+        if (kill(0, $pid) != 1) { # see if we can signal the process
+            print "job ", $job->id, " pid $pid died, unlocking\n";
+            $job->busy(0);
+            $job->locker("");
+            $job->update;
+        }
+    }
+});
+
+
+while (1) {
+
+    print "looking for runnable jobs...\n";
+
+    my $job;
+
+    $db->txn_do(sub {
+    
+        my @jobs = $db->resultset('Jobs')->search({ busy => 0 }, {order_by => ["priority", "timestamp"]});
+
+        print "# of available jobs: ", scalar(@jobs), "\n";
+
+        if (scalar @jobs > 0) {
+            $job = $jobs[0];
+            $job->busy(1);
+            $job->locker($$);
+            $job->update;
+        }
+
+    });
+
+    # Start the job.  We need to do this outside the transaction in
+    # case it aborts or something.
+    if (defined $job) {
+        print "starting job ", $job->id, "\n";
+        eval {
+            system("perl -I HydraFrontend/lib -w ./build.pl " . $job->id);
+        };
+        if ($@) {
+            warn $@;
+            $db->txn_do(sub {
+                $job->busy(0);
+                $job->locker($$);
+                $job->update;
+            });
+        }
+    }
+
+    print "sleeping...\n";
+    sleep(10);
+}

src/scheduler.pl

@@ -2,7 +2,6 @@
 
 use strict;
 use XML::Simple;
-use File::Basename;
 use HydraFrontend::Schema;
 
 
@@ -15,131 +14,6 @@ sub isValidPath {
 }
 
 
-sub buildJob {
-    my ($project, $jobset, $jobName, $description, $drvPath, $outPath, $usedInputs, $system) = @_;
-
-    if (scalar($db->resultset('Builds')->search({project => $project->name, jobset => $jobset->name, attrname => $jobName, outPath => $outPath})) > 0) {
-        print "      already done\n";
-        return;
-    }
-
-    my $isCachedBuild = 1;
-    my $outputCreated = 1; # i.e., the Nix build succeeded (but it could be a positive failure)
-    my $startTime = 0;
-    my $stopTime = 0;
-    
-    if (!isValidPath($outPath)) {
-        $isCachedBuild = 0;
-
-        $startTime = time();
-
-        print "      BUILDING\n";
-
-        my $res = system("nix-store --realise $drvPath");
-
-        $stopTime = time();
-
-        $outputCreated = $res == 0;
-    }
-
-    my $buildStatus;
-    
-    if ($outputCreated) {
-        # "Positive" failures, e.g. the builder returned exit code 0
-        # but flagged some error condition.
-        $buildStatus = -e "$outPath/nix-support/failed" ? 2 : 0;
-    } else {
-        $buildStatus = 1; # = Nix failure
-    }
-
-    $db->txn_do(sub {
-        my $build = $db->resultset('Builds')->create(
-            { timestamp => time()
-            , project => $project->name
-            , jobset => $jobset->name
-            , attrname => $jobName
-            , description => $description
-            , drvpath => $drvPath
-            , outpath => $outPath
-            , iscachedbuild => $isCachedBuild
-            , buildstatus => $buildStatus
-            , starttime => $startTime
-            , stoptime => $stopTime
-            , system => $system
-            });
-        print "      build ID = ", $build->id, "\n";
-
-        foreach my $inputName (keys %{$usedInputs}) {
-            my $input = $usedInputs->{$inputName};
-            $db->resultset('Inputs')->create(
-                { build => $build->id
-                , name => $inputName
-                , type => $input->{type}
-                , uri => $input->{uri}
-                #, revision => $input->{orig}->revision
-                #, tag => $input->{orig}->tag
-                , value => $input->{value}
-                , dependency => $input->{id}
-                , path => ($input->{storePath} or "") # !!! temporary hack
-                });
-        }
-
-        my $logPath = "/nix/var/log/nix/drvs/" . basename $drvPath;
-        if (-e $logPath) {
-            print "      LOG $logPath\n";
-            $db->resultset('Buildlogs')->create(
-                { build => $build->id
-                , logphase => "full"
-                , path => $logPath
-                , type => "raw"
-                });
-        }
-
-        if ($outputCreated) {
-
-            if (-e "$outPath/log") {
-                foreach my $logPath (glob "$outPath/log/*") {
-                    print "      LOG $logPath\n";
-                    $db->resultset('Buildlogs')->create(
-                        { build => $build->id
-                        , logphase => basename($logPath)
-                        , path => $logPath
-                        , type => "raw"
-                        });
-                }
-            }
-
-            if (-e "$outPath/nix-support/hydra-build-products") {
-                open LIST, "$outPath/nix-support/hydra-build-products" or die;
-                while (<LIST>) {
-                    /^(\w+)\s+([\w-]+)\s+(\S+)$/ or die;
-                    my $type = $1;
-                    my $subtype = $2;
-                    my $path = $3;
-                    die unless -e $path;
-                    $db->resultset('Buildproducts')->create(
-                        { build => $build->id
-                        , type => $type
-                        , subtype => $subtype
-                        , path => $path
-                        });
-                }
-                close LIST;
-            } else {
-                $db->resultset('Buildproducts')->create(
-                    { build => $build->id
-                    , type => "nix-build"
-                    , subtype => ""
-                    , path => $outPath
-                    });
-            }
-        }
-        
-    });
-
-}
-
-
 sub fetchInput {
     my ($input, $alt, $inputInfo) = @_;
     my $type = $input->type;
@@ -186,7 +60,53 @@ sub checkJob {
     die unless $job->{drvPath} eq $drvPath;
     my $outPath = $job->{outPath};
 
-    buildJob($project, $jobset, $jobName, $description, $drvPath, $outPath, $inputInfo, $job->{system});
+    $db->txn_do(sub {
+        if (scalar($db->resultset('Builds')->search(
+                { project => $project->name, jobset => $jobset->name
+                , attrname => $jobName, outPath => $outPath })) > 0)
+        {
+            print "      already done\n";
+            return;
+        }
+
+        if (scalar($db->resultset('Jobs')->search(
+                { project => $project->name, jobset => $jobset->name
+                , attrname => $jobName, outPath => $outPath })) > 0)
+        {
+            print "      already queued\n";
+            return;
+        }
+        
+        print "      adding to queue\n";
+        my $job = $db->resultset('Jobs')->create(
+            { timestamp => time()
+            , priority => 0
+            , busy => 0
+            , locker => ""
+            , project => $project->name
+            , jobset => $jobset->name
+            , attrname => $jobName
+            , description => $description
+            , drvpath => $drvPath
+            , outpath => $outPath
+            , system => $job->{system}
+            });
+
+        foreach my $inputName (keys %{$inputInfo}) {
+            my $input = $inputInfo->{$inputName};
+            $db->resultset('Inputs')->create(
+                { job => $job->id
+                , name => $inputName
+                , type => $input->{type}
+                , uri => $input->{uri}
+                #, revision => $input->{orig}->revision
+                #, tag => $input->{orig}->tag
+                , value => $input->{value}
+                , dependency => $input->{id}
+                , path => ($input->{storePath} or "") # !!! temporary hack
+                });
+        }
+    });
 };