hydra: add some admin for adding/enabling/etc build machines

src/lib/Hydra/Controller/Admin.pm

@@ -6,15 +6,178 @@ use base 'Catalyst::Controller';
 use Hydra::Helper::Nix;
 use Hydra::Helper::CatalystUtils;
 use Hydra::Helper::AddBuilds;
+use Data::Dump qw(dump);
+
+sub nixMachines {
+    my ($c) = @_;
+    my $result = '';
+	
+    foreach my $machine ($c->model("DB::BuildMachines")->all) {
+    	if($machine->enabled) {
+	        $result = $result . $machine->username . '@'. $machine->hostname . ' ';
+	        foreach my $system ($machine->buildmachinesystemtypes) {
+	            $result = $result . $system->system .',';
+	        }
+	        chop $result;
+	        $result = $result . ' '. $machine->ssh_key . ' ' . $machine->maxconcurrent . ' '. $machine->speedfactor . ' ' . $machine->options . "\n";
+    	}
+    }
+    return $result;
+}
+
+sub saveNixMachines {
+    my ($c) = @_;
+
+    die("File not writable: /etc/nix.machines") if ! -w "/etc/nix.machines" ;
+
+    open (NIXMACHINES, '>/etc/nix.machines') or die("Could not write to /etc/nix.machines");
+    print NIXMACHINES nixMachines($c);
+    close (NIXMACHINES);     
+}
 
 sub admin : Chained('/') PathPart('admin') CaptureArgs(0) {
     my ($self, $c) = @_;
     requireAdmin($c);
+    $c->stash->{admin} = 1;
 }
 
 sub index : Chained('admin') PathPart('') Args(0) {
     my ($self, $c) = @_;
+    $c->stash->{machines} = [$c->model('DB::BuildMachines')->search(
+        {}, 
+        { order_by => "hostname"
+        , '+select' => ["(select bs.stoptime from buildsteps as bs where bs.machine = (me.username || '\@' || me.hostname) and not bs.stoptime is null order by bs.stoptime desc limit 1)"]
+        , '+as' => ['idle']
+        })];
+    $c->stash->{steps} = [ $c->model('DB::BuildSteps')->search(
+        { 'me.busy' => 1, 'schedulingInfo.busy' => 1 },
+        { join => [ 'schedulingInfo', 'build' ] 
+        , order_by => [ 'machine', 'outpath' ]
+        } ) ];
     $c->stash->{template} = 'admin.tt';
+ }
+
+sub machines : Chained('admin') PathPart('machines') Args(0) {
+    my ($self, $c) = @_;
+    $c->stash->{machines} = [$c->model('DB::BuildMachines')->search({}, {order_by => "hostname"})];
+    $c->stash->{systems} = [$c->model('DB::SystemTypes')->search({}, {select => ["system"], order_by => "system" })];
+    $c->stash->{nixMachines} = nixMachines($c);
+    $c->stash->{nixMachinesWritable} = (-e "/etc/nix.machines" && -w "/etc/nix.machines");
+
+    $c->stash->{template} = 'machines.tt';
+ }
+
+sub machine : Chained('admin') PathPart('machine') CaptureArgs(1) {
+    my ($self, $c, $machineName) = @_;
+
+    requireAdmin($c);
+
+    my $machine = $c->model('DB::BuildMachines')->find($machineName)
+        or notFound($c, "Machine $machineName doesn't exist.");
+
+    $c->stash->{machine} = $machine;
+}
+
+sub machine_edit : Chained('machine') PathPart('edit') Args(0) {
+    my ($self, $c) = @_;
+    $c->stash->{template} = 'machine.tt';
+    $c->stash->{systemtypes} = [$c->model('DB::SystemTypes')->search({}, {order_by => "system"})];
+    $c->stash->{edit} = 1;
+}
+
+sub machine_edit_submit : Chained('machine') PathPart('submit') Args(0) {
+    my ($self, $c) = @_;
+    requirePost($c);
+
+    txn_do($c->model('DB')->schema, sub {
+    	updateMachine($c, $c->stash->{machine}) ;
+    });
+    saveNixMachines($c);
+    $c->res->redirect("/admin/machines");
+}
+
+sub updateMachine {
+	my ($c, $machine) = @_;
+
+    my $hostname      = trim $c->request->params->{"hostname"};
+    my $username      = trim $c->request->params->{"username"};
+    my $maxconcurrent = trim $c->request->params->{"maxconcurrent"};
+    my $speedfactor   = trim $c->request->params->{"speedfactor"};
+    my $ssh_key       = trim $c->request->params->{"ssh_key"};
+    my $systems       = $c->request->params->{"systems"} ; 
+	
+    error($c, "Invalid or empty username.") if $username eq "";
+    error($c, "Max concurrent builds should be an integer > 0.") if $maxconcurrent eq "" || ! $maxconcurrent =~ m/[0-9]+/;
+    error($c, "Speed factor should be an integer > 0.") if $speedfactor eq "" || ! $speedfactor =~ m/[0-9]+/;
+    error($c, "Invalid or empty SSH key.") if $ssh_key eq "";
+    	
+    $machine->update(
+        { username => $username
+        , maxconcurrent => $maxconcurrent
+        , speedfactor => $speedfactor
+        , ssh_key => $ssh_key
+        });
+    $machine->buildmachinesystemtypes->delete_all;
+    if(ref($systems) eq 'ARRAY') {
+	    for my $s (@$systems) {
+	        $machine->buildmachinesystemtypes->create({ system => $s}) ;
+	    }       
+    } else {
+        $machine->buildmachinesystemtypes->create({ system => $systems}) ;
+    }   
+}
+
+sub create_machine : Chained('admin') PathPart('create-machine') Args(0) {
+    my ($self, $c) = @_;
+
+    requireAdmin($c);
+    
+    $c->stash->{template} = 'machine.tt';
+    $c->stash->{systemtypes} = [$c->model('DB::SystemTypes')->search({}, {order_by => "system"})];
+    $c->stash->{edit} = 1;
+    $c->stash->{create} = 1;
+}
+
+
+sub create_machine_submit : Chained('admin') PathPart('create-machine/submit') Args(0) {
+    my ($self, $c) = @_;
+
+    requireAdmin($c);
+    
+    my $hostname      = trim $c->request->params->{"hostname"};
+    error($c, "Invalid or empty hostname.") if $hostname eq "";
+    
+    txn_do($c->model('DB')->schema, sub {
+        my $machine = $c->model('DB::BuildMachines')->create(
+            { hostname => $hostname });
+        updateMachine($c, $machine);           
+    });
+    $c->res->redirect("/admin/machines");
+}
+
+sub machine_delete : Chained('machine') PathPart('delete') Args(0) {
+    my ($self, $c) = @_;
+    requirePost($c);
+
+    txn_do($c->model('DB')->schema, sub {
+        $c->stash->{machine}->delete;
+    });    
+    saveNixMachines($c);
+    $c->res->redirect("/admin/machines");
+}
+
+sub machine_enable : Chained('machine') PathPart('enable') Args(0) {
+    my ($self, $c) = @_;
+    $c->stash->{machine}->update({ enabled => 1});
+    saveNixMachines($c);
+    $c->res->redirect("/admin/machines");
+}
+
+sub machine_disable : Chained('machine') PathPart('disable') Args(0) {
+    my ($self, $c) = @_;
+    $c->stash->{machine}->update({ enabled => 0});
+    saveNixMachines($c);
+    $c->res->redirect("/admin/machines");
 }
 
 sub clearfailedcache : Chained('admin') Path('clear-failed-cache') Args(0) {
@@ -31,7 +194,7 @@ sub clearevalcache : Chained('admin') Path('clear-eval-cache') Args(0) {
     print "Clearing evaluation cache\n";
     $c->model('DB::JobsetInputHashes')->delete_all;
 
-    $c->res->redirect("/admin");
+    $c->res->redirect("/admin")
 }
 
 sub clearvcscache : Chained('admin') Path('clear-vcs-cache') Args(0) {

src/lib/Hydra/Controller/Root.pm

@@ -98,7 +98,7 @@ sub status :Local {
     my ($self, $c) = @_;
     $c->stash->{steps} = [ $c->model('DB::BuildSteps')->search(
         { 'me.busy' => 1, 'schedulingInfo.busy' => 1 },
-        { join => [ 'schedulingInfo' ] 
+        { join => [ 'schedulingInfo', 'build' ] 
         , order_by => [ 'machine', 'outpath' ]
         } ) ];
 }

src/lib/Hydra/Schema/BuildMachineSystemTypes.pm

@@ -0,0 +1,81 @@
+package Hydra::Schema::BuildMachineSystemTypes;
+
+# Created by DBIx::Class::Schema::Loader
+# DO NOT MODIFY THE FIRST PART OF THIS FILE
+
+use strict;
+use warnings;
+
+use base 'DBIx::Class::Core';
+
+
+=head1 NAME
+
+Hydra::Schema::BuildMachineSystemTypes
+
+=cut
+
+__PACKAGE__->table("BuildMachineSystemTypes");
+
+=head1 ACCESSORS
+
+=head2 hostname
+
+  data_type: text
+  default_value: undef
+  is_foreign_key: 1
+  is_nullable: 0
+  size: undef
+
+=head2 system
+
+  data_type: text
+  default_value: undef
+  is_nullable: 0
+  size: undef
+
+=cut
+
+__PACKAGE__->add_columns(
+  "hostname",
+  {
+    data_type => "text",
+    default_value => undef,
+    is_foreign_key => 1,
+    is_nullable => 0,
+    size => undef,
+  },
+  "system",
+  {
+    data_type => "text",
+    default_value => undef,
+    is_nullable => 0,
+    size => undef,
+  },
+);
+__PACKAGE__->set_primary_key("hostname", "system");
+
+=head1 RELATIONS
+
+=head2 hostname
+
+Type: belongs_to
+
+Related object: L<Hydra::Schema::BuildMachines>
+
+=cut
+
+__PACKAGE__->belongs_to(
+  "hostname",
+  "Hydra::Schema::BuildMachines",
+  { hostname => "hostname" },
+  {},
+);
+
+
+# Created by DBIx::Class::Schema::Loader v0.05000 @ 2010-10-08 13:47:26
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:F/voQZLNESTotUOWRbg4WA
+
+
+# You can replace this text with custom content, and it will be preserved on regeneration
+1;

src/lib/Hydra/Schema/BuildMachines.pm

@@ -0,0 +1,133 @@
+package Hydra::Schema::BuildMachines;
+
+# Created by DBIx::Class::Schema::Loader
+# DO NOT MODIFY THE FIRST PART OF THIS FILE
+
+use strict;
+use warnings;
+
+use base 'DBIx::Class::Core';
+
+
+=head1 NAME
+
+Hydra::Schema::BuildMachines
+
+=cut
+
+__PACKAGE__->table("BuildMachines");
+
+=head1 ACCESSORS
+
+=head2 hostname
+
+  data_type: text
+  default_value: undef
+  is_nullable: 0
+  size: undef
+
+=head2 username
+
+  data_type: text
+  default_value: undef
+  is_nullable: 0
+  size: undef
+
+=head2 ssh_key
+
+  data_type: text
+  default_value: undef
+  is_nullable: 0
+  size: undef
+
+=head2 options
+
+  data_type: text
+  default_value: undef
+  is_nullable: 0
+  size: undef
+
+=head2 maxconcurrent
+
+  data_type: integer
+  default_value: 2
+  is_nullable: 0
+  size: undef
+
+=head2 speedfactor
+
+  data_type: integer
+  default_value: 1
+  is_nullable: 0
+  size: undef
+
+=head2 enabled
+
+  data_type: integer
+  default_value: 1
+  is_nullable: 0
+  size: undef
+
+=cut
+
+__PACKAGE__->add_columns(
+  "hostname",
+  {
+    data_type => "text",
+    default_value => undef,
+    is_nullable => 0,
+    size => undef,
+  },
+  "username",
+  {
+    data_type => "text",
+    default_value => undef,
+    is_nullable => 0,
+    size => undef,
+  },
+  "ssh_key",
+  {
+    data_type => "text",
+    default_value => undef,
+    is_nullable => 0,
+    size => undef,
+  },
+  "options",
+  {
+    data_type => "text",
+    default_value => undef,
+    is_nullable => 0,
+    size => undef,
+  },
+  "maxconcurrent",
+  { data_type => "integer", default_value => 2, is_nullable => 0, size => undef },
+  "speedfactor",
+  { data_type => "integer", default_value => 1, is_nullable => 0, size => undef },
+  "enabled",
+  { data_type => "integer", default_value => 1, is_nullable => 0, size => undef },
+);
+__PACKAGE__->set_primary_key("hostname");
+
+=head1 RELATIONS
+
+=head2 buildmachinesystemtypes
+
+Type: has_many
+
+Related object: L<Hydra::Schema::BuildMachineSystemTypes>
+
+=cut
+
+__PACKAGE__->has_many(
+  "buildmachinesystemtypes",
+  "Hydra::Schema::BuildMachineSystemTypes",
+  { "foreign.hostname" => "self.hostname" },
+);
+
+
+# Created by DBIx::Class::Schema::Loader v0.05000 @ 2010-10-11 12:58:16
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:gje+VA73Hghl7JXp+Fl8pw
+
+
+# You can replace this text with custom content, and it will be preserved on regeneration
+1;

src/root/admin.tt

@@ -12,8 +12,45 @@
             <li>[% INCLUDE maybeLink uri = c.uri_for(c.controller('Admin').action_for('clearvcscache')) content = "Clear VCS caches" confirmmsg = "Are you sure you want to clear the VCS caches?" %]</li>
         </ul>
     </li>    
-    <li>[% INCLUDE maybeLink uri = c.uri_for(c.controller('Admin').action_for('managenews')) content = "News" %]</li>    
 </ul>
 
+<h2>Status</h2>
+
+[% FOREACH m IN machines %]
+<table style="width: 40em;">
+    <thead>
+        <tr>
+            <th colspan="5">
+                [% IF m.enabled == 1 %]
+                    [% INCLUDE maybeLink uri = c.uri_for('/admin/machine' m.hostname 'disable' ) content='-' %]
+                [% ELSE %]
+                    [% INCLUDE maybeLink uri = c.uri_for('/admin/machine' m.hostname 'enable' ) content='+' %]
+                [% END %] 
+                [% m.hostname %] <tt>[% FOREACH ms IN m.buildmachinesystemtypes %] [% ms.system %][% END %]</tt>
+            </th>
+        </tr>
+    </thead>
+    <tbody>
+    [% idle = 1 %]
+    [% FOREACH step IN steps %]
+        [% IF step.machine.match('@(.*)').0 == m.hostname %]
+            [% idle = 0 %]
+            <tr>
+                <td><tt>[% INCLUDE renderFullJobName project = step.build.project.name jobset = step.build.jobset.name job = step.build.job.name %]</tt></td>
+                <td><tt>[% step.system %]</tt></td>
+                <td><a href="[% c.uri_for('/build' step.build.id) %]">[% step.build.id %]</a></td>
+                <td><tt>[% step.outpath.match('-(.*)').0 %]</tt></td>
+                <td class='right'>[% INCLUDE renderDuration duration = curTime - step.starttime %] </td>
+            </tr>
+        [% END %]
+    [% END %]
+    [% IF idle == 1 %]
+        <tr><td colspan="5">Idle since [% INCLUDE renderDuration duration = curTime - m.get_column('idle') %]</td></tr>
+    [% END %]
+    </tbody>
+</table>
+
+[% END %]
+
 
 [% END %]

src/root/common.tt

@@ -334,3 +334,23 @@
 
 [% END %]
 
+[% BLOCK hydraStatus %]
+<table class="tablesorter">
+  <thead>
+    <tr><th>Machine</th><th>Job</th><th>Type</th><th>Build</th><th>Step</th><th>What</th><th>Since</th></tr>
+  </thead>
+  <tbody>
+    [% FOREACH step IN steps %]
+      <tr>
+        <td><tt>[% IF step.machine; step.machine.match('@(.*)').0; ELSE; 'localhost'; END %]</tt></td>
+        <td><tt>[% INCLUDE renderFullJobName project = step.build.project.name jobset = step.build.jobset.name job = step.build.job.name %]</tt></td>
+        <td><tt>[% step.system %]</tt></td>
+        <td><a href="[% c.uri_for('/build' step.build.id) %]">[% step.build.id %]</a></td>
+        <td><a href="[% c.uri_for('/build' step.build.id 'nixlog' step.stepnr) %]">[% step.stepnr %]</a></td>
+        <td><tt>[% step.outpath.match('-(.*)').0 %]</tt></td>
+        <td class='right'>[% INCLUDE renderDuration duration = curTime - step.starttime %] </td>
+      </tr>
+    [% END %]
+  </tbody>
+</table>
+[% END %]

src/root/layout.tt

@@ -16,7 +16,7 @@
     <link rel="stylesheet" href="/static/css/nixos-site.css" type="text/css" />
     <link rel="stylesheet" href="/static/css/logfile.css" type="text/css" />
     <script type="text/javascript" src="/static/js/jquery/js/jquery-1.4.2.min.js"></script> 
-    <script type="text/javascript" src="/static/js/jquery/js/jquery-ui-1.8.4.custom.min.js"></script> 
+    <script type="text/javascript" src="/static/js/jquery/js/jquery-ui-1.8.5.custom.min.js"></script> 
     <script type="text/javascript" src="/static/js/tablesorter/jquery.tablesorter.js"></script>
     
     <script type="text/javascript">

src/root/navbar.tt

@@ -24,7 +24,7 @@
   </ul>
 [% END %]
 
-[% IF project %]
+[% IF project || admin %]
 
 <div id="left-bar">  
 
@@ -73,6 +73,17 @@
         title = "Errors" %]
     [% END %]
   [% END %]
+
+  [% IF admin %]
+    [% WRAPPER makeSubMenu title="Admin" %]
+      [% INCLUDE makeLink
+        uri = c.uri_for(c.controller('Admin').action_for('machines'))
+        title = "Machines" %]
+      [% INCLUDE makeLink
+        uri = c.uri_for(c.controller('Admin').action_for('managenews'))
+        title = "News" %]
+    [% END %]
+  [% END %]
   
 </div>
 [% END %]

src/root/status.tt

@@ -3,22 +3,6 @@
 
 <h1>Hydra Status</h1>
 
-<table class="tablesorter">
-  <thead>
-    <tr><th>Machine</th><th>Type</th><th>Build</th><th>Step</th><th>What</th><th>Since</th></tr>
-  </thead>
-  <tbody>
-    [% FOREACH step IN steps %]
-      <tr>
-        <td><tt>[% IF step.machine; step.machine.match('@(.*)').0; ELSE; 'localhost'; END %]</tt></td>
-        <td><tt>[% step.system %]</tt></td>
-        <td><a href="[% c.uri_for('/build' step.build.id) %]">[% step.build.id %]</a></td>
-        <td><a href="[% c.uri_for('/build' step.build.id 'nixlog' step.stepnr) %]">[% step.stepnr %]</a></td>
-        <td><tt>[% step.outpath.match('-(.*)').0 %]</tt></td>
-        <td class='right'>[% INCLUDE renderDuration duration = curTime - step.starttime %] </td>
-      </tr>
-    [% END %]
-  </tbody>
-</table>
+[% INCLUDE hydraStatus %]
 
 [% END %]

src/sql/hydra.sql

@@ -485,6 +485,22 @@ create table NewsItems (
     foreign key   (author) references Users(userName) on delete cascade on update cascade
 );
 
+create table BuildMachines (
+    hostname text primary key NOT NULL,
+    username text NOT NULL,
+    ssh_key text NOT NULL,
+    options text NOT NULL,
+    maxconcurrent integer DEFAULT 2 NOT NULL,
+    speedfactor integer DEFAULT 1 NOT NULL,
+    enabled integer DEFAULT 1 NOT NULL
+);
+
+create table BuildMachineSystemTypes (
+    hostname text NOT NULL,
+    system text NOT NULL,
+    primary key (hostname, system),
+    foreign key (hostname) references BuildMachines(hostname) on delete cascade
+);
 
 -- Some indices.
 create index IndexBuildInputsOnBuild on BuildInputs(build);