From 398993f688c34d1842d21310a496ab4c954420c9 Mon Sep 17 00:00:00 2001 From: Rob Vermaas Date: Wed, 13 Oct 2010 12:32:57 +0000 Subject: [PATCH] hydra: add some admin for adding/enabling/etc build machines --- src/lib/Hydra/Controller/Admin.pm | 165 +++++++++++++++++- src/lib/Hydra/Controller/Root.pm | 2 +- .../Hydra/Schema/BuildMachineSystemTypes.pm | 81 +++++++++ src/lib/Hydra/Schema/BuildMachines.pm | 133 ++++++++++++++ src/root/admin.tt | 39 ++++- src/root/common.tt | 20 +++ src/root/layout.tt | 2 +- src/root/navbar.tt | 13 +- src/root/status.tt | 18 +- src/sql/hydra.sql | 16 ++ 10 files changed, 467 insertions(+), 22 deletions(-) create mode 100644 src/lib/Hydra/Schema/BuildMachineSystemTypes.pm create mode 100644 src/lib/Hydra/Schema/BuildMachines.pm diff --git a/src/lib/Hydra/Controller/Admin.pm b/src/lib/Hydra/Controller/Admin.pm index ac1a2ce4..d9cdc9bf 100644 --- a/src/lib/Hydra/Controller/Admin.pm +++ b/src/lib/Hydra/Controller/Admin.pm @@ -6,15 +6,178 @@ use base 'Catalyst::Controller'; use Hydra::Helper::Nix; use Hydra::Helper::CatalystUtils; use Hydra::Helper::AddBuilds; +use Data::Dump qw(dump); + +sub nixMachines { + my ($c) = @_; + my $result = ''; + + foreach my $machine ($c->model("DB::BuildMachines")->all) { + if($machine->enabled) { + $result = $result . $machine->username . '@'. $machine->hostname . ' '; + foreach my $system ($machine->buildmachinesystemtypes) { + $result = $result . $system->system .','; + } + chop $result; + $result = $result . ' '. $machine->ssh_key . ' ' . $machine->maxconcurrent . ' '. $machine->speedfactor . ' ' . $machine->options . "\n"; + } + } + return $result; +} + +sub saveNixMachines { + my ($c) = @_; + + die("File not writable: /etc/nix.machines") if ! -w "/etc/nix.machines" ; + + open (NIXMACHINES, '>/etc/nix.machines') or die("Could not write to /etc/nix.machines"); + print NIXMACHINES nixMachines($c); + close (NIXMACHINES); +} sub admin : Chained('/') PathPart('admin') CaptureArgs(0) { my ($self, $c) = @_; requireAdmin($c); + $c->stash->{admin} = 1; } sub index : Chained('admin') PathPart('') Args(0) { my ($self, $c) = @_; + $c->stash->{machines} = [$c->model('DB::BuildMachines')->search( + {}, + { order_by => "hostname" + , '+select' => ["(select bs.stoptime from buildsteps as bs where bs.machine = (me.username || '\@' || me.hostname) and not bs.stoptime is null order by bs.stoptime desc limit 1)"] + , '+as' => ['idle'] + })]; + $c->stash->{steps} = [ $c->model('DB::BuildSteps')->search( + { 'me.busy' => 1, 'schedulingInfo.busy' => 1 }, + { join => [ 'schedulingInfo', 'build' ] + , order_by => [ 'machine', 'outpath' ] + } ) ]; $c->stash->{template} = 'admin.tt'; + } + +sub machines : Chained('admin') PathPart('machines') Args(0) { + my ($self, $c) = @_; + $c->stash->{machines} = [$c->model('DB::BuildMachines')->search({}, {order_by => "hostname"})]; + $c->stash->{systems} = [$c->model('DB::SystemTypes')->search({}, {select => ["system"], order_by => "system" })]; + $c->stash->{nixMachines} = nixMachines($c); + $c->stash->{nixMachinesWritable} = (-e "/etc/nix.machines" && -w "/etc/nix.machines"); + + $c->stash->{template} = 'machines.tt'; + } + +sub machine : Chained('admin') PathPart('machine') CaptureArgs(1) { + my ($self, $c, $machineName) = @_; + + requireAdmin($c); + + my $machine = $c->model('DB::BuildMachines')->find($machineName) + or notFound($c, "Machine $machineName doesn't exist."); + + $c->stash->{machine} = $machine; +} + +sub machine_edit : Chained('machine') PathPart('edit') Args(0) { + my ($self, $c) = @_; + $c->stash->{template} = 'machine.tt'; + $c->stash->{systemtypes} = [$c->model('DB::SystemTypes')->search({}, {order_by => "system"})]; + $c->stash->{edit} = 1; +} + +sub machine_edit_submit : Chained('machine') PathPart('submit') Args(0) { + my ($self, $c) = @_; + requirePost($c); + + txn_do($c->model('DB')->schema, sub { + updateMachine($c, $c->stash->{machine}) ; + }); + saveNixMachines($c); + $c->res->redirect("/admin/machines"); +} + +sub updateMachine { + my ($c, $machine) = @_; + + my $hostname = trim $c->request->params->{"hostname"}; + my $username = trim $c->request->params->{"username"}; + my $maxconcurrent = trim $c->request->params->{"maxconcurrent"}; + my $speedfactor = trim $c->request->params->{"speedfactor"}; + my $ssh_key = trim $c->request->params->{"ssh_key"}; + my $systems = $c->request->params->{"systems"} ; + + error($c, "Invalid or empty username.") if $username eq ""; + error($c, "Max concurrent builds should be an integer > 0.") if $maxconcurrent eq "" || ! $maxconcurrent =~ m/[0-9]+/; + error($c, "Speed factor should be an integer > 0.") if $speedfactor eq "" || ! $speedfactor =~ m/[0-9]+/; + error($c, "Invalid or empty SSH key.") if $ssh_key eq ""; + + $machine->update( + { username => $username + , maxconcurrent => $maxconcurrent + , speedfactor => $speedfactor + , ssh_key => $ssh_key + }); + $machine->buildmachinesystemtypes->delete_all; + if(ref($systems) eq 'ARRAY') { + for my $s (@$systems) { + $machine->buildmachinesystemtypes->create({ system => $s}) ; + } + } else { + $machine->buildmachinesystemtypes->create({ system => $systems}) ; + } +} + +sub create_machine : Chained('admin') PathPart('create-machine') Args(0) { + my ($self, $c) = @_; + + requireAdmin($c); + + $c->stash->{template} = 'machine.tt'; + $c->stash->{systemtypes} = [$c->model('DB::SystemTypes')->search({}, {order_by => "system"})]; + $c->stash->{edit} = 1; + $c->stash->{create} = 1; +} + + +sub create_machine_submit : Chained('admin') PathPart('create-machine/submit') Args(0) { + my ($self, $c) = @_; + + requireAdmin($c); + + my $hostname = trim $c->request->params->{"hostname"}; + error($c, "Invalid or empty hostname.") if $hostname eq ""; + + txn_do($c->model('DB')->schema, sub { + my $machine = $c->model('DB::BuildMachines')->create( + { hostname => $hostname }); + updateMachine($c, $machine); + }); + $c->res->redirect("/admin/machines"); +} + +sub machine_delete : Chained('machine') PathPart('delete') Args(0) { + my ($self, $c) = @_; + requirePost($c); + + txn_do($c->model('DB')->schema, sub { + $c->stash->{machine}->delete; + }); + saveNixMachines($c); + $c->res->redirect("/admin/machines"); +} + +sub machine_enable : Chained('machine') PathPart('enable') Args(0) { + my ($self, $c) = @_; + $c->stash->{machine}->update({ enabled => 1}); + saveNixMachines($c); + $c->res->redirect("/admin/machines"); +} + +sub machine_disable : Chained('machine') PathPart('disable') Args(0) { + my ($self, $c) = @_; + $c->stash->{machine}->update({ enabled => 0}); + saveNixMachines($c); + $c->res->redirect("/admin/machines"); } sub clearfailedcache : Chained('admin') Path('clear-failed-cache') Args(0) { @@ -31,7 +194,7 @@ sub clearevalcache : Chained('admin') Path('clear-eval-cache') Args(0) { print "Clearing evaluation cache\n"; $c->model('DB::JobsetInputHashes')->delete_all; - $c->res->redirect("/admin"); + $c->res->redirect("/admin") } sub clearvcscache : Chained('admin') Path('clear-vcs-cache') Args(0) { diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm index 259263e8..3d3a7994 100644 --- a/src/lib/Hydra/Controller/Root.pm +++ b/src/lib/Hydra/Controller/Root.pm @@ -98,7 +98,7 @@ sub status :Local { my ($self, $c) = @_; $c->stash->{steps} = [ $c->model('DB::BuildSteps')->search( { 'me.busy' => 1, 'schedulingInfo.busy' => 1 }, - { join => [ 'schedulingInfo' ] + { join => [ 'schedulingInfo', 'build' ] , order_by => [ 'machine', 'outpath' ] } ) ]; } diff --git a/src/lib/Hydra/Schema/BuildMachineSystemTypes.pm b/src/lib/Hydra/Schema/BuildMachineSystemTypes.pm new file mode 100644 index 00000000..ed328010 --- /dev/null +++ b/src/lib/Hydra/Schema/BuildMachineSystemTypes.pm @@ -0,0 +1,81 @@ +package Hydra::Schema::BuildMachineSystemTypes; + +# Created by DBIx::Class::Schema::Loader +# DO NOT MODIFY THE FIRST PART OF THIS FILE + +use strict; +use warnings; + +use base 'DBIx::Class::Core'; + + +=head1 NAME + +Hydra::Schema::BuildMachineSystemTypes + +=cut + +__PACKAGE__->table("BuildMachineSystemTypes"); + +=head1 ACCESSORS + +=head2 hostname + + data_type: text + default_value: undef + is_foreign_key: 1 + is_nullable: 0 + size: undef + +=head2 system + + data_type: text + default_value: undef + is_nullable: 0 + size: undef + +=cut + +__PACKAGE__->add_columns( + "hostname", + { + data_type => "text", + default_value => undef, + is_foreign_key => 1, + is_nullable => 0, + size => undef, + }, + "system", + { + data_type => "text", + default_value => undef, + is_nullable => 0, + size => undef, + }, +); +__PACKAGE__->set_primary_key("hostname", "system"); + +=head1 RELATIONS + +=head2 hostname + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "hostname", + "Hydra::Schema::BuildMachines", + { hostname => "hostname" }, + {}, +); + + +# Created by DBIx::Class::Schema::Loader v0.05000 @ 2010-10-08 13:47:26 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:F/voQZLNESTotUOWRbg4WA + + +# You can replace this text with custom content, and it will be preserved on regeneration +1; diff --git a/src/lib/Hydra/Schema/BuildMachines.pm b/src/lib/Hydra/Schema/BuildMachines.pm new file mode 100644 index 00000000..ab615f33 --- /dev/null +++ b/src/lib/Hydra/Schema/BuildMachines.pm @@ -0,0 +1,133 @@ +package Hydra::Schema::BuildMachines; + +# Created by DBIx::Class::Schema::Loader +# DO NOT MODIFY THE FIRST PART OF THIS FILE + +use strict; +use warnings; + +use base 'DBIx::Class::Core'; + + +=head1 NAME + +Hydra::Schema::BuildMachines + +=cut + +__PACKAGE__->table("BuildMachines"); + +=head1 ACCESSORS + +=head2 hostname + + data_type: text + default_value: undef + is_nullable: 0 + size: undef + +=head2 username + + data_type: text + default_value: undef + is_nullable: 0 + size: undef + +=head2 ssh_key + + data_type: text + default_value: undef + is_nullable: 0 + size: undef + +=head2 options + + data_type: text + default_value: undef + is_nullable: 0 + size: undef + +=head2 maxconcurrent + + data_type: integer + default_value: 2 + is_nullable: 0 + size: undef + +=head2 speedfactor + + data_type: integer + default_value: 1 + is_nullable: 0 + size: undef + +=head2 enabled + + data_type: integer + default_value: 1 + is_nullable: 0 + size: undef + +=cut + +__PACKAGE__->add_columns( + "hostname", + { + data_type => "text", + default_value => undef, + is_nullable => 0, + size => undef, + }, + "username", + { + data_type => "text", + default_value => undef, + is_nullable => 0, + size => undef, + }, + "ssh_key", + { + data_type => "text", + default_value => undef, + is_nullable => 0, + size => undef, + }, + "options", + { + data_type => "text", + default_value => undef, + is_nullable => 0, + size => undef, + }, + "maxconcurrent", + { data_type => "integer", default_value => 2, is_nullable => 0, size => undef }, + "speedfactor", + { data_type => "integer", default_value => 1, is_nullable => 0, size => undef }, + "enabled", + { data_type => "integer", default_value => 1, is_nullable => 0, size => undef }, +); +__PACKAGE__->set_primary_key("hostname"); + +=head1 RELATIONS + +=head2 buildmachinesystemtypes + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "buildmachinesystemtypes", + "Hydra::Schema::BuildMachineSystemTypes", + { "foreign.hostname" => "self.hostname" }, +); + + +# Created by DBIx::Class::Schema::Loader v0.05000 @ 2010-10-11 12:58:16 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:gje+VA73Hghl7JXp+Fl8pw + + +# You can replace this text with custom content, and it will be preserved on regeneration +1; diff --git a/src/root/admin.tt b/src/root/admin.tt index b548dc5f..71a1b4df 100644 --- a/src/root/admin.tt +++ b/src/root/admin.tt @@ -12,8 +12,45 @@
  • [% INCLUDE maybeLink uri = c.uri_for(c.controller('Admin').action_for('clearvcscache')) content = "Clear VCS caches" confirmmsg = "Are you sure you want to clear the VCS caches?" %]
    • -
  • [% INCLUDE maybeLink uri = c.uri_for(c.controller('Admin').action_for('managenews')) content = "News" %]
    • +

    Status

    + +[% FOREACH m IN machines %] + + + + + + + + [% idle = 1 %] + [% FOREACH step IN steps %] + [% IF step.machine.match('@(.*)').0 == m.hostname %] + [% idle = 0 %] + + + + + + + + [% END %] + [% END %] + [% IF idle == 1 %] + + [% END %] + +
    + [% IF m.enabled == 1 %] + [% INCLUDE maybeLink uri = c.uri_for('/admin/machine' m.hostname 'disable' ) content='-' %] + [% ELSE %] + [% INCLUDE maybeLink uri = c.uri_for('/admin/machine' m.hostname 'enable' ) content='+' %] + [% END %] + [% m.hostname %] [% FOREACH ms IN m.buildmachinesystemtypes %] [% ms.system %][% END %] +
    [% INCLUDE renderFullJobName project = step.build.project.name jobset = step.build.jobset.name job = step.build.job.name %][% step.system %][% step.build.id %][% step.outpath.match('-(.*)').0 %][% INCLUDE renderDuration duration = curTime - step.starttime %]
    Idle since [% INCLUDE renderDuration duration = curTime - m.get_column('idle') %]
    + +[% END %] + [% END %] diff --git a/src/root/common.tt b/src/root/common.tt index 5b4c8425..9977d97e 100644 --- a/src/root/common.tt +++ b/src/root/common.tt @@ -334,3 +334,23 @@ [% END %] +[% BLOCK hydraStatus %] + + + + + + [% FOREACH step IN steps %] + + + + + + + + + + [% END %] + +
    MachineJobTypeBuildStepWhatSince
    [% IF step.machine; step.machine.match('@(.*)').0; ELSE; 'localhost'; END %][% INCLUDE renderFullJobName project = step.build.project.name jobset = step.build.jobset.name job = step.build.job.name %][% step.system %][% step.build.id %][% step.stepnr %][% step.outpath.match('-(.*)').0 %][% INCLUDE renderDuration duration = curTime - step.starttime %]
    +[% END %] diff --git a/src/root/layout.tt b/src/root/layout.tt index 58f91ba9..b77993b3 100644 --- a/src/root/layout.tt +++ b/src/root/layout.tt @@ -16,7 +16,7 @@ - +