From 42d2015357c18a7137ba65e68e67940cb545bc72 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 25 Feb 2013 21:04:10 +0100 Subject: [PATCH] Support push notification of repository changes External machines can now notify Hydra that it should check a repository by sending a GET or PUSH request to /api/push, providing a list of jobsets to be checked and/or a list of repository URLs. In the latter case, all jobsets that have any of the specified repositories as an input will be checked. For instance, you can configure GitHub or BitBucket to send a request to the URL http://hydra.example.org/api/push?repos=git://github.com/NixOS/nixpkgs.git to trigger evaluation of all jobsets that have git://github.com/NixOS/nixpkgs.git as an input, or to the URL http://hydra.example.org/api/push?jobsets=patchelf:trunk,nixpkgs:trunk to trigger evaluation of just the specified jobsets. --- src/lib/Hydra.pm | 2 +- src/lib/Hydra/Controller/API.pm | 46 +++++++++++++++++++++++++++ src/lib/Hydra/Controller/Build.pm | 7 ++-- src/lib/Hydra/Helper/CatalystUtils.pm | 8 +++++ src/lib/Hydra/Schema/Jobsets.pm | 11 +++++-- src/script/hydra-evaluator | 30 ++++++++++++++--- src/sql/hydra.sql | 1 + src/sql/upgrade-10.sql | 1 + 8 files changed, 95 insertions(+), 11 deletions(-) create mode 100644 src/sql/upgrade-10.sql diff --git a/src/lib/Hydra.pm b/src/lib/Hydra.pm index c43cf560..44c899b4 100644 --- a/src/lib/Hydra.pm +++ b/src/lib/Hydra.pm @@ -45,7 +45,7 @@ __PACKAGE__->config( expires => 3600 }, 'View::JSON' => { - expose_stash => qr/^json/, + expose_stash => 'json' }, 'Plugin::Session' => { expires => 3600 * 24 * 2, diff --git a/src/lib/Hydra/Controller/API.pm b/src/lib/Hydra/Controller/API.pm index d60df4fb..6b962a0f 100644 --- a/src/lib/Hydra/Controller/API.pm +++ b/src/lib/Hydra/Controller/API.pm @@ -1,5 +1,6 @@ package Hydra::Controller::API; +use utf8; use strict; use warnings; use base 'Catalyst::Controller'; @@ -22,6 +23,16 @@ sub api : Chained('/') PathPart('api') CaptureArgs(0) { } +sub end : ActionClass('RenderView') { + my ($self, $c) = @_; + if (scalar @{$c->error}) { + $c->stash->{json}->{error} = join "\n", @{$c->error}; + $c->forward('View::JSON'); + $c->clear_errors; + } +} + + sub projectToHash { my ($project) = @_; return { @@ -274,4 +285,39 @@ sub logdiff : Chained('api') PathPart('logdiff') Args(2) { } +sub triggerJobset { + my ($self, $c, $jobset) = @_; + txn_do($c->model('DB')->schema, sub { + $jobset->update({ triggertime => time }); + }); + push @{$c->{stash}->{json}->{jobsetsTriggered}}, $jobset->project->name . ":" . $jobset->name; +} + + +sub push : Chained('api') PathPart('push') Args(0) { + my ($self, $c) = @_; + + $c->{stash}->{json}->{jobsetsTriggered} = []; + + my @jobsets = split /,/, ($c->request->params->{jobsets} // ""); + foreach my $s (@jobsets) { + my ($p, $j) = parseJobsetName($s); + my $jobset = $c->model('DB::Jobsets')->find($p, $j) or notFound($c, "Jobset ‘$p:$j’ does not exist."); + next unless $jobset->project->enabled && $jobset->enabled; + triggerJobset($self, $c, $jobset); + } + + my @repos = split /,/, ($c->request->params->{repos} // ""); + foreach my $r (@repos) { + triggerJobset($self, $c, $_) foreach $c->model('DB::Jobsets')->search( + { 'project.enabled' => 1, 'me.enabled' => 1 }, + { join => 'project' + , where => \ [ 'exists (select 1 from JobsetInputAlts where project = me.project and jobset = me.name and value = ?)', [ 'value', $r ] ] + }); + } + + $c->forward('View::JSON'); +} + + 1; diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm index 81f170d7..857e4a5d 100644 --- a/src/lib/Hydra/Controller/Build.pm +++ b/src/lib/Hydra/Controller/Build.pm @@ -563,11 +563,10 @@ sub clone_submit : Chained('build') PathPart('clone/submit') Args(0) { sub get_info : Chained('build') PathPart('api/get-info') Args(0) { my ($self, $c) = @_; my $build = $c->stash->{build}; - # !!! strip the json prefix - $c->stash->{jsonBuildId} = $build->id; - $c->stash->{jsonDrvPath} = $build->drvpath; + $c->stash->{json}->{buildId} = $build->id; + $c->stash->{json}->{drvPath} = $build->drvpath; my $out = getMainOutput($build); - $c->stash->{jsonOutPath} = $out->path if defined $out; + $c->stash->{json}->{outPath} = $out->path if defined $out; $c->forward('View::JSON'); } diff --git a/src/lib/Hydra/Helper/CatalystUtils.pm b/src/lib/Hydra/Helper/CatalystUtils.pm index 6e90071e..65398954 100644 --- a/src/lib/Hydra/Helper/CatalystUtils.pm +++ b/src/lib/Hydra/Helper/CatalystUtils.pm @@ -13,6 +13,7 @@ our @EXPORT = qw( requireLogin requireProjectOwner requireAdmin requirePost isAdmin isProjectOwner trim getLatestFinishedEval + parseJobsetName $pathCompRE $relPathRE $relNameRE $projectNameRE $jobsetNameRE $jobNameRE $systemRE @buildListColumns ); @@ -171,4 +172,11 @@ Readonly our $jobNameRE => "(?:$attrNameRE(?:\\.$attrNameRE)*)"; Readonly our $systemRE => "(?:[a-z0-9_]+-[a-z0-9_]+)"; +sub parseJobsetName { + my ($s) = @_; + $s =~ /^($projectNameRE):($jobsetNameRE)$/ or die "invalid jobset specifier ‘$s’\n"; + return ($1, $2); +} + + 1; diff --git a/src/lib/Hydra/Schema/Jobsets.pm b/src/lib/Hydra/Schema/Jobsets.pm index 2a399fc6..649043e4 100644 --- a/src/lib/Hydra/Schema/Jobsets.pm +++ b/src/lib/Hydra/Schema/Jobsets.pm @@ -66,6 +66,11 @@ __PACKAGE__->table("Jobsets"); data_type: 'integer' is_nullable: 1 +=head2 triggertime + + data_type: 'integer' + is_nullable: 1 + =head2 enabled data_type: 'integer' @@ -114,6 +119,8 @@ __PACKAGE__->add_columns( { data_type => "integer", is_nullable => 1 }, "lastcheckedtime", { data_type => "integer", is_nullable => 1 }, + "triggertime", + { data_type => "integer", is_nullable => 1 }, "enabled", { data_type => "integer", default_value => 1, is_nullable => 0 }, "enableemail", @@ -245,7 +252,7 @@ __PACKAGE__->belongs_to( ); -# Created by DBIx::Class::Schema::Loader v0.07033 @ 2013-01-22 13:29:36 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:9smV/zbSSxQNLiBcnADFXA +# Created by DBIx::Class::Schema::Loader v0.07033 @ 2013-02-25 19:10:12 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:SvBgR0iH9NjVH4jvBATYPA 1; diff --git a/src/script/hydra-evaluator b/src/script/hydra-evaluator index 404bf989..c26856ff 100755 --- a/src/script/hydra-evaluator +++ b/src/script/hydra-evaluator @@ -21,7 +21,7 @@ my $db = Hydra::Model::DB->new(); my $config = getHydraConfig(); # Don't check a jobset more than once every five minutes. -my $minCheckInterval = 1; +my $minCheckInterval = 5 * 60; @@ -232,6 +232,8 @@ sub checkJobset { ? " (last checked " . (time() - $jobset->lastcheckedtime) . "s ago)\n" : " (never checked)\n"; + my $triggerTime = $jobset->triggertime; + eval { checkJobsetWrapped($jobset); }; @@ -244,15 +246,35 @@ sub checkJobset { setJobsetError($jobset, $msg); }); } + + if (defined $triggerTime) { + txn_do($db, sub { + # Only clear the trigger time if the jobset hasn't been + # triggered in the meantime. In that case, we need to + # evaluate again. + my $new = $jobset->get_from_storage(); + $jobset->update({ triggertime => undef }) + if $new->triggertime == $triggerTime; + }); + } } -# Check the jobset that hasn't been checked for the longest time. sub checkSomeJobset { + # If any jobset has been triggered by a push, check it. my ($jobset) = $db->resultset('Jobsets')->search( - { 'project.enabled' => 1, 'me.enabled' => 1 + { 'project.enabled' => 1, 'me.enabled' => 1, 'triggertime' => { '!=', undef }, , -or => [ 'lastcheckedtime' => undef, 'lastcheckedtime' => { '<', time() - $minCheckInterval } ] }, - { join => 'project', order_by => [ 'lastcheckedtime nulls first' ], rows => 1 }); + { join => 'project', order_by => [ 'triggertime' ], rows => 1 }); + + # Otherwise, check the jobset that hasn't been checked for the + # longest time (but don't check more often than the minimal check + # interval). + ($jobset) = $db->resultset('Jobsets')->search( + { 'project.enabled' => 1, 'me.enabled' => 1, + , -or => [ 'lastcheckedtime' => undef, 'lastcheckedtime' => { '<', time() - $minCheckInterval } ] }, + { join => 'project', order_by => [ 'lastcheckedtime nulls first' ], rows => 1 }) + unless defined $jobset; return 0 unless defined $jobset; diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index 3f236c78..4ff9d9c5 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -54,6 +54,7 @@ create table Jobsets ( errorMsg text, -- used to signal the last evaluation error etc. for this jobset errorTime integer, -- timestamp associated with errorMsg lastCheckedTime integer, -- last time the evaluator looked at this jobset + triggerTime integer, -- set if we were triggered by a push event enabled integer not null default 1, enableEmail integer not null default 1, hidden integer not null default 0, diff --git a/src/sql/upgrade-10.sql b/src/sql/upgrade-10.sql new file mode 100644 index 00000000..aeab7873 --- /dev/null +++ b/src/sql/upgrade-10.sql @@ -0,0 +1 @@ +alter table Jobsets add column triggerTime integer;