From 9b37cb89aea86b0262bea27dec8d651b410d6360 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Fri, 11 Mar 2016 21:48:31 -0500 Subject: [PATCH 1/3] Add buildStarted plugin hook --- src/hydra-queue-runner/builder.cc | 17 ++++++++++++----- src/hydra-queue-runner/hydra-queue-runner.cc | 8 ++++---- src/hydra-queue-runner/state.hh | 11 ++++++++++- src/lib/Hydra/Helper/PluginHooks.pm | 13 +++++++++++++ src/lib/Hydra/Plugin.pm | 5 +++++ src/script/hydra-notify | 12 +++++++----- 6 files changed, 51 insertions(+), 15 deletions(-) diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc index 2a31ff2e..9fe2e45b 100644 --- a/src/hydra-queue-runner/builder.cc +++ b/src/hydra-queue-runner/builder.cc @@ -84,9 +84,16 @@ State::StepResult State::doBuildStep(nix::ref destStore, Step::ptr step, return sMaybeCancelled; } - for (auto build2 : dependents) - if (build2->drvPath == step->drvPath) { build = build2; break; } - + for (auto build2 : dependents) { + if (build2->drvPath == step->drvPath) { + build = build2; + { + auto notificationSenderQueue_(notificationSenderQueue.lock()); + notificationSenderQueue_->push(NotificationItem{NotificationItem::Type::Started, build->id}); + } + notificationSenderWakeup.notify_one(); + } + } if (!build) build = *dependents.begin(); printMsg(lvlInfo, format("performing step ‘%1%’ on ‘%2%’ (needed by build %3% and %4% others)") @@ -249,7 +256,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, Step::ptr step, for (auto id : buildIDs) { { auto notificationSenderQueue_(notificationSenderQueue.lock()); - notificationSenderQueue_->push(NotificationItem(id, std::vector())); + notificationSenderQueue_->push(NotificationItem{NotificationItem::Type::Finished, id}); } notificationSenderWakeup.notify_one(); } @@ -369,7 +376,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, Step::ptr step, /* Send notification about this build and its dependents. */ { auto notificationSenderQueue_(notificationSenderQueue.lock()); - notificationSenderQueue_->push(NotificationItem(build->id, dependentIDs)); + notificationSenderQueue_->push(NotificationItem{NotificationItem::Type::Finished, build->id, dependentIDs}); } notificationSenderWakeup.notify_one(); diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index b1842523..d0305441 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -482,11 +482,11 @@ void State::notificationSender() notificationSenderQueue_->pop(); } - printMsg(lvlChatty, format("sending notification about build %1%") % item.first); + printMsg(lvlChatty, format("sending notification about build %1%") % item.id); Pid pid = startProcess([&]() { - Strings argv({"hydra-notify", "build", std::to_string(item.first)}); - for (auto id : item.second) + Strings argv({"hydra-notify", item.type == NotificationItem::Type::Started ? "build-started" : "build-finished", std::to_string(item.id)}); + for (auto id : item.dependentIds) argv.push_back(std::to_string(id)); execvp("hydra-notify", (char * *) stringsToCharPtrs(argv).data()); // FIXME: remove cast throw SysError("cannot start hydra-notify"); @@ -496,7 +496,7 @@ void State::notificationSender() if (res != 0) throw Error(format("hydra-build returned exit code %1% notifying about build %2%") - % res % item.first); + % res % item.id); } catch (std::exception & e) { printMsg(lvlError, format("notification sender: %1%") % e.what()); diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh index 34f6eaa1..d8e6caed 100644 --- a/src/hydra-queue-runner/state.hh +++ b/src/hydra-queue-runner/state.hh @@ -318,7 +318,16 @@ private: killed before it has finished sending notifications about a build, then the notifications may be lost. It would be better to mark builds with pending notification in the database. */ - typedef std::pair> NotificationItem; + struct NotificationItem + { + enum class Type : char { + Started, + Finished + }; + Type type; + BuildID id; + std::vector dependentIds; + }; nix::Sync> notificationSenderQueue; std::condition_variable notificationSenderWakeup; diff --git a/src/lib/Hydra/Helper/PluginHooks.pm b/src/lib/Hydra/Helper/PluginHooks.pm index 4000045b..2813ac65 100644 --- a/src/lib/Hydra/Helper/PluginHooks.pm +++ b/src/lib/Hydra/Helper/PluginHooks.pm @@ -5,8 +5,21 @@ use Exporter; our @ISA = qw(Exporter); our @EXPORT = qw( + notifyBuildStarted notifyBuildFinished); +sub notifyBuildStarted { + my ($plugins, $build) = @_; + foreach my $plugin (@{$plugins}) { + eval { + $plugin->buildStarted($build); + }; + if ($@) { + print STDERR "$plugin->buildStarted: $@\n": + } + } +} + sub notifyBuildFinished { my ($plugins, $build, $dependents) = @_; foreach my $plugin (@{$plugins}) { diff --git a/src/lib/Hydra/Plugin.pm b/src/lib/Hydra/Plugin.pm index 4a8ef69e..7783fb9e 100644 --- a/src/lib/Hydra/Plugin.pm +++ b/src/lib/Hydra/Plugin.pm @@ -20,6 +20,11 @@ sub instantiate { return @$plugins; } +# Called when build $build has started. +sub buildStarted { + my ($self, $build) = @_; +} + # Called when build $build has finished. If the build failed, then # $dependents is an array ref to a list of builds that have also # failed as a result (i.e. because they depend on $build or a failed diff --git a/src/script/hydra-notify b/src/script/hydra-notify index 1e68dde7..4ecce21b 100755 --- a/src/script/hydra-notify +++ b/src/script/hydra-notify @@ -15,12 +15,12 @@ my $db = Hydra::Model::DB->new(); my @plugins = Hydra::Plugin->instantiate(db => $db, config => $config); -my $cmd = shift @ARGV or die "Syntax: hydra-notify build BUILD-ID [BUILD-IDs...]\n"; +my $cmd = shift @ARGV or die "Syntax: hydra-notify CMD BUILD-ID [BUILD-IDs...]\n"; -if ($cmd eq "build") { - my $buildId = shift @ARGV or die; - my $build = $db->resultset('Builds')->find($buildId) - or die "build $buildId does not exist\n"; +my $buildId = shift @ARGV or die; +my $build = $db->resultset('Builds')->find($buildId) + or die "build $buildId does not exist\n"; +if ($cmd eq "build-finished") { my @dependents; foreach my $id (@ARGV) { my $dep = $db->resultset('Builds')->find($id) @@ -28,6 +28,8 @@ if ($cmd eq "build") { push @dependents, $dep; } notifyBuildFinished(\@plugins, $build, [@dependents]); +} elsif ($cmd eq "build-started") { + notifyBuildStarted(\@plugins, $build); } else { From 0d0b925af1e4ffcfc6a561ca505abb22746b6580 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Sat, 12 Mar 2016 11:03:05 -0500 Subject: [PATCH 2/3] Add a plugin to interact with the github status API. Mutliple sections are possible: * jobs: regexp for jobs to match * inputs: the input which corresponds to the github repo/rev whose status we want to report. Can be repeated * authorization: Verbatim contents of the Authorization header. See https://developer.github.com/v3/#authentication. --- src/lib/Hydra/Helper/PluginHooks.pm | 2 +- src/lib/Hydra/Plugin/GithubStatus.pm | 76 ++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 src/lib/Hydra/Plugin/GithubStatus.pm diff --git a/src/lib/Hydra/Helper/PluginHooks.pm b/src/lib/Hydra/Helper/PluginHooks.pm index 2813ac65..8ff59bd2 100644 --- a/src/lib/Hydra/Helper/PluginHooks.pm +++ b/src/lib/Hydra/Helper/PluginHooks.pm @@ -15,7 +15,7 @@ sub notifyBuildStarted { $plugin->buildStarted($build); }; if ($@) { - print STDERR "$plugin->buildStarted: $@\n": + print STDERR "$plugin->buildStarted: $@\n"; } } } diff --git a/src/lib/Hydra/Plugin/GithubStatus.pm b/src/lib/Hydra/Plugin/GithubStatus.pm new file mode 100644 index 00000000..e84cec3d --- /dev/null +++ b/src/lib/Hydra/Plugin/GithubStatus.pm @@ -0,0 +1,76 @@ +package Hydra::Plugin::GithubStatus; + +use strict; +use parent 'Hydra::Plugin'; +use HTTP::Request; +use JSON; +use LWP::UserAgent; +use Hydra::Helper::CatalystUtils; + +sub toGithubState { + my ($buildStatus) = @_; + if ($buildStatus == 0) { + return "success"; + } elsif ($buildStatus == 3 || $buildStatus == 4 || $buildStatus == 8 || $buildStatus == 10 || $buildStatus == 11) { + return "error"; + } else { + return "failure"; + } +} + +sub common { + my ($self, $build, $dependents, $finished) = @_; + my $cfg = $self->{config}->{githubstatus}; + my @config = defined $cfg ? ref $cfg eq "ARRAY" ? @$cfg : ($cfg) : (); + my $baseurl = $self->{config}->{'base_uri'} || "http://localhost:3000"; + + # Find matching configs + foreach my $b ($build, @{$dependents}) { + my $jobName = showJobName $b; + my $evals = $build->jobsetevals; + my $ua = LWP::UserAgent->new(); + my $body = encode_json( + { + state => $finished ? toGithubState($b->buildstatus) : "pending", + target_url => "$baseurl/build/" . $b->id, + description => "Hydra build #" . $b->id . " of $jobName", + context => "continuous-integration/hydra" + }); + foreach my $conf (@config) { + next unless $jobName =~ /^$conf->{jobs}$/; + + my $inputs_cfg = $conf->{inputs}; + my @inputs = defined $inputs_cfg ? ref $inputs_cfg eq "ARRAY" ? @$inputs_cfg : ($inputs_cfg) : (); + my %seen = map { $_ => {} } @inputs; + while (my $eval = $evals->next) { + foreach my $input (@inputs) { + my $i = $eval->jobsetevalinputs->find({ name => $input, altnr => 0 }); + next unless defined $i; + my $uri = $i->uri; + my $rev = $i->revision; + my $key = $uri . "-" . $rev; + next if exists $seen{$input}->{$key}; + $seen{$input}->{$key} = 1; + $uri =~ m![:/]([^/]+)/([^/]+?)(?:.git)?$!; + my $req = HTTP::Request->new('POST', "https://api.github.com/repos/$1/$2/statuses/$rev"); + $req->header('Content-Type' => 'application/json'); + $req->header('Accept' => 'application/vnd.github.v3+json'); + $req->header('Authorization' => $conf->{authorization}); + $req->content($body); + my $res = $ua->request($req); + print STDERR $res->status_line, ": ", $res->decoded_content, "\n" unless $res->is_success; + } + } + } + } +} + +sub buildStarted { + common(@_, [], 0); +} + +sub buildFinished { + common(@_, 1); +} + +1; From ae24e7fb316a2c25b508df23f2ac60fe3a440644 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Tue, 12 Apr 2016 15:19:05 -0400 Subject: [PATCH 3/3] GithubStatus: Include the full job name and build ID in the context. Build ID can be omitted by setting excludeBuildFromContext in config --- src/lib/Hydra/Plugin/GithubStatus.pm | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/lib/Hydra/Plugin/GithubStatus.pm b/src/lib/Hydra/Plugin/GithubStatus.pm index e84cec3d..7677cbd8 100644 --- a/src/lib/Hydra/Plugin/GithubStatus.pm +++ b/src/lib/Hydra/Plugin/GithubStatus.pm @@ -29,16 +29,18 @@ sub common { my $jobName = showJobName $b; my $evals = $build->jobsetevals; my $ua = LWP::UserAgent->new(); - my $body = encode_json( - { - state => $finished ? toGithubState($b->buildstatus) : "pending", - target_url => "$baseurl/build/" . $b->id, - description => "Hydra build #" . $b->id . " of $jobName", - context => "continuous-integration/hydra" - }); + foreach my $conf (@config) { next unless $jobName =~ /^$conf->{jobs}$/; + my $contextTrailer = $conf->{excludeBuildFromContext} ? "" : (":" . $b->id); + my $body = encode_json( + { + state => $finished ? toGithubState($b->buildstatus) : "pending", + target_url => "$baseurl/build/" . $b->id, + description => "Hydra build #" . $b->id . " of $jobName", + context => "continuous-integration/hydra:" . $jobName . $contextTrailer + }); my $inputs_cfg = $conf->{inputs}; my @inputs = defined $inputs_cfg ? ref $inputs_cfg eq "ARRAY" ? @$inputs_cfg : ($inputs_cfg) : (); my %seen = map { $_ => {} } @inputs;