From 68aad22d1930020a6e4417604950c6522e6a4245 Mon Sep 17 00:00:00 2001 From: Antoine Eiche Date: Mon, 3 Sep 2018 23:41:30 +0200 Subject: [PATCH 1/2] Add GitlabPulls input plugin --- src/lib/Hydra/Plugin/GitlabPulls.pm | 94 +++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 src/lib/Hydra/Plugin/GitlabPulls.pm diff --git a/src/lib/Hydra/Plugin/GitlabPulls.pm b/src/lib/Hydra/Plugin/GitlabPulls.pm new file mode 100644 index 00000000..de371f9a --- /dev/null +++ b/src/lib/Hydra/Plugin/GitlabPulls.pm @@ -0,0 +1,94 @@ +# This plugin allows to build Gitlab merge requests. +# +# The declarative project spec.json file must contains an input such as +# "pulls": { +# "type": "gitlabpulls", +# "value": "https://gitlab.com 42", +# "emailresponsible": false +# } +# where 42 is the project id of a repository. +# +# The values source_repo_url and source_branch can then be used to +# build the git input value. + +package Hydra::Plugin::GitlabPulls; + +use strict; +use parent 'Hydra::Plugin'; +use HTTP::Request; +use LWP::UserAgent; +use JSON; +use Hydra::Helper::CatalystUtils; +use File::Temp; +use POSIX qw(strftime); + +sub supportedInputTypes { + my ($self, $inputTypes) = @_; + $inputTypes->{'gitlabpulls'} = 'Open Gitlab Merge Requests'; +} + +sub _query { + my ($url, $ua) = @_; + my $req = HTTP::Request->new('GET', $url); + my $res = $ua->request($req); + my $content = $res->decoded_content; + die "Error pulling from the gitlab pulls API: $content\n" + unless $res->is_success; + return (decode_json $content, $res); +} + +# We need to query the Gitlab API for each merge request to get the +# source repository URL. +sub _enhanceGitlabPull { + my ($pull, $baseUrl, $ua) = @_; + my $projectId = $pull->{source_project_id}; + (my $repo, my $res) = _query("$baseUrl/api/v4/projects/$projectId", $ua); + $pull->{source_repo_url} = $repo->{http_url_to_repo}; +} + +sub _iterate { + my ($url, $baseUrl, $pulls, $ua) = @_; + my ($pulls_list, $res) = _query($url, $ua); + + foreach my $pull (@$pulls_list) { + _enhanceGitlabPull($pull, $baseUrl, $ua); + $pulls->{$pull->{iid}} = $pull; + } + # TODO Make Link header parsing more robust!!! + my @links = split ',', $res->header("Link"); + my $next = ""; + foreach my $link (@links) { + my ($url, $rel) = split ";", $link; + if (trim($rel) eq 'rel="next"') { + $next = substr trim($url), 1, -1; + last; + } + } + _iterate($next, $baseUrl, $pulls, $ua) unless $next eq ""; +} + +sub fetchInput { + my ($self, $type, $name, $value, $project, $jobset) = @_; + return undef if $type ne "gitlabpulls"; + + (my $baseUrl, my $projectId) = split ' ', $value; + my $url = "$baseUrl/api/v4/projects/$projectId/merge_requests?per_page=100&state=opened"; + + my %pulls; + my $ua = LWP::UserAgent->new(); + _iterate($url, $baseUrl, \%pulls, $ua); + + my $tempdir = File::Temp->newdir("gitlab-pulls" . "XXXXX", TMPDIR => 1); + my $filename = "$tempdir/gitlab-pulls.json"; + open(my $fh, ">", $filename) or die "Cannot open $filename for writing: $!"; + print $fh encode_json \%pulls; + close $fh; + system("jq -S . < $filename > $tempdir/gitlab-pulls-sorted.json"); + my $storePath = trim(`nix-store --add "$tempdir/gitlab-pulls-sorted.json"` + or die "cannot copy path $filename to the Nix store.\n"); + chomp $storePath; + my $timestamp = time; + return { storePath => $storePath, revision => strftime "%Y%m%d%H%M%S", gmtime($timestamp) }; +} + +1; From d9253543e4fe29f57d9d47eaf85fea1f709237d8 Mon Sep 17 00:00:00 2001 From: Antoine Eiche Date: Tue, 20 Nov 2018 16:27:40 +0100 Subject: [PATCH 2/2] plugin/GitLabPulls: support for using a personal access token (PAT) In order to access protected or private repositories. Using the target repository URL along with the merge-request ref instead of the source repository url and branch is necessary to avoid running into issues if the source repository is not actually accessible to the user Hydra is authenticating as. Thanks Alexei Robyn for this patch. --- src/lib/Hydra/Plugin/GitlabPulls.pm | 33 +++++++++++++++-------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/src/lib/Hydra/Plugin/GitlabPulls.pm b/src/lib/Hydra/Plugin/GitlabPulls.pm index de371f9a..b44a1a98 100644 --- a/src/lib/Hydra/Plugin/GitlabPulls.pm +++ b/src/lib/Hydra/Plugin/GitlabPulls.pm @@ -8,8 +8,9 @@ # } # where 42 is the project id of a repository. # -# The values source_repo_url and source_branch can then be used to -# build the git input value. +# The values `target_repo_url` and `iid` can then be used to +# build the git input value, e.g.: +# "${target_repo_url} merge-requests/${iid}/head". package Hydra::Plugin::GitlabPulls; @@ -37,21 +38,12 @@ sub _query { return (decode_json $content, $res); } -# We need to query the Gitlab API for each merge request to get the -# source repository URL. -sub _enhanceGitlabPull { - my ($pull, $baseUrl, $ua) = @_; - my $projectId = $pull->{source_project_id}; - (my $repo, my $res) = _query("$baseUrl/api/v4/projects/$projectId", $ua); - $pull->{source_repo_url} = $repo->{http_url_to_repo}; -} - sub _iterate { - my ($url, $baseUrl, $pulls, $ua) = @_; + my ($url, $baseUrl, $pulls, $ua, $target_repo_url) = @_; my ($pulls_list, $res) = _query($url, $ua); foreach my $pull (@$pulls_list) { - _enhanceGitlabPull($pull, $baseUrl, $ua); + $pull->{target_repo_url} = $target_repo_url; $pulls->{$pull->{iid}} = $pull; } # TODO Make Link header parsing more robust!!! @@ -64,7 +56,7 @@ sub _iterate { last; } } - _iterate($next, $baseUrl, $pulls, $ua) unless $next eq ""; + _iterate($next, $baseUrl, $pulls, $ua, $target_repo_url) unless $next eq ""; } sub fetchInput { @@ -74,10 +66,19 @@ sub fetchInput { (my $baseUrl, my $projectId) = split ' ', $value; my $url = "$baseUrl/api/v4/projects/$projectId/merge_requests?per_page=100&state=opened"; + my $accessToken = $self->{config}->{gitlab_authorization}->{$projectId}; + my %pulls; my $ua = LWP::UserAgent->new(); - _iterate($url, $baseUrl, \%pulls, $ua); - + $ua->default_header('Private-Token' => $accessToken) if defined $accessToken; + + # Get the target project URL, as it is the one we need to build the pull + # urls from later + (my $repo, my $res) = _query("$baseUrl/api/v4/projects/$projectId", $ua); + my $target_repo_url = $repo->{http_url_to_repo}; + + _iterate($url, $baseUrl, \%pulls, $ua, $target_repo_url); + my $tempdir = File::Temp->newdir("gitlab-pulls" . "XXXXX", TMPDIR => 1); my $filename = "$tempdir/gitlab-pulls.json"; open(my $fh, ">", $filename) or die "Cannot open $filename for writing: $!";