diff --git a/doc/manual/src/projects.md b/doc/manual/src/projects.md index 4228c2dc..b69dad85 100644 --- a/doc/manual/src/projects.md +++ b/doc/manual/src/projects.md @@ -468,3 +468,34 @@ notifications, add it to the path option of the Hydra services in your systemd.services.hydra-queue-runner.path = [ pkgs.ssmtp ]; systemd.services.hydra-server.path = [ pkgs.ssmtp ]; +Gitea Integration +----------------- + +Hydra can notify Git servers (such as [GitLab](https://gitlab.com/), [GitHub](https://github.com) +or [Gitea](https://gitea.io/en-us/)) about the result of a build from a Git checkout. + +This section describes how it can be implemented for `gitea`, but the approach for `gitlab` is +analogous: + +* [Obtain an API token for your user](https://docs.gitea.io/en-us/api-usage/#authentication) +* Add it to your `hydra.conf` like this: + ``` nix + { + services.hydra-dev.extraConfig = '' + + your_username=your_token + + ''; + } + ``` + +* For a jobset with a `Git`-input which points to a `gitea`-instance, add the following + additional inputs: + + | Type | Name | Value | + | -------------- | ------------------- | ---------------------------------- | + | `String value` | `gitea_repo_name` | *Name of the repository to build* | + | `String value` | `gitea_repo_owner` | *Owner of the repository* | + | `String value` | `gitea_status_repo` | *Name of the `Git checkout` input* | + | `String value` | `gitea_http_url` | *Public URL of `gitea`*, optional | + diff --git a/flake.nix b/flake.nix index 2e2a6574..6444af6c 100644 --- a/flake.nix +++ b/flake.nix @@ -311,7 +311,7 @@ ]; checkInputs = [ - foreman + foreman python3 ]; hydraPath = lib.makeBinPath ( @@ -494,6 +494,199 @@ ''; }; + tests.gitea.x86_64-linux = + with import (nixpkgs + "/nixos/lib/testing-python.nix") { system = "x86_64-linux"; }; + makeTest { + machine = { pkgs, ... }: { + imports = [ hydraServer ]; + services.hydra-dev.extraConfig = '' + + root=d7f16a3412e01a43a414535b16007c6931d3a9c7 + + ''; + nix = { + distributedBuilds = true; + buildMachines = [{ + hostName = "localhost"; + systems = [ "x86_64-linux" ]; + }]; + binaryCaches = []; + }; + services.gitea = { + enable = true; + database.type = "postgres"; + disableRegistration = true; + httpPort = 3001; + }; + services.openssh.enable = true; + environment.systemPackages = with pkgs; [ gitea git jq gawk ]; + networking.firewall.allowedTCPPorts = [ 3000 ]; + }; + skipLint = true; + testScript = let + scripts.mktoken = pkgs.writeText "token.sql" '' + INSERT INTO access_token (id, uid, name, created_unix, updated_unix, token_hash, token_salt, token_last_eight) VALUES (1, 1, 'hydra', 1617107360, 1617107360, 'a930f319ca362d7b49a4040ac0af74521c3a3c3303a86f327b01994430672d33b6ec53e4ea774253208686c712495e12a486', 'XRjWE9YW0g', '31d3a9c7'); + ''; + + scripts.git-setup = pkgs.writeShellScript "setup.sh" '' + set -x + mkdir -p /tmp/repo $HOME/.ssh + cat ${snakeoilKeypair.privkey} > $HOME/.ssh/privk + chmod 0400 $HOME/.ssh/privk + git -C /tmp/repo init + cp ${smallDrv} /tmp/repo/jobset.nix + git -C /tmp/repo add . + git config --global user.email test@localhost + git config --global user.name test + git -C /tmp/repo commit -m 'Initial import' + git -C /tmp/repo remote add origin gitea@machine:root/repo + GIT_SSH_COMMAND='ssh -i $HOME/.ssh/privk -o StrictHostKeyChecking=no' \ + git -C /tmp/repo push origin master + git -C /tmp/repo log >&2 + ''; + + scripts.hydra-setup = pkgs.writeShellScript "hydra.sh" '' + set -x + su -l hydra -c "hydra-create-user root --email-address \ + 'alice@example.org' --password foobar --role admin" + + URL=http://localhost:3000 + USERNAME="root" + PASSWORD="foobar" + PROJECT_NAME="trivial" + JOBSET_NAME="trivial" + mycurl() { + curl --referer $URL -H "Accept: application/json" \ + -H "Content-Type: application/json" $@ + } + + cat >data.json <data.json <data.json < $out; exit 0"]; + }; + } + ''; + in '' + import json + + machine.start() + machine.wait_for_unit("multi-user.target") + machine.wait_for_open_port(3000) + machine.wait_for_open_port(3001) + + machine.succeed( + "su -l gitea -c 'GITEA_WORK_DIR=/var/lib/gitea gitea admin create-user " + + "--username root --password root --email test@localhost'" + ) + machine.succeed("su -l postgres -c 'psql gitea < ${scripts.mktoken}'") + + machine.succeed( + "curl --fail -X POST http://localhost:3001/api/v1/user/repos " + + "-H 'Accept: application/json' -H 'Content-Type: application/json' " + + f"-H 'Authorization: token ${api_token}'" + + ' -d \'{"auto_init":false, "description":"string", "license":"mit", "name":"repo", "private":false}\''' + ) + + machine.succeed( + "curl --fail -X POST http://localhost:3001/api/v1/user/keys " + + "-H 'Accept: application/json' -H 'Content-Type: application/json' " + + f"-H 'Authorization: token ${api_token}'" + + ' -d \'{"key":"${snakeoilKeypair.pubkey}","read_only":true,"title":"SSH"}\''' + ) + + machine.succeed( + "${scripts.git-setup}" + ) + + machine.succeed( + "${scripts.hydra-setup}" + ) + + machine.wait_until_succeeds( + 'curl -Lf -s http://localhost:3000/build/1 -H "Accept: application/json" ' + + '| jq .buildstatus | xargs test 0 -eq' + ) + + data = machine.succeed( + 'curl -Lf -s "http://localhost:3001/api/v1/repos/root/repo/statuses/$(cd /tmp/repo && git show | head -n1 | awk "{print \\$2}")" ' + + "-H 'Accept: application/json' -H 'Content-Type: application/json' " + + f"-H 'Authorization: token ${api_token}'" + ) + + response = json.loads(data) + + assert len(response) == 2, "Expected exactly two status updates for latest commit!" + assert response[0]['status'] == "success", "Expected latest status to be success!" + assert response[1]['status'] == "pending", "Expected first status to be pending!" + + machine.shutdown() + ''; + }; + tests.ldap.x86_64-linux = with import (nixpkgs + "/nixos/lib/testing-python.nix") { system = "x86_64-linux"; }; makeTest { diff --git a/src/lib/Hydra/Plugin/GiteaStatus.pm b/src/lib/Hydra/Plugin/GiteaStatus.pm new file mode 100644 index 00000000..b8da1d0f --- /dev/null +++ b/src/lib/Hydra/Plugin/GiteaStatus.pm @@ -0,0 +1,98 @@ +package Hydra::Plugin::GiteaStatus; + +use strict; +use parent 'Hydra::Plugin'; + +use HTTP::Request; +use JSON; +use LWP::UserAgent; +use Hydra::Helper::CatalystUtils; +use List::Util qw(max); + +sub isEnabled { + my ($self) = @_; + return defined $self->{config}->{gitea_authorization}; +} + +sub toGiteaState { + # See https://try.gitea.io/api/swagger#/repository/repoCreateStatus + my ($status, $buildStatus) = @_; + if ($status == 0 || $status == 1) { + return "pending"; + } elsif ($buildStatus == 0) { + return "success"; + } elsif ($buildStatus == 3 || $buildStatus == 4 || $buildStatus == 8 || $buildStatus == 10 || $buildStatus == 11) { + return "error"; + } else { + return "failure"; + } +} + +sub common { + my ($self, $build, $dependents, $status) = @_; + my $baseurl = $self->{config}->{'base_uri'} || "http://localhost:3000"; + + # Find matching configs + foreach my $b ($build, @{$dependents}) { + my $jobName = showJobName $b; + my $evals = $build->jobsetevals; + my $ua = LWP::UserAgent->new(); + + # Don't send out "pending/running" status updates if the build is already finished + next if $status < 2 && $b->finished == 1; + + my $state = toGiteaState($status, $b->buildstatus); + my $body = encode_json( + { + state => $state, + target_url => "$baseurl/build/" . $b->id, + description => "Hydra build #" . $b->id . " of $jobName", + context => "Hydra " . $b->get_column('job'), + }); + + while (my $eval = $evals->next) { + my $giteastatusInput = $eval->jobsetevalinputs->find({ name => "gitea_status_repo" }); + next unless defined $giteastatusInput && defined $giteastatusInput->value; + my $i = $eval->jobsetevalinputs->find({ name => $giteastatusInput->value, altnr => 0 }); + next unless defined $i; + my $gitea_url = $eval->jobsetevalinputs->find({ name => "gitea_http_url" }); + + my $repoOwner = $eval->jobsetevalinputs->find({ name => "gitea_repo_owner" })->value; + my $repoName = $eval->jobsetevalinputs->find({ name => "gitea_repo_name" })->value; + my $accessToken = $self->{config}->{gitea_authorization}->{$repoOwner}; + + my $rev = $i->revision; + my $domain = URI->new($i->uri)->host; + my $host; + unless (defined $gitea_url) { + $host = "https://$domain"; + } else { + $host = $gitea_url->value; + } + + my $url = "$host/api/v1/repos/$repoOwner/$repoName/statuses/$rev"; + + print STDERR "GiteaStatus POSTing $state to $url\n"; + my $req = HTTP::Request->new('POST', $url); + $req->header('Content-Type' => 'application/json'); + $req->header('Authorization' => "token $accessToken"); + $req->content($body); + my $res = $ua->request($req); + print STDERR $res->status_line, ": ", $res->decoded_content, "\n" unless $res->is_success; + } + } +} + +sub buildQueued { + common(@_, [], 0); +} + +sub buildStarted { + common(@_, [], 1); +} + +sub buildFinished { + common(@_, 2); +} + +1; diff --git a/t/jobs/server.py b/t/jobs/server.py new file mode 100644 index 00000000..85a5c666 --- /dev/null +++ b/t/jobs/server.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 + +from http.server import BaseHTTPRequestHandler, HTTPServer +from sys import argv + + +def factory(file): + h = handler + h.file = file + return h + + +class handler(BaseHTTPRequestHandler): + def do_POST(self): + self.send_response(200) + self.send_header('Content-type', 'application/json') + with open(self.file, 'w+') as f: + f.write(f"{self.path}\n") + length = int(self.headers.get('content-length', 0)) + body = str(self.rfile.read(length).decode("utf-8")) + + f.write(f"{body}") + self.end_headers() + + message = "{}" + self.wfile.write(bytes(message, "utf8")) + + +if __name__ == '__main__': + try: + assert len(argv) > 1 + with HTTPServer(('localhost', 8282), factory(argv[1])) as server: + server.serve_forever() + except KeyboardInterrupt: + pass diff --git a/t/plugins/gitea.t b/t/plugins/gitea.t new file mode 100644 index 00000000..8180d5b3 --- /dev/null +++ b/t/plugins/gitea.t @@ -0,0 +1,76 @@ +use feature 'unicode_strings'; +use strict; +use warnings; +use JSON; +use Setup; + +my %ctx = test_init( + hydra_config => q| + + root=d7f16a3412e01a43a414535b16007c6931d3a9c7 + +|); + +require Hydra::Schema; +require Hydra::Model::DB; + +use Test2::V0; + +my $db = Hydra::Model::DB->new; +hydra_setup($db); + +my $scratch = "$ctx{tmpdir}/scratch"; +mkdir $scratch; + +my $uri = "file://$scratch/git-repo"; + +my $jobset = createJobsetWithOneInput('gitea', 'git-input.nix', 'src', 'git', $uri, $ctx{jobsdir}); + +sub addStringInput { + my ($jobset, $name, $value) = @_; + my $input = $jobset->jobsetinputs->create({name => $name, type => "string"}); + $input->jobsetinputalts->create({value => $value, altnr => 0}); +} + +addStringInput($jobset, "gitea_repo_owner", "root"); +addStringInput($jobset, "gitea_repo_name", "foo"); +addStringInput($jobset, "gitea_status_repo", "src"); +addStringInput($jobset, "gitea_http_url", "http://localhost:8282/gitea"); + +updateRepository('gitea', "$ctx{testdir}/jobs/git-update.sh", $scratch); + +ok(evalSucceeds($jobset), "Evaluating nix expression"); +is(nrQueuedBuildsForJobset($jobset), 1, "Evaluating jobs/runcommand.nix should result in 1 build1"); + +(my $build) = queuedBuildsForJobset($jobset); +ok(runBuild($build), "Build should succeed with exit code 0"); + +my $filename = $ENV{'HYDRA_DATA'} . "/giteaout.json"; +my $pid; +if (!defined($pid = fork())) { + die "Cannot fork(): $!"; +} elsif ($pid == 0) { + exec("python3 $ctx{jobsdir}/server.py $filename"); +} else { + my $newbuild = $db->resultset('Builds')->find($build->id); + is($newbuild->finished, 1, "Build should be finished."); + is($newbuild->buildstatus, 0, "Build should have buildstatus 0."); + ok(sendNotifications(), "Sent notifications"); + + kill('INT', $pid); +} + +open my $fh, $filename or die ("Can't open(): $!\n"); +my $i = 0; +my $uri = <$fh>; +my $data = <$fh>; + +ok(index($uri, "gitea/api/v1/repos/root/foo/statuses") != -1, "Correct URL"); + +my $json = JSON->new; +my $content; +$content = $json->decode($data); + +is($content->{state}, "success", "Success notification"); + +done_testing;