Merge pull request #904 from Ma27/gitea-integration

Add `GiteaStatus`-Plugin

doc/manual/src/projects.md

@@ -468,3 +468,34 @@ notifications, add it to the path option of the Hydra services in your
     systemd.services.hydra-queue-runner.path = [ pkgs.ssmtp ];
     systemd.services.hydra-server.path = [ pkgs.ssmtp ];
 
+Gitea Integration
+-----------------
+
+Hydra can notify Git servers (such as [GitLab](https://gitlab.com/), [GitHub](https://github.com)
+or [Gitea](https://gitea.io/en-us/)) about the result of a build from a Git checkout.
+
+This section describes how it can be implemented for `gitea`, but the approach for `gitlab` is
+analogous:
+
+* [Obtain an API token for your user](https://docs.gitea.io/en-us/api-usage/#authentication)
+* Add it to your `hydra.conf` like this:
+  ``` nix
+  {
+    services.hydra-dev.extraConfig = ''
+      <gitea_authorization>
+      your_username=your_token
+      </gitea_authorization>
+    '';
+  }
+  ```
+
+* For a jobset with a `Git`-input which points to a `gitea`-instance, add the following
+  additional inputs:
+
+  | Type           | Name                | Value                              |
+  | -------------- | ------------------- | ---------------------------------- |
+  | `String value` | `gitea_repo_name`   | *Name of the repository to build*  |
+  | `String value` | `gitea_repo_owner`  | *Owner of the repository*          |
+  | `String value` | `gitea_status_repo` | *Name of the `Git checkout` input* |
+  | `String value` | `gitea_http_url`    | *Public URL of `gitea`*, optional  |
+

flake.nix

@@ -311,7 +311,7 @@
             ];
 
           checkInputs = [
-            foreman
+            foreman python3
           ];
 
           hydraPath = lib.makeBinPath (
@@ -494,6 +494,199 @@
             '';
         };
 
+        tests.gitea.x86_64-linux =
+          with import (nixpkgs + "/nixos/lib/testing-python.nix") { system = "x86_64-linux"; };
+          makeTest {
+            machine = { pkgs, ... }: {
+              imports = [ hydraServer ];
+              services.hydra-dev.extraConfig = ''
+                <gitea_authorization>
+                root=d7f16a3412e01a43a414535b16007c6931d3a9c7
+                </gitea_authorization>
+              '';
+              nix = {
+                distributedBuilds = true;
+                buildMachines = [{
+                  hostName = "localhost";
+                  systems = [ "x86_64-linux" ];
+                }];
+                binaryCaches = [];
+              };
+              services.gitea = {
+                enable = true;
+                database.type = "postgres";
+                disableRegistration = true;
+                httpPort = 3001;
+              };
+              services.openssh.enable = true;
+              environment.systemPackages = with pkgs; [ gitea git jq gawk ];
+              networking.firewall.allowedTCPPorts = [ 3000 ];
+            };
+            skipLint = true;
+            testScript = let
+              scripts.mktoken = pkgs.writeText "token.sql" ''
+                INSERT INTO access_token (id, uid, name, created_unix, updated_unix, token_hash, token_salt, token_last_eight) VALUES (1, 1, 'hydra', 1617107360, 1617107360, 'a930f319ca362d7b49a4040ac0af74521c3a3c3303a86f327b01994430672d33b6ec53e4ea774253208686c712495e12a486', 'XRjWE9YW0g', '31d3a9c7');
+              '';
+
+              scripts.git-setup = pkgs.writeShellScript "setup.sh" ''
+                set -x
+                mkdir -p /tmp/repo $HOME/.ssh
+                cat ${snakeoilKeypair.privkey} > $HOME/.ssh/privk
+                chmod 0400 $HOME/.ssh/privk
+                git -C /tmp/repo init
+                cp ${smallDrv} /tmp/repo/jobset.nix
+                git -C /tmp/repo add .
+                git config --global user.email test@localhost
+                git config --global user.name test
+                git -C /tmp/repo commit -m 'Initial import'
+                git -C /tmp/repo remote add origin gitea@machine:root/repo
+                GIT_SSH_COMMAND='ssh -i $HOME/.ssh/privk -o StrictHostKeyChecking=no' \
+                  git -C /tmp/repo push origin master
+                git -C /tmp/repo log >&2
+              '';
+
+              scripts.hydra-setup = pkgs.writeShellScript "hydra.sh" ''
+                set -x
+                su -l hydra -c "hydra-create-user root --email-address \
+                  'alice@example.org' --password foobar --role admin"
+
+                URL=http://localhost:3000
+                USERNAME="root"
+                PASSWORD="foobar"
+                PROJECT_NAME="trivial"
+                JOBSET_NAME="trivial"
+                mycurl() {
+                  curl --referer $URL -H "Accept: application/json" \
+                    -H "Content-Type: application/json" $@
+                }
+
+                cat >data.json <<EOF
+                { "username": "$USERNAME", "password": "$PASSWORD" }
+                EOF
+                mycurl -X POST -d '@data.json' $URL/login -c hydra-cookie.txt
+
+                cat >data.json <<EOF
+                {
+                  "displayname":"Trivial",
+                  "enabled":"1",
+                  "visible":"1"
+                }
+                EOF
+                mycurl --silent -X PUT $URL/project/$PROJECT_NAME \
+                  -d @data.json -b hydra-cookie.txt
+
+                cat >data.json <<EOF
+                {
+                  "description": "Trivial",
+                  "checkinterval": "60",
+                  "enabled": "1",
+                  "visible": "1",
+                  "keepnr": "1",
+                  "enableemail": true,
+                  "emailoverride": "hydra@localhost",
+                  "type": 0,
+                  "nixexprinput": "git",
+                  "nixexprpath": "jobset.nix",
+                  "inputs": {
+                    "git": {"value": "http://localhost:3001/root/repo.git", "type": "git"},
+                    "gitea_repo_name": {"value": "repo", "type": "string"},
+                    "gitea_repo_owner": {"value": "root", "type": "string"},
+                    "gitea_status_repo": {"value": "git", "type": "string"},
+                    "gitea_http_url": {"value": "http://localhost:3001", "type": "string"}
+                  }
+                }
+                EOF
+
+                mycurl --silent -X PUT $URL/jobset/$PROJECT_NAME/$JOBSET_NAME \
+                  -d @data.json -b hydra-cookie.txt
+              '';
+
+              api_token = "d7f16a3412e01a43a414535b16007c6931d3a9c7";
+
+              snakeoilKeypair = {
+                privkey = pkgs.writeText "privkey.snakeoil" ''
+                  -----BEGIN EC PRIVATE KEY-----
+                  MHcCAQEEIHQf/khLvYrQ8IOika5yqtWvI0oquHlpRLTZiJy5dRJmoAoGCCqGSM49
+                  AwEHoUQDQgAEKF0DYGbBwbj06tA3fd/+yP44cvmwmHBWXZCKbS+RQlAKvLXMWkpN
+                  r1lwMyJZoSGgBHoUahoYjTh9/sJL7XLJtA==
+                  -----END EC PRIVATE KEY-----
+                '';
+
+                pubkey = pkgs.lib.concatStrings [
+                  "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHA"
+                  "yNTYAAABBBChdA2BmwcG49OrQN33f/sj+OHL5sJhwVl2Qim0vkUJQCry1zFpKTa"
+                  "9ZcDMiWaEhoAR6FGoaGI04ff7CS+1yybQ= sakeoil"
+                ];
+              };
+
+              smallDrv = pkgs.writeText "jobset.nix" ''
+                { trivial = builtins.derivation {
+                    name = "trivial";
+                    system = "x86_64-linux";
+                    builder = "/bin/sh";
+                    allowSubstitutes = false;
+                    preferLocalBuild = true;
+                    args = ["-c" "echo success > $out; exit 0"];
+                  };
+                 }
+              '';
+            in ''
+              import json
+
+              machine.start()
+              machine.wait_for_unit("multi-user.target")
+              machine.wait_for_open_port(3000)
+              machine.wait_for_open_port(3001)
+
+              machine.succeed(
+                  "su -l gitea -c 'GITEA_WORK_DIR=/var/lib/gitea gitea admin create-user "
+                  + "--username root --password root --email test@localhost'"
+              )
+              machine.succeed("su -l postgres -c 'psql gitea < ${scripts.mktoken}'")
+
+              machine.succeed(
+                  "curl --fail -X POST http://localhost:3001/api/v1/user/repos "
+                  + "-H 'Accept: application/json' -H 'Content-Type: application/json' "
+                  + f"-H 'Authorization: token ${api_token}'"
+                  + ' -d \'{"auto_init":false, "description":"string", "license":"mit", "name":"repo", "private":false}\'''
+              )
+
+              machine.succeed(
+                  "curl --fail -X POST http://localhost:3001/api/v1/user/keys "
+                  + "-H 'Accept: application/json' -H 'Content-Type: application/json' "
+                  + f"-H 'Authorization: token ${api_token}'"
+                  + ' -d \'{"key":"${snakeoilKeypair.pubkey}","read_only":true,"title":"SSH"}\'''
+              )
+
+              machine.succeed(
+                  "${scripts.git-setup}"
+              )
+
+              machine.succeed(
+                  "${scripts.hydra-setup}"
+              )
+
+              machine.wait_until_succeeds(
+                  'curl -Lf -s http://localhost:3000/build/1 -H "Accept: application/json" '
+                  + '|  jq .buildstatus | xargs test 0 -eq'
+              )
+
+              data = machine.succeed(
+                  'curl -Lf -s "http://localhost:3001/api/v1/repos/root/repo/statuses/$(cd /tmp/repo && git show | head -n1 | awk "{print \\$2}")" '
+                  + "-H 'Accept: application/json' -H 'Content-Type: application/json' "
+                  + f"-H 'Authorization: token ${api_token}'"
+              )
+
+              response = json.loads(data)
+
+              assert len(response) == 2, "Expected exactly two status updates for latest commit!"
+              assert response[0]['status'] == "success", "Expected latest status to be success!"
+              assert response[1]['status'] == "pending", "Expected first status to be pending!"
+
+              machine.shutdown()
+            '';
+          };
+
         tests.ldap.x86_64-linux =
           with import (nixpkgs + "/nixos/lib/testing-python.nix") { system = "x86_64-linux"; };
           makeTest {

src/lib/Hydra/Plugin/GiteaStatus.pm

@@ -0,0 +1,98 @@
+package Hydra::Plugin::GiteaStatus;
+
+use strict;
+use parent 'Hydra::Plugin';
+
+use HTTP::Request;
+use JSON;
+use LWP::UserAgent;
+use Hydra::Helper::CatalystUtils;
+use List::Util qw(max);
+
+sub isEnabled {
+    my ($self) = @_;
+    return defined $self->{config}->{gitea_authorization};
+}
+
+sub toGiteaState {
+    # See https://try.gitea.io/api/swagger#/repository/repoCreateStatus
+    my ($status, $buildStatus) = @_;
+    if ($status == 0 || $status == 1) {
+        return "pending";
+    } elsif ($buildStatus == 0) {
+        return "success";
+    } elsif ($buildStatus == 3 || $buildStatus == 4 || $buildStatus == 8 || $buildStatus == 10 || $buildStatus == 11) {
+        return "error";
+    } else {
+        return "failure";
+    }
+}
+
+sub common {
+    my ($self, $build, $dependents, $status) = @_;
+    my $baseurl = $self->{config}->{'base_uri'} || "http://localhost:3000";
+
+    # Find matching configs
+    foreach my $b ($build, @{$dependents}) {
+        my $jobName = showJobName $b;
+        my $evals = $build->jobsetevals;
+        my $ua = LWP::UserAgent->new();
+
+        # Don't send out "pending/running" status updates if the build is already finished
+        next if $status < 2 && $b->finished == 1;
+
+        my $state = toGiteaState($status, $b->buildstatus);
+        my $body = encode_json(
+            {
+                state => $state,
+                target_url => "$baseurl/build/" . $b->id,
+                description => "Hydra build #" . $b->id . " of $jobName",
+                context => "Hydra " . $b->get_column('job'),
+            });
+
+        while (my $eval = $evals->next) {
+            my $giteastatusInput = $eval->jobsetevalinputs->find({ name => "gitea_status_repo" });
+            next unless defined $giteastatusInput && defined $giteastatusInput->value;
+            my $i = $eval->jobsetevalinputs->find({ name => $giteastatusInput->value, altnr => 0 });
+            next unless defined $i;
+            my $gitea_url = $eval->jobsetevalinputs->find({ name => "gitea_http_url" });
+
+            my $repoOwner = $eval->jobsetevalinputs->find({ name => "gitea_repo_owner" })->value;
+            my $repoName = $eval->jobsetevalinputs->find({ name => "gitea_repo_name" })->value;
+            my $accessToken = $self->{config}->{gitea_authorization}->{$repoOwner};
+
+            my $rev = $i->revision;
+            my $domain = URI->new($i->uri)->host;
+            my $host;
+            unless (defined $gitea_url) {
+                $host = "https://$domain";
+            } else {
+                $host = $gitea_url->value;
+            }
+
+            my $url = "$host/api/v1/repos/$repoOwner/$repoName/statuses/$rev";
+
+            print STDERR "GiteaStatus POSTing $state to $url\n";
+            my $req = HTTP::Request->new('POST', $url);
+            $req->header('Content-Type' => 'application/json');
+            $req->header('Authorization' => "token $accessToken");
+            $req->content($body);
+            my $res = $ua->request($req);
+            print STDERR $res->status_line, ": ", $res->decoded_content, "\n" unless $res->is_success;
+        }
+    }
+}
+
+sub buildQueued {
+    common(@_, [], 0);
+}
+
+sub buildStarted {
+    common(@_, [], 1);
+}
+
+sub buildFinished {
+    common(@_, 2);
+}
+
+1;

t/jobs/server.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from sys import argv
+
+
+def factory(file):
+    h = handler
+    h.file = file
+    return h
+
+
+class handler(BaseHTTPRequestHandler):
+    def do_POST(self):
+        self.send_response(200)
+        self.send_header('Content-type', 'application/json')
+        with open(self.file, 'w+') as f:
+            f.write(f"{self.path}\n")
+            length = int(self.headers.get('content-length', 0))
+            body = str(self.rfile.read(length).decode("utf-8"))
+
+            f.write(f"{body}")
+        self.end_headers()
+
+        message = "{}"
+        self.wfile.write(bytes(message, "utf8"))
+
+
+if __name__ == '__main__':
+    try:
+        assert len(argv) > 1
+        with HTTPServer(('localhost', 8282), factory(argv[1])) as server:
+            server.serve_forever()
+    except KeyboardInterrupt:
+        pass

t/plugins/gitea.t

@@ -0,0 +1,76 @@
+use feature 'unicode_strings';
+use strict;
+use warnings;
+use JSON;
+use Setup;
+
+my %ctx = test_init(
+    hydra_config => q|
+    <gitea_authorization>
+    root=d7f16a3412e01a43a414535b16007c6931d3a9c7
+    </gitea_authorization>
+|);
+
+require Hydra::Schema;
+require Hydra::Model::DB;
+
+use Test2::V0;
+
+my $db = Hydra::Model::DB->new;
+hydra_setup($db);
+
+my $scratch = "$ctx{tmpdir}/scratch";
+mkdir $scratch;
+
+my $uri = "file://$scratch/git-repo";
+
+my $jobset = createJobsetWithOneInput('gitea', 'git-input.nix', 'src', 'git', $uri, $ctx{jobsdir});
+
+sub addStringInput {
+    my ($jobset, $name, $value) = @_;
+    my $input = $jobset->jobsetinputs->create({name => $name, type => "string"});
+    $input->jobsetinputalts->create({value => $value, altnr => 0});
+}
+
+addStringInput($jobset, "gitea_repo_owner", "root");
+addStringInput($jobset, "gitea_repo_name", "foo");
+addStringInput($jobset, "gitea_status_repo", "src");
+addStringInput($jobset, "gitea_http_url", "http://localhost:8282/gitea");
+
+updateRepository('gitea', "$ctx{testdir}/jobs/git-update.sh", $scratch);
+
+ok(evalSucceeds($jobset), "Evaluating nix expression");
+is(nrQueuedBuildsForJobset($jobset), 1, "Evaluating jobs/runcommand.nix should result in 1 build1");
+ 
+(my $build) = queuedBuildsForJobset($jobset);
+ok(runBuild($build), "Build should succeed with exit code 0");
+
+my $filename = $ENV{'HYDRA_DATA'} . "/giteaout.json";
+my $pid;
+if (!defined($pid = fork())) {
+    die "Cannot fork(): $!";
+} elsif ($pid == 0) {
+    exec("python3 $ctx{jobsdir}/server.py $filename");
+} else {
+    my $newbuild = $db->resultset('Builds')->find($build->id);
+    is($newbuild->finished, 1, "Build should be finished.");
+    is($newbuild->buildstatus, 0, "Build should have buildstatus 0.");
+    ok(sendNotifications(), "Sent notifications");
+
+    kill('INT', $pid);
+}
+
+open my $fh, $filename or die ("Can't open(): $!\n");
+my $i = 0;
+my $uri = <$fh>;
+my $data = <$fh>;
+
+ok(index($uri, "gitea/api/v1/repos/root/foo/statuses") != -1, "Correct URL");
+
+my $json = JSON->new;
+my $content;
+$content = $json->decode($data);
+
+is($content->{state}, "success", "Success notification");
+
+done_testing;