diff --git a/.gitignore b/.gitignore index ab5603e2..e53ae6a8 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ Makefile.in /inst hydra-config.h hydra-config.h.in +result diff --git a/doc/manual/projects.xml b/doc/manual/projects.xml index 44b6238f..02b9811d 100644 --- a/doc/manual/projects.xml +++ b/doc/manual/projects.xml @@ -205,7 +205,7 @@ in shows what a release.nix file for GNU Hello - would you like. GNU Hello is representative of many GNU + would look like. GNU Hello is representative of many GNU and non-GNU free software projects: diff --git a/hydra-module.nix b/hydra-module.nix index ce5dc266..52eccd40 100644 --- a/hydra-module.nix +++ b/hydra-module.nix @@ -273,6 +273,7 @@ in runuser -u ${config.services.postgresql.superUser} -- ${config.services.postgresql.package}/bin/createdb -O hydra hydra touch ${baseDir}/.db-created fi + echo "create extension if not exists pg_trgm" | runuser -u ${config.services.postgresql.superUser} -- ${config.services.postgresql.package}/bin/psql hydra ''} if [ ! -e ${cfg.gcRootsDir} ]; then @@ -415,6 +416,8 @@ in hydra-users hydra-queue-runner hydra hydra-users hydra-www hydra hydra-users root hydra + # The postgres user is used to create the pg_trgm extension for the hydra database + hydra-users postgres postgres ''; services.postgresql.authentication = optionalString haveLocalDB diff --git a/release.nix b/release.nix index c01518b2..4ae73091 100644 --- a/release.nix +++ b/release.nix @@ -28,6 +28,22 @@ let services.postgresql.package = pkgs.postgresql95; environment.systemPackages = [ pkgs.perlPackages.LWP pkgs.perlPackages.JSON ]; + + # The following is to work around the following error from hydra-server: + # [error] Caught exception in engine "Cannot determine local time zone" + time.timeZone = "UTC"; + + nix = { + # The following is to work around: https://github.com/NixOS/hydra/pull/432 + buildMachines = [ + { hostName = "localhost"; + system = "x86_64-linux"; + } + ]; + # Without this nix tries to fetch packages from the default + # cache.nixos.org which is not reachable from this sandboxed NixOS test. + binaryCaches = []; + }; }; # FIXME: use commit date. @@ -216,10 +232,62 @@ rec { $machine->waitForOpenPort("3000"); # Run the API tests. - $machine->mustSucceed("su - hydra -c 'perl ${./tests/api-test.pl}' >&2"); + $machine->mustSucceed("su - hydra -c 'perl -I ${build.${system}.perlDeps}/lib/perl5/site_perl ${./tests/api-test.pl}' >&2"); ''; }); + tests.notifications = genAttrs' (system: + with import (nixpkgs + "/nixos/lib/testing.nix") { inherit system; }; + simpleTest { + machine = { pkgs, ... }: { + imports = [ (hydraServer build.${system}) ]; + services.hydra-dev.extraConfig = '' + + url = http://127.0.0.1:8086 + db = hydra + + ''; + services.influxdb.enable = true; + }; + testScript = '' + $machine->waitForJob("hydra-init"); + + # Create an admin account and some other state. + $machine->succeed + ( "su - hydra -c \"hydra-create-user root --email-address 'alice\@example.org' --password foobar --role admin\"" + , "mkdir /run/jobset" + , "chmod 755 /run/jobset" + , "cp ${./tests/api-test.nix} /run/jobset/default.nix" + , "chmod 644 /run/jobset/default.nix" + , "chown -R hydra /run/jobset" + ); + + # Wait until InfluxDB can receive web requests + $machine->waitForJob("influxdb"); + $machine->waitForOpenPort("8086"); + + # Create an InfluxDB database where hydra will write to + $machine->succeed( + "curl -XPOST 'http://127.0.0.1:8086/query' \\ + --data-urlencode 'q=CREATE DATABASE hydra'"); + + # Wait until hydra-server can receive HTTP requests + $machine->waitForJob("hydra-server"); + $machine->waitForOpenPort("3000"); + + # Setup the project and jobset + $machine->mustSucceed( + "su - hydra -c 'perl -I ${build.${system}.perlDeps}/lib/perl5/site_perl ${./tests/setup-notifications-jobset.pl}' >&2"); + + # Wait until hydra has build the job and + # the InfluxDBNotification plugin uploaded its notification to InfluxDB + $machine->waitUntilSucceeds( + "curl -s -H 'Accept: application/csv' \\ + -G 'http://127.0.0.1:8086/query?db=hydra' \\ + --data-urlencode 'q=SELECT * FROM hydra_build_status' | grep success"); + ''; + }); + /* tests.s3backup = genAttrs' (system: with import (nixpkgs + "/nixos/lib/testing.nix") { inherit system; }; diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm index fcc62138..188d73bd 100644 --- a/src/lib/Hydra/Controller/Root.pm +++ b/src/lib/Hydra/Controller/Root.pm @@ -391,7 +391,12 @@ sub search :Local Args(0) { error($c, "Invalid character in query.") unless $query =~ /^[a-zA-Z0-9_\-\/.]+$/; - $c->stash->{limit} = 500; + my $limit = trim $c->request->params->{"limit"}; + if ($limit eq "") { + $c->stash->{limit} = 500; + } else { + $c->stash->{limit} = $limit; + } $c->stash->{projects} = [ $c->model('DB::Projects')->search( { -and => @@ -422,12 +427,16 @@ sub search :Local Args(0) { # Perform build search in separate queries to prevent seq scan on buildoutputs table. $c->stash->{builds} = [ $c->model('DB::Builds')->search( - { "buildoutputs.path" => trim($query) }, - { order_by => ["id desc"], join => ["buildoutputs"] } ) ]; + { "buildoutputs.path" => { ilike => "%$query%" } }, + { order_by => ["id desc"], join => ["buildoutputs"] + , rows => $c->stash->{limit} + } ) ]; $c->stash->{buildsdrv} = [ $c->model('DB::Builds')->search( - { "drvpath" => trim($query) }, - { order_by => ["id desc"] } ) ]; + { "drvpath" => { ilike => "%$query%" } }, + { order_by => ["id desc"] + , rows => $c->stash->{limit} + } ) ]; $c->stash->{resource} = { projects => $c->stash->{projects}, jobsets => $c->stash->{jobsets}, diff --git a/src/lib/Hydra/Plugin/InfluxDBNotification.pm b/src/lib/Hydra/Plugin/InfluxDBNotification.pm new file mode 100644 index 00000000..bef83d1c --- /dev/null +++ b/src/lib/Hydra/Plugin/InfluxDBNotification.pm @@ -0,0 +1,137 @@ +package Hydra::Plugin::InfluxDBNotification; + +use strict; +use parent 'Hydra::Plugin'; +use HTTP::Request; +# use JSON; +use LWP::UserAgent; +# use Hydra::Helper::CatalystUtils; + +sub toBuildStatusDetailed { + my ($buildStatus) = @_; + if ($buildStatus == 0) { + return "success"; + } + elsif ($buildStatus == 1) { + return "failure"; + } + elsif ($buildStatus == 2) { + return "dependency-failed"; + } + elsif ($buildStatus == 4) { + return "cancelled"; + } + elsif ($buildStatus == 6) { + return "failed-with-output"; + } + elsif ($buildStatus == 7) { + return "timed-out"; + } + elsif ($buildStatus == 9) { + return "unsupported-system"; + } + elsif ($buildStatus == 10) { + return "log-limit-exceeded"; + } + elsif ($buildStatus == 11) { + return "output-limit-exceeded"; + } + elsif ($buildStatus == 12) { + return "non-deterministic-build"; + } + else { + return "aborted"; + } +} + +sub toBuildStatusClass { + my ($buildStatus) = @_; + if ($buildStatus == 0) { + return "success"; + } + elsif ($buildStatus == 3 + || $buildStatus == 4 + || $buildStatus == 8 + || $buildStatus == 10 + || $buildStatus == 11) + { + return "canceled"; + } + else { + return "failed"; + } +} + +# Syntax +# build_status,job=my-job status=failed,result=dependency-failed duration=123i +# | -------------------- -------------- | +# | | | | +# | | | | +# +-----------+--------+-+---------+-+---------+ +# |measurement|,tag_set| |field_set| |timestamp| +# +-----------+--------+-+---------+-+---------+ +sub createLine { + my ($measurement, $tagSet, $fieldSet, $timestamp) = @_; + my @tags = (); + foreach my $tag (sort keys %$tagSet) { + push @tags, "$tag=$tagSet->{$tag}"; + } + my @fields = (); + foreach my $field (sort keys %$fieldSet) { + push @fields, "$field=$fieldSet->{$field}"; + } + my $tags = join(",", @tags); + my $fields = join(",", @fields); + return "$measurement,$tags $fields $timestamp"; +} + +sub buildFinished { + my ($self, $build, $dependents) = @_; + my $influxdb = $self->{config}->{influxdb}; + + # skip if we didn't configure + return unless defined $influxdb; + # skip if we didn't set the URL and the DB + return unless ref $influxdb eq 'HASH' and exists $influxdb->{url} and exists $influxdb->{db}; + + my @lines = (); + foreach my $b ($build, @{$dependents}) { + my $tagSet = { + status => toBuildStatusClass($b->buildstatus), + result => toBuildStatusDetailed($b->buildstatus), + project => $b->project->name, + jobset => $b->jobset->name, + repo => ($b->jobset->name =~ /^(.*)\.pr-/) ? $1 : $b->jobset->name, + job => $b->job->name, + system => $b->system, + cached => $b->iscachedbuild ? "true" : "false", + }; + my $fieldSet = { + # this line is needed to be able to query the statuses + build_status => $b->buildstatus . "i", + build_id => '"' . $b->id . '"', + main_build_id => '"' . $build->id . '"', + duration => ($b->stoptime - $b->starttime) . "i", + queued => ($b->starttime - $b->timestamp > 0 ? $b->starttime - $b->timestamp : 0) . "i", + closure_size => ($b->closuresize // 0) . "i", + size => ($b->size // 0) . "i", + }; + my $line = + createLine("hydra_build_status", $tagSet, $fieldSet, $b->stoptime); + push @lines, $line; + } + + my $payload = join("\n", @lines); + print STDERR "sending InfluxDB measurements to server $influxdb->{url}:\n$payload\n"; + + my $ua = LWP::UserAgent->new(); + my $req = HTTP::Request->new('POST', + "$influxdb->{url}/write?db=$influxdb->{db}&precision=s"); + $req->header('Content-Type' => 'application/x-www-form-urlencoded'); + $req->content($payload); + my $res = $ua->request($req); + print STDERR $res->status_line, ": ", $res->decoded_content, "\n" + unless $res->is_success; +} + +1; diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index f81bbad4..33ecab4d 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -695,3 +695,22 @@ create index IndexBuildsOnKeep on Builds(keep) where keep = 1; create index IndexJobsetEvalsOnJobsetId on JobsetEvals(project, jobset, id desc) where hasNewBuilds = 1; create index IndexBuildsOnNotificationPendingSince on Builds(notificationPendingSince) where notificationPendingSince is not null; + +#ifdef POSTGRESQL +-- The pg_trgm extension has to be created by a superuser. The NixOS +-- module creates this extension in the systemd prestart script. We +-- then ensure the extension has been created before creating the +-- index. If it is not possible to create the extension, a warning +-- message is emitted to inform the user the index creation is skipped +-- (slower complex queries on builds.drvpath). +do $$ +begin + create extension if not exists pg_trgm; + -- Provide an index used by LIKE operator on builds.drvpath (search query) + create index IndexTrgmBuildsOnDrvpath on builds using gin (drvpath gin_trgm_ops); +exception when others then + raise warning 'Can not create extension pg_trgm: %', SQLERRM; + raise warning 'HINT: Temporary provide superuser role to your Hydra Postgresql user and run the script src/sql/upgrade-57.sql'; + raise warning 'The pg_trgm index on builds.drvpath has been skipped (slower complex queries on builds.drvpath)'; +end$$; +#endif diff --git a/src/sql/upgrade-57.sql b/src/sql/upgrade-57.sql index b59ee949..cae873d4 100644 --- a/src/sql/upgrade-57.sql +++ b/src/sql/upgrade-57.sql @@ -1,7 +1,16 @@ -alter table Jobsets alter column nixExprInput drop not null; -alter table Jobsets alter column nixExprPath drop not null; -alter table Jobsets add column type integer default 0; -alter table Jobsets add column flake text; -alter table Jobsets add check ((type = 0) = (nixExprInput is not null and nixExprPath is not null)); -alter table Jobsets add check ((type = 1) = (flake is not null)); -alter table JobsetEvals add column flake text; +-- The pg_trgm extension has to be created by a superuser. The NixOS +-- module creates this extension in the systemd prestart script. We +-- then ensure the extension has been created before creating the +-- index. If it is not possible to create the extension, a warning +-- message is emitted to inform the user the index creation is skipped +-- (slower complex queries on builds.drvpath). +do $$ +begin + create extension if not exists pg_trgm; + -- Provide an index used by LIKE operator on builds.drvpath (search query) + create index IndexTrgmBuildsOnDrvpath on builds using gin (drvpath gin_trgm_ops); +exception when others then + raise warning 'Can not create extension pg_trgm: %', SQLERRM; + raise warning 'HINT: Temporary provide superuser role to your Hydra Postgresql user and run the script src/sql/upgrade-57.sql'; + raise warning 'The pg_trgm index on builds.drvpath has been skipped (slower complex queries on builds.drvpath)'; +end$$; diff --git a/src/sql/upgrade-58.sql b/src/sql/upgrade-58.sql new file mode 100644 index 00000000..b59ee949 --- /dev/null +++ b/src/sql/upgrade-58.sql @@ -0,0 +1,7 @@ +alter table Jobsets alter column nixExprInput drop not null; +alter table Jobsets alter column nixExprPath drop not null; +alter table Jobsets add column type integer default 0; +alter table Jobsets add column flake text; +alter table Jobsets add check ((type = 0) = (nixExprInput is not null and nixExprPath is not null)); +alter table Jobsets add check ((type = 1) = (flake is not null)); +alter table JobsetEvals add column flake text; diff --git a/tests/Makefile.am b/tests/Makefile.am index cff12f09..e56e528e 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -12,6 +12,7 @@ TESTS_ENVIRONMENT = \ NIX_STORE_DIR="$(abs_builddir)/nix/store" \ NIX_LOG_DIR="$(abs_builddir)/nix/var/log/nix" \ NIX_BUILD_HOOK= \ + PGHOST=/tmp \ PERL5LIB="$(srcdir):$(abs_top_srcdir)/src/lib:$$PERL5LIB" \ PATH=$(abs_top_srcdir)/src/hydra-evaluator:$(abs_top_srcdir)/src/script:$(abs_top_srcdir)/src/hydra-eval-jobs:$(abs_top_srcdir)/src/hydra-queue-runner:$$PATH \ perl -w diff --git a/tests/set-up.pl b/tests/set-up.pl index 63679b63..d7aa35cc 100644 --- a/tests/set-up.pl +++ b/tests/set-up.pl @@ -1,5 +1,5 @@ use strict; system("initdb -D postgres") == 0 or die; -system("pg_ctl -D postgres -o \"-F -p 6433 -h ''\" -w start") == 0 or die; +system("pg_ctl -D postgres -o \"-F -p 6433 -h '' -k /tmp \" -w start") == 0 or die; system("createdb -p 6433 hydra-test-suite") == 0 or die; system("hydra-init") == 0 or die; diff --git a/tests/setup-notifications-jobset.pl b/tests/setup-notifications-jobset.pl new file mode 100644 index 00000000..8eef78e6 --- /dev/null +++ b/tests/setup-notifications-jobset.pl @@ -0,0 +1,56 @@ +use LWP::UserAgent; +use JSON; + +my $ua = LWP::UserAgent->new; +$ua->cookie_jar({}); + +sub request_json { + my ($opts) = @_; + my $req = HTTP::Request->new; + $req->method($opts->{method} or "GET"); + $req->uri("http://localhost:3000$opts->{uri}"); + $req->header(Accept => "application/json"); + $req->header(Referer => "http://localhost:3000/") if $opts->{method} eq "POST"; + $req->content(encode_json($opts->{data})) if defined $opts->{data}; + my $res = $ua->request($req); + print $res->as_string(); + return $res; +} + +my $result = request_json({ + uri => "/login", + method => "POST", + data => { + username => "root", + password => "foobar" + } +}); + +$result = request_json({ + uri => '/project/sample', + method => 'PUT', + data => { + displayname => "Sample", + enabled => "1", + visible => "1", + } +}); + +$result = request_json({ + uri => '/jobset/sample/default', + method => 'PUT', + data => { + nixexprpath => "default.nix", + nixexprinput => "my-src", + inputs => { + "my-src" => { + type => "path", + value => "/run/jobset" + } + }, + enabled => "1", + visible => "1", + checkinterval => "5", + keepnr => 1 + } +});