From 1f49a5af66316ebc9e6bb4f34555ba8bd7799509 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Sun, 9 Jan 2022 11:51:49 -0500 Subject: [PATCH 1/6] Fixup project name, jobset name, in buildToHash --- src/lib/Hydra/Controller/API.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/Hydra/Controller/API.pm b/src/lib/Hydra/Controller/API.pm index 76c54bdf..b816ed53 100644 --- a/src/lib/Hydra/Controller/API.pm +++ b/src/lib/Hydra/Controller/API.pm @@ -24,8 +24,8 @@ sub buildToHash { my ($build) = @_; my $result = { id => $build->id, - project => $build->get_column("project"), - jobset => $build->get_column("jobset"), + project => $build->jobset->get_column("project"), + jobset => $build->jobset->get_column("name"), job => $build->get_column("job"), system => $build->system, nixname => $build->nixname, From e5c8a3542341edda3ad32e10680a6d3b4cf55d05 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Fri, 14 Jan 2022 13:33:44 -0500 Subject: [PATCH 2/6] API: test fetching the queue and latestbuilds --- t/Hydra/Controller/API/checks.t | 80 +++++++++++++++++++++++++++++++++ t/jobs/one-job.nix | 8 ++++ 2 files changed, 88 insertions(+) create mode 100644 t/Hydra/Controller/API/checks.t create mode 100644 t/jobs/one-job.nix diff --git a/t/Hydra/Controller/API/checks.t b/t/Hydra/Controller/API/checks.t new file mode 100644 index 00000000..d47f0947 --- /dev/null +++ b/t/Hydra/Controller/API/checks.t @@ -0,0 +1,80 @@ +use strict; +use warnings; +use Setup; +use Test2::V0; +use Catalyst::Test (); +use HTTP::Request::Common; +use JSON::MaybeXS qw(decode_json); + +sub is_json { + my ($response, $message) = @_; + + my $data; + my $valid_json = lives { $data = decode_json($response->content); }; + ok($valid_json, $message // "We get back valid JSON."); + if (!$valid_json) { + use Data::Dumper; + print STDERR Dumper $response->content; + } + + return $data; +} + +my $ctx = test_context(); + +Catalyst::Test->import('Hydra'); + +my $finishedBuilds = $ctx->makeAndEvaluateJobset( + expression => "one-job.nix", + build => 1 +); + +my $queuedBuilds = $ctx->makeAndEvaluateJobset( + expression => "one-job.nix", + build => 0 +); + +subtest "/api/queue" => sub { + my $response = request(GET '/api/queue?nr=1'); + ok($response->is_success, "The API enpdoint showing the queue returns 200."); + + my $data = is_json($response); + my $build = $queuedBuilds->{"one_job"}; + like($data, [{ + priority => $build->priority, + id => $build->id, + }]); +}; + +subtest "/api/latestbuilds" => sub { + subtest "with no specific parameters" => sub { + my $response = request(GET '/api/latestbuilds?nr=1'); + ok($response->is_success, "The API enpdoint showing the latest builds returns 200."); + + my $data = is_json($response); + my $build = $finishedBuilds->{"one_job"}; + like($data, [{ + buildstatus => $build->buildstatus, + id => $build->id, + }]); + }; + + subtest "with very specific parameters" => sub { + my $build = $finishedBuilds->{"one_job"}; + my $projectName = $build->project->name; + my $jobsetName = $build->jobset->name; + my $jobName = $build->job; + my $system = $build->system; + my $response = request(GET "/api/latestbuilds?nr=1&project=$projectName&jobset=$jobsetName&job=$jobName&system=$system"); + ok($response->is_success, "The API enpdoint showing the latest builds returns 200."); + + my $data = is_json($response); + + like($data, [{ + buildstatus => $build->buildstatus, + id => $build->id, + }]); + }; +}; + +done_testing; diff --git a/t/jobs/one-job.nix b/t/jobs/one-job.nix new file mode 100644 index 00000000..92f4f98b --- /dev/null +++ b/t/jobs/one-job.nix @@ -0,0 +1,8 @@ +with import ./config.nix; +{ + one_job = + mkDerivation { + name = "empty-dir"; + builder = ./empty-dir-builder.sh; + }; +} From 86473f4b3cfe2be075210dea9dcb3fe3b89d3f1f Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Fri, 14 Jan 2022 13:34:00 -0500 Subject: [PATCH 3/6] API: fixup filtering latestbuilds by project and jobset --- src/lib/Hydra/Controller/API.pm | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/lib/Hydra/Controller/API.pm b/src/lib/Hydra/Controller/API.pm index b816ed53..0a8cfd65 100644 --- a/src/lib/Hydra/Controller/API.pm +++ b/src/lib/Hydra/Controller/API.pm @@ -54,12 +54,18 @@ sub latestbuilds : Chained('api') PathPart('latestbuilds') Args(0) { my $system = $c->request->params->{system}; my $filter = {finished => 1}; - $filter->{project} = $project if ! $project eq ""; - $filter->{jobset} = $jobset if ! $jobset eq ""; + $filter->{"jobset.project"} = $project if ! $project eq ""; + $filter->{"jobset.name"} = $jobset if ! $jobset eq ""; $filter->{job} = $job if !$job eq ""; $filter->{system} = $system if !$system eq ""; - my @latest = $c->model('DB::Builds')->search($filter, {rows => $nr, order_by => ["id DESC"] }); + my @latest = $c->model('DB::Builds')->search( + $filter, + { + rows => $nr, + order_by => ["id DESC"], + join => [ "jobset" ] + }); my @list; push @list, buildToHash($_) foreach @latest; From fe095a56c54623a4bafb2c589e897c5953c04018 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Fri, 14 Jan 2022 13:42:00 -0500 Subject: [PATCH 4/6] API: test /nrbuilds and fix jobset / project references --- src/lib/Hydra/Controller/API.pm | 16 +++++++++++++--- t/Hydra/Controller/API/checks.t | 23 +++++++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/src/lib/Hydra/Controller/API.pm b/src/lib/Hydra/Controller/API.pm index 0a8cfd65..3ddee97a 100644 --- a/src/lib/Hydra/Controller/API.pm +++ b/src/lib/Hydra/Controller/API.pm @@ -160,15 +160,25 @@ sub nrbuilds : Chained('api') PathPart('nrbuilds') Args(0) { my $system = $c->request->params->{system}; my $filter = {finished => 1}; - $filter->{project} = $project if ! $project eq ""; - $filter->{jobset} = $jobset if ! $jobset eq ""; + $filter->{"jobset.project"} = $project if ! $project eq ""; + $filter->{"jobset.name"} = $jobset if ! $jobset eq ""; $filter->{job} = $job if !$job eq ""; $filter->{system} = $system if !$system eq ""; $base = 60*60 if($period eq "hour"); $base = 24*60*60 if($period eq "day"); - my @stats = $c->model('DB::Builds')->search($filter, {select => [{ count => "*" }], as => ["nr"], group_by => ["timestamp - timestamp % $base"], order_by => "timestamp - timestamp % $base DESC", rows => $nr}); + my @stats = $c->model('DB::Builds')->search( + $filter, + { + select => [{ count => "*" }], + as => ["nr"], + group_by => ["timestamp - timestamp % $base"], + order_by => "timestamp - timestamp % $base DESC", + rows => $nr, + join => [ "jobset" ] + } + ); my @arr; push @arr, int($_->get_column("nr")) foreach @stats; @arr = reverse(@arr); diff --git a/t/Hydra/Controller/API/checks.t b/t/Hydra/Controller/API/checks.t index d47f0947..c133f098 100644 --- a/t/Hydra/Controller/API/checks.t +++ b/t/Hydra/Controller/API/checks.t @@ -77,4 +77,27 @@ subtest "/api/latestbuilds" => sub { }; }; +subtest "/api/nrbuilds" => sub { + subtest "with no specific parameters" => sub { + my $response = request(GET '/api/nrbuilds?nr=1&period=hour'); + ok($response->is_success, "The API enpdoint showing the latest builds returns 200."); + + my $data = is_json($response); + is($data, [1]); + }; + + subtest "with very specific parameters" => sub { + my $build = $finishedBuilds->{"one_job"}; + my $projectName = $build->project->name; + my $jobsetName = $build->jobset->name; + my $jobName = $build->job; + my $system = $build->system; + my $response = request(GET "/api/nrbuilds?nr=1&period=hour&project=$projectName&jobset=$jobsetName&job=$jobName&system=$system"); + ok($response->is_success, "The API enpdoint showing the latest builds returns 200."); + + my $data = is_json($response); + is($data, [1]); + }; +}; + done_testing; From 20db82b00177ce534fb4ed840b345be241ae066b Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Fri, 14 Jan 2022 14:04:03 -0500 Subject: [PATCH 5/6] API test /api/push --- src/lib/Hydra/Controller/API.pm | 6 +++-- t/Hydra/Controller/API/checks.t | 41 +++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/src/lib/Hydra/Controller/API.pm b/src/lib/Hydra/Controller/API.pm index 3ddee97a..f4936982 100644 --- a/src/lib/Hydra/Controller/API.pm +++ b/src/lib/Hydra/Controller/API.pm @@ -254,8 +254,10 @@ sub push : Chained('api') PathPart('push') Args(0) { foreach my $r (@repos) { triggerJobset($self, $c, $_, $force) foreach $c->model('DB::Jobsets')->search( { 'project.enabled' => 1, 'me.enabled' => 1 }, - { join => 'project' - , where => \ [ 'exists (select 1 from JobsetInputAlts where project = me.project and jobset = me.name and value = ?)', [ 'value', $r ] ] + { + join => 'project', + where => \ [ 'exists (select 1 from JobsetInputAlts where project = me.project and jobset = me.name and value = ?)', [ 'value', $r ] ], + order_by => 'me.id DESC' }); } diff --git a/t/Hydra/Controller/API/checks.t b/t/Hydra/Controller/API/checks.t index c133f098..4aac3fbf 100644 --- a/t/Hydra/Controller/API/checks.t +++ b/t/Hydra/Controller/API/checks.t @@ -100,4 +100,45 @@ subtest "/api/nrbuilds" => sub { }; }; +subtest "/api/push" => sub { + subtest "with a specific jobset" => sub { + my $build = $finishedBuilds->{"one_job"}; + my $jobset = $build->jobset; + my $projectName = $jobset->project->name; + my $jobsetName = $jobset->name; + is($jobset->forceeval, undef, "The existing jobset is not set to be forced to eval"); + + my $response = request(GET "/api/push?jobsets=$projectName:$jobsetName&force=1"); + ok($response->is_success, "The API enpdoint for triggering jobsets returns 200."); + + my $data = is_json($response); + is($data, { jobsetsTriggered => [ "$projectName:$jobsetName" ] }); + + my $updatedJobset = $ctx->db->resultset('Jobsets')->find({ id => $jobset->id }); + is($updatedJobset->forceeval, 1, "The jobset is now forced to eval"); + }; + + subtest "with a specific source" => sub { + my $repo = $ctx->jobsdir; + my $jobsetA = $queuedBuilds->{"one_job"}->jobset; + my $jobsetB = $finishedBuilds->{"one_job"}->jobset; + + is($jobsetA->forceeval, undef, "The existing jobset is not set to be forced to eval"); + + print STDERR $repo; + + my $response = request(GET "/api/push?repos=$repo&force=1"); + ok($response->is_success, "The API enpdoint for triggering jobsets returns 200."); + + my $data = is_json($response); + is($data, { jobsetsTriggered => [ + "${\$jobsetA->project->name}:${\$jobsetA->name}", + "${\$jobsetB->project->name}:${\$jobsetB->name}" + ] }); + + my $updatedJobset = $ctx->db->resultset('Jobsets')->find({ id => $jobsetA->id }); + is($updatedJobset->forceeval, 1, "The jobset is now forced to eval"); + }; +}; + done_testing; From a81e3580167e73184e5df6a0b7bffe9446a08cbb Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Fri, 14 Jan 2022 14:41:37 -0500 Subject: [PATCH 6/6] API: test api/push-github --- src/lib/Hydra/Controller/API.pm | 1 - t/Hydra/Controller/API/checks.t | 73 ++++++++++++++++++++++++++++++++- 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/src/lib/Hydra/Controller/API.pm b/src/lib/Hydra/Controller/API.pm index f4936982..6f10ef57 100644 --- a/src/lib/Hydra/Controller/API.pm +++ b/src/lib/Hydra/Controller/API.pm @@ -267,7 +267,6 @@ sub push : Chained('api') PathPart('push') Args(0) { ); } - sub push_github : Chained('api') PathPart('push-github') Args(0) { my ($self, $c) = @_; diff --git a/t/Hydra/Controller/API/checks.t b/t/Hydra/Controller/API/checks.t index 4aac3fbf..2b97b489 100644 --- a/t/Hydra/Controller/API/checks.t +++ b/t/Hydra/Controller/API/checks.t @@ -3,8 +3,9 @@ use warnings; use Setup; use Test2::V0; use Catalyst::Test (); +use HTTP::Request; use HTTP::Request::Common; -use JSON::MaybeXS qw(decode_json); +use JSON::MaybeXS qw(decode_json encode_json); sub is_json { my ($response, $message) = @_; @@ -141,4 +142,74 @@ subtest "/api/push" => sub { }; }; +subtest "/api/push-github" => sub { + # Create a project and jobset which looks like it comes from GitHub + my $user = $ctx->db()->resultset('Users')->create({ + username => "api-push-github", + emailaddress => 'api-push-github@example.org', + password => '' + }); + + my $project = $ctx->db()->resultset('Projects')->create({ + name => "api-push-github", + displayname => "api-push-github", + owner => $user->username + }); + + subtest "with a legacy input type" => sub { + my $jobset = $project->jobsets->create({ + name => "legacy-input-type", + nixexprinput => "src", + nixexprpath => "default.nix", + emailoverride => "" + }); + + my $jobsetinput = $jobset->jobsetinputs->create({name => "src", type => "git"}); + $jobsetinput->jobsetinputalts->create({altnr => 0, value => "https://github.com/OWNER/LEGACY-REPO.git"}); + + my $req = POST '/api/push-github', + "Content-Type" => "application/json", + "Content" => encode_json({ + repository => { + owner => { + name => "OWNER", + }, + name => "LEGACY-REPO", + } + }); + + my $response = request($req); + ok($response->is_success, "The API enpdoint for triggering jobsets returns 200."); + + my $data = is_json($response); + is($data, { jobsetsTriggered => [ "api-push-github:legacy-input-type" ] }, "The correct jobsets are triggered."); + }; + + subtest "with a flake input type" => sub { + my $jobset = $project->jobsets->create({ + name => "flake-input-type", + type => 1, + flake => "github:OWNER/FLAKE-REPO", + emailoverride => "" + }); + + my $req = POST '/api/push-github', + "Content-Type" => "application/json", + "Content" => encode_json({ + repository => { + owner => { + name => "OWNER", + }, + name => "FLAKE-REPO", + } + }); + + my $response = request($req); + ok($response->is_success, "The API enpdoint for triggering jobsets returns 200."); + + my $data = is_json($response); + is($data, { jobsetsTriggered => [ "api-push-github:flake-input-type" ] }, "The correct jobsets are triggered."); + }; +}; + done_testing;