From 323b556dc8c348f1f9d5bf5b8a35f2617bfe600c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jan 2024 21:03:08 -0500 Subject: [PATCH] Minimal CA support This verison has a worse UI, but also chnages the schema less: One non-null constraint is removed, but no new columns are added. Co-Authored-By: Andrea Ciceri Co-Authored-By: regnat --- doc/manual/src/projects.md | 7 +++ src/lib/Hydra/Controller/Root.pm | 29 +++++++++ src/lib/Hydra/Schema/Result/BuildOutputs.pm | 8 +-- .../Hydra/Schema/Result/BuildStepOutputs.pm | 8 +-- src/sql/hydra.sql | 4 +- t/content-addressed/basic.t | 61 +++++++++++++++++++ .../without-experimental-feature.t | 28 +++++++++ t/jobs/config.nix.in | 5 ++ t/jobs/content-addressed.nix | 35 +++++++++++ t/jobs/dir-with-file-builder.sh | 7 +++ t/queue-runner/notifications.t | 2 +- 11 files changed, 183 insertions(+), 11 deletions(-) create mode 100644 t/content-addressed/basic.t create mode 100644 t/content-addressed/without-experimental-feature.t create mode 100644 t/jobs/content-addressed.nix create mode 100755 t/jobs/dir-with-file-builder.sh diff --git a/doc/manual/src/projects.md b/doc/manual/src/projects.md index a399406d..f7c4975f 100644 --- a/doc/manual/src/projects.md +++ b/doc/manual/src/projects.md @@ -404,3 +404,10 @@ analogous: | `String value` | `gitea_status_repo` | *Name of the `Git checkout` input* | | `String value` | `gitea_http_url` | *Public URL of `gitea`*, optional | +Content-addressed derivations +----------------------------- + +Hydra can to a certain extent use the [`ca-derivations` experimental Nix feature](https://github.com/NixOS/rfcs/pull/62). +To use it, make sure that the Nix version you use is at least as recent as the one used in hydra's flake. + +Be warned that this support is still highly experimental, and anything beyond the basic functionality might be broken at that point. diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm index c6843d29..548cfac3 100644 --- a/src/lib/Hydra/Controller/Root.pm +++ b/src/lib/Hydra/Controller/Root.pm @@ -18,6 +18,8 @@ use Net::Prometheus; use Types::Standard qw/StrMatch/; use constant NARINFO_REGEX => qr{^([a-z0-9]{32})\.narinfo$}; +# e.g.: https://hydra.example.com/realisations/sha256:a62128132508a3a32eef651d6467695944763602f226ac630543e947d9feb140!out.doi +use constant REALISATIONS_REGEX => qr{^(sha256:[a-z0-9]{64}![a-z]+)\.doi$}; # Put this controller at top-level. __PACKAGE__->config->{namespace} = ''; @@ -355,6 +357,33 @@ sub nix_cache_info :Path('nix-cache-info') :Args(0) { } +sub realisations :Path('realisations') :Args(StrMatch[REALISATIONS_REGEX]) { + my ($self, $c, $realisation) = @_; + + if (!isLocalStore) { + notFound($c, "There is no binary cache here."); + } + + else { + my ($rawDrvOutput) = $realisation =~ REALISATIONS_REGEX; + my $rawRealisation = queryRawRealisation($rawDrvOutput); + + if (!$rawRealisation) { + $c->response->status(404); + $c->response->content_type('text/plain'); + $c->stash->{plain}->{data} = "does not exist\n"; + $c->forward('Hydra::View::Plain'); + setCacheHeaders($c, 60 * 60); + return; + } + + $c->response->content_type('text/plain'); + $c->stash->{plain}->{data} = $rawRealisation; + $c->forward('Hydra::View::Plain'); + } +} + + sub narinfo :Path :Args(StrMatch[NARINFO_REGEX]) { my ($self, $c, $narinfo) = @_; diff --git a/src/lib/Hydra/Schema/Result/BuildOutputs.pm b/src/lib/Hydra/Schema/Result/BuildOutputs.pm index 9fc4f7c7..3997b497 100644 --- a/src/lib/Hydra/Schema/Result/BuildOutputs.pm +++ b/src/lib/Hydra/Schema/Result/BuildOutputs.pm @@ -49,7 +49,7 @@ __PACKAGE__->table("buildoutputs"); =head2 path data_type: 'text' - is_nullable: 0 + is_nullable: 1 =cut @@ -59,7 +59,7 @@ __PACKAGE__->add_columns( "name", { data_type => "text", is_nullable => 0 }, "path", - { data_type => "text", is_nullable => 0 }, + { data_type => "text", is_nullable => 1 }, ); =head1 PRIMARY KEY @@ -94,8 +94,8 @@ __PACKAGE__->belongs_to( ); -# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-08-26 12:02:36 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:gU+kZ6A0ISKpaXGRGve8mg +# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-06-30 12:02:32 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Jsabm3YTcI7YvCuNdKP5Ng my %hint = ( columns => [ diff --git a/src/lib/Hydra/Schema/Result/BuildStepOutputs.pm b/src/lib/Hydra/Schema/Result/BuildStepOutputs.pm index 016a35fe..6d997a8c 100644 --- a/src/lib/Hydra/Schema/Result/BuildStepOutputs.pm +++ b/src/lib/Hydra/Schema/Result/BuildStepOutputs.pm @@ -55,7 +55,7 @@ __PACKAGE__->table("buildstepoutputs"); =head2 path data_type: 'text' - is_nullable: 0 + is_nullable: 1 =cut @@ -67,7 +67,7 @@ __PACKAGE__->add_columns( "name", { data_type => "text", is_nullable => 0 }, "path", - { data_type => "text", is_nullable => 0 }, + { data_type => "text", is_nullable => 1 }, ); =head1 PRIMARY KEY @@ -119,8 +119,8 @@ __PACKAGE__->belongs_to( ); -# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-08-26 12:02:36 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:gxp8rOjpRVen4YbIjomHTw +# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-06-30 12:02:32 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Bad70CRTt7zb2GGuRoQ++Q # You can replace this text with custom code or comments, and it will be preserved on regeneration diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index eaae6da3..e9457972 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -247,7 +247,7 @@ create trigger BuildBumped after update on Builds for each row create table BuildOutputs ( build integer not null, name text not null, - path text not null, + path text, primary key (build, name), foreign key (build) references Builds(id) on delete cascade ); @@ -303,7 +303,7 @@ create table BuildStepOutputs ( build integer not null, stepnr integer not null, name text not null, - path text not null, + path text, primary key (build, stepnr, name), foreign key (build) references Builds(id) on delete cascade, foreign key (build, stepnr) references BuildSteps(build, stepnr) on delete cascade diff --git a/t/content-addressed/basic.t b/t/content-addressed/basic.t new file mode 100644 index 00000000..6597e727 --- /dev/null +++ b/t/content-addressed/basic.t @@ -0,0 +1,61 @@ +use feature 'unicode_strings'; +use strict; +use warnings; +use Setup; + +my %ctx = test_init( + nix_config => qq| + experimental-features = ca-derivations + |, +); + +require Hydra::Schema; +require Hydra::Model::DB; + +use JSON::MaybeXS; + +use HTTP::Request::Common; +use Test2::V0; +require Catalyst::Test; +Catalyst::Test->import('Hydra'); + +my $db = Hydra::Model::DB->new; +hydra_setup($db); + +my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"}); + +my $jobset = createBaseJobset("content-addressed", "content-addressed.nix", $ctx{jobsdir}); + +ok(evalSucceeds($jobset), "Evaluating jobs/content-addressed.nix should exit with return code 0"); +is(nrQueuedBuildsForJobset($jobset), 5, "Evaluating jobs/content-addressed.nix should result in 4 builds"); + +for my $build (queuedBuildsForJobset($jobset)) { + ok(runBuild($build), "Build '".$build->job."' from jobs/content-addressed.nix should exit with code 0"); + my $newbuild = $db->resultset('Builds')->find($build->id); + is($newbuild->finished, 1, "Build '".$build->job."' from jobs/content-addressed.nix should be finished."); + my $expected = $build->job eq "fails" ? 1 : $build->job =~ /with_failed/ ? 6 : 0; + is($newbuild->buildstatus, $expected, "Build '".$build->job."' from jobs/content-addressed.nix should have buildstatus $expected."); + + my $response = request("/build/".$build->id); + ok($response->is_success, "The 'build' page for build '".$build->job."' should load properly"); + + if ($newbuild->buildstatus == 0) { + my $buildOutputs = $newbuild->buildoutputs; + for my $output ($newbuild->buildoutputs) { + # XXX: This hardcodes /nix/store/. + # It's fine because in practice the nix store for the tests will be of + # the form `/some/thing/nix/store/`, but it would be cleaner if there + # was a way to query Nix for its store dir? + like( + $output->path, qr|/nix/store/|, + "Output '".$output->name."' of build '".$build->job."' should be a valid store path" + ); + } + } + +} + +isnt(<$ctx{deststoredir}/realisations/*>, "", "The destination store should have the realisations of the built derivations registered"); + +done_testing; + diff --git a/t/content-addressed/without-experimental-feature.t b/t/content-addressed/without-experimental-feature.t new file mode 100644 index 00000000..a37d138e --- /dev/null +++ b/t/content-addressed/without-experimental-feature.t @@ -0,0 +1,28 @@ +use feature 'unicode_strings'; +use strict; +use warnings; +use Setup; + +my %ctx = test_init(); + +require Hydra::Schema; +require Hydra::Model::DB; + +use JSON::MaybeXS; + +use HTTP::Request::Common; +use Test2::V0; +require Catalyst::Test; +Catalyst::Test->import('Hydra'); + +my $db = Hydra::Model::DB->new; +hydra_setup($db); + +my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"}); + +my $jobset = createBaseJobset("content-addressed", "content-addressed.nix", $ctx{jobsdir}); + +ok(evalSucceeds($jobset), "Evaluating jobs/content-addressed.nix without the experimental feature should exit with return code 0"); +is(nrQueuedBuildsForJobset($jobset), 0, "Evaluating jobs/content-addressed.nix without the experimental Nix feature should result in 0 build"); + +done_testing; diff --git a/t/jobs/config.nix.in b/t/jobs/config.nix.in index 51b6c06f..41776341 100644 --- a/t/jobs/config.nix.in +++ b/t/jobs/config.nix.in @@ -6,4 +6,9 @@ rec { system = builtins.currentSystem; PATH = path; } // args); + mkContentAddressedDerivation = args: mkDerivation ({ + __contentAddressed = true; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + } // args); } diff --git a/t/jobs/content-addressed.nix b/t/jobs/content-addressed.nix new file mode 100644 index 00000000..65496df5 --- /dev/null +++ b/t/jobs/content-addressed.nix @@ -0,0 +1,35 @@ +let cfg = import ./config.nix; in +rec { + empty_dir = + cfg.mkContentAddressedDerivation { + name = "empty-dir"; + builder = ./empty-dir-builder.sh; + }; + + fails = + cfg.mkContentAddressedDerivation { + name = "fails"; + builder = ./fail.sh; + }; + + succeed_with_failed = + cfg.mkContentAddressedDerivation { + name = "succeed-with-failed"; + builder = ./succeed-with-failed.sh; + }; + + caDependingOnCA = + cfg.mkContentAddressedDerivation { + name = "ca-depending-on-ca"; + builder = ./dir-with-file-builder.sh; + FOO = empty_dir; + }; + + nonCaDependingOnCA = + cfg.mkDerivation { + name = "non-ca-depending-on-ca"; + builder = ./dir-with-file-builder.sh; + FOO = empty_dir; + }; +} + diff --git a/t/jobs/dir-with-file-builder.sh b/t/jobs/dir-with-file-builder.sh new file mode 100755 index 00000000..e51c6558 --- /dev/null +++ b/t/jobs/dir-with-file-builder.sh @@ -0,0 +1,7 @@ +#! /bin/sh + +# Workaround for https://github.com/NixOS/nix/pull/6051 +echo "some output" + +mkdir $out +echo foo > $out/a-file diff --git a/t/queue-runner/notifications.t b/t/queue-runner/notifications.t index 1966cde1..d0e72409 100644 --- a/t/queue-runner/notifications.t +++ b/t/queue-runner/notifications.t @@ -8,7 +8,7 @@ my $binarycachedir = File::Temp->newdir(); my $ctx = test_context( nix_config => qq| - experimental-features = nix-command + experimental-features = nix-command ca-derivations substituters = file://${binarycachedir}?trusted=1 |, hydra_config => q|