Merge pull request #1349 from NixOS/ca-no-new-col

Allow building content-addressed derivations with hydra, minimally
This commit is contained in:
John Ericson 2024-01-26 17:54:02 -05:00 committed by GitHub
commit 838648c0ce
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 184 additions and 11 deletions

View file

@ -404,3 +404,10 @@ analogous:
| `String value` | `gitea_status_repo` | *Name of the `Git checkout` input* | | `String value` | `gitea_status_repo` | *Name of the `Git checkout` input* |
| `String value` | `gitea_http_url` | *Public URL of `gitea`*, optional | | `String value` | `gitea_http_url` | *Public URL of `gitea`*, optional |
Content-addressed derivations
-----------------------------
Hydra can to a certain extent use the [`ca-derivations` experimental Nix feature](https://github.com/NixOS/rfcs/pull/62).
To use it, make sure that the Nix version you use is at least as recent as the one used in hydra's flake.
Be warned that this support is still highly experimental, and anything beyond the basic functionality might be broken at that point.

View file

@ -18,6 +18,8 @@ use Net::Prometheus;
use Types::Standard qw/StrMatch/; use Types::Standard qw/StrMatch/;
use constant NARINFO_REGEX => qr{^([a-z0-9]{32})\.narinfo$}; use constant NARINFO_REGEX => qr{^([a-z0-9]{32})\.narinfo$};
# e.g.: https://hydra.example.com/realisations/sha256:a62128132508a3a32eef651d6467695944763602f226ac630543e947d9feb140!out.doi
use constant REALISATIONS_REGEX => qr{^(sha256:[a-z0-9]{64}![a-z]+)\.doi$};
# Put this controller at top-level. # Put this controller at top-level.
__PACKAGE__->config->{namespace} = ''; __PACKAGE__->config->{namespace} = '';
@ -355,6 +357,33 @@ sub nix_cache_info :Path('nix-cache-info') :Args(0) {
} }
sub realisations :Path('realisations') :Args(StrMatch[REALISATIONS_REGEX]) {
my ($self, $c, $realisation) = @_;
if (!isLocalStore) {
notFound($c, "There is no binary cache here.");
}
else {
my ($rawDrvOutput) = $realisation =~ REALISATIONS_REGEX;
my $rawRealisation = queryRawRealisation($rawDrvOutput);
if (!$rawRealisation) {
$c->response->status(404);
$c->response->content_type('text/plain');
$c->stash->{plain}->{data} = "does not exist\n";
$c->forward('Hydra::View::Plain');
setCacheHeaders($c, 60 * 60);
return;
}
$c->response->content_type('text/plain');
$c->stash->{plain}->{data} = $rawRealisation;
$c->forward('Hydra::View::Plain');
}
}
sub narinfo :Path :Args(StrMatch[NARINFO_REGEX]) { sub narinfo :Path :Args(StrMatch[NARINFO_REGEX]) {
my ($self, $c, $narinfo) = @_; my ($self, $c, $narinfo) = @_;

View file

@ -49,7 +49,7 @@ __PACKAGE__->table("buildoutputs");
=head2 path =head2 path
data_type: 'text' data_type: 'text'
is_nullable: 0 is_nullable: 1
=cut =cut
@ -59,7 +59,7 @@ __PACKAGE__->add_columns(
"name", "name",
{ data_type => "text", is_nullable => 0 }, { data_type => "text", is_nullable => 0 },
"path", "path",
{ data_type => "text", is_nullable => 0 }, { data_type => "text", is_nullable => 1 },
); );
=head1 PRIMARY KEY =head1 PRIMARY KEY
@ -94,8 +94,8 @@ __PACKAGE__->belongs_to(
); );
# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-08-26 12:02:36 # Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-06-30 12:02:32
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:gU+kZ6A0ISKpaXGRGve8mg # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Jsabm3YTcI7YvCuNdKP5Ng
my %hint = ( my %hint = (
columns => [ columns => [

View file

@ -55,7 +55,7 @@ __PACKAGE__->table("buildstepoutputs");
=head2 path =head2 path
data_type: 'text' data_type: 'text'
is_nullable: 0 is_nullable: 1
=cut =cut
@ -67,7 +67,7 @@ __PACKAGE__->add_columns(
"name", "name",
{ data_type => "text", is_nullable => 0 }, { data_type => "text", is_nullable => 0 },
"path", "path",
{ data_type => "text", is_nullable => 0 }, { data_type => "text", is_nullable => 1 },
); );
=head1 PRIMARY KEY =head1 PRIMARY KEY
@ -119,8 +119,8 @@ __PACKAGE__->belongs_to(
); );
# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-08-26 12:02:36 # Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-06-30 12:02:32
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:gxp8rOjpRVen4YbIjomHTw # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Bad70CRTt7zb2GGuRoQ++Q
# You can replace this text with custom code or comments, and it will be preserved on regeneration # You can replace this text with custom code or comments, and it will be preserved on regeneration

View file

@ -247,7 +247,7 @@ create trigger BuildBumped after update on Builds for each row
create table BuildOutputs ( create table BuildOutputs (
build integer not null, build integer not null,
name text not null, name text not null,
path text not null, path text,
primary key (build, name), primary key (build, name),
foreign key (build) references Builds(id) on delete cascade foreign key (build) references Builds(id) on delete cascade
); );
@ -303,7 +303,7 @@ create table BuildStepOutputs (
build integer not null, build integer not null,
stepnr integer not null, stepnr integer not null,
name text not null, name text not null,
path text not null, path text,
primary key (build, stepnr, name), primary key (build, stepnr, name),
foreign key (build) references Builds(id) on delete cascade, foreign key (build) references Builds(id) on delete cascade,
foreign key (build, stepnr) references BuildSteps(build, stepnr) on delete cascade foreign key (build, stepnr) references BuildSteps(build, stepnr) on delete cascade

4
src/sql/upgrade-84.sql Normal file
View file

@ -0,0 +1,4 @@
-- CA derivations do not have statically known output paths. The values
-- are only filled in after the build runs.
ALTER TABLE BuildStepOutputs ALTER COLUMN path DROP NOT NULL;
ALTER TABLE BuildOutputs ALTER COLUMN path DROP NOT NULL;

View file

@ -0,0 +1,61 @@
use feature 'unicode_strings';
use strict;
use warnings;
use Setup;
my %ctx = test_init(
nix_config => qq|
experimental-features = ca-derivations
|,
);
require Hydra::Schema;
require Hydra::Model::DB;
use JSON::MaybeXS;
use HTTP::Request::Common;
use Test2::V0;
require Catalyst::Test;
Catalyst::Test->import('Hydra');
my $db = Hydra::Model::DB->new;
hydra_setup($db);
my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"});
my $jobset = createBaseJobset("content-addressed", "content-addressed.nix", $ctx{jobsdir});
ok(evalSucceeds($jobset), "Evaluating jobs/content-addressed.nix should exit with return code 0");
is(nrQueuedBuildsForJobset($jobset), 5, "Evaluating jobs/content-addressed.nix should result in 4 builds");
for my $build (queuedBuildsForJobset($jobset)) {
ok(runBuild($build), "Build '".$build->job."' from jobs/content-addressed.nix should exit with code 0");
my $newbuild = $db->resultset('Builds')->find($build->id);
is($newbuild->finished, 1, "Build '".$build->job."' from jobs/content-addressed.nix should be finished.");
my $expected = $build->job eq "fails" ? 1 : $build->job =~ /with_failed/ ? 6 : 0;
is($newbuild->buildstatus, $expected, "Build '".$build->job."' from jobs/content-addressed.nix should have buildstatus $expected.");
my $response = request("/build/".$build->id);
ok($response->is_success, "The 'build' page for build '".$build->job."' should load properly");
if ($newbuild->buildstatus == 0) {
my $buildOutputs = $newbuild->buildoutputs;
for my $output ($newbuild->buildoutputs) {
# XXX: This hardcodes /nix/store/.
# It's fine because in practice the nix store for the tests will be of
# the form `/some/thing/nix/store/`, but it would be cleaner if there
# was a way to query Nix for its store dir?
like(
$output->path, qr|/nix/store/|,
"Output '".$output->name."' of build '".$build->job."' should be a valid store path"
);
}
}
}
isnt(<$ctx{deststoredir}/realisations/*>, "", "The destination store should have the realisations of the built derivations registered");
done_testing;

View file

@ -0,0 +1,28 @@
use feature 'unicode_strings';
use strict;
use warnings;
use Setup;
my %ctx = test_init();
require Hydra::Schema;
require Hydra::Model::DB;
use JSON::MaybeXS;
use HTTP::Request::Common;
use Test2::V0;
require Catalyst::Test;
Catalyst::Test->import('Hydra');
my $db = Hydra::Model::DB->new;
hydra_setup($db);
my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"});
my $jobset = createBaseJobset("content-addressed", "content-addressed.nix", $ctx{jobsdir});
ok(evalSucceeds($jobset), "Evaluating jobs/content-addressed.nix without the experimental feature should exit with return code 0");
is(nrQueuedBuildsForJobset($jobset), 0, "Evaluating jobs/content-addressed.nix without the experimental Nix feature should result in 0 build");
done_testing;

View file

@ -6,4 +6,9 @@ rec {
system = builtins.currentSystem; system = builtins.currentSystem;
PATH = path; PATH = path;
} // args); } // args);
mkContentAddressedDerivation = args: mkDerivation ({
__contentAddressed = true;
outputHashMode = "recursive";
outputHashAlgo = "sha256";
} // args);
} }

View file

@ -0,0 +1,35 @@
let cfg = import ./config.nix; in
rec {
empty_dir =
cfg.mkContentAddressedDerivation {
name = "empty-dir";
builder = ./empty-dir-builder.sh;
};
fails =
cfg.mkContentAddressedDerivation {
name = "fails";
builder = ./fail.sh;
};
succeed_with_failed =
cfg.mkContentAddressedDerivation {
name = "succeed-with-failed";
builder = ./succeed-with-failed.sh;
};
caDependingOnCA =
cfg.mkContentAddressedDerivation {
name = "ca-depending-on-ca";
builder = ./dir-with-file-builder.sh;
FOO = empty_dir;
};
nonCaDependingOnCA =
cfg.mkDerivation {
name = "non-ca-depending-on-ca";
builder = ./dir-with-file-builder.sh;
FOO = empty_dir;
};
}

View file

@ -0,0 +1,4 @@
#! /bin/sh
mkdir $out
echo foo > $out/a-file

View file

@ -8,7 +8,7 @@ my $binarycachedir = File::Temp->newdir();
my $ctx = test_context( my $ctx = test_context(
nix_config => qq| nix_config => qq|
experimental-features = nix-command experimental-features = nix-command ca-derivations
substituters = file://${binarycachedir}?trusted=1 substituters = file://${binarycachedir}?trusted=1
|, |,
hydra_config => q| hydra_config => q|