forked from lix-project/hydra
Merge pull request #1349 from NixOS/ca-no-new-col
Allow building content-addressed derivations with hydra, minimally
This commit is contained in:
commit
838648c0ce
|
@ -404,3 +404,10 @@ analogous:
|
||||||
| `String value` | `gitea_status_repo` | *Name of the `Git checkout` input* |
|
| `String value` | `gitea_status_repo` | *Name of the `Git checkout` input* |
|
||||||
| `String value` | `gitea_http_url` | *Public URL of `gitea`*, optional |
|
| `String value` | `gitea_http_url` | *Public URL of `gitea`*, optional |
|
||||||
|
|
||||||
|
Content-addressed derivations
|
||||||
|
-----------------------------
|
||||||
|
|
||||||
|
Hydra can to a certain extent use the [`ca-derivations` experimental Nix feature](https://github.com/NixOS/rfcs/pull/62).
|
||||||
|
To use it, make sure that the Nix version you use is at least as recent as the one used in hydra's flake.
|
||||||
|
|
||||||
|
Be warned that this support is still highly experimental, and anything beyond the basic functionality might be broken at that point.
|
||||||
|
|
|
@ -18,6 +18,8 @@ use Net::Prometheus;
|
||||||
use Types::Standard qw/StrMatch/;
|
use Types::Standard qw/StrMatch/;
|
||||||
|
|
||||||
use constant NARINFO_REGEX => qr{^([a-z0-9]{32})\.narinfo$};
|
use constant NARINFO_REGEX => qr{^([a-z0-9]{32})\.narinfo$};
|
||||||
|
# e.g.: https://hydra.example.com/realisations/sha256:a62128132508a3a32eef651d6467695944763602f226ac630543e947d9feb140!out.doi
|
||||||
|
use constant REALISATIONS_REGEX => qr{^(sha256:[a-z0-9]{64}![a-z]+)\.doi$};
|
||||||
|
|
||||||
# Put this controller at top-level.
|
# Put this controller at top-level.
|
||||||
__PACKAGE__->config->{namespace} = '';
|
__PACKAGE__->config->{namespace} = '';
|
||||||
|
@ -355,6 +357,33 @@ sub nix_cache_info :Path('nix-cache-info') :Args(0) {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
sub realisations :Path('realisations') :Args(StrMatch[REALISATIONS_REGEX]) {
|
||||||
|
my ($self, $c, $realisation) = @_;
|
||||||
|
|
||||||
|
if (!isLocalStore) {
|
||||||
|
notFound($c, "There is no binary cache here.");
|
||||||
|
}
|
||||||
|
|
||||||
|
else {
|
||||||
|
my ($rawDrvOutput) = $realisation =~ REALISATIONS_REGEX;
|
||||||
|
my $rawRealisation = queryRawRealisation($rawDrvOutput);
|
||||||
|
|
||||||
|
if (!$rawRealisation) {
|
||||||
|
$c->response->status(404);
|
||||||
|
$c->response->content_type('text/plain');
|
||||||
|
$c->stash->{plain}->{data} = "does not exist\n";
|
||||||
|
$c->forward('Hydra::View::Plain');
|
||||||
|
setCacheHeaders($c, 60 * 60);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
$c->response->content_type('text/plain');
|
||||||
|
$c->stash->{plain}->{data} = $rawRealisation;
|
||||||
|
$c->forward('Hydra::View::Plain');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
sub narinfo :Path :Args(StrMatch[NARINFO_REGEX]) {
|
sub narinfo :Path :Args(StrMatch[NARINFO_REGEX]) {
|
||||||
my ($self, $c, $narinfo) = @_;
|
my ($self, $c, $narinfo) = @_;
|
||||||
|
|
||||||
|
|
|
@ -49,7 +49,7 @@ __PACKAGE__->table("buildoutputs");
|
||||||
=head2 path
|
=head2 path
|
||||||
|
|
||||||
data_type: 'text'
|
data_type: 'text'
|
||||||
is_nullable: 0
|
is_nullable: 1
|
||||||
|
|
||||||
=cut
|
=cut
|
||||||
|
|
||||||
|
@ -59,7 +59,7 @@ __PACKAGE__->add_columns(
|
||||||
"name",
|
"name",
|
||||||
{ data_type => "text", is_nullable => 0 },
|
{ data_type => "text", is_nullable => 0 },
|
||||||
"path",
|
"path",
|
||||||
{ data_type => "text", is_nullable => 0 },
|
{ data_type => "text", is_nullable => 1 },
|
||||||
);
|
);
|
||||||
|
|
||||||
=head1 PRIMARY KEY
|
=head1 PRIMARY KEY
|
||||||
|
@ -94,8 +94,8 @@ __PACKAGE__->belongs_to(
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|
||||||
# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-08-26 12:02:36
|
# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-06-30 12:02:32
|
||||||
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:gU+kZ6A0ISKpaXGRGve8mg
|
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Jsabm3YTcI7YvCuNdKP5Ng
|
||||||
|
|
||||||
my %hint = (
|
my %hint = (
|
||||||
columns => [
|
columns => [
|
||||||
|
|
|
@ -55,7 +55,7 @@ __PACKAGE__->table("buildstepoutputs");
|
||||||
=head2 path
|
=head2 path
|
||||||
|
|
||||||
data_type: 'text'
|
data_type: 'text'
|
||||||
is_nullable: 0
|
is_nullable: 1
|
||||||
|
|
||||||
=cut
|
=cut
|
||||||
|
|
||||||
|
@ -67,7 +67,7 @@ __PACKAGE__->add_columns(
|
||||||
"name",
|
"name",
|
||||||
{ data_type => "text", is_nullable => 0 },
|
{ data_type => "text", is_nullable => 0 },
|
||||||
"path",
|
"path",
|
||||||
{ data_type => "text", is_nullable => 0 },
|
{ data_type => "text", is_nullable => 1 },
|
||||||
);
|
);
|
||||||
|
|
||||||
=head1 PRIMARY KEY
|
=head1 PRIMARY KEY
|
||||||
|
@ -119,8 +119,8 @@ __PACKAGE__->belongs_to(
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|
||||||
# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-08-26 12:02:36
|
# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-06-30 12:02:32
|
||||||
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:gxp8rOjpRVen4YbIjomHTw
|
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Bad70CRTt7zb2GGuRoQ++Q
|
||||||
|
|
||||||
|
|
||||||
# You can replace this text with custom code or comments, and it will be preserved on regeneration
|
# You can replace this text with custom code or comments, and it will be preserved on regeneration
|
||||||
|
|
|
@ -247,7 +247,7 @@ create trigger BuildBumped after update on Builds for each row
|
||||||
create table BuildOutputs (
|
create table BuildOutputs (
|
||||||
build integer not null,
|
build integer not null,
|
||||||
name text not null,
|
name text not null,
|
||||||
path text not null,
|
path text,
|
||||||
primary key (build, name),
|
primary key (build, name),
|
||||||
foreign key (build) references Builds(id) on delete cascade
|
foreign key (build) references Builds(id) on delete cascade
|
||||||
);
|
);
|
||||||
|
@ -303,7 +303,7 @@ create table BuildStepOutputs (
|
||||||
build integer not null,
|
build integer not null,
|
||||||
stepnr integer not null,
|
stepnr integer not null,
|
||||||
name text not null,
|
name text not null,
|
||||||
path text not null,
|
path text,
|
||||||
primary key (build, stepnr, name),
|
primary key (build, stepnr, name),
|
||||||
foreign key (build) references Builds(id) on delete cascade,
|
foreign key (build) references Builds(id) on delete cascade,
|
||||||
foreign key (build, stepnr) references BuildSteps(build, stepnr) on delete cascade
|
foreign key (build, stepnr) references BuildSteps(build, stepnr) on delete cascade
|
||||||
|
|
4
src/sql/upgrade-84.sql
Normal file
4
src/sql/upgrade-84.sql
Normal file
|
@ -0,0 +1,4 @@
|
||||||
|
-- CA derivations do not have statically known output paths. The values
|
||||||
|
-- are only filled in after the build runs.
|
||||||
|
ALTER TABLE BuildStepOutputs ALTER COLUMN path DROP NOT NULL;
|
||||||
|
ALTER TABLE BuildOutputs ALTER COLUMN path DROP NOT NULL;
|
61
t/content-addressed/basic.t
Normal file
61
t/content-addressed/basic.t
Normal file
|
@ -0,0 +1,61 @@
|
||||||
|
use feature 'unicode_strings';
|
||||||
|
use strict;
|
||||||
|
use warnings;
|
||||||
|
use Setup;
|
||||||
|
|
||||||
|
my %ctx = test_init(
|
||||||
|
nix_config => qq|
|
||||||
|
experimental-features = ca-derivations
|
||||||
|
|,
|
||||||
|
);
|
||||||
|
|
||||||
|
require Hydra::Schema;
|
||||||
|
require Hydra::Model::DB;
|
||||||
|
|
||||||
|
use JSON::MaybeXS;
|
||||||
|
|
||||||
|
use HTTP::Request::Common;
|
||||||
|
use Test2::V0;
|
||||||
|
require Catalyst::Test;
|
||||||
|
Catalyst::Test->import('Hydra');
|
||||||
|
|
||||||
|
my $db = Hydra::Model::DB->new;
|
||||||
|
hydra_setup($db);
|
||||||
|
|
||||||
|
my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"});
|
||||||
|
|
||||||
|
my $jobset = createBaseJobset("content-addressed", "content-addressed.nix", $ctx{jobsdir});
|
||||||
|
|
||||||
|
ok(evalSucceeds($jobset), "Evaluating jobs/content-addressed.nix should exit with return code 0");
|
||||||
|
is(nrQueuedBuildsForJobset($jobset), 5, "Evaluating jobs/content-addressed.nix should result in 4 builds");
|
||||||
|
|
||||||
|
for my $build (queuedBuildsForJobset($jobset)) {
|
||||||
|
ok(runBuild($build), "Build '".$build->job."' from jobs/content-addressed.nix should exit with code 0");
|
||||||
|
my $newbuild = $db->resultset('Builds')->find($build->id);
|
||||||
|
is($newbuild->finished, 1, "Build '".$build->job."' from jobs/content-addressed.nix should be finished.");
|
||||||
|
my $expected = $build->job eq "fails" ? 1 : $build->job =~ /with_failed/ ? 6 : 0;
|
||||||
|
is($newbuild->buildstatus, $expected, "Build '".$build->job."' from jobs/content-addressed.nix should have buildstatus $expected.");
|
||||||
|
|
||||||
|
my $response = request("/build/".$build->id);
|
||||||
|
ok($response->is_success, "The 'build' page for build '".$build->job."' should load properly");
|
||||||
|
|
||||||
|
if ($newbuild->buildstatus == 0) {
|
||||||
|
my $buildOutputs = $newbuild->buildoutputs;
|
||||||
|
for my $output ($newbuild->buildoutputs) {
|
||||||
|
# XXX: This hardcodes /nix/store/.
|
||||||
|
# It's fine because in practice the nix store for the tests will be of
|
||||||
|
# the form `/some/thing/nix/store/`, but it would be cleaner if there
|
||||||
|
# was a way to query Nix for its store dir?
|
||||||
|
like(
|
||||||
|
$output->path, qr|/nix/store/|,
|
||||||
|
"Output '".$output->name."' of build '".$build->job."' should be a valid store path"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
isnt(<$ctx{deststoredir}/realisations/*>, "", "The destination store should have the realisations of the built derivations registered");
|
||||||
|
|
||||||
|
done_testing;
|
||||||
|
|
28
t/content-addressed/without-experimental-feature.t
Normal file
28
t/content-addressed/without-experimental-feature.t
Normal file
|
@ -0,0 +1,28 @@
|
||||||
|
use feature 'unicode_strings';
|
||||||
|
use strict;
|
||||||
|
use warnings;
|
||||||
|
use Setup;
|
||||||
|
|
||||||
|
my %ctx = test_init();
|
||||||
|
|
||||||
|
require Hydra::Schema;
|
||||||
|
require Hydra::Model::DB;
|
||||||
|
|
||||||
|
use JSON::MaybeXS;
|
||||||
|
|
||||||
|
use HTTP::Request::Common;
|
||||||
|
use Test2::V0;
|
||||||
|
require Catalyst::Test;
|
||||||
|
Catalyst::Test->import('Hydra');
|
||||||
|
|
||||||
|
my $db = Hydra::Model::DB->new;
|
||||||
|
hydra_setup($db);
|
||||||
|
|
||||||
|
my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"});
|
||||||
|
|
||||||
|
my $jobset = createBaseJobset("content-addressed", "content-addressed.nix", $ctx{jobsdir});
|
||||||
|
|
||||||
|
ok(evalSucceeds($jobset), "Evaluating jobs/content-addressed.nix without the experimental feature should exit with return code 0");
|
||||||
|
is(nrQueuedBuildsForJobset($jobset), 0, "Evaluating jobs/content-addressed.nix without the experimental Nix feature should result in 0 build");
|
||||||
|
|
||||||
|
done_testing;
|
|
@ -6,4 +6,9 @@ rec {
|
||||||
system = builtins.currentSystem;
|
system = builtins.currentSystem;
|
||||||
PATH = path;
|
PATH = path;
|
||||||
} // args);
|
} // args);
|
||||||
|
mkContentAddressedDerivation = args: mkDerivation ({
|
||||||
|
__contentAddressed = true;
|
||||||
|
outputHashMode = "recursive";
|
||||||
|
outputHashAlgo = "sha256";
|
||||||
|
} // args);
|
||||||
}
|
}
|
||||||
|
|
35
t/jobs/content-addressed.nix
Normal file
35
t/jobs/content-addressed.nix
Normal file
|
@ -0,0 +1,35 @@
|
||||||
|
let cfg = import ./config.nix; in
|
||||||
|
rec {
|
||||||
|
empty_dir =
|
||||||
|
cfg.mkContentAddressedDerivation {
|
||||||
|
name = "empty-dir";
|
||||||
|
builder = ./empty-dir-builder.sh;
|
||||||
|
};
|
||||||
|
|
||||||
|
fails =
|
||||||
|
cfg.mkContentAddressedDerivation {
|
||||||
|
name = "fails";
|
||||||
|
builder = ./fail.sh;
|
||||||
|
};
|
||||||
|
|
||||||
|
succeed_with_failed =
|
||||||
|
cfg.mkContentAddressedDerivation {
|
||||||
|
name = "succeed-with-failed";
|
||||||
|
builder = ./succeed-with-failed.sh;
|
||||||
|
};
|
||||||
|
|
||||||
|
caDependingOnCA =
|
||||||
|
cfg.mkContentAddressedDerivation {
|
||||||
|
name = "ca-depending-on-ca";
|
||||||
|
builder = ./dir-with-file-builder.sh;
|
||||||
|
FOO = empty_dir;
|
||||||
|
};
|
||||||
|
|
||||||
|
nonCaDependingOnCA =
|
||||||
|
cfg.mkDerivation {
|
||||||
|
name = "non-ca-depending-on-ca";
|
||||||
|
builder = ./dir-with-file-builder.sh;
|
||||||
|
FOO = empty_dir;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
4
t/jobs/dir-with-file-builder.sh
Executable file
4
t/jobs/dir-with-file-builder.sh
Executable file
|
@ -0,0 +1,4 @@
|
||||||
|
#! /bin/sh
|
||||||
|
|
||||||
|
mkdir $out
|
||||||
|
echo foo > $out/a-file
|
|
@ -8,7 +8,7 @@ my $binarycachedir = File::Temp->newdir();
|
||||||
|
|
||||||
my $ctx = test_context(
|
my $ctx = test_context(
|
||||||
nix_config => qq|
|
nix_config => qq|
|
||||||
experimental-features = nix-command
|
experimental-features = nix-command ca-derivations
|
||||||
substituters = file://${binarycachedir}?trusted=1
|
substituters = file://${binarycachedir}?trusted=1
|
||||||
|,
|
|,
|
||||||
hydra_config => q|
|
hydra_config => q|
|
||||||
|
|
Loading…
Reference in a new issue