Merge pull request #1349 from NixOS/ca-no-new-col

Allow building content-addressed derivations with hydra, minimally

doc/manual/src/projects.md

@@ -404,3 +404,10 @@ analogous:
   | `String value` | `gitea_status_repo` | *Name of the `Git checkout` input* |
   | `String value` | `gitea_http_url`    | *Public URL of `gitea`*, optional  |
 
+Content-addressed derivations
+-----------------------------
+
+Hydra can to a certain extent use the [`ca-derivations` experimental Nix feature](https://github.com/NixOS/rfcs/pull/62).
+To use it, make sure that the Nix version you use is at least as recent as the one used in hydra's flake.
+
+Be warned that this support is still highly experimental, and anything beyond the basic functionality might be broken at that point.

src/lib/Hydra/Controller/Root.pm

@@ -18,6 +18,8 @@ use Net::Prometheus;
 use Types::Standard qw/StrMatch/;
 
 use constant NARINFO_REGEX => qr{^([a-z0-9]{32})\.narinfo$};
+# e.g.: https://hydra.example.com/realisations/sha256:a62128132508a3a32eef651d6467695944763602f226ac630543e947d9feb140!out.doi
+use constant REALISATIONS_REGEX => qr{^(sha256:[a-z0-9]{64}![a-z]+)\.doi$};
 
 # Put this controller at top-level.
 __PACKAGE__->config->{namespace} = '';
@@ -355,6 +357,33 @@ sub nix_cache_info :Path('nix-cache-info') :Args(0) {
 }
 
 
+sub realisations :Path('realisations') :Args(StrMatch[REALISATIONS_REGEX]) {
+    my ($self, $c, $realisation) = @_;
+
+    if (!isLocalStore) {
+        notFound($c, "There is no binary cache here.");
+    }
+
+    else {
+        my ($rawDrvOutput) = $realisation =~ REALISATIONS_REGEX;
+        my $rawRealisation = queryRawRealisation($rawDrvOutput);
+
+        if (!$rawRealisation) {
+            $c->response->status(404);
+            $c->response->content_type('text/plain');
+            $c->stash->{plain}->{data} = "does not exist\n";
+            $c->forward('Hydra::View::Plain');
+            setCacheHeaders($c, 60 * 60);
+            return;
+        }
+
+        $c->response->content_type('text/plain');
+        $c->stash->{plain}->{data} = $rawRealisation;
+        $c->forward('Hydra::View::Plain');
+    }
+}
+
+
 sub narinfo :Path :Args(StrMatch[NARINFO_REGEX]) {
     my ($self, $c, $narinfo) = @_;
 

src/lib/Hydra/Schema/Result/BuildOutputs.pm

@@ -49,7 +49,7 @@ __PACKAGE__->table("buildoutputs");
 =head2 path
 
   data_type: 'text'
-  is_nullable: 0
+  is_nullable: 1
 
 =cut
 
@@ -59,7 +59,7 @@ __PACKAGE__->add_columns(
   "name",
   { data_type => "text", is_nullable => 0 },
   "path",
-  { data_type => "text", is_nullable => 0 },
+  { data_type => "text", is_nullable => 1 },
 );
 
 =head1 PRIMARY KEY
@@ -94,8 +94,8 @@ __PACKAGE__->belongs_to(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-08-26 12:02:36
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:gU+kZ6A0ISKpaXGRGve8mg
+# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-06-30 12:02:32
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Jsabm3YTcI7YvCuNdKP5Ng
 
 my %hint = (
     columns => [

src/lib/Hydra/Schema/Result/BuildStepOutputs.pm

@@ -55,7 +55,7 @@ __PACKAGE__->table("buildstepoutputs");
 =head2 path
 
   data_type: 'text'
-  is_nullable: 0
+  is_nullable: 1
 
 =cut
 
@@ -67,7 +67,7 @@ __PACKAGE__->add_columns(
   "name",
   { data_type => "text", is_nullable => 0 },
   "path",
-  { data_type => "text", is_nullable => 0 },
+  { data_type => "text", is_nullable => 1 },
 );
 
 =head1 PRIMARY KEY
@@ -119,8 +119,8 @@ __PACKAGE__->belongs_to(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-08-26 12:02:36
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:gxp8rOjpRVen4YbIjomHTw
+# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-06-30 12:02:32
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Bad70CRTt7zb2GGuRoQ++Q
 
 
 # You can replace this text with custom code or comments, and it will be preserved on regeneration

src/sql/hydra.sql

@@ -247,7 +247,7 @@ create trigger BuildBumped after update on Builds for each row
 create table BuildOutputs (
     build         integer not null,
     name          text not null,
-    path          text not null,
+    path          text,
     primary key   (build, name),
     foreign key   (build) references Builds(id) on delete cascade
 );
@@ -303,7 +303,7 @@ create table BuildStepOutputs (
     build         integer not null,
     stepnr        integer not null,
     name          text not null,
-    path          text not null,
+    path          text,
     primary key   (build, stepnr, name),
     foreign key   (build) references Builds(id) on delete cascade,
     foreign key   (build, stepnr) references BuildSteps(build, stepnr) on delete cascade

src/sql/upgrade-84.sql

@@ -0,0 +1,4 @@
+-- CA derivations do not have statically known output paths. The values
+-- are only filled in after the build runs.
+ALTER TABLE BuildStepOutputs ALTER COLUMN path DROP NOT NULL;
+ALTER TABLE BuildOutputs ALTER COLUMN path DROP NOT NULL;

t/content-addressed/basic.t

@@ -0,0 +1,61 @@
+use feature 'unicode_strings';
+use strict;
+use warnings;
+use Setup;
+
+my %ctx = test_init(
+    nix_config => qq|
+    experimental-features = ca-derivations
+    |,
+);
+
+require Hydra::Schema;
+require Hydra::Model::DB;
+
+use JSON::MaybeXS;
+
+use HTTP::Request::Common;
+use Test2::V0;
+require Catalyst::Test;
+Catalyst::Test->import('Hydra');
+
+my $db = Hydra::Model::DB->new;
+hydra_setup($db);
+
+my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"});
+
+my $jobset = createBaseJobset("content-addressed", "content-addressed.nix", $ctx{jobsdir});
+
+ok(evalSucceeds($jobset), "Evaluating jobs/content-addressed.nix should exit with return code 0");
+is(nrQueuedBuildsForJobset($jobset), 5, "Evaluating jobs/content-addressed.nix should result in 4 builds");
+
+for my $build (queuedBuildsForJobset($jobset)) {
+    ok(runBuild($build), "Build '".$build->job."' from jobs/content-addressed.nix should exit with code 0");
+    my $newbuild = $db->resultset('Builds')->find($build->id);
+    is($newbuild->finished, 1, "Build '".$build->job."' from jobs/content-addressed.nix should be finished.");
+    my $expected = $build->job eq "fails" ? 1 : $build->job =~ /with_failed/ ? 6 : 0;
+    is($newbuild->buildstatus, $expected, "Build '".$build->job."' from jobs/content-addressed.nix should have buildstatus $expected.");
+
+    my $response = request("/build/".$build->id);
+    ok($response->is_success, "The 'build' page for build '".$build->job."' should load properly");
+
+    if ($newbuild->buildstatus == 0) {
+      my $buildOutputs = $newbuild->buildoutputs;
+      for my $output ($newbuild->buildoutputs) {
+        # XXX: This hardcodes /nix/store/.
+        # It's fine because in practice the nix store for the tests will be of
+        # the form `/some/thing/nix/store/`, but it would be cleaner if there
+        # was a way to query Nix for its store dir?
+        like(
+          $output->path, qr|/nix/store/|,
+          "Output '".$output->name."' of build '".$build->job."' should be a valid store path"
+        );
+      }
+    }
+
+}
+
+isnt(<$ctx{deststoredir}/realisations/*>, "", "The destination store should have the realisations of the built derivations registered");
+
+done_testing;
+

t/content-addressed/without-experimental-feature.t

@@ -0,0 +1,28 @@
+use feature 'unicode_strings';
+use strict;
+use warnings;
+use Setup;
+
+my %ctx = test_init();
+
+require Hydra::Schema;
+require Hydra::Model::DB;
+
+use JSON::MaybeXS;
+
+use HTTP::Request::Common;
+use Test2::V0;
+require Catalyst::Test;
+Catalyst::Test->import('Hydra');
+
+my $db = Hydra::Model::DB->new;
+hydra_setup($db);
+
+my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"});
+
+my $jobset = createBaseJobset("content-addressed", "content-addressed.nix", $ctx{jobsdir});
+
+ok(evalSucceeds($jobset),                  "Evaluating jobs/content-addressed.nix without the experimental feature should exit with return code 0");
+is(nrQueuedBuildsForJobset($jobset), 0, "Evaluating jobs/content-addressed.nix without the experimental Nix feature should result in 0 build");
+
+done_testing;

t/jobs/config.nix.in

@@ -6,4 +6,9 @@ rec {
       system = builtins.currentSystem;
       PATH = path;
     } // args);
+  mkContentAddressedDerivation = args: mkDerivation ({
+    __contentAddressed = true;
+    outputHashMode = "recursive";
+    outputHashAlgo = "sha256";
+  } // args);
 }

t/jobs/content-addressed.nix

@@ -0,0 +1,35 @@
+let cfg =  import ./config.nix; in
+rec {
+  empty_dir =
+    cfg.mkContentAddressedDerivation {
+      name = "empty-dir";
+      builder = ./empty-dir-builder.sh;
+    };
+
+  fails =
+    cfg.mkContentAddressedDerivation {
+      name = "fails";
+      builder = ./fail.sh;
+    };
+
+  succeed_with_failed =
+    cfg.mkContentAddressedDerivation {
+      name = "succeed-with-failed";
+      builder = ./succeed-with-failed.sh;
+    };
+
+  caDependingOnCA =
+    cfg.mkContentAddressedDerivation {
+      name = "ca-depending-on-ca";
+      builder = ./dir-with-file-builder.sh;
+      FOO = empty_dir;
+    };
+
+  nonCaDependingOnCA =
+    cfg.mkDerivation {
+      name = "non-ca-depending-on-ca";
+      builder = ./dir-with-file-builder.sh;
+      FOO = empty_dir;
+    };
+}
+

t/jobs/dir-with-file-builder.sh

@@ -0,0 +1,4 @@
+#! /bin/sh
+
+mkdir $out
+echo foo > $out/a-file

t/queue-runner/notifications.t

@@ -8,7 +8,7 @@ my $binarycachedir = File::Temp->newdir();
 
 my $ctx = test_context(
     nix_config => qq|
-    experimental-features = nix-command
+    experimental-features = nix-command ca-derivations
     substituters = file://${binarycachedir}?trusted=1
     |,
     hydra_config => q|