Merge pull request #1348 from NixOS/ca-prep

More CA derivations prep

src/hydra-eval-jobs/hydra-eval-jobs.cc

@@ -250,11 +250,7 @@ static void worker(
                         // See the `queryOutputs` call above; we should
                         // not encounter missing output paths otherwise.
                         assert(experimentalFeatureSettings.isEnabled(Xp::CaDerivations));
-                        // TODO it would be better to set `null` than an
+                        out[outputName] = nullptr;
-                        // empty string here, to force the consumer of
-                        // this JSON to more explicitly handle this
-                        // case.
-                        out[outputName] = "";
                     }
                 }
                 job["outputs"] = std::move(out);

src/hydra-queue-runner/hydra-queue-runner.cc

@@ -336,7 +336,10 @@ unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID
     for (auto & [name, output] : getDestStore()->queryPartialDerivationOutputMap(step->drvPath, &*localStore))
         txn.exec_params0
             ("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)",
-            buildId, stepNr, name, output ? localStore->printStorePath(*output) : "");
+            buildId, stepNr, name,
+            output
+                ? std::optional { localStore->printStorePath(*output)}
+                : std::nullopt);
 
     if (status == bsBusy)
         txn.exec(fmt("notify step_started, '%d\t%d'", buildId, stepNr));

src/hydra-queue-runner/queue-monitor.cc

@@ -193,14 +193,18 @@ bool State::getQueuedBuilds(Connection & conn,
 
                 if (!propagatedFrom) {
                     for (auto & [outputName, optOutputPath] : destStore->queryPartialDerivationOutputMap(ex.step->drvPath, &*localStore)) {
-                          // ca-derivations not actually supported yet
+                        constexpr std::string_view common = "select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where startTime != 0 and stopTime != 0 and status = 1";
-                          assert(optOutputPath);
+                        auto res = optOutputPath
-                          auto res = txn.exec_params
+                            ? txn.exec_params(
-                              ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where path = $1 and startTime != 0 and stopTime != 0 and status = 1",
+                                std::string { common } + " and path = $1",
-                               localStore->printStorePath(*optOutputPath));
+                                localStore->printStorePath(*optOutputPath))
-                          if (!res[0][0].is_null()) {
+                            : txn.exec_params(
-                              propagatedFrom = res[0][0].as<BuildID>();
+                                std::string { common } + " and drvPath = $1 and name = $2",
-                              break;
+                                localStore->printStorePath(ex.step->drvPath),
+                                outputName);
+                        if (!res[0][0].is_null()) {
+                            propagatedFrom = res[0][0].as<BuildID>();
+                            break;
                         }
                     }
                 }

src/lib/Hydra/Controller/Build.pm

@@ -78,14 +78,16 @@ sub build_GET {
 
     $c->stash->{template} = 'build.tt';
     $c->stash->{isLocalStore} = isLocalStore();
+    # XXX: If the derivation is content-addressed then this will always return
+    # false because `$_->path` will be empty
     $c->stash->{available} =
         $c->stash->{isLocalStore}
-        ? all { isValidPath($_->path) } $build->buildoutputs->all
+        ? all { $_->path && isValidPath($_->path) } $build->buildoutputs->all
         : 1;
     $c->stash->{drvAvailable} = isValidPath $build->drvpath;
 
     if ($build->finished && $build->iscachedbuild) {
-        my $path = ($build->buildoutputs)[0]->path or die;
+        my $path = ($build->buildoutputs)[0]->path or undef;
         my $cachedBuildStep = findBuildStepByOutPath($self, $c, $path);
         if (defined $cachedBuildStep) {
             $c->stash->{cachedBuild} = $cachedBuildStep->build;

src/script/hydra-eval-jobset

@@ -438,13 +438,17 @@ sub checkBuild {
         # new build to be scheduled if the meta.maintainers field is
         # changed?
         if (defined $prevEval) {
+            my $pathOrDrvConstraint = defined $firstOutputPath
+                ? { path => $firstOutputPath }
+                : { drvPath => $drvPath };
+
             my ($prevBuild) = $prevEval->builds->search(
                 # The "project" and "jobset" constraints are
                 # semantically unnecessary (because they're implied by
                 # the eval), but they give a factor 1000 speedup on
                 # the Nixpkgs jobset with PostgreSQL.
                 { jobset_id => $jobset->get_column('id'), job => $jobName,
-                  name => $firstOutputName, path => $firstOutputPath },
+                  name => $firstOutputName, %$pathOrDrvConstraint },
                 { rows => 1, columns => ['id', 'finished'], join => ['buildoutputs'] });
             if (defined $prevBuild) {
                 #print STDERR "    already scheduled/built as build ", $prevBuild->id, "\n";

t/lib/HydraTestContext.pm

@@ -39,6 +39,8 @@ use Hydra::Helper::Exec;
 sub new {
     my ($class, %opts) = @_;
 
+    my $deststoredir;
+
     # Cleanup will be managed by yath. By the default it will be cleaned
     # up, but can be kept to aid in debugging test failures.
     my $dir = File::Temp->newdir(CLEANUP => 0);
@@ -55,6 +57,7 @@ sub new {
     my $hydra_config = $opts{'hydra_config'} || "";
     $hydra_config = "queue_runner_metrics_address = 127.0.0.1:0\n" . $hydra_config;
     if ($opts{'use_external_destination_store'} // 1) {
+        $deststoredir = "$dir/nix/dest-store";
         $hydra_config = "store_uri = file://$dir/nix/dest-store\n" . $hydra_config;
     }
 
@@ -81,7 +84,8 @@ sub new {
         nix_state_dir => $nix_state_dir,
         nix_log_dir => $nix_log_dir,
         testdir => abs_path(dirname(__FILE__) . "/.."),
-        jobsdir => abs_path(dirname(__FILE__) . "/../jobs")
+        jobsdir => abs_path(dirname(__FILE__) . "/../jobs"),
+        deststoredir => $deststoredir,
     }, $class;
 
     if ($opts{'before_init'}) {