From fcde5908d8e51f975b883329b34d24a9f30ea4b3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jan 2024 21:32:22 -0500 Subject: [PATCH] More CA derivations prep Again, with care not to change the schema in any way. --- src/hydra-eval-jobs/hydra-eval-jobs.cc | 6 +----- src/hydra-queue-runner/hydra-queue-runner.cc | 5 ++++- src/hydra-queue-runner/queue-monitor.cc | 20 ++++++++++++-------- src/lib/Hydra/Controller/Build.pm | 6 ++++-- src/script/hydra-eval-jobset | 6 +++++- t/lib/HydraTestContext.pm | 6 +++++- 6 files changed, 31 insertions(+), 18 deletions(-) diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index 2794cc62..5cfc3a52 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -250,11 +250,7 @@ static void worker( // See the `queryOutputs` call above; we should // not encounter missing output paths otherwise. assert(experimentalFeatureSettings.isEnabled(Xp::CaDerivations)); - // TODO it would be better to set `null` than an - // empty string here, to force the consumer of - // this JSON to more explicitly handle this - // case. - out[outputName] = ""; + out[outputName] = nullptr; } } job["outputs"] = std::move(out); diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index df79e1dc..0f4e759c 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -336,7 +336,10 @@ unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID for (auto & [name, output] : getDestStore()->queryPartialDerivationOutputMap(step->drvPath, &*localStore)) txn.exec_params0 ("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)", - buildId, stepNr, name, output ? localStore->printStorePath(*output) : ""); + buildId, stepNr, name, + output + ? std::optional { localStore->printStorePath(*output)} + : std::nullopt); if (status == bsBusy) txn.exec(fmt("notify step_started, '%d\t%d'", buildId, stepNr)); diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index 513d399b..d7214c43 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -193,14 +193,18 @@ bool State::getQueuedBuilds(Connection & conn, if (!propagatedFrom) { for (auto & [outputName, optOutputPath] : destStore->queryPartialDerivationOutputMap(ex.step->drvPath, &*localStore)) { - // ca-derivations not actually supported yet - assert(optOutputPath); - auto res = txn.exec_params - ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where path = $1 and startTime != 0 and stopTime != 0 and status = 1", - localStore->printStorePath(*optOutputPath)); - if (!res[0][0].is_null()) { - propagatedFrom = res[0][0].as(); - break; + constexpr std::string_view common = "select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where startTime != 0 and stopTime != 0 and status = 1"; + auto res = optOutputPath + ? txn.exec_params( + std::string { common } + " and path = $1", + localStore->printStorePath(*optOutputPath)) + : txn.exec_params( + std::string { common } + " and drvPath = $1 and name = $2", + localStore->printStorePath(ex.step->drvPath), + outputName); + if (!res[0][0].is_null()) { + propagatedFrom = res[0][0].as(); + break; } } } diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm index 2d74f86a..c3869838 100644 --- a/src/lib/Hydra/Controller/Build.pm +++ b/src/lib/Hydra/Controller/Build.pm @@ -78,14 +78,16 @@ sub build_GET { $c->stash->{template} = 'build.tt'; $c->stash->{isLocalStore} = isLocalStore(); + # XXX: If the derivation is content-addressed then this will always return + # false because `$_->path` will be empty $c->stash->{available} = $c->stash->{isLocalStore} - ? all { isValidPath($_->path) } $build->buildoutputs->all + ? all { $_->path && isValidPath($_->path) } $build->buildoutputs->all : 1; $c->stash->{drvAvailable} = isValidPath $build->drvpath; if ($build->finished && $build->iscachedbuild) { - my $path = ($build->buildoutputs)[0]->path or die; + my $path = ($build->buildoutputs)[0]->path or undef; my $cachedBuildStep = findBuildStepByOutPath($self, $c, $path); if (defined $cachedBuildStep) { $c->stash->{cachedBuild} = $cachedBuildStep->build; diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset index c6f6c275..7ed7ebe8 100755 --- a/src/script/hydra-eval-jobset +++ b/src/script/hydra-eval-jobset @@ -438,13 +438,17 @@ sub checkBuild { # new build to be scheduled if the meta.maintainers field is # changed? if (defined $prevEval) { + my $pathOrDrvConstraint = defined $firstOutputPath + ? { path => $firstOutputPath } + : { drvPath => $drvPath }; + my ($prevBuild) = $prevEval->builds->search( # The "project" and "jobset" constraints are # semantically unnecessary (because they're implied by # the eval), but they give a factor 1000 speedup on # the Nixpkgs jobset with PostgreSQL. { jobset_id => $jobset->get_column('id'), job => $jobName, - name => $firstOutputName, path => $firstOutputPath }, + name => $firstOutputName, %$pathOrDrvConstraint }, { rows => 1, columns => ['id', 'finished'], join => ['buildoutputs'] }); if (defined $prevBuild) { #print STDERR " already scheduled/built as build ", $prevBuild->id, "\n"; diff --git a/t/lib/HydraTestContext.pm b/t/lib/HydraTestContext.pm index a22c3df1..e1a5b226 100644 --- a/t/lib/HydraTestContext.pm +++ b/t/lib/HydraTestContext.pm @@ -39,6 +39,8 @@ use Hydra::Helper::Exec; sub new { my ($class, %opts) = @_; + my $deststoredir; + # Cleanup will be managed by yath. By the default it will be cleaned # up, but can be kept to aid in debugging test failures. my $dir = File::Temp->newdir(CLEANUP => 0); @@ -55,6 +57,7 @@ sub new { my $hydra_config = $opts{'hydra_config'} || ""; $hydra_config = "queue_runner_metrics_address = 127.0.0.1:0\n" . $hydra_config; if ($opts{'use_external_destination_store'} // 1) { + $deststoredir = "$dir/nix/dest-store"; $hydra_config = "store_uri = file://$dir/nix/dest-store\n" . $hydra_config; } @@ -81,7 +84,8 @@ sub new { nix_state_dir => $nix_state_dir, nix_log_dir => $nix_log_dir, testdir => abs_path(dirname(__FILE__) . "/.."), - jobsdir => abs_path(dirname(__FILE__) . "/../jobs") + jobsdir => abs_path(dirname(__FILE__) . "/../jobs"), + deststoredir => $deststoredir, }, $class; if ($opts{'before_init'}) {