More CA derivations prep

Again, with care not to change the schema in any way.
This commit is contained in:
John Ericson 2024-01-25 21:32:22 -05:00
parent 083ef46c12
commit fcde5908d8
6 changed files with 31 additions and 18 deletions

View file

@ -250,11 +250,7 @@ static void worker(
// See the `queryOutputs` call above; we should // See the `queryOutputs` call above; we should
// not encounter missing output paths otherwise. // not encounter missing output paths otherwise.
assert(experimentalFeatureSettings.isEnabled(Xp::CaDerivations)); assert(experimentalFeatureSettings.isEnabled(Xp::CaDerivations));
// TODO it would be better to set `null` than an out[outputName] = nullptr;
// empty string here, to force the consumer of
// this JSON to more explicitly handle this
// case.
out[outputName] = "";
} }
} }
job["outputs"] = std::move(out); job["outputs"] = std::move(out);

View file

@ -336,7 +336,10 @@ unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID
for (auto & [name, output] : getDestStore()->queryPartialDerivationOutputMap(step->drvPath, &*localStore)) for (auto & [name, output] : getDestStore()->queryPartialDerivationOutputMap(step->drvPath, &*localStore))
txn.exec_params0 txn.exec_params0
("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)", ("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)",
buildId, stepNr, name, output ? localStore->printStorePath(*output) : ""); buildId, stepNr, name,
output
? std::optional { localStore->printStorePath(*output)}
: std::nullopt);
if (status == bsBusy) if (status == bsBusy)
txn.exec(fmt("notify step_started, '%d\t%d'", buildId, stepNr)); txn.exec(fmt("notify step_started, '%d\t%d'", buildId, stepNr));

View file

@ -193,11 +193,15 @@ bool State::getQueuedBuilds(Connection & conn,
if (!propagatedFrom) { if (!propagatedFrom) {
for (auto & [outputName, optOutputPath] : destStore->queryPartialDerivationOutputMap(ex.step->drvPath, &*localStore)) { for (auto & [outputName, optOutputPath] : destStore->queryPartialDerivationOutputMap(ex.step->drvPath, &*localStore)) {
// ca-derivations not actually supported yet constexpr std::string_view common = "select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where startTime != 0 and stopTime != 0 and status = 1";
assert(optOutputPath); auto res = optOutputPath
auto res = txn.exec_params ? txn.exec_params(
("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where path = $1 and startTime != 0 and stopTime != 0 and status = 1", std::string { common } + " and path = $1",
localStore->printStorePath(*optOutputPath)); localStore->printStorePath(*optOutputPath))
: txn.exec_params(
std::string { common } + " and drvPath = $1 and name = $2",
localStore->printStorePath(ex.step->drvPath),
outputName);
if (!res[0][0].is_null()) { if (!res[0][0].is_null()) {
propagatedFrom = res[0][0].as<BuildID>(); propagatedFrom = res[0][0].as<BuildID>();
break; break;

View file

@ -78,14 +78,16 @@ sub build_GET {
$c->stash->{template} = 'build.tt'; $c->stash->{template} = 'build.tt';
$c->stash->{isLocalStore} = isLocalStore(); $c->stash->{isLocalStore} = isLocalStore();
# XXX: If the derivation is content-addressed then this will always return
# false because `$_->path` will be empty
$c->stash->{available} = $c->stash->{available} =
$c->stash->{isLocalStore} $c->stash->{isLocalStore}
? all { isValidPath($_->path) } $build->buildoutputs->all ? all { $_->path && isValidPath($_->path) } $build->buildoutputs->all
: 1; : 1;
$c->stash->{drvAvailable} = isValidPath $build->drvpath; $c->stash->{drvAvailable} = isValidPath $build->drvpath;
if ($build->finished && $build->iscachedbuild) { if ($build->finished && $build->iscachedbuild) {
my $path = ($build->buildoutputs)[0]->path or die; my $path = ($build->buildoutputs)[0]->path or undef;
my $cachedBuildStep = findBuildStepByOutPath($self, $c, $path); my $cachedBuildStep = findBuildStepByOutPath($self, $c, $path);
if (defined $cachedBuildStep) { if (defined $cachedBuildStep) {
$c->stash->{cachedBuild} = $cachedBuildStep->build; $c->stash->{cachedBuild} = $cachedBuildStep->build;

View file

@ -438,13 +438,17 @@ sub checkBuild {
# new build to be scheduled if the meta.maintainers field is # new build to be scheduled if the meta.maintainers field is
# changed? # changed?
if (defined $prevEval) { if (defined $prevEval) {
my $pathOrDrvConstraint = defined $firstOutputPath
? { path => $firstOutputPath }
: { drvPath => $drvPath };
my ($prevBuild) = $prevEval->builds->search( my ($prevBuild) = $prevEval->builds->search(
# The "project" and "jobset" constraints are # The "project" and "jobset" constraints are
# semantically unnecessary (because they're implied by # semantically unnecessary (because they're implied by
# the eval), but they give a factor 1000 speedup on # the eval), but they give a factor 1000 speedup on
# the Nixpkgs jobset with PostgreSQL. # the Nixpkgs jobset with PostgreSQL.
{ jobset_id => $jobset->get_column('id'), job => $jobName, { jobset_id => $jobset->get_column('id'), job => $jobName,
name => $firstOutputName, path => $firstOutputPath }, name => $firstOutputName, %$pathOrDrvConstraint },
{ rows => 1, columns => ['id', 'finished'], join => ['buildoutputs'] }); { rows => 1, columns => ['id', 'finished'], join => ['buildoutputs'] });
if (defined $prevBuild) { if (defined $prevBuild) {
#print STDERR " already scheduled/built as build ", $prevBuild->id, "\n"; #print STDERR " already scheduled/built as build ", $prevBuild->id, "\n";

View file

@ -39,6 +39,8 @@ use Hydra::Helper::Exec;
sub new { sub new {
my ($class, %opts) = @_; my ($class, %opts) = @_;
my $deststoredir;
# Cleanup will be managed by yath. By the default it will be cleaned # Cleanup will be managed by yath. By the default it will be cleaned
# up, but can be kept to aid in debugging test failures. # up, but can be kept to aid in debugging test failures.
my $dir = File::Temp->newdir(CLEANUP => 0); my $dir = File::Temp->newdir(CLEANUP => 0);
@ -55,6 +57,7 @@ sub new {
my $hydra_config = $opts{'hydra_config'} || ""; my $hydra_config = $opts{'hydra_config'} || "";
$hydra_config = "queue_runner_metrics_address = 127.0.0.1:0\n" . $hydra_config; $hydra_config = "queue_runner_metrics_address = 127.0.0.1:0\n" . $hydra_config;
if ($opts{'use_external_destination_store'} // 1) { if ($opts{'use_external_destination_store'} // 1) {
$deststoredir = "$dir/nix/dest-store";
$hydra_config = "store_uri = file://$dir/nix/dest-store\n" . $hydra_config; $hydra_config = "store_uri = file://$dir/nix/dest-store\n" . $hydra_config;
} }
@ -81,7 +84,8 @@ sub new {
nix_state_dir => $nix_state_dir, nix_state_dir => $nix_state_dir,
nix_log_dir => $nix_log_dir, nix_log_dir => $nix_log_dir,
testdir => abs_path(dirname(__FILE__) . "/.."), testdir => abs_path(dirname(__FILE__) . "/.."),
jobsdir => abs_path(dirname(__FILE__) . "/../jobs") jobsdir => abs_path(dirname(__FILE__) . "/../jobs"),
deststoredir => $deststoredir,
}, $class; }, $class;
if ($opts{'before_init'}) { if ($opts{'before_init'}) {