delroth/hydra
0
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Switch to new Nix bindings, update Nix for that

Browse source 
Implements support for Nix's new Perl bindings[1]. The current state
basically does `openStore()`, but always uses `auto` and doesn't support
stores at other URIs.

Even though the stores are cached inside the Perl implementation, I
decided to instantiate those once in the Nix helper module. That way
store openings aren't cluttered across the entire codebase. Also, there
are two stores used later on - MACHINE_LOCAL_STORE for `auto`,
BINARY_CACHE_STORE for the one from `store_uri` in `hydra.conf` - and
using consistent names should make the intent clearer then.

This doesn't contain any behavioral changes, i.e. the build product
availability issue from #1352 isn't fixed. This patch only contains the
migration to the new API.

[1] https://github.com/NixOS/nix/pull/9863
This commit is contained in:
Maximilian Bosch 2024-02-12 17:12:49 +01:00
parent 878c0f240e
commit e499509595
Signed by: ma27
SSH key fingerprint: SHA256:d7dmwHmpai66L6KIXA+wxzVbkPq0nGLrcHK3ZNroqZY
18 changed files with 50 additions and 58 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
flake.lock
View file

@ -42,11 +42,11 @@
"nixpkgs-regression": "nixpkgs-regression"
},
"locked": {
"lastModified": 1706629374,
"narHash": "sha256-KyAiLGxJ39fSY0cuq8EWAZQ4vaDdqAItSRP4+vjYvq8=",
"lastModified": 1707750141,
"narHash": "sha256-9qSzGQs/Rjf2i3UQjyaZUznZzYDHkLYro/1FTT1easg=",
"owner": "NixOS",
"repo": "nix",
"rev": "75ebb90a70f6320c1c7a1fca87a0a8adb0716143",
"rev": "c4ebb82da4eade975e874da600dc50e9dec610cb",
"type": "github"
},
"original": {

6
src/hydra-eval-jobs/hydra-eval-jobs.cc
View file

@ -185,7 +185,7 @@ static void worker(
!experimentalFeatureSettings.isEnabled(Xp::CaDerivations));
if (drv->querySystem() == "unknown")
throw EvalError("derivation must have a 'system' attribute");
state.error<EvalError>("derivation must have a 'system' attribute").debugThrow();
auto drvPath = state.store->printStorePath(drv->requireDrvPath());
@ -208,7 +208,7 @@ static void worker(
if (a && state.forceBool(*a->value, a->pos, "while evaluating the `_hydraAggregate` attribute")) {
auto a = v->attrs->get(state.symbols.create("constituents"));
if (!a)
throw EvalError("derivation must have a ‘constituents’ attribute");
state.error<EvalError>("derivation must have a ‘constituents’ attribute").debugThrow();
NixStringContext context;
state.coerceToString(a->pos, *a->value, context, "while evaluating the `constituents` attribute", true, false);
@ -274,7 +274,7 @@ static void worker(
else if (v->type() == nNull)
;
else throw TypeError("attribute '%s' is %s, which is not supported", attrPath, showType(*v));
else state.error<TypeError>("attribute '%s' is %s, which is not supported", attrPath, showType(*v)).debugThrow();
} catch (EvalError & e) {
auto msg = e.msg();

2
src/hydra-evaluator/hydra-evaluator.cc
View file

@ -38,7 +38,7 @@ class JobsetId {
friend bool operator!= (const JobsetId & lhs, const JobsetName & rhs);
std::string display() const {
return str(format("%1%:%2% (jobset#%3%)") % project % jobset % id);
return boost::str(boost::format("%1%:%2% (jobset#%3%)") % project % jobset % id);
}
};
bool operator==(const JobsetId & lhs, const JobsetId & rhs)

2
src/hydra-queue-runner/queue-monitor.cc
View file

@ -294,7 +294,7 @@ bool State::getQueuedBuilds(Connection & conn,
try {
createBuild(build);
} catch (Error & e) {
e.addTrace({}, hintfmt("while loading build %d: ", build->id));
e.addTrace({}, HintFmt("while loading build %d: ", build->id));
throw;
}

3
src/lib/Hydra/Base/Controller/NixChannel.pm
View file

@ -4,7 +4,6 @@ use strict;
use warnings;
use base 'Hydra::Base::Controller::REST';
use List::SomeUtils qw(any);
use Nix::Store;
use Hydra::Helper::Nix;
use Hydra::Helper::CatalystUtils;
@ -30,7 +29,7 @@ sub getChannelData {
my $outputs = {};
foreach my $output (@outputs) {
my $outPath = $output->get_column("outpath");
next if $checkValidity && !isValidPath($outPath);
next if $checkValidity && !$MACHINE_LOCAL_STORE->isValidPath($outPath);
$outputs->{$output->get_column("outname")} = $outPath;
push @storePaths, $outPath;
# Put the system type in the manifest (for top-level

18
src/lib/Hydra/Controller/Build.pm
View file

@ -10,8 +10,6 @@ use File::Basename;
use File::LibMagic;
use File::stat;
use Data::Dump qw(dump);
use Nix::Store;
use Nix::Config;
use List::SomeUtils qw(all);
use Encode;
use JSON::PP;
@ -82,9 +80,9 @@ sub build_GET {
# false because `$_->path` will be empty
$c->stash->{available} =
$c->stash->{isLocalStore}
? all { $_->path && isValidPath($_->path) } $build->buildoutputs->all
? all { $_->path && $MACHINE_LOCAL_STORE->isValidPath($_->path) } $build->buildoutputs->all
: 1;
$c->stash->{drvAvailable} = isValidPath $build->drvpath;
$c->stash->{drvAvailable} = $MACHINE_LOCAL_STORE->isValidPath($build->drvpath);
if ($build->finished && $build->iscachedbuild) {
my $path = ($build->buildoutputs)[0]->path or undef;
@ -308,7 +306,7 @@ sub output : Chained('buildChain') PathPart Args(1) {
error($c, "This build is not finished yet.") unless $build->finished;
my $output = $build->buildoutputs->find({name => $outputName});
notFound($c, "This build has no output named ‘$outputName’") unless defined $output;
gone($c, "Output is no longer available.") unless isValidPath $output->path;
gone($c, "Output is no longer available.") unless $MACHINE_LOCAL_STORE->isValidPath($output->path);
$c->response->header('Content-Disposition', "attachment; filename=\"build-${\$build->id}-${\$outputName}.nar.bz2\"");
$c->stash->{current_view} = 'NixNAR';
@ -425,7 +423,7 @@ sub getDependencyGraph {
};
$$done{$path} = $node;
my @refs;
foreach my $ref (queryReferences($path)) {
foreach my $ref ($MACHINE_LOCAL_STORE->queryReferences($path)) {
next if $ref eq $path;
next unless $runtime || $ref =~ /\.drv$/;
getDependencyGraph($self, $c, $runtime, $done, $ref);
@ -433,7 +431,7 @@ sub getDependencyGraph {
}
# Show in reverse topological order to flatten the graph.
# Should probably do a proper BFS.
my @sorted = reverse topoSortPaths(@refs);
my @sorted = reverse $MACHINE_LOCAL_STORE->topoSortPaths(@refs);
$node->{refs} = [map { $$done{$_} } @sorted];
}
@ -446,7 +444,7 @@ sub build_deps : Chained('buildChain') PathPart('build-deps') {
my $build = $c->stash->{build};
my $drvPath = $build->drvpath;
error($c, "Derivation no longer available.") unless isValidPath $drvPath;
error($c, "Derivation no longer available.") unless $MACHINE_LOCAL_STORE->isValidPath($drvPath);
$c->stash->{buildTimeGraph} = getDependencyGraph($self, $c, 0, {}, $drvPath);
@ -461,7 +459,7 @@ sub runtime_deps : Chained('buildChain') PathPart('runtime-deps') {
requireLocalStore($c);
error($c, "Build outputs no longer available.") unless all { isValidPath($_) } @outPaths;
error($c, "Build outputs no longer available.") unless all { $MACHINE_LOCAL_STORE->isValidPath($_) } @outPaths;
my $done = {};
$c->stash->{runtimeGraph} = [ map { getDependencyGraph($self, $c, 1, $done, $_) } @outPaths ];
@ -481,7 +479,7 @@ sub nix : Chained('buildChain') PathPart('nix') CaptureArgs(0) {
if (isLocalStore) {
foreach my $out ($build->buildoutputs) {
notFound($c, "Path " . $out->path . " is no longer available.")
unless isValidPath($out->path);
unless $MACHINE_LOCAL_STORE->isValidPath($out->path);
}
}

2
src/lib/Hydra/Controller/Root.pm
View file

@ -395,7 +395,7 @@ sub narinfo :Path :Args(StrMatch[NARINFO_REGEX]) {
my ($hash) = $narinfo =~ NARINFO_REGEX;
die("Hash length was not 32") if length($hash) != 32;
my $path = queryPathFromHashPart($hash);
my $path = $MACHINE_LOCAL_STORE->queryPathFromHashPart($hash);
if (!$path) {
$c->response->status(404);

5
src/lib/Hydra/Helper/Nix.pm
View file

@ -40,8 +40,11 @@ our @EXPORT = qw(
registerRoot
restartBuilds
run
$MACHINE_LOCAL_STORE
);
our $MACHINE_LOCAL_STORE = Nix::Store->new();
sub getHydraHome {
my $dir = $ENV{"HYDRA_HOME"} or die "The HYDRA_HOME directory does not exist!\n";
@ -494,7 +497,7 @@ sub restartBuilds {
$builds = $builds->search({ finished => 1 });
foreach my $build ($builds->search({}, { columns => ["drvpath"] })) {
next if !isValidPath($build->drvpath);
next if !$MACHINE_LOCAL_STORE->isValidPath($build->drvpath);
registerRoot $build->drvpath;
}

7
src/lib/Hydra/Plugin/BazaarInput.pm
View file

@ -7,7 +7,6 @@ use Digest::SHA qw(sha256_hex);
use File::Path;
use Hydra::Helper::Exec;
use Hydra::Helper::Nix;
use Nix::Store;
sub supportedInputTypes {
my ($self, $inputTypes) = @_;
@ -38,9 +37,9 @@ sub fetchInput {
(my $cachedInput) = $self->{db}->resultset('CachedBazaarInputs')->search(
{uri => $uri, revision => $revision});
addTempRoot($cachedInput->storepath) if defined $cachedInput;
$MACHINE_LOCAL_STORE->addTempRoot($cachedInput->storepath) if defined $cachedInput;
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) {
$storePath = $cachedInput->storepath;
$sha256 = $cachedInput->sha256hash;
} else {
@ -58,7 +57,7 @@ sub fetchInput {
($sha256, $storePath) = split ' ', $stdout;
# FIXME: time window between nix-prefetch-bzr and addTempRoot.
addTempRoot($storePath);
$MACHINE_LOCAL_STORE->addTempRoot($storePath);
$self->{db}->txn_do(sub {
$self->{db}->resultset('CachedBazaarInputs')->create(

7
src/lib/Hydra/Plugin/DarcsInput.pm
View file

@ -7,7 +7,6 @@ use Digest::SHA qw(sha256_hex);
use File::Path;
use Hydra::Helper::Exec;
use Hydra::Helper::Nix;
use Nix::Store;
sub supportedInputTypes {
my ($self, $inputTypes) = @_;
@ -58,7 +57,7 @@ sub fetchInput {
{uri => $uri, revision => $revision},
{rows => 1});
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) {
$storePath = $cachedInput->storepath;
$sha256 = $cachedInput->sha256hash;
$revision = $cachedInput->revision;
@ -75,8 +74,8 @@ sub fetchInput {
die "darcs changes --count failed" if $? != 0;
system "rm", "-rf", "$tmpDir/export/_darcs";
$storePath = addToStore("$tmpDir/export", 1, "sha256");
$sha256 = queryPathHash($storePath);
$storePath = $MACHINE_LOCAL_STORE->addToStore("$tmpDir/export", 1, "sha256");
$sha256 = $MACHINE_LOCAL_STORE->queryPathHash($storePath);
$sha256 =~ s/sha256://;
$self->{db}->txn_do(sub {

6
src/lib/Hydra/Plugin/GitInput.pm
View file

@ -186,9 +186,9 @@ sub fetchInput {
{uri => $uri, branch => $branch, revision => $revision, isdeepclone => defined($deepClone) ? 1 : 0},
{rows => 1});
addTempRoot($cachedInput->storepath) if defined $cachedInput;
$MACHINE_LOCAL_STORE->addTempRoot($cachedInput->storepath) if defined $cachedInput;
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) {
$storePath = $cachedInput->storepath;
$sha256 = $cachedInput->sha256hash;
$revision = $cachedInput->revision;
@ -217,7 +217,7 @@ sub fetchInput {
($sha256, $storePath) = split ' ', grab(cmd => ["nix-prefetch-git", $clonePath, $revision], chomp => 1);
# FIXME: time window between nix-prefetch-git and addTempRoot.
addTempRoot($storePath);
$MACHINE_LOCAL_STORE->addTempRoot($storePath);
$self->{db}->txn_do(sub {
$self->{db}->resultset('CachedGitInputs')->update_or_create(

7
src/lib/Hydra/Plugin/MercurialInput.pm
View file

@ -7,7 +7,6 @@ use Digest::SHA qw(sha256_hex);
use File::Path;
use Hydra::Helper::Nix;
use Hydra::Helper::Exec;
use Nix::Store;
use Fcntl qw(:flock);
sub supportedInputTypes {
@ -68,9 +67,9 @@ sub fetchInput {
(my $cachedInput) = $self->{db}->resultset('CachedHgInputs')->search(
{uri => $uri, branch => $branch, revision => $revision});
addTempRoot($cachedInput->storepath) if defined $cachedInput;
$MACHINE_LOCAL_STORE->addTempRoot($cachedInput->storepath) if defined $cachedInput;
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) {
$storePath = $cachedInput->storepath;
$sha256 = $cachedInput->sha256hash;
} else {
@ -85,7 +84,7 @@ sub fetchInput {
($sha256, $storePath) = split ' ', $stdout;
# FIXME: time window between nix-prefetch-hg and addTempRoot.
addTempRoot($storePath);
$MACHINE_LOCAL_STORE->addTempRoot($storePath);
$self->{db}->txn_do(sub {
$self->{db}->resultset('CachedHgInputs')->update_or_create(

5
src/lib/Hydra/Plugin/PathInput.pm
View file

@ -5,7 +5,6 @@ use warnings;
use parent 'Hydra::Plugin';
use POSIX qw(strftime);
use Hydra::Helper::Nix;
use Nix::Store;
sub supportedInputTypes {
my ($self, $inputTypes) = @_;
@ -30,7 +29,7 @@ sub fetchInput {
{srcpath => $uri, lastseen => {">", $timestamp - $timeout}},
{rows => 1, order_by => "lastseen DESC"});
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) {
$storePath = $cachedInput->storepath;
$sha256 = $cachedInput->sha256hash;
$timestamp = $cachedInput->timestamp;
@ -46,7 +45,7 @@ sub fetchInput {
}
chomp $storePath;
$sha256 = (queryPathInfo($storePath, 0))[1] or die;
$sha256 = ($MACHINE_LOCAL_STORE->queryPathInfo($storePath, 0))[1] or die;
($cachedInput) = $self->{db}->resultset('CachedPathInputs')->search(
{srcpath => $uri, sha256hash => $sha256});

9
src/lib/Hydra/Plugin/SubversionInput.pm
View file

@ -7,7 +7,6 @@ use Digest::SHA qw(sha256_hex);
use Hydra::Helper::Exec;
use Hydra::Helper::Nix;
use IPC::Run;
use Nix::Store;
sub supportedInputTypes {
my ($self, $inputTypes) = @_;
@ -45,7 +44,7 @@ sub fetchInput {
(my $cachedInput) = $self->{db}->resultset('CachedSubversionInputs')->search(
{uri => $uri, revision => $revision});
addTempRoot($cachedInput->storepath) if defined $cachedInput;
$MACHINE_LOCAL_STORE->addTempRoot($cachedInput->storepath) if defined $cachedInput;
if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
$storePath = $cachedInput->storepath;
@ -62,16 +61,16 @@ sub fetchInput {
die "error checking out Subversion repo at `$uri':\n$stderr" if $res;
if ($type eq "svn-checkout") {
$storePath = addToStore($wcPath, 1, "sha256");
$storePath = $MACHINE_LOCAL_STORE->addToStore($wcPath, 1, "sha256");
} else {
# Hm, if the Nix Perl bindings supported filters in
# addToStore(), then we wouldn't need to make a copy here.
my $tmpDir = File::Temp->newdir("hydra-svn-export.XXXXXX", CLEANUP => 1, TMPDIR => 1) or die;
(system "svn", "export", $wcPath, "$tmpDir/source", "--quiet") == 0 or die "svn export failed";
$storePath = addToStore("$tmpDir/source", 1, "sha256");
$storePath = $MACHINE_LOCAL_STORE->addToStore("$tmpDir/source", 1, "sha256");
}
$sha256 = queryPathHash($storePath); $sha256 =~ s/sha256://;
$sha256 = $MACHINE_LOCAL_STORE->queryPathHash($storePath); $sha256 =~ s/sha256://;
$self->{db}->txn_do(sub {
$self->{db}->resultset('CachedSubversionInputs')->update_or_create(

9
src/lib/Hydra/View/NARInfo.pm
View file

@ -6,8 +6,7 @@ use File::Basename;
use Hydra::Helper::CatalystUtils;
use MIME::Base64;
use Nix::Manifest;
use Nix::Store;
use Nix::Utils;
use Hydra::Helper::Nix;
use base qw/Catalyst::View/;
sub process {
@ -17,7 +16,7 @@ sub process {
$c->response->content_type('text/x-nix-narinfo'); # !!! check MIME type
my ($deriver, $narHash, $time, $narSize, $refs) = queryPathInfo($storePath, 1);
my ($deriver, $narHash, $time, $narSize, $refs) = $MACHINE_LOCAL_STORE->queryPathInfo($storePath, 1);
my $info;
$info .= "StorePath: $storePath\n";
@ -28,8 +27,8 @@ sub process {
$info .= "References: " . join(" ", map { basename $_ } @{$refs}) . "\n";
if (defined $deriver) {
$info .= "Deriver: " . basename $deriver . "\n";
if (isValidPath($deriver)) {
my $drv = derivationFromPath($deriver);
if ($MACHINE_LOCAL_STORE->isValidPath($deriver)) {
my $drv = $MACHINE_LOCAL_STORE->derivationFromPath($deriver);
$info .= "System: $drv->{platform}\n";
}
}

8
src/script/hydra-eval-jobset
View file

@ -85,14 +85,14 @@ sub attrsToSQL {
# Fetch a store path from 'eval_substituter' if not already present.
sub getPath {
my ($path) = @_;
return 1 if isValidPath($path);
return 1 if $MACHINE_LOCAL_STORE->isValidPath($path);
my $substituter = $config->{eval_substituter};
system("nix", "--experimental-features", "nix-command", "copy", "--from", $substituter, "--", $path)
if defined $substituter;
return isValidPath($path);
return $MACHINE_LOCAL_STORE->isValidPath($path);
}
@ -143,7 +143,7 @@ sub fetchInputBuild {
, version => $version
, outputName => $mainOutput->name
};
if (isValidPath($prevBuild->drvpath)) {
if ($MACHINE_LOCAL_STORE->isValidPath($prevBuild->drvpath)) {
$result->{drvPath} = $prevBuild->drvpath;
}
@ -233,7 +233,7 @@ sub fetchInputEval {
my $out = $build->buildoutputs->find({ name => "out" });
next unless defined $out;
# FIXME: Should we fail if the path is not valid?
next unless isValidPath($out->path);
next unless $MACHINE_LOCAL_STORE->isValidPath($out->path);
$jobs->{$build->get_column('job')} = $out->path;
}

5
src/script/hydra-update-gc-roots
View file

@ -5,7 +5,6 @@ use warnings;
use File::Path;
use File::stat;
use File::Basename;
use Nix::Store;
use Hydra::Config;
use Hydra::Schema;
use Hydra::Helper::Nix;
@ -47,7 +46,7 @@ sub keepBuild {
$build->finished && ($build->buildstatus == 0 || $build->buildstatus == 6))
{
foreach my $path (split / /, $build->get_column('outpaths')) {
if (isValidPath($path)) {
if ($MACHINE_LOCAL_STORE->isValidPath($path)) {
addRoot $path;
} else {
print STDERR " warning: output ", $path, " has disappeared\n" if $build->finished;
@ -55,7 +54,7 @@ sub keepBuild {
}
}
if (!$build->finished || ($keepFailedDrvs && $build->buildstatus != 0)) {
if (isValidPath($build->drvpath)) {
if ($MACHINE_LOCAL_STORE->isValidPath($build->drvpath)) {
addRoot $build->drvpath;
} else {
print STDERR " warning: derivation ", $build->drvpath, " has disappeared\n";

1
t/s3-backup-test.pl
View file

@ -3,7 +3,6 @@ use warnings;
use File::Basename;
use Hydra::Model::DB;
use Hydra::Helper::Nix;
use Nix::Store;
use Cwd;
my $db = Hydra::Model::DB->new;