ma27/hydra
0
0
Fork 0
forked from lix-project/hydra
Code Pull requests Activity

Cache flake-based jobset evaluations

Browse source
This commit is contained in:
Eelco Dolstra 2019-05-11 00:41:13 +02:00
parent 30e8fe951b
commit ed00f0b25e
No known key found for this signature in database
GPG key ID: 8170B4726D7198DE
4 changed files with 25 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
src/lib/Hydra/Schema/JobsetEvals.pm
View file

@ -88,6 +88,11 @@ __PACKAGE__->table("JobsetEvals");
data_type: 'integer' data_type: 'integer'
is_nullable: 1 is_nullable: 1
=head2 flake
data_type: 'text'
is_nullable: 1
=cut =cut
__PACKAGE__->add_columns( __PACKAGE__->add_columns(
@ -111,6 +116,8 @@ __PACKAGE__->add_columns(
{ data_type => "integer", is_nullable => 1 }, { data_type => "integer", is_nullable => 1 },
"nrsucceeded", "nrsucceeded",
{ data_type => "integer", is_nullable => 1 }, { data_type => "integer", is_nullable => 1 },
"flake",
{ data_type => "text", is_nullable => 1 },
); );
=head1 PRIMARY KEY =head1 PRIMARY KEY
@ -188,8 +195,8 @@ __PACKAGE__->belongs_to(
); );
# Created by DBIx::Class::Schema::Loader v0.07033 @ 2013-06-13 01:54:50 # Created by DBIx::Class::Schema::Loader v0.07049 @ 2019-05-11 00:16:10
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:SlEiF8oN6FBK262uSiMKiw # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:XwlJFCJiS0LHsLg2fFqfUg
__PACKAGE__->has_many( __PACKAGE__->has_many(
"buildIds", "buildIds",

15
src/script/hydra-eval-jobset
View file

@ -607,6 +607,16 @@ sub checkJobsetWrapped {
}; };
my $fetchError = $@; my $fetchError = $@;
my $flakeRef = $jobset->flake;
if (defined $flakeRef) {
(my $res, my $json, my $stderr) = captureStdoutStderr(
600, "nix", "flake", "info", "--tarball-ttl", 0, "--json", "--", $flakeRef);
die "'nix flake info' returned " . ($res & 127 ? "signal $res" : "exit code " . ($res >> 8))
. ":\n" . ($stderr ? decode("utf-8", $stderr) : "(no output)\n")
if $res;
$flakeRef = decode_json($json)->{'uri'};
}
Net::Statsd::increment("hydra.evaluator.checkouts"); Net::Statsd::increment("hydra.evaluator.checkouts");
my $checkoutStop = clock_gettime(CLOCK_MONOTONIC); my $checkoutStop = clock_gettime(CLOCK_MONOTONIC);
Net::Statsd::timing("hydra.evaluator.checkout_time", int(($checkoutStop - $checkoutStart) * 1000)); Net::Statsd::timing("hydra.evaluator.checkout_time", int(($checkoutStop - $checkoutStart) * 1000));
@ -626,7 +636,7 @@ sub checkJobsetWrapped {
my @args = ($jobset->nixexprinput, $jobset->nixexprpath, inputsToArgs($inputInfo, $exprType)); my @args = ($jobset->nixexprinput, $jobset->nixexprpath, inputsToArgs($inputInfo, $exprType));
my $argsHash = sha256_hex("@args"); my $argsHash = sha256_hex("@args");
my $prevEval = getPrevJobsetEval($db, $jobset, 0); my $prevEval = getPrevJobsetEval($db, $jobset, 0);
if (defined $prevEval && $prevEval->hash eq $argsHash && !$dryRun && !$jobset->forceeval) { if (defined $prevEval && $prevEval->hash eq $argsHash && !$dryRun && !$jobset->forceeval && $prevEval->flake eq $flakeRef) {
print STDERR " jobset is unchanged, skipping\n"; print STDERR " jobset is unchanged, skipping\n";
Net::Statsd::increment("hydra.evaluator.unchanged_checkouts"); Net::Statsd::increment("hydra.evaluator.unchanged_checkouts");
txn_do($db, sub { txn_do($db, sub {
@ -637,7 +647,7 @@ sub checkJobsetWrapped {
# Evaluate the job expression. # Evaluate the job expression.
my $evalStart = clock_gettime(CLOCK_MONOTONIC); my $evalStart = clock_gettime(CLOCK_MONOTONIC);
my $jobs = evalJobs($inputInfo, $exprType, $jobset->nixexprinput, $jobset->nixexprpath, $jobset->flake); my $jobs = evalJobs($inputInfo, $exprType, $jobset->nixexprinput, $jobset->nixexprpath, $flakeRef);
my $evalStop = clock_gettime(CLOCK_MONOTONIC); my $evalStop = clock_gettime(CLOCK_MONOTONIC);
if ($jobsetsJobset) { if ($jobsetsJobset) {
@ -697,6 +707,7 @@ sub checkJobsetWrapped {
, evaltime => abs(int($evalStop - $evalStart)) , evaltime => abs(int($evalStop - $evalStart))
, hasnewbuilds => $jobsetChanged ? 1 : 0 , hasnewbuilds => $jobsetChanged ? 1 : 0
, nrbuilds => $jobsetChanged ? scalar(keys %buildMap) : undef , nrbuilds => $jobsetChanged ? scalar(keys %buildMap) : undef
, flake => $flakeRef
}); });
if ($jobsetChanged) { if ($jobsetChanged) {

2
src/sql/hydra.sql
View file

@ -527,6 +527,8 @@ create table JobsetEvals (
nrBuilds integer, nrBuilds integer,
nrSucceeded integer, -- set lazily when all builds are finished nrSucceeded integer, -- set lazily when all builds are finished
flake text, -- immutable flake reference
foreign key (project) references Projects(name) on delete cascade on update cascade, foreign key (project) references Projects(name) on delete cascade on update cascade,
foreign key (project, jobset) references Jobsets(project, name) on delete cascade on update cascade foreign key (project, jobset) references Jobsets(project, name) on delete cascade on update cascade
); );

1
src/sql/upgrade-57.sql
View file

@ -4,3 +4,4 @@ alter table Jobsets add column type integer default 0;
alter table Jobsets add column flake text; alter table Jobsets add column flake text;
alter table Jobsets add check ((type = 0) = (nixExprInput is not null and nixExprPath is not null)); alter table Jobsets add check ((type = 0) = (nixExprInput is not null and nixExprPath is not null));
alter table Jobsets add check ((type = 1) = (flake is not null)); alter table Jobsets add check ((type = 1) = (flake is not null));
alter table JobsetEvals add column flake text;