Merge pull request #1105 from DeterminateSystems/hydra-eval-jobset-notifications-sub-send_cached_finished_queued

hydra-eval-jobset: send cached_build_finished, cached_build_queued
This commit is contained in:
Graham Christensen 2022-01-14 09:25:51 -05:00 committed by GitHub
commit 526a21ef7e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 425 additions and 19 deletions

View file

@ -8,6 +8,19 @@ Notifications are passed from `hydra-queue-runner` to `hydra-notify` through Pos
Note that the notification format is subject to change and should not be considered an API. Integrate with `hydra-notify` instead of listening directly. Note that the notification format is subject to change and should not be considered an API. Integrate with `hydra-notify` instead of listening directly.
### `cached_build_finished`
* **Payload:** Exactly two values, tab separated: The ID of the evaluation which contains the finished build, followed by the ID of the finished build.
* **When:** Issued directly after an evaluation completes, when that evaluation includes this finished build.
* **Delivery Semantics:** At most once per evaluation.
### `cached_build_queued`
* **Payload:** Exactly two values, tab separated: The ID of the evaluation which contains the finished build, followed by the ID of the queued build.
* **When:** Issued directly after an evaluation completes, when that evaluation includes this queued build.
* **Delivery Semantics:** At most once per evaluation.
### `build_queued` ### `build_queued`
* **Payload:** Exactly one value, the ID of the build. * **Payload:** Exactly one value, the ID of the build.

View file

@ -2,6 +2,8 @@ package Hydra::Event;
use strict; use strict;
use warnings; use warnings;
use Hydra::Event::CachedBuildFinished;
use Hydra::Event::CachedBuildQueued;
use Hydra::Event::BuildFinished; use Hydra::Event::BuildFinished;
use Hydra::Event::BuildQueued; use Hydra::Event::BuildQueued;
use Hydra::Event::BuildStarted; use Hydra::Event::BuildStarted;
@ -12,6 +14,8 @@ my %channels_to_events = (
build_started => \&Hydra::Event::BuildStarted::parse, build_started => \&Hydra::Event::BuildStarted::parse,
step_finished => \&Hydra::Event::StepFinished::parse, step_finished => \&Hydra::Event::StepFinished::parse,
build_finished => \&Hydra::Event::BuildFinished::parse, build_finished => \&Hydra::Event::BuildFinished::parse,
cached_build_finished => \&Hydra::Event::CachedBuildFinished::parse,
cached_build_queued => \&Hydra::Event::CachedBuildQueued::parse,
); );

View file

@ -0,0 +1,59 @@
package Hydra::Event::CachedBuildFinished;
use strict;
use warnings;
sub parse :prototype(@) {
if (@_ != 2) {
die "cached_build_finished: payload takes two arguments, but ", scalar(@_), " were given";
}
my @failures = grep(!/^\d+$/, @_);
if (@failures > 0) {
die "cached_build_finished: payload arguments should be integers, but we received the following non-integers:", @failures;
}
my ($evaluation_id, $build_id) = map int, @_;
return Hydra::Event::CachedBuildFinished->new($evaluation_id, $build_id);
}
sub new {
my ($self, $evaluation_id, $build_id) = @_;
return bless {
"evaluation_id" => $evaluation_id,
"build_id" => $build_id,
"evaluation" => undef,
"build" => undef,
}, $self;
}
sub interestedIn {
my ($self, $plugin) = @_;
return int(defined($plugin->can('cachedBuildFinished')));
}
sub load {
my ($self, $db) = @_;
if (!defined($self->{"build"})) {
$self->{"build"} = $db->resultset('Builds')->find($self->{"build_id"})
or die "build $self->{'build_id'} does not exist\n";
}
if (!defined($self->{"evaluation"})) {
$self->{"evaluation"} = $db->resultset('JobsetEvals')->find($self->{"evaluation_id"})
or die "evaluation $self->{'evaluation_id'} does not exist\n";
}
}
sub execute {
my ($self, $db, $plugin) = @_;
$self->load($db);
$plugin->cachedBuildFinished($self->{"evaluation"}, $self->{"build"});
return 1;
}
1;

View file

@ -0,0 +1,59 @@
package Hydra::Event::CachedBuildQueued;
use strict;
use warnings;
sub parse :prototype(@) {
if (@_ != 2) {
die "cached_build_queued: payload takes two arguments, but ", scalar(@_), " were given";
}
my @failures = grep(!/^\d+$/, @_);
if (@failures > 0) {
die "cached_build_queued: payload arguments should be integers, but we received the following non-integers:", @failures;
}
my ($evaluation_id, $build_id) = map int, @_;
return Hydra::Event::CachedBuildQueued->new($evaluation_id, $build_id);
}
sub new {
my ($self, $evaluation_id, $build_id) = @_;
return bless {
"evaluation_id" => $evaluation_id,
"build_id" => $build_id,
"evaluation" => undef,
"build" => undef,
}, $self;
}
sub interestedIn {
my ($self, $plugin) = @_;
return int(defined($plugin->can('cachedBuildQueued')));
}
sub load {
my ($self, $db) = @_;
if (!defined($self->{"build"})) {
$self->{"build"} = $db->resultset('Builds')->find($self->{"build_id"})
or die "build $self->{'build_id'} does not exist\n";
}
if (!defined($self->{"evaluation"})) {
$self->{"evaluation"} = $db->resultset('JobsetEvals')->find($self->{"evaluation_id"})
or die "evaluation $self->{'evaluation_id'} does not exist\n";
}
}
sub execute {
my ($self, $db, $plugin) = @_;
$self->load($db);
$plugin->cachedBuildQueued($self->{"evaluation"}, $self->{"build"});
return 1;
}
1;

View file

@ -36,6 +36,18 @@ sub instantiate {
# my ($self, $build) = @_; # my ($self, $build) = @_;
# } # }
# # Called when build $build has been queued again by evaluation $evaluation
# where $build has not yet finished.
# sub cachedBuildQueued {
# my ($self, $evaluation, $build) = @_;
# }
# # Called when build $build is a finished build, and is
# part evaluation $evaluation
# sub cachedBuildFinished {
# my ($self, $evaluation, $build) = @_;
# }
# # Called when build $build has started. # # Called when build $build has started.
# sub buildStarted { # sub buildStarted {
# my ($self, $build) = @_; # my ($self, $build) = @_;

View file

@ -399,7 +399,7 @@ sub getPrevJobsetEval {
# Check whether to add the build described by $buildInfo. # Check whether to add the build described by $buildInfo.
sub checkBuild { sub checkBuild {
my ($db, $jobset, $inputInfo, $buildInfo, $buildMap, $prevEval, $jobOutPathMap, $plugins) = @_; my ($db, $jobset, $eval, $inputInfo, $buildInfo, $buildMap, $prevEval, $jobOutPathMap, $plugins) = @_;
my @outputNames = sort keys %{$buildInfo->{outputs}}; my @outputNames = sort keys %{$buildInfo->{outputs}};
die unless scalar @outputNames; die unless scalar @outputNames;
@ -435,10 +435,17 @@ sub checkBuild {
# the Nixpkgs jobset with PostgreSQL. # the Nixpkgs jobset with PostgreSQL.
{ jobset_id => $jobset->get_column('id'), job => $jobName, { jobset_id => $jobset->get_column('id'), job => $jobName,
name => $firstOutputName, path => $firstOutputPath }, name => $firstOutputName, path => $firstOutputPath },
{ rows => 1, columns => ['id'], join => ['buildoutputs'] }); { rows => 1, columns => ['id', 'finished'], join => ['buildoutputs'] });
if (defined $prevBuild) { if (defined $prevBuild) {
#print STDERR " already scheduled/built as build ", $prevBuild->id, "\n"; #print STDERR " already scheduled/built as build ", $prevBuild->id, "\n";
$buildMap->{$prevBuild->id} = { id => $prevBuild->id, jobName => $jobName, new => 0, drvPath => $drvPath }; $buildMap->{$prevBuild->id} = { id => $prevBuild->id, jobName => $jobName, new => 0, drvPath => $drvPath };
if ($prevBuild->finished) {
$db->storage->dbh->do("notify cached_build_finished, ?", undef, "${\$eval->id}\t${\$prevBuild->id}");
} else {
$db->storage->dbh->do("notify cached_build_queued, ?", undef, "${\$eval->id}\t${\$prevBuild->id}");
}
return; return;
} }
} }
@ -723,11 +730,24 @@ sub checkJobsetWrapped {
# current builds have been added. # current builds have been added.
$jobset->builds->search({iscurrent => 1})->update({iscurrent => 0}); $jobset->builds->search({iscurrent => 1})->update({iscurrent => 0});
my $ev = $jobset->jobsetevals->create(
{ hash => $argsHash
, evaluationerror => $evaluationErrorRecord
, timestamp => time
, checkouttime => abs(int($checkoutStop - $checkoutStart))
, evaltime => abs(int($evalStop - $evalStart))
, hasnewbuilds => 0
, nrbuilds => 0
, flake => $flakeRef
, nixexprinput => $jobset->nixexprinput
, nixexprpath => $jobset->nixexprpath
});
# Schedule each successfully evaluated job. # Schedule each successfully evaluated job.
foreach my $job (permute(values %{$jobs})) { foreach my $job (permute(values %{$jobs})) {
next if defined $job->{error}; next if defined $job->{error};
#print STDERR "considering job " . $project->name, ":", $jobset->name, ":", $job->{jobName} . "\n"; #print STDERR "considering job " . $project->name, ":", $jobset->name, ":", $job->{jobName} . "\n";
checkBuild($db, $jobset, $inputInfo, $job, \%buildMap, $prevEval, $jobOutPathMap, $plugins); checkBuild($db, $jobset, $ev, $inputInfo, $job, \%buildMap, $prevEval, $jobOutPathMap, $plugins);
} }
# Have any builds been added or removed since last time? # Have any builds been added or removed since last time?
@ -735,19 +755,10 @@ sub checkJobsetWrapped {
(scalar(grep { $_->{new} } values(%buildMap)) > 0) (scalar(grep { $_->{new} } values(%buildMap)) > 0)
|| (defined $prevEval && $prevEval->jobsetevalmembers->count != scalar(keys %buildMap)); || (defined $prevEval && $prevEval->jobsetevalmembers->count != scalar(keys %buildMap));
$ev->update({
my $ev = $jobset->jobsetevals->create( hasnewbuilds => $jobsetChanged ? 1 : 0,
{ hash => $argsHash nrbuilds => $jobsetChanged ? scalar(keys %buildMap) : undef
, evaluationerror => $evaluationErrorRecord });
, timestamp => time
, checkouttime => abs(int($checkoutStop - $checkoutStart))
, evaltime => abs(int($evalStop - $evalStart))
, hasnewbuilds => $jobsetChanged ? 1 : 0
, nrbuilds => $jobsetChanged ? scalar(keys %buildMap) : undef
, flake => $flakeRef
, nixexprinput => $jobset->nixexprinput
, nixexprpath => $jobset->nixexprpath
});
$db->storage->dbh->do("notify eval_added, ?", undef, $db->storage->dbh->do("notify eval_added, ?", undef,
join('\t', $tmpId, $ev->id)); join('\t', $tmpId, $ev->id));

View file

@ -93,11 +93,13 @@ my $task_dispatcher = Hydra::TaskDispatcher->new(
my $dbh = $db->storage->dbh; my $dbh = $db->storage->dbh;
my $listener = Hydra::PostgresListener->new($dbh); my $listener = Hydra::PostgresListener->new($dbh);
$listener->subscribe("build_finished");
$listener->subscribe("build_queued"); $listener->subscribe("build_queued");
$listener->subscribe("build_started"); $listener->subscribe("build_started");
$listener->subscribe("build_finished"); $listener->subscribe("cached_build_finished");
$listener->subscribe("step_finished"); $listener->subscribe("cached_build_queued");
$listener->subscribe("hydra_notify_dump_metrics"); $listener->subscribe("hydra_notify_dump_metrics");
$listener->subscribe("step_finished");
# Process builds that finished while hydra-notify wasn't running. # Process builds that finished while hydra-notify wasn't running.
for my $build ($db->resultset('Builds')->search( for my $build ($db->resultset('Builds')->search(

View file

@ -0,0 +1,111 @@
use strict;
use warnings;
use Setup;
my %ctx = test_init();
require Hydra::Schema;
require Hydra::Model::DB;
use Hydra::Event;
use Hydra::Event::CachedBuildFinished;
use Test2::V0;
use Test2::Tools::Exception;
use Test2::Tools::Mock qw(mock_obj);
my $db = Hydra::Model::DB->new;
hydra_setup($db);
subtest "Parsing" => sub {
like(
dies { Hydra::Event::parse_payload("cached_build_finished", "") },
qr/takes two arguments/,
"empty payload"
);
like(
dies { Hydra::Event::parse_payload("cached_build_finished", "abc123") },
qr/takes two arguments/,
"missing the build ID"
);
like(
dies { Hydra::Event::parse_payload("cached_build_finished", "123\t456\t789\t012\t345") },
qr/takes two arguments/,
"too many arguments"
);
like(
dies { Hydra::Event::parse_payload("cached_build_finished", "abc123\tdef456") },
qr/should be integers/,
"evaluation ID should be an integer"
);
like(
dies { Hydra::Event::parse_payload("cached_build_finished", "123\tabc123") },
qr/should be integers/,
"build ID should be an integer"
);
is(
Hydra::Event::parse_payload("cached_build_finished", "123\t456"),
Hydra::Event::CachedBuildFinished->new(123, 456),
"one dependent build"
);
};
my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"});
my $jobset = createBaseJobset("basic", "basic.nix", $ctx{jobsdir});
ok(evalSucceeds($jobset), "Evaluating jobs/basic.nix should exit with return code 0");
is(nrQueuedBuildsForJobset($jobset), 3, "Evaluating jobs/basic.nix should result in 3 builds");
subtest "interested" => sub {
my $event = Hydra::Event::CachedBuildFinished->new(123, 456);
subtest "A plugin which does not implement the API" => sub {
my $plugin = {};
my $mock = mock_obj $plugin => ();
is($event->interestedIn($plugin), 0, "The plugin is not interesting.");
};
subtest "A plugin which does implement the API" => sub {
my $plugin = {};
my $mock = mock_obj $plugin => (
add => [
"cachedBuildFinished" => sub {}
]
);
is($event->interestedIn($plugin), 1, "The plugin is interesting.");
};
};
subtest "load" => sub {
my ($build) = $db->resultset('Builds')->search({ }, { limit => 1 })->single;
my $evaluation = $build->jobsetevals->search({}, { limit => 1 })->single;
my $event = Hydra::Event::CachedBuildFinished->new($evaluation->id, $build->id);
$event->load($db);
is($event->{"evaluation"}->id, $evaluation->id, "The evaluation record matches.");
is($event->{"build"}->id, $build->id, "The build record matches.");
# Create a fake "plugin" with a cachedBuildFinished sub, the sub sets this
# global passedEvaluation and passedBuild variables for verifying.
my $passedEvaluation;
my $passedBuild;
my $plugin = {};
my $mock = mock_obj $plugin => (
add => [
"cachedBuildFinished" => sub {
my ($self, $evaluation, $build) = @_;
$passedEvaluation = $evaluation;
$passedBuild = $build;
}
]
);
$event->execute($db, $plugin);
is($passedEvaluation->id, $evaluation->id, "The plugin's cachedBuildFinished hook is called with a matching evaluation");
is($passedBuild->id, $build->id, "The plugin's cachedBuildFinished hook is called with a matching build");
};
done_testing;

111
t/Event/CachedBuildQueued.t Normal file
View file

@ -0,0 +1,111 @@
use strict;
use warnings;
use Setup;
my %ctx = test_init();
require Hydra::Schema;
require Hydra::Model::DB;
use Hydra::Event;
use Hydra::Event::CachedBuildQueued;
use Test2::V0;
use Test2::Tools::Exception;
use Test2::Tools::Mock qw(mock_obj);
my $db = Hydra::Model::DB->new;
hydra_setup($db);
subtest "Parsing" => sub {
like(
dies { Hydra::Event::parse_payload("cached_build_queued", "") },
qr/takes two arguments/,
"empty payload"
);
like(
dies { Hydra::Event::parse_payload("cached_build_queued", "abc123") },
qr/takes two arguments/,
"missing the build ID"
);
like(
dies { Hydra::Event::parse_payload("cached_build_queued", "123\t456\t789\t012\t345") },
qr/takes two arguments/,
"too many arguments"
);
like(
dies { Hydra::Event::parse_payload("cached_build_queued", "abc123\tdef456") },
qr/should be integers/,
"evaluation ID should be an integer"
);
like(
dies { Hydra::Event::parse_payload("cached_build_queued", "123\tabc123") },
qr/should be integers/,
"build ID should be an integer"
);
is(
Hydra::Event::parse_payload("cached_build_queued", "123\t456"),
Hydra::Event::CachedBuildQueued->new(123, 456),
"one dependent build"
);
};
my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"});
my $jobset = createBaseJobset("basic", "basic.nix", $ctx{jobsdir});
ok(evalSucceeds($jobset), "Evaluating jobs/basic.nix should exit with return code 0");
is(nrQueuedBuildsForJobset($jobset), 3, "Evaluating jobs/basic.nix should result in 3 builds");
subtest "interested" => sub {
my $event = Hydra::Event::CachedBuildQueued->new(123, 456);
subtest "A plugin which does not implement the API" => sub {
my $plugin = {};
my $mock = mock_obj $plugin => ();
is($event->interestedIn($plugin), 0, "The plugin is not interesting.");
};
subtest "A plugin which does implement the API" => sub {
my $plugin = {};
my $mock = mock_obj $plugin => (
add => [
"cachedBuildQueued" => sub {}
]
);
is($event->interestedIn($plugin), 1, "The plugin is interesting.");
};
};
subtest "load" => sub {
my ($build) = $db->resultset('Builds')->search({ }, { limit => 1 })->single;
my $evaluation = $build->jobsetevals->search({}, { limit => 1 })->single;
my $event = Hydra::Event::CachedBuildQueued->new($evaluation->id, $build->id);
$event->load($db);
is($event->{"evaluation"}->id, $evaluation->id, "The evaluation record matches.");
is($event->{"build"}->id, $build->id, "The build record matches.");
# Create a fake "plugin" with a cachedBuildQueued sub, the sub sets this
# global passedEvaluation and passedBuild variables for verifying.
my $passedEvaluation;
my $passedBuild;
my $plugin = {};
my $mock = mock_obj $plugin => (
add => [
"cachedBuildQueued" => sub {
my ($self, $evaluation, $build) = @_;
$passedEvaluation = $evaluation;
$passedBuild = $build;
}
]
);
$event->execute($db, $plugin);
is($passedEvaluation->id, $evaluation->id, "The plugin's cachedBuildQueued hook is called with a matching evaluation");
is($passedBuild->id, $build->id, "The plugin's cachedBuildQueued hook is called with a matching build");
};
done_testing;

View file

@ -69,7 +69,31 @@ subtest "on a fresh evaluation with changed sources" => sub {
ok(evalSucceeds($builds->{"variable-job"}->jobset), "evaluating for the third time"); ok(evalSucceeds($builds->{"variable-job"}->jobset), "evaluating for the third time");
is($listener->block_for_messages(0)->()->{"channel"}, "eval_started", "the evaluation started"); is($listener->block_for_messages(0)->()->{"channel"}, "eval_started", "the evaluation started");
is($listener->block_for_messages(0)->()->{"channel"}, "build_queued", "expect only one new build being queued"); # The order of builds is randomized when writing to the database,
# so we can't expect the list in any specific order here.
is(
[sort(
$listener->block_for_messages(0)->()->{"channel"},
$listener->block_for_messages(0)->()->{"channel"},
$listener->block_for_messages(0)->()->{"channel"},
$listener->block_for_messages(0)->()->{"channel"}
)],
[
# The `variable-job` build since it is the only one that is
# totally different in this evaluation.
"build_queued",
# The next two are `stable-job-passing` and `stable-job-failing`,
# since those are the two we explicitly built above
"cached_build_finished",
"cached_build_finished",
# Finally, this should be `stable-job-queued` since we never
# built it.
"cached_build_queued",
],
"we get a notice that a build is queued, one is still queued from a previous eval"
);
is($listener->block_for_messages(0)->()->{"channel"}, "eval_added", "a new evaluation was added"); is($listener->block_for_messages(0)->()->{"channel"}, "eval_added", "a new evaluation was added");
is($listener->block_for_messages(0)->()->{"channel"}, "builds_added", "a new build was added"); is($listener->block_for_messages(0)->()->{"channel"}, "builds_added", "a new build was added");