Merge pull request #1105 from DeterminateSystems/hydra-eval-jobset-notifications-sub-send_cached_finished_queued

hydra-eval-jobset: send cached_build_finished, cached_build_queued

doc/manual/src/notifications.md

@@ -8,6 +8,19 @@ Notifications are passed from `hydra-queue-runner` to `hydra-notify` through Pos
 
 Note that the notification format is subject to change and should not be considered an API. Integrate with `hydra-notify` instead of listening directly.
 
+### `cached_build_finished`
+
+* **Payload:** Exactly two values, tab separated: The ID of the evaluation which contains the finished build, followed by the ID of the finished build.
+* **When:** Issued directly after an evaluation completes, when that evaluation includes this finished build.
+* **Delivery Semantics:** At most once per evaluation.
+
+
+### `cached_build_queued`
+
+* **Payload:** Exactly two values, tab separated: The ID of the evaluation which contains the finished build, followed by the ID of the queued build.
+* **When:** Issued directly after an evaluation completes, when that evaluation includes this queued build.
+* **Delivery Semantics:** At most once per evaluation.
+
 ### `build_queued`
 
 * **Payload:** Exactly one value, the ID of the build.

src/lib/Hydra/Event.pm

@@ -2,6 +2,8 @@ package Hydra::Event;
 
 use strict;
 use warnings;
+use Hydra::Event::CachedBuildFinished;
+use Hydra::Event::CachedBuildQueued;
 use Hydra::Event::BuildFinished;
 use Hydra::Event::BuildQueued;
 use Hydra::Event::BuildStarted;
@@ -12,6 +14,8 @@ my %channels_to_events = (
   build_started => \&Hydra::Event::BuildStarted::parse,
   step_finished => \&Hydra::Event::StepFinished::parse,
   build_finished => \&Hydra::Event::BuildFinished::parse,
+  cached_build_finished => \&Hydra::Event::CachedBuildFinished::parse,
+  cached_build_queued => \&Hydra::Event::CachedBuildQueued::parse,
 );
 
 

src/lib/Hydra/Event/CachedBuildFinished.pm

@@ -0,0 +1,59 @@
+package Hydra::Event::CachedBuildFinished;
+
+use strict;
+use warnings;
+
+sub parse :prototype(@) {
+    if (@_ != 2) {
+        die "cached_build_finished: payload takes two arguments, but ", scalar(@_), " were given";
+    }
+
+    my @failures = grep(!/^\d+$/, @_);
+    if (@failures > 0) {
+        die "cached_build_finished: payload arguments should be integers, but we received the following non-integers:", @failures;
+    }
+
+    my ($evaluation_id, $build_id) = map int, @_;
+    return Hydra::Event::CachedBuildFinished->new($evaluation_id, $build_id);
+}
+
+sub new {
+    my ($self, $evaluation_id, $build_id) = @_;
+    return bless {
+        "evaluation_id" => $evaluation_id,
+        "build_id" => $build_id,
+        "evaluation" => undef,
+        "build" => undef,
+    }, $self;
+}
+
+sub interestedIn {
+    my ($self, $plugin) = @_;
+    return int(defined($plugin->can('cachedBuildFinished')));
+}
+
+sub load {
+    my ($self, $db) = @_;
+
+    if (!defined($self->{"build"})) {
+        $self->{"build"} = $db->resultset('Builds')->find($self->{"build_id"})
+            or die "build $self->{'build_id'} does not exist\n";
+    }
+
+    if (!defined($self->{"evaluation"})) {
+        $self->{"evaluation"} = $db->resultset('JobsetEvals')->find($self->{"evaluation_id"})
+            or die "evaluation $self->{'evaluation_id'} does not exist\n";
+    }
+}
+
+sub execute {
+    my ($self, $db, $plugin) = @_;
+
+    $self->load($db);
+
+    $plugin->cachedBuildFinished($self->{"evaluation"}, $self->{"build"});
+
+    return 1;
+}
+
+1;

src/lib/Hydra/Event/CachedBuildQueued.pm

@@ -0,0 +1,59 @@
+package Hydra::Event::CachedBuildQueued;
+
+use strict;
+use warnings;
+
+sub parse :prototype(@) {
+    if (@_ != 2) {
+        die "cached_build_queued: payload takes two arguments, but ", scalar(@_), " were given";
+    }
+
+    my @failures = grep(!/^\d+$/, @_);
+    if (@failures > 0) {
+        die "cached_build_queued: payload arguments should be integers, but we received the following non-integers:", @failures;
+    }
+
+    my ($evaluation_id, $build_id) = map int, @_;
+    return Hydra::Event::CachedBuildQueued->new($evaluation_id, $build_id);
+}
+
+sub new {
+    my ($self, $evaluation_id, $build_id) = @_;
+    return bless {
+        "evaluation_id" => $evaluation_id,
+        "build_id" => $build_id,
+        "evaluation" => undef,
+        "build" => undef,
+    }, $self;
+}
+
+sub interestedIn {
+    my ($self, $plugin) = @_;
+    return int(defined($plugin->can('cachedBuildQueued')));
+}
+
+sub load {
+    my ($self, $db) = @_;
+
+    if (!defined($self->{"build"})) {
+        $self->{"build"} = $db->resultset('Builds')->find($self->{"build_id"})
+            or die "build $self->{'build_id'} does not exist\n";
+    }
+
+    if (!defined($self->{"evaluation"})) {
+        $self->{"evaluation"} = $db->resultset('JobsetEvals')->find($self->{"evaluation_id"})
+            or die "evaluation $self->{'evaluation_id'} does not exist\n";
+    }
+}
+
+sub execute {
+    my ($self, $db, $plugin) = @_;
+
+    $self->load($db);
+
+    $plugin->cachedBuildQueued($self->{"evaluation"}, $self->{"build"});
+
+    return 1;
+}
+
+1;

src/lib/Hydra/Plugin.pm

@@ -36,6 +36,18 @@ sub instantiate {
 #     my ($self, $build) = @_;
 # }
 
+# # Called when build $build has been queued again by evaluation $evaluation
+# where $build has not yet finished.
+# sub cachedBuildQueued {
+#     my ($self, $evaluation, $build) = @_;
+# }
+
+# # Called when build $build is a finished build, and is
+# part evaluation $evaluation
+# sub cachedBuildFinished {
+#     my ($self, $evaluation, $build) = @_;
+# }
+
 # # Called when build $build has started.
 # sub buildStarted {
 #     my ($self, $build) = @_;

src/script/hydra-eval-jobset

@@ -399,7 +399,7 @@ sub getPrevJobsetEval {
 
 # Check whether to add the build described by $buildInfo.
 sub checkBuild {
-    my ($db, $jobset, $inputInfo, $buildInfo, $buildMap, $prevEval, $jobOutPathMap, $plugins) = @_;
+    my ($db, $jobset, $eval, $inputInfo, $buildInfo, $buildMap, $prevEval, $jobOutPathMap, $plugins) = @_;
 
     my @outputNames = sort keys %{$buildInfo->{outputs}};
     die unless scalar @outputNames;
@@ -435,10 +435,17 @@ sub checkBuild {
                 # the Nixpkgs jobset with PostgreSQL.
                 { jobset_id => $jobset->get_column('id'), job => $jobName,
                   name => $firstOutputName, path => $firstOutputPath },
-                { rows => 1, columns => ['id'], join => ['buildoutputs'] });
+                { rows => 1, columns => ['id', 'finished'], join => ['buildoutputs'] });
             if (defined $prevBuild) {
                 #print STDERR "    already scheduled/built as build ", $prevBuild->id, "\n";
                 $buildMap->{$prevBuild->id} = { id => $prevBuild->id, jobName => $jobName, new => 0, drvPath => $drvPath };
+
+                if ($prevBuild->finished) {
+                    $db->storage->dbh->do("notify cached_build_finished, ?", undef, "${\$eval->id}\t${\$prevBuild->id}");
+                } else {
+                    $db->storage->dbh->do("notify cached_build_queued, ?", undef, "${\$eval->id}\t${\$prevBuild->id}");
+                }
+
                 return;
             }
         }
@@ -723,11 +730,24 @@ sub checkJobsetWrapped {
         # current builds have been added.
         $jobset->builds->search({iscurrent => 1})->update({iscurrent => 0});
 
+        my $ev = $jobset->jobsetevals->create(
+            { hash => $argsHash
+            , evaluationerror => $evaluationErrorRecord
+            , timestamp => time
+            , checkouttime => abs(int($checkoutStop - $checkoutStart))
+            , evaltime => abs(int($evalStop - $evalStart))
+            , hasnewbuilds => 0
+            , nrbuilds => 0
+            , flake => $flakeRef
+            , nixexprinput => $jobset->nixexprinput
+            , nixexprpath => $jobset->nixexprpath
+            });
+
         # Schedule each successfully evaluated job.
         foreach my $job (permute(values %{$jobs})) {
             next if defined $job->{error};
             #print STDERR "considering job " . $project->name, ":", $jobset->name, ":", $job->{jobName} . "\n";
-            checkBuild($db, $jobset, $inputInfo, $job, \%buildMap, $prevEval, $jobOutPathMap, $plugins);
+            checkBuild($db, $jobset, $ev, $inputInfo, $job, \%buildMap, $prevEval, $jobOutPathMap, $plugins);
         }
 
         # Have any builds been added or removed since last time?
@@ -735,18 +755,9 @@ sub checkJobsetWrapped {
             (scalar(grep { $_->{new} } values(%buildMap)) > 0)
             || (defined $prevEval && $prevEval->jobsetevalmembers->count != scalar(keys %buildMap));
 
-
+        $ev->update({
-        my $ev = $jobset->jobsetevals->create(
+            hasnewbuilds => $jobsetChanged ? 1 : 0,
-            { hash => $argsHash
+            nrbuilds => $jobsetChanged ? scalar(keys %buildMap) : undef
-            , evaluationerror => $evaluationErrorRecord
-            , timestamp => time
-            , checkouttime => abs(int($checkoutStop - $checkoutStart))
-            , evaltime => abs(int($evalStop - $evalStart))
-            , hasnewbuilds => $jobsetChanged ? 1 : 0
-            , nrbuilds => $jobsetChanged ? scalar(keys %buildMap) : undef
-            , flake => $flakeRef
-            , nixexprinput => $jobset->nixexprinput
-            , nixexprpath => $jobset->nixexprpath
         });
 
         $db->storage->dbh->do("notify eval_added, ?", undef,

src/script/hydra-notify

@@ -93,11 +93,13 @@ my $task_dispatcher = Hydra::TaskDispatcher->new(
 my $dbh = $db->storage->dbh;
 
 my $listener = Hydra::PostgresListener->new($dbh);
+$listener->subscribe("build_finished");
 $listener->subscribe("build_queued");
 $listener->subscribe("build_started");
-$listener->subscribe("build_finished");
+$listener->subscribe("cached_build_finished");
-$listener->subscribe("step_finished");
+$listener->subscribe("cached_build_queued");
 $listener->subscribe("hydra_notify_dump_metrics");
+$listener->subscribe("step_finished");
 
 # Process builds that finished while hydra-notify wasn't running.
 for my $build ($db->resultset('Builds')->search(

t/Event/CachedBuildFinished.t

@@ -0,0 +1,111 @@
+use strict;
+use warnings;
+use Setup;
+
+my %ctx = test_init();
+
+require Hydra::Schema;
+require Hydra::Model::DB;
+use Hydra::Event;
+use Hydra::Event::CachedBuildFinished;
+
+use Test2::V0;
+use Test2::Tools::Exception;
+use Test2::Tools::Mock qw(mock_obj);
+
+my $db = Hydra::Model::DB->new;
+hydra_setup($db);
+
+subtest "Parsing" => sub {
+    like(
+        dies { Hydra::Event::parse_payload("cached_build_finished", "") },
+        qr/takes two arguments/,
+        "empty payload"
+    );
+    like(
+        dies { Hydra::Event::parse_payload("cached_build_finished", "abc123") },
+        qr/takes two arguments/,
+        "missing the build ID"
+    );
+
+    like(
+        dies { Hydra::Event::parse_payload("cached_build_finished", "123\t456\t789\t012\t345") },
+        qr/takes two arguments/,
+        "too many arguments"
+    );
+    like(
+        dies { Hydra::Event::parse_payload("cached_build_finished", "abc123\tdef456") },
+        qr/should be integers/,
+        "evaluation ID should be an integer"
+    );
+    like(
+        dies { Hydra::Event::parse_payload("cached_build_finished", "123\tabc123") },
+        qr/should be integers/,
+        "build ID should be an integer"
+    );
+    is(
+        Hydra::Event::parse_payload("cached_build_finished", "123\t456"),
+        Hydra::Event::CachedBuildFinished->new(123, 456),
+        "one dependent build"
+    );
+};
+
+my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"});
+my $jobset = createBaseJobset("basic", "basic.nix", $ctx{jobsdir});
+ok(evalSucceeds($jobset),               "Evaluating jobs/basic.nix should exit with return code 0");
+is(nrQueuedBuildsForJobset($jobset), 3, "Evaluating jobs/basic.nix should result in 3 builds");
+
+subtest "interested" => sub {
+    my $event = Hydra::Event::CachedBuildFinished->new(123, 456);
+
+    subtest "A plugin which does not implement the API" => sub {
+        my $plugin = {};
+        my $mock = mock_obj $plugin => ();
+
+        is($event->interestedIn($plugin), 0, "The plugin is not interesting.");
+    };
+
+    subtest "A plugin which does implement the API" => sub {
+        my $plugin = {};
+        my $mock = mock_obj $plugin => (
+            add => [
+                "cachedBuildFinished" => sub {}
+            ]
+        );
+
+        is($event->interestedIn($plugin), 1, "The plugin is interesting.");
+    };
+};
+
+subtest "load" => sub {
+    my ($build) = $db->resultset('Builds')->search({ }, { limit => 1 })->single;
+    my $evaluation = $build->jobsetevals->search({}, { limit => 1 })->single;
+
+    my $event = Hydra::Event::CachedBuildFinished->new($evaluation->id, $build->id);
+
+    $event->load($db);
+    is($event->{"evaluation"}->id, $evaluation->id, "The evaluation record matches.");
+    is($event->{"build"}->id, $build->id, "The build record matches.");
+
+    # Create a fake "plugin" with a cachedBuildFinished sub, the sub sets this
+    # global passedEvaluation and passedBuild variables for verifying.
+    my $passedEvaluation;
+    my $passedBuild;
+    my $plugin = {};
+    my $mock = mock_obj $plugin => (
+        add => [
+            "cachedBuildFinished" => sub {
+                my ($self, $evaluation, $build) = @_;
+                $passedEvaluation = $evaluation;
+                $passedBuild = $build;
+            }
+        ]
+    );
+
+    $event->execute($db, $plugin);
+
+    is($passedEvaluation->id, $evaluation->id, "The plugin's cachedBuildFinished hook is called with a matching evaluation");
+    is($passedBuild->id, $build->id, "The plugin's cachedBuildFinished hook is called with a matching build");
+};
+
+done_testing;

t/Event/CachedBuildQueued.t

@@ -0,0 +1,111 @@
+use strict;
+use warnings;
+use Setup;
+
+my %ctx = test_init();
+
+require Hydra::Schema;
+require Hydra::Model::DB;
+use Hydra::Event;
+use Hydra::Event::CachedBuildQueued;
+
+use Test2::V0;
+use Test2::Tools::Exception;
+use Test2::Tools::Mock qw(mock_obj);
+
+my $db = Hydra::Model::DB->new;
+hydra_setup($db);
+
+subtest "Parsing" => sub {
+    like(
+        dies { Hydra::Event::parse_payload("cached_build_queued", "") },
+        qr/takes two arguments/,
+        "empty payload"
+    );
+    like(
+        dies { Hydra::Event::parse_payload("cached_build_queued", "abc123") },
+        qr/takes two arguments/,
+        "missing the build ID"
+    );
+
+    like(
+        dies { Hydra::Event::parse_payload("cached_build_queued", "123\t456\t789\t012\t345") },
+        qr/takes two arguments/,
+        "too many arguments"
+    );
+    like(
+        dies { Hydra::Event::parse_payload("cached_build_queued", "abc123\tdef456") },
+        qr/should be integers/,
+        "evaluation ID should be an integer"
+    );
+    like(
+        dies { Hydra::Event::parse_payload("cached_build_queued", "123\tabc123") },
+        qr/should be integers/,
+        "build ID should be an integer"
+    );
+    is(
+        Hydra::Event::parse_payload("cached_build_queued", "123\t456"),
+        Hydra::Event::CachedBuildQueued->new(123, 456),
+        "one dependent build"
+    );
+};
+
+my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"});
+my $jobset = createBaseJobset("basic", "basic.nix", $ctx{jobsdir});
+ok(evalSucceeds($jobset),               "Evaluating jobs/basic.nix should exit with return code 0");
+is(nrQueuedBuildsForJobset($jobset), 3, "Evaluating jobs/basic.nix should result in 3 builds");
+
+subtest "interested" => sub {
+    my $event = Hydra::Event::CachedBuildQueued->new(123, 456);
+
+    subtest "A plugin which does not implement the API" => sub {
+        my $plugin = {};
+        my $mock = mock_obj $plugin => ();
+
+        is($event->interestedIn($plugin), 0, "The plugin is not interesting.");
+    };
+
+    subtest "A plugin which does implement the API" => sub {
+        my $plugin = {};
+        my $mock = mock_obj $plugin => (
+            add => [
+                "cachedBuildQueued" => sub {}
+            ]
+        );
+
+        is($event->interestedIn($plugin), 1, "The plugin is interesting.");
+    };
+};
+
+subtest "load" => sub {
+    my ($build) = $db->resultset('Builds')->search({ }, { limit => 1 })->single;
+    my $evaluation = $build->jobsetevals->search({}, { limit => 1 })->single;
+
+    my $event = Hydra::Event::CachedBuildQueued->new($evaluation->id, $build->id);
+
+    $event->load($db);
+    is($event->{"evaluation"}->id, $evaluation->id, "The evaluation record matches.");
+    is($event->{"build"}->id, $build->id, "The build record matches.");
+
+    # Create a fake "plugin" with a cachedBuildQueued sub, the sub sets this
+    # global passedEvaluation and passedBuild variables for verifying.
+    my $passedEvaluation;
+    my $passedBuild;
+    my $plugin = {};
+    my $mock = mock_obj $plugin => (
+        add => [
+            "cachedBuildQueued" => sub {
+                my ($self, $evaluation, $build) = @_;
+                $passedEvaluation = $evaluation;
+                $passedBuild = $build;
+            }
+        ]
+    );
+
+    $event->execute($db, $plugin);
+
+    is($passedEvaluation->id, $evaluation->id, "The plugin's cachedBuildQueued hook is called with a matching evaluation");
+    is($passedBuild->id, $build->id, "The plugin's cachedBuildQueued hook is called with a matching build");
+};
+
+done_testing;

t/scripts/hydra-eval-jobset/notifications.t

@@ -69,7 +69,31 @@ subtest "on a fresh evaluation with changed sources" => sub {
     ok(evalSucceeds($builds->{"variable-job"}->jobset), "evaluating for the third time");
     is($listener->block_for_messages(0)->()->{"channel"}, "eval_started", "the evaluation started");
 
-    is($listener->block_for_messages(0)->()->{"channel"}, "build_queued", "expect only one new build being queued");
+    # The order of builds is randomized when writing to the database,
+    # so we can't expect the list in any specific order here.
+    is(
+        [sort(
+            $listener->block_for_messages(0)->()->{"channel"},
+            $listener->block_for_messages(0)->()->{"channel"},
+            $listener->block_for_messages(0)->()->{"channel"},
+            $listener->block_for_messages(0)->()->{"channel"}
+        )],
+        [
+            # The `variable-job` build since it is the only one that is
+            # totally different in this evaluation.
+            "build_queued",
+
+            # The next two are `stable-job-passing` and `stable-job-failing`,
+            # since those are the two we explicitly built above
+            "cached_build_finished",
+            "cached_build_finished",
+
+            # Finally, this should be `stable-job-queued` since we never
+            # built it.
+            "cached_build_queued",
+        ],
+        "we get a notice that a build is queued, one is still queued from a previous eval"
+    );
 
     is($listener->block_for_messages(0)->()->{"channel"}, "eval_added", "a new evaluation was added");
     is($listener->block_for_messages(0)->()->{"channel"}, "builds_added", "a new build was added");