Merge pull request #1103 from DeterminateSystems/runcommand/dynamic

Dynamic RunCommand

doc/manual/src/SUMMARY.md

@@ -7,6 +7,7 @@
 - [Hydra jobs](./jobs.md)
 - [Plugins](./plugins/README.md)
   - [Declarative Projects](./plugins/declarative-projects.md)
+  - [RunCommand](./plugins/RunCommand.md)
 - [Using the external API](api.md)
 - [Webhooks](webhooks.md)
 - [Monitoring Hydra](./monitoring/README.md)

doc/manual/src/plugins/README.md

@@ -192,10 +192,12 @@ Writes InfluxDB events when a builds finished.
 - `influxdb.url`
 - `influxdb.db`
 
-## Run command
+## RunCommand
 
 Runs a shell command when the build is finished.
 
+See [The RunCommand Plugin](./RunCommand.md) for more information.
+
 ### Configuration options:
 
 - `runcommand.[].job`

doc/manual/src/plugins/RunCommand.md

@@ -0,0 +1,83 @@
+## The RunCommand Plugin
+
+Hydra supports executing a program after certain builds finish.
+This behavior is disabled by default.
+
+Hydra executes these commands under the `hydra-notify` service.
+
+### Static Commands
+
+Configure specific commands to execute after the specified matching job finishes.
+
+#### Configuration
+
+- `runcommand.[].job`
+
+A matcher for jobs to match in the format `project:jobset:job`. Defaults to `*:*:*`.
+
+**Note:** This matcher format is not a regular expression.
+The `*` is a wildcard for that entire section, partial matches are not supported.
+
+- `runcommand.[].command`
+
+Command to run. Can use the `$HYDRA_JSON` environment variable to access information about the build.
+
+### Example
+
+```xml
+<runcommand>
+  job = myProject:*:*
+  command = cat $HYDRA_JSON > /tmp/hydra-output
+</runcommand>
+```
+
+### Dynamic Commands
+
+Hydra can optionally run RunCommand hooks defined dynamically by the jobset. In
+order to enable dynamic commands, you must enable this feature in your
+`hydra.conf`, *as well as* in the parent project and jobset configuration.
+
+#### Behavior
+
+Hydra will execute any program defined under the `runCommandHook` attribute set. These jobs must have a single output named `out`, and that output must be an executable file located directly at `$out`.
+
+#### Security Properties
+
+Safely deploying dynamic commands requires careful design of your Hydra jobs. Allowing arbitrary users to define attributes in your top level attribute set will allow that user to execute code on your Hydra.
+
+If a jobset has dynamic commands enabled, you must ensure only trusted users can define top level attributes.
+
+
+#### Configuration
+
+- `dynamicruncommand.enable`
+
+Set to 1 to enable dynamic RunCommand program execution.
+
+#### Example
+
+In your Hydra configuration, specify:
+
+```xml
+<dynamicruncommand>
+  enable = 1
+</dynamicruncommand>
+```
+
+Then create a job named `runCommandHook.example` in your jobset:
+
+```
+{ pkgs, ... }: {
+    runCommandHook = {
+        recurseForDerivations = true;
+
+        example = pkgs.writeScript "run-me" ''
+          #!${pkgs.runtimeShell}
+
+          ${pkgs.jq}/bin/jq . "$HYDRA_JSON"
+        '';
+    };
+}
+```
+
+After the `runcommandHook.example` build finishes that script will execute.

doc/manual/src/plugins/declarative-projects.md

@@ -34,6 +34,7 @@ To configure a static declarative project, take the following steps:
         "checkinterval": 300,
         "schedulingshares": 100,
         "enableemail": false,
+        "enable_dynamic_run_command": false,
         "emailoverride": "",
         "keepnr": 3,
         "inputs": {
@@ -53,6 +54,7 @@ To configure a static declarative project, take the following steps:
         "checkinterval": 300,
         "schedulingshares": 100,
         "enableemail": false,
+        "enable_dynamic_run_command": false,
         "emailoverride": "",
         "keepnr": 3,
         "inputs": {
@@ -92,6 +94,7 @@ containing the configuration of the jobset, for example:
         "checkinterval": 300,
         "schedulingshares": 100,
         "enableemail": false,
+        "enable_dynamic_run_command": false,
         "emailoverride": "",
         "keepnr": 3,
         "inputs": {

hydra-api.yaml

@@ -178,6 +178,9 @@ paths:
                 enabled:
                   description: when set to true the project gets scheduled for evaluation
                   type: boolean
+                enable_dynamic_run_command:
+                  description: when true the project's jobsets support executing dynamically defined RunCommand hooks. Requires the server and project's configuration to also enable dynamic RunCommand.
+                  type: boolean
                 visible:
                   description: when set to true the project is displayed in the web interface
                   type: boolean
@@ -607,6 +610,9 @@ components:
         enabled:
           description: when set to true the project gets scheduled for evaluation
           type: boolean
+        enable_dynamic_run_command:
+          description: when true the project's jobsets support executing dynamically defined RunCommand hooks. Requires the server and project's configuration to also enable dynamic RunCommand.
+          type: boolean
         declarative:
           description: declarative input configured for this project
           type: object
@@ -689,6 +695,9 @@ components:
         enableemail:
           description: when true the jobset sends emails when previously-successful builds fail
           type: boolean
+        enable_dynamic_run_command:
+          description: when true the jobset supports executing dynamically defined RunCommand hooks. Requires the server and project's configuration to also enable dynamic RunCommand.
+          type: boolean
         visible:
           description: when true the jobset is visible in the web frontend
           type: boolean

src/lib/Hydra/Controller/Build.pm

@@ -38,6 +38,17 @@ sub buildChain :Chained('/') :PathPart('build') :CaptureArgs(1) {
     $c->stash->{jobset} = $c->stash->{build}->jobset;
     $c->stash->{job} = $c->stash->{build}->job;
     $c->stash->{runcommandlogs} = [$c->stash->{build}->runcommandlogs->search({}, {order_by => ["id DESC"]})];
+
+    $c->stash->{runcommandlogProblem} = undef;
+    if ($c->stash->{job} =~ qr/^runCommandHook\..*/) {
+        if (!$c->config->{dynamicruncommand}->{enable}) {
+            $c->stash->{runcommandlogProblem} = "disabled-server";
+        } elsif (!$c->stash->{project}->enable_dynamic_run_command) {
+            $c->stash->{runcommandlogProblem} = "disabled-project";
+        } elsif (!$c->stash->{jobset}->enable_dynamic_run_command) {
+            $c->stash->{runcommandlogProblem} = "disabled-jobset";
+        }
+    }
 }
 
 

src/lib/Hydra/Controller/Jobset.pm

@@ -261,6 +261,14 @@ sub updateJobset {
 
     my $checkinterval = int(trim($c->stash->{params}->{checkinterval}));
 
+    my $enable_dynamic_run_command = defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0;
+    if ($enable_dynamic_run_command
+        && !($c->config->{dynamicruncommand}->{enable}
+            && $jobset->project->enable_dynamic_run_command))
+    {
+        badRequest($c, "Dynamic RunCommand is not enabled by the server or the parent project.");
+    }
+
     $jobset->update(
         { name => $jobsetName
         , description => trim($c->stash->{params}->{"description"})
@@ -268,6 +276,7 @@ sub updateJobset {
         , nixexprinput => $nixExprInput
         , enabled => $enabled
         , enableemail => defined $c->stash->{params}->{enableemail} ? 1 : 0
+        , enable_dynamic_run_command => $enable_dynamic_run_command
         , emailoverride => trim($c->stash->{params}->{emailoverride}) || ""
         , hidden => defined $c->stash->{params}->{visible} ? 0 : 1
         , keepnr => int(trim($c->stash->{params}->{keepnr} // "0"))

src/lib/Hydra/Controller/Project.pm

@@ -149,6 +149,11 @@ sub updateProject {
     my $displayName = trim $c->stash->{params}->{displayname};
     error($c, "You must specify a display name.") if $displayName eq "";
 
+    my $enable_dynamic_run_command = defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0;
+    if ($enable_dynamic_run_command && !$c->config->{dynamicruncommand}->{enable}) {
+        badRequest($c, "Dynamic RunCommand is not enabled by the server.");
+    }
+
     $project->update(
         { name => $projectName
         , displayname => $displayName
@@ -157,6 +162,7 @@ sub updateProject {
         , enabled => defined $c->stash->{params}->{enabled} ? 1 : 0
         , hidden => defined $c->stash->{params}->{visible} ? 0 : 1
         , owner => $owner
+        , enable_dynamic_run_command => $enable_dynamic_run_command
         , declfile => trim($c->stash->{params}->{declarative}->{file})
         , decltype => trim($c->stash->{params}->{declarative}->{type})
         , declvalue => trim($c->stash->{params}->{declarative}->{value})

src/lib/Hydra/Helper/AddBuilds.pm

@@ -19,14 +19,16 @@ use Hydra::Helper::CatalystUtils;
 
 our @ISA = qw(Exporter);
 our @EXPORT = qw(
+    validateDeclarativeJobset
+    createJobsetInputsRowAndData
     updateDeclarativeJobset
     handleDeclarativeJobsetBuild
     handleDeclarativeJobsetJson
 );
 
 
-sub updateDeclarativeJobset {
+sub validateDeclarativeJobset {
-    my ($db, $project, $jobsetName, $declSpec) = @_;
+    my ($config, $project, $jobsetName, $declSpec) = @_;
 
     my @allowed_keys = qw(
         enabled
@@ -39,6 +41,7 @@ sub updateDeclarativeJobset {
         checkinterval
         schedulingshares
         enableemail
+        enable_dynamic_run_command
         emailoverride
         keepnr
     );
@@ -61,16 +64,39 @@ sub updateDeclarativeJobset {
         }
     }
 
-    $db->txn_do(sub {
+    my $enable_dynamic_run_command = defined $update{enable_dynamic_run_command} ? 1 : 0;
-        my $jobset = $project->jobsets->update_or_create(\%update);
+    if ($enable_dynamic_run_command
-        $jobset->jobsetinputs->delete;
+        && !($config->{dynamicruncommand}->{enable}
-        foreach my $name (keys %{$declSpec->{"inputs"}}) {
+            && $project->{enable_dynamic_run_command}))
+    {
+        die "Dynamic RunCommand is not enabled by the server or the parent project.";
+    }
+
+    return %update;
+}
+
+sub createJobsetInputsRowAndData {
+    my ($name, $declSpec) = @_;
     my $data = $declSpec->{"inputs"}->{$name};
     my $row = {
         name => $name,
         type => $data->{type}
     };
     $row->{emailresponsible} = $data->{emailresponsible} // 0;
+
+    return ($row, $data);
+}
+
+sub updateDeclarativeJobset {
+    my ($config, $db, $project, $jobsetName, $declSpec) = @_;
+
+    my %update = validateDeclarativeJobset($config, $project, $jobsetName, $declSpec);
+
+    $db->txn_do(sub {
+        my $jobset = $project->jobsets->update_or_create(\%update);
+        $jobset->jobsetinputs->delete;
+        foreach my $name (keys %{$declSpec->{"inputs"}}) {
+            my ($row, $data) = createJobsetInputsRowAndData($name, $declSpec);
             my $input = $jobset->jobsetinputs->create($row);
             $input->jobsetinputalts->create({altnr => 0, value => $data->{value}});
         }
@@ -81,6 +107,7 @@ sub updateDeclarativeJobset {
 
 sub handleDeclarativeJobsetJson {
     my ($db, $project, $declSpec) = @_;
+    my $config = getHydraConfig();
     $db->txn_do(sub {
             my @kept = keys %$declSpec;
             push @kept, ".jobsets";
@@ -88,7 +115,7 @@ sub handleDeclarativeJobsetJson {
             foreach my $jobsetName (keys %$declSpec) {
                 my $spec = $declSpec->{$jobsetName};
                 eval {
-                    updateDeclarativeJobset($db, $project, $jobsetName, $spec);
+                    updateDeclarativeJobset($config, $db, $project, $jobsetName, $spec);
                     1;
                 } or do {
                     print STDERR "ERROR: failed to process declarative jobset ", $project->name, ":${jobsetName}, ", $@, "\n";

src/lib/Hydra/Plugin/RunCommand.pm

@@ -12,7 +12,74 @@ use Try::Tiny;
 
 sub isEnabled {
     my ($self) = @_;
-    return defined $self->{config}->{runcommand};
+
+    return areStaticCommandsEnabled($self->{config}) || areDynamicCommandsEnabled($self->{config});
+}
+
+sub areStaticCommandsEnabled {
+    my ($config) = @_;
+
+    if (defined $config->{runcommand}) {
+        return 1;
+    }
+
+    return 0;
+}
+
+sub areDynamicCommandsEnabled {
+    my ($config) = @_;
+
+    if ((defined $config->{dynamicruncommand})
+        && $config->{dynamicruncommand}->{enable}) {
+        return 1;
+    }
+
+    return 0;
+}
+
+sub isBuildEligibleForDynamicRunCommand {
+    my ($build) = @_;
+
+    if ($build->get_column("buildstatus") != 0) {
+        return 0;
+    }
+
+    if ($build->get_column("job") =~ "^runCommandHook\..+") {
+        my $out = $build->buildoutputs->find({name => "out"});
+        if (!defined $out) {
+            warn "DynamicRunCommand hook on " . $build->job . " (" . $build->id . ") rejected: no output named 'out'.";
+            return 0;
+        }
+
+        my $path = $out->path;
+        if (-l $path) {
+            $path = readlink($path);
+        }
+
+        if (! -e $path) {
+            warn "DynamicRunCommand hook on " . $build->job . " (" . $build->id . ") rejected: The 'out' output doesn't exist locally. This is a bug.";
+            return 0;
+        }
+
+        if (! -x $path) {
+            warn "DynamicRunCommand hook on " . $build->job . " (" . $build->id . ") rejected: The 'out' output is not executable.";
+            return 0;
+        }
+
+        if (! -f $path) {
+            warn "DynamicRunCommand hook on " . $build->job . " (" . $build->id . ") rejected: The 'out' output is not a regular file or symlink.";
+            return 0;
+        }
+
+        if (! $build->jobset->supportsDynamicRunCommand()) {
+            warn "DynamicRunCommand hook on " . $build->job . " (" . $build->id . ") rejected: The project or jobset don't have dynamic runcommand enabled.";
+            return 0;
+        }
+
+        return 1;
+    }
+
+    return 0;
 }
 
 sub configSectionMatches {
@@ -43,10 +110,11 @@ sub eventMatches {
 }
 
 sub fanoutToCommands {
-    my ($config, $event, $project, $jobset, $job) = @_;
+    my ($config, $event, $build) = @_;
 
     my @commands;
 
+    # Calculate all the statically defined commands to execute
     my $cfg = $config->{runcommand};
     my @config = defined $cfg ? ref $cfg eq "ARRAY" ? @$cfg : ($cfg) : ();
 
@@ -55,9 +123,10 @@ sub fanoutToCommands {
         next unless eventMatches($conf, $event);
         next unless configSectionMatches(
             $matcher,
-            $project,
+            $build->jobset->get_column('project'),
-            $jobset,
+            $build->jobset->get_column('name'),
-            $job);
+            $build->get_column('job')
+        );
 
         if (!defined($conf->{command})) {
             warn "<runcommand> section for '$matcher' lacks a 'command' option";
@@ -70,6 +139,18 @@ sub fanoutToCommands {
         })
     }
 
+    # Calculate all dynamically defined commands to execute
+    if (areDynamicCommandsEnabled($config)) {
+        if (isBuildEligibleForDynamicRunCommand($build)) {
+            my $job = $build->get_column('job');
+            my $out = $build->buildoutputs->find({name => "out"});
+            push(@commands, {
+                matcher => "DynamicRunCommand($job)",
+                command => $out->path
+            })
+        }
+    }
+
     return \@commands;
 }
 
@@ -138,9 +219,7 @@ sub buildFinished {
     my $commandsToRun = fanoutToCommands(
         $self->{config},
         $event,
-        $build->project->get_column('name'),
+        $build
-        $build->jobset->get_column('name'),
-        $build->get_column('job')
     );
 
     if (@$commandsToRun == 0) {

src/lib/Hydra/Schema/Result/Jobsets.pm

@@ -155,6 +155,12 @@ __PACKAGE__->table("jobsets");
   data_type: 'text'
   is_nullable: 1
 
+=head2 enable_dynamic_run_command
+
+  data_type: 'boolean'
+  default_value: false
+  is_nullable: 0
+
 =cut
 
 __PACKAGE__->add_columns(
@@ -207,6 +213,8 @@ __PACKAGE__->add_columns(
   { data_type => "integer", default_value => 0, is_nullable => 0 },
   "flake",
   { data_type => "text", is_nullable => 1 },
+  "enable_dynamic_run_command",
+  { data_type => "boolean", default_value => \"false", is_nullable => 0 },
 );
 
 =head1 PRIMARY KEY
@@ -354,8 +362,8 @@ __PACKAGE__->has_many(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-01-08 22:24:10
+# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-01-24 14:17:33
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:cQOnMitrWGMoJX6kZGNW+w
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:7wPE5ebeVTkenMCWG9Sgcg
 
 use JSON::MaybeXS;
 
@@ -378,6 +386,13 @@ __PACKAGE__->add_column(
     "+id" => { retrieve_on_insert => 1 }
 );
 
+sub supportsDynamicRunCommand {
+  my ($self) = @_;
+
+  return $self->get_column('enable_dynamic_run_command') == 1
+    && $self->project->supportsDynamicRunCommand();
+}
+
 sub as_json {
     my $self = shift;
 
@@ -406,6 +421,7 @@ sub as_json {
 
         # boolean_columns
         "enableemail" => $self->get_column("enableemail") ? JSON::MaybeXS::true : JSON::MaybeXS::false,
+        "enable_dynamic_run_command" => $self->get_column("enable_dynamic_run_command") ? JSON::MaybeXS::true : JSON::MaybeXS::false,
         "visible" => $self->get_column("hidden") ? JSON::MaybeXS::false : JSON::MaybeXS::true,
 
         "inputs" => { map { $_->name => $_ } $self->jobsetinputs }

src/lib/Hydra/Schema/Result/Projects.pm

@@ -88,6 +88,12 @@ __PACKAGE__->table("projects");
   data_type: 'text'
   is_nullable: 1
 
+=head2 enable_dynamic_run_command
+
+  data_type: 'boolean'
+  default_value: false
+  is_nullable: 0
+
 =cut
 
 __PACKAGE__->add_columns(
@@ -111,6 +117,8 @@ __PACKAGE__->add_columns(
   { data_type => "text", is_nullable => 1 },
   "declvalue",
   { data_type => "text", is_nullable => 1 },
+  "enable_dynamic_run_command",
+  { data_type => "boolean", default_value => \"false", is_nullable => 0 },
 );
 
 =head1 PRIMARY KEY
@@ -228,8 +236,8 @@ Composing rels: L</projectmembers> -> username
 __PACKAGE__->many_to_many("usernames", "projectmembers", "username");
 
 
-# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-01-08 22:24:10
+# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-01-24 14:20:32
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:r/wbX3FAm5/OFrrwOQL5fA
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:PtXDyT8Pc7LYhhdEG39EKQ
 
 use JSON::MaybeXS;
 
@@ -238,6 +246,12 @@ sub builds {
   return $self->jobsets->related_resultset('builds');
 };
 
+sub supportsDynamicRunCommand {
+  my ($self) = @_;
+
+  return $self->get_column('enable_dynamic_run_command') == 1;
+}
+
 sub as_json {
     my $self = shift;
 
@@ -251,6 +265,7 @@ sub as_json {
 
         # boolean_columns
         "enabled" => $self->get_column("enabled") ? JSON::MaybeXS::true : JSON::MaybeXS::false,
+        "enable_dynamic_run_command" => $self->get_column("enable_dynamic_run_command") ? JSON::MaybeXS::true : JSON::MaybeXS::false,
         "hidden" => $self->get_column("hidden") ? JSON::MaybeXS::true : JSON::MaybeXS::false,
 
         "jobsets" => [ map { $_->name } $self->jobsets ]

src/root/build.tt

@@ -149,7 +149,7 @@ END;
   [% IF build.dependents %]<li class="nav-item"><a class="nav-link" href="#tabs-usedby" data-toggle="tab">Used By</a></li>[% END%]
   [% IF drvAvailable %]<li class="nav-item"><a class="nav-link" href="#tabs-build-deps" data-toggle="tab">Build Dependencies</a></li>[% END %]
   [% IF localStore && available %]<li class="nav-item"><a class="nav-link" href="#tabs-runtime-deps" data-toggle="tab">Runtime Dependencies</a></li>[% END %]
-  [% IF runcommandlogs.size() > 0 %]<li class="nav-item"><a class="nav-link" href="#tabs-runcommandlogs" data-toggle="tab">RunCommand Logs</a></li>[% END %]
+  [% IF runcommandlogProblem || runcommandlogs.size() > 0 %]<li class="nav-item"><a class="nav-link" href="#tabs-runcommandlogs" data-toggle="tab">RunCommand Logs[% IF runcommandlogProblem %] <span class="badge badge-warning">Disabled</span>[% END %]</a></li>[% END %]
 </ul>
 
 <div id="generic-tabs" class="tab-content">
@@ -489,6 +489,19 @@ END;
   [% END %]
 
   <div id="tabs-runcommandlogs" class="tab-pane">
+    [% IF runcommandlogProblem %]
+      <div class="alert alert-warning" role="alert">
+        [% IF runcommandlogProblem == "disabled-server" %]
+          This server does not enable Dynamic RunCommand support.
+        [% ELSIF runcommandlogProblem == "disabled-project" %]
+          This project does not enable Dynamic RunCommand support.
+        [% ELSIF runcommandlogProblem == "disabled-jobset" %]
+          This jobset does not enable Dynamic RunCommand support.
+        [% ELSE %]
+          Dynamic RunCommand is not enabled: [% runcommandlogProblem %].
+        [% END %]
+      </div>
+    [% END %]
     <div class="d-flex flex-column">
     [% FOREACH runcommandlog IN runcommandlogs %]
       <div class="p-2 border-bottom">

src/root/edit-jobset.tt

@@ -157,6 +157,21 @@
     </div>
   </div>
 
+  <div class="form-group row">
+    <label class="col-sm-3" for="editjobsetenable_dynamic_run_command">Enable Dynamic RunCommand Hooks</label>
+    <div class="col-sm-9">
+      <input type="checkbox" id="editjobsetenable_dynamic_run_command" name="enable_dynamic_run_command"
+        [% IF !c.config.dynamicruncommand.enable %]
+          title="The server has not enabled dynamic RunCommands" disabled
+        [% ELSIF !project.enable_dynamic_run_command %]
+          title="The parent project has not enabled dynamic RunCommands" disabled
+        [% ELSIF jobset.enable_dynamic_run_command %]
+          checked
+        [% END %]
+      />
+    </div>
+  </div>
+
   <div class="form-group row">
     <label class="col-sm-3" for="editjobsetenableemail">Email notification</label>
     <div class="col-sm-9">

src/root/edit-project.tt

@@ -52,6 +52,20 @@
     </div>
   </div>
 
+
+  <div class="form-group row">
+    <label class="col-sm-3" for="editprojectenable_dynamic_run_command">Enable Dynamic RunCommand Hooks for Jobsets</label>
+    <div class="col-sm-9">
+      <input type="checkbox" id="editprojectenable_dynamic_run_command" name="enable_dynamic_run_command"
+        [% IF !c.config.dynamicruncommand.enable %]
+          title="The server has not enabled dynamic RunCommands" disabled
+        [% ELSIF project.enable_dynamic_run_command %]
+          checked
+        [% END %]
+      />
+    </div>
+  </div>
+
   <div class="form-group row">
     <label class="col-sm-3" for="editprojectdeclfile">
       Declarative spec file

src/root/jobset.tt

@@ -160,6 +160,10 @@
         <th>Scheduling shares:</th>
         <td>[% jobset.schedulingshares %] [% IF totalShares %] ([% f = format("%.2f"); f(jobset.schedulingshares / totalShares * 100) %]% out of [% totalShares %] shares)[% END %]</td>
       </tr>
+      <tr>
+        <th>Enable Dynamic RunCommand Hooks:</th>
+        <td>[% c.config.dynamicruncommand.enable ? project.enable_dynamic_run_command ? jobset.enable_dynamic_run_command ? "Yes" : "No (not enabled by jobset)" : "No (not enabled by project)" : "No (not enabled by server)" %]</td>
+      </tr>
       [% IF emailNotification %]
       <tr>
         <th>Enable email notification:</th>

src/root/project.tt

@@ -92,6 +92,10 @@
         <th>Enabled:</th>
         <td>[% project.enabled ? "Yes" : "No" %]</td>
       </tr>
+      <tr>
+        <th>Enable Dynamic RunCommand Hooks:</th>
+        <td>[% c.config.dynamicruncommand.enable ? project.enable_dynamic_run_command ? "Yes" : "No (not enabled by project)" : "No (not enabled by server)" %]</td>
+      </tr>
     </table>
   </div>
 

src/script/hydra-eval-jobset

@@ -619,7 +619,7 @@ sub checkJobsetWrapped {
             } else {
                 # Update the jobset with the spec's inputs, and the continue
                 # evaluating the .jobsets jobset.
-                updateDeclarativeJobset($db, $project, ".jobsets", $declSpec);
+                updateDeclarativeJobset($config, $db, $project, ".jobsets", $declSpec);
                 $jobset->discard_changes;
                 $inputInfo->{"declInput"} = [ $declInput ];
                 $inputInfo->{"projectName"} = [ fetchInput($plugins, $db, $project, $jobset, "projectName", "string", $project->name, 0) ];

src/sql/hydra.sql

@@ -49,6 +49,7 @@ create table Projects (
     declfile      text, -- File containing declarative jobset specification
     decltype      text, -- Type of the input containing declarative jobset specification
     declvalue     text, -- Value of the input containing declarative jobset specification
+    enable_dynamic_run_command boolean not null default false,
     foreign key   (owner) references Users(userName) on update cascade
 );
 
@@ -88,6 +89,7 @@ create table Jobsets (
     startTime     integer, -- if jobset is currently running
     type          integer not null default 0, -- 0 == legacy, 1 == flake
     flake         text,
+    enable_dynamic_run_command boolean not null default false,
     constraint jobsets_schedulingshares_nonzero_check check (schedulingShares > 0),
     constraint jobsets_type_known_check   check (type = 0 or type = 1),
     -- If the type is 0, then nixExprInput and nixExprPath should be non-null and other type-specific fields should be null

src/sql/upgrade-82.sql

@@ -0,0 +1,4 @@
+ALTER TABLE Jobsets
+    ADD COLUMN enable_dynamic_run_command boolean not null default false;
+ALTER TABLE Projects
+    ADD COLUMN enable_dynamic_run_command boolean not null default false;

t/Helper/AddBuilds/dynamic-disabled.t

@@ -0,0 +1,85 @@
+use strict;
+use warnings;
+use Setup;
+use Test2::V0;
+
+require Catalyst::Test;
+use HTTP::Request::Common qw(POST PUT GET DELETE);
+use JSON::MaybeXS qw(decode_json encode_json);
+use Hydra::Helper::AddBuilds qw(validateDeclarativeJobset);
+use Hydra::Helper::Nix qw(getHydraConfig);
+
+my $ctx = test_context();
+
+sub makeJobsetSpec {
+    my ($dynamic) = @_;
+
+    return {
+        enabled => 2,
+        enable_dynamic_run_command => $dynamic ? JSON::MaybeXS::true : undef,
+        visible => JSON::MaybeXS::true,
+        name => "job",
+        type => 1,
+        description => "test jobset",
+        flake => "github:nixos/nix",
+        checkinterval => 0,
+        schedulingshares => 100,
+        keepnr => 3
+    };
+};
+
+subtest "validate declarative jobset with dynamic RunCommand disabled by server" => sub {
+    my $config = getHydraConfig();
+
+    subtest "project enabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub {
+        like(
+            dies {
+              validateDeclarativeJobset(
+                  $config,
+                  { enable_dynamic_run_command => 1 },
+                  "test-jobset",
+                  makeJobsetSpec(JSON::MaybeXS::true),
+              ),
+            },
+            qr/Dynamic RunCommand is not enabled/,
+        );
+    };
+
+    subtest "project enabled dynamic runcommand, declarative jobset disabled dynamic runcommand" => sub {
+        ok(
+            validateDeclarativeJobset(
+                $config,
+                { enable_dynamic_run_command => 1 },
+                "test-jobset",
+                makeJobsetSpec(JSON::MaybeXS::false)
+            ),
+        );
+    };
+
+    subtest "project disabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub {
+        like(
+            dies {
+              validateDeclarativeJobset(
+                  $config,
+                  { enable_dynamic_run_command => 0 },
+                  "test-jobset",
+                  makeJobsetSpec(JSON::MaybeXS::true),
+              ),
+            },
+            qr/Dynamic RunCommand is not enabled/,
+        );
+    };
+
+    subtest "project disabled dynamic runcommand, declarative jobset disabled dynamic runcommand" => sub {
+        ok(
+            validateDeclarativeJobset(
+                $config,
+                { enable_dynamic_run_command => 0 },
+                "test-jobset",
+                makeJobsetSpec(JSON::MaybeXS::false)
+            ),
+        );
+    };
+};
+
+done_testing;

t/Helper/AddBuilds/dynamic-enabled.t

@@ -0,0 +1,88 @@
+use strict;
+use warnings;
+use Setup;
+use Test2::V0;
+
+require Catalyst::Test;
+use HTTP::Request::Common qw(POST PUT GET DELETE);
+use JSON::MaybeXS qw(decode_json encode_json);
+use Hydra::Helper::AddBuilds qw(validateDeclarativeJobset);
+use Hydra::Helper::Nix qw(getHydraConfig);
+
+my $ctx = test_context(
+    hydra_config => q|
+    <dynamicruncommand>
+    enable = 1
+    </dynamicruncommand>
+    |
+);
+
+sub makeJobsetSpec {
+    my ($dynamic) = @_;
+
+    return {
+        enabled => 2,
+        enable_dynamic_run_command => $dynamic ? JSON::MaybeXS::true : undef,
+        visible => JSON::MaybeXS::true,
+        name => "job",
+        type => 1,
+        description => "test jobset",
+        flake => "github:nixos/nix",
+        checkinterval => 0,
+        schedulingshares => 100,
+        keepnr => 3
+    };
+};
+
+subtest "validate declarative jobset with dynamic RunCommand enabled by server" => sub {
+    my $config = getHydraConfig();
+
+    subtest "project enabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub {
+        ok(
+            validateDeclarativeJobset(
+                $config,
+                { enable_dynamic_run_command => 1 },
+                "test-jobset",
+                makeJobsetSpec(JSON::MaybeXS::true)
+            ),
+        );
+    };
+
+    subtest "project enabled dynamic runcommand, declarative jobset disabled dynamic runcommand" => sub {
+        ok(
+            validateDeclarativeJobset(
+                $config,
+                { enable_dynamic_run_command => 1 },
+                "test-jobset",
+                makeJobsetSpec(JSON::MaybeXS::false)
+            ),
+        );
+    };
+
+    subtest "project disabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub {
+        like(
+            dies {
+              validateDeclarativeJobset(
+                  $config,
+                  { enable_dynamic_run_command => 0 },
+                  "test-jobset",
+                  makeJobsetSpec(JSON::MaybeXS::true),
+              ),
+            },
+            qr/Dynamic RunCommand is not enabled/,
+        );
+    };
+
+    subtest "project disabled dynamic runcommand, declarative jobset disabled dynamic runcommand" => sub {
+        ok(
+            validateDeclarativeJobset(
+                $config,
+                { enable_dynamic_run_command => 0 },
+                "test-jobset",
+                makeJobsetSpec(JSON::MaybeXS::false)
+            ),
+        );
+    };
+};
+
+done_testing;

t/Hydra/Controller/Jobset/http.t

@@ -73,6 +73,7 @@ subtest 'Read newly-created jobset "job"' => sub {
      emailoverride => "",
      enabled => 2,
      enableemail => JSON::MaybeXS::false,
+     enable_dynamic_run_command => JSON::MaybeXS::false,
      errortime => undef,
      errormsg => "",
      fetcherrormsg => "",
@@ -131,6 +132,7 @@ subtest 'Update jobset "job" to legacy type' => sub {
     emailoverride => "",
     enabled => 3,
     enableemail => JSON::MaybeXS::false,
+    enable_dynamic_run_command => JSON::MaybeXS::false,
     errortime => undef,
     errormsg => "",
     fetcherrormsg => "",

t/Hydra/Controller/projects.t

@@ -46,6 +46,7 @@ subtest "Read project 'tests'" => sub {
         description => "",
         displayname => "Tests",
         enabled => JSON::MaybeXS::true,
+        enable_dynamic_run_command => JSON::MaybeXS::false,
         hidden => JSON::MaybeXS::false,
         homepage => "",
         jobsets => [],
@@ -85,6 +86,7 @@ subtest "Transitioning from declarative project to normal" => sub {
             description => "",
             displayname => "Tests",
             enabled => JSON::MaybeXS::true,
+            enable_dynamic_run_command => JSON::MaybeXS::false,
             hidden => JSON::MaybeXS::false,
             homepage => "",
             jobsets => [".jobsets"],
@@ -128,6 +130,7 @@ subtest "Transitioning from declarative project to normal" => sub {
             description => "",
             displayname => "Tests",
             enabled => JSON::MaybeXS::true,
+            enable_dynamic_run_command => JSON::MaybeXS::false,
             hidden => JSON::MaybeXS::false,
             homepage => "",
             jobsets => [],

t/Hydra/Plugin/RunCommand/dynamic-disabled.t

@@ -0,0 +1,110 @@
+use strict;
+use warnings;
+use Setup;
+use Test2::V0;
+
+require Catalyst::Test;
+use HTTP::Request::Common qw(POST PUT GET DELETE);
+use JSON::MaybeXS qw(decode_json encode_json);
+
+my $ctx = test_context();
+Catalyst::Test->import('Hydra');
+
+# Create a user to log in to
+my $user = $ctx->db->resultset('Users')->create({ username => 'alice', emailaddress => 'root@invalid.org', password => '!' });
+$user->setPassword('foobar');
+$user->userroles->update_or_create({ role => 'admin' });
+
+subtest "can't enable dynamic RunCommand when disabled by server" => sub {
+    my $builds = $ctx->makeAndEvaluateJobset(
+        expression => "runcommand-dynamic.nix",
+        build => 1
+    );
+
+    my $build = $builds->{"runCommandHook.example"};
+    my $project = $build->project;
+    my $project_name = $project->name;
+    my $jobset = $build->jobset;
+    my $jobset_name = $jobset->name;
+
+    is($project->enable_dynamic_run_command, 0, "dynamic RunCommand is disabled on projects by default");
+    is($jobset->enable_dynamic_run_command, 0, "dynamic RunCommand is disabled on jobsets by default");
+
+    my $req = request(POST '/login',
+        Referer => 'http://localhost/',
+        Content => {
+            username => 'alice',
+            password => 'foobar'
+        }
+    );
+    is($req->code, 302, "logged in successfully");
+    my $cookie = $req->header("set-cookie");
+
+    subtest "can't enable dynamic RunCommand on project" => sub {
+        my $projectresponse = request(GET "/project/$project_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+        );
+
+        my $projectjson = decode_json($projectresponse->content);
+        $projectjson->{enable_dynamic_run_command} = 1;
+
+        my $projectupdate = request(PUT "/project/$project_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+            Content => encode_json($projectjson)
+        );
+
+        $projectresponse = request(GET "/project/$project_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+        );
+        $projectjson = decode_json($projectresponse->content);
+
+        is($projectupdate->code, 400);
+        like(
+            $projectupdate->content,
+            qr/Dynamic RunCommand is not/,
+            "failed to change enable_dynamic_run_command, not any other error"
+        );
+        is($projectjson->{enable_dynamic_run_command}, JSON::MaybeXS::false);
+    };
+
+    subtest "can't enable dynamic RunCommand on jobset" => sub {
+        my $jobsetresponse = request(GET "/jobset/$project_name/$jobset_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+        );
+
+        my $jobsetjson = decode_json($jobsetresponse->content);
+        $jobsetjson->{enable_dynamic_run_command} = 1;
+
+        my $jobsetupdate = request(PUT "/jobset/$project_name/$jobset_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+            Content => encode_json($jobsetjson)
+        );
+
+        $jobsetresponse = request(GET "/jobset/$project_name/$jobset_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+        );
+        $jobsetjson = decode_json($jobsetresponse->content);
+
+        is($jobsetupdate->code, 400);
+        like(
+            $jobsetupdate->content,
+            qr/Dynamic RunCommand is not/,
+            "failed to change enable_dynamic_run_command, not any other error"
+        );
+        is($jobsetjson->{enable_dynamic_run_command}, JSON::MaybeXS::false);
+    };
+};
+
+done_testing;

t/Hydra/Plugin/RunCommand/dynamic-enabled.t

@@ -0,0 +1,106 @@
+use strict;
+use warnings;
+use Setup;
+use Test2::V0;
+
+require Catalyst::Test;
+use HTTP::Request::Common qw(POST PUT GET DELETE);
+use JSON::MaybeXS qw(decode_json encode_json);
+
+my $ctx = test_context(
+    hydra_config => q|
+    <dynamicruncommand>
+    enable = 1
+    </dynamicruncommand>
+    |
+);
+Catalyst::Test->import('Hydra');
+
+# Create a user to log in to
+my $user = $ctx->db->resultset('Users')->create({ username => 'alice', emailaddress => 'root@invalid.org', password => '!' });
+$user->setPassword('foobar');
+$user->userroles->update_or_create({ role => 'admin' });
+
+subtest "can enable dynamic RunCommand when enabled by server" => sub {
+    my $builds = $ctx->makeAndEvaluateJobset(
+        expression => "runcommand-dynamic.nix",
+        build => 1
+    );
+
+    my $build = $builds->{"runCommandHook.example"};
+    my $project = $build->project;
+    my $project_name = $project->name;
+    my $jobset = $build->jobset;
+    my $jobset_name = $jobset->name;
+
+    is($project->enable_dynamic_run_command, 0, "dynamic RunCommand is disabled on projects by default");
+    is($jobset->enable_dynamic_run_command, 0, "dynamic RunCommand is disabled on jobsets by default");
+
+    my $req = request(POST '/login',
+        Referer => 'http://localhost/',
+        Content => {
+            username => 'alice',
+            password => 'foobar'
+        }
+    );
+    is($req->code, 302, "logged in successfully");
+    my $cookie = $req->header("set-cookie");
+
+    subtest "can enable dynamic RunCommand on project" => sub {
+        my $projectresponse = request(GET "/project/$project_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+        );
+
+        my $projectjson = decode_json($projectresponse->content);
+        $projectjson->{enable_dynamic_run_command} = 1;
+
+        my $projectupdate = request(PUT "/project/$project_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+            Content => encode_json($projectjson)
+        );
+
+        $projectresponse = request(GET "/project/$project_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+        );
+        $projectjson = decode_json($projectresponse->content);
+
+        is($projectupdate->code, 200);
+        is($projectjson->{enable_dynamic_run_command}, JSON::MaybeXS::true);
+    };
+
+    subtest "can enable dynamic RunCommand on jobset" => sub {
+        my $jobsetresponse = request(GET "/jobset/$project_name/$jobset_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+        );
+
+        my $jobsetjson = decode_json($jobsetresponse->content);
+        $jobsetjson->{enable_dynamic_run_command} = 1;
+
+        my $jobsetupdate = request(PUT "/jobset/$project_name/$jobset_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+            Content => encode_json($jobsetjson)
+        );
+
+        $jobsetresponse = request(GET "/jobset/$project_name/$jobset_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+        );
+        $jobsetjson = decode_json($jobsetresponse->content);
+
+        is($jobsetupdate->code, 200);
+        is($jobsetjson->{enable_dynamic_run_command}, JSON::MaybeXS::true);
+    };
+};
+
+done_testing;

t/Hydra/Plugin/RunCommand/fanout.t

@@ -0,0 +1,233 @@
+use strict;
+use warnings;
+use Setup;
+use Test2::V0;
+use Hydra::Plugin::RunCommand;
+
+my $ctx = test_context();
+
+my $builds = $ctx->makeAndEvaluateJobset(
+    expression => "runcommand-dynamic.nix",
+    build => 1
+);
+
+my $build = $builds->{"runCommandHook.example"};
+
+# Enable dynamic runcommand on the project and jobset
+$build->project->update({enable_dynamic_run_command => 1});
+$build->jobset->update({enable_dynamic_run_command => 1});
+
+is($build->job, "runCommandHook.example", "The only job should be runCommandHook.example");
+is($build->finished, 1, "Build should be finished.");
+is($build->buildstatus, 0, "Build should have buildstatus 0.");
+
+subtest "fanoutToCommands" => sub {
+    my $config = {
+        runcommand => [
+            {
+                job => "",
+                command => "foo"
+            },
+            {
+                job => "*:*:*",
+                command => "bar"
+            },
+            {
+                job => "tests:basic:nomatch",
+                command => "baz"
+            }
+        ]
+    };
+
+    is(
+        Hydra::Plugin::RunCommand::fanoutToCommands(
+            $config,
+            "buildFinished",
+            $build
+        ),
+        [
+            {
+                matcher => "",
+                command => "foo"
+            },
+            {
+                matcher => "*:*:*",
+                command => "bar"
+            }
+        ],
+        "fanoutToCommands returns a command per matching job"
+    );
+};
+
+subtest "fanoutToCommandsWithDynamicRunCommandSupport" => sub {
+    like(
+        $build->buildoutputs->find({name => "out"})->path,
+        qr/my-build-product$/,
+        "The way we find the out path is reasonable"
+    );
+
+    my $config = {
+        dynamicruncommand => { enable => 1 },
+        runcommand => [
+            {
+                job => "*:*:*",
+                command => "baz"
+            }
+        ]
+    };
+
+    is(
+        Hydra::Plugin::RunCommand::fanoutToCommands(
+            $config,
+            "buildFinished",
+            $build
+        ),
+        [
+            {
+                matcher => "*:*:*",
+                command => "baz"
+            },
+            {
+                matcher => "DynamicRunCommand(runCommandHook.example)",
+                command => $build->buildoutputs->find({name => "out"})->path
+            }
+        ],
+        "fanoutToCommands returns a command per matching job"
+    );
+};
+
+subtest "isBuildEligibleForDynamicRunCommand" => sub {
+    subtest "Non-matches based on name alone ..." => sub {
+        my $build = $builds->{"foo-bar-baz"};
+        is(
+            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($build),
+            0,
+            "The job name does not match"
+        );
+
+        $build->set_column("job", "runCommandHook");
+        is(
+            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($build),
+            0,
+            "The job name does not match"
+        );
+
+        $build->set_column("job", "runCommandHook.");
+        is(
+            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($build),
+            0,
+            "The job name does not match"
+        );
+    };
+
+    subtest "On outputs ..." => sub {
+        ok(!warns {
+            is(
+                Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}),
+                1,
+                "out is an executable file"
+            );
+        }, "No warnings for an executable file.");
+
+        ok(!warns {
+            is(
+                Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.symlink"}),
+                1,
+                "out is a symlink to an executable file"
+            );
+        }, "No warnings for a symlink to an executable file.");
+
+        like(warning {
+            is(
+                Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.no-out"}),
+                0,
+                "No output named out"
+            );
+        }, qr/rejected: no output named 'out'/, "A relevant warning is provided for a missing output");
+
+        like(warning {
+            is(
+                Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.out-is-directory"}),
+                0,
+                "out is a directory"
+            );
+        }, qr/output is not a regular file or symlink/, "A relevant warning is provided for a directory output");
+
+        like(warning {
+            is(
+                Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.out-is-not-executable-file"}),
+                0,
+                "out is a file which is not a regular file or symlink"
+            );
+        }, qr/output is not executable/, "A relevant warning is provided if the file isn't executable");
+
+        like(warning {
+            is(
+                Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.symlink-non-executable"}),
+                0,
+                "out is a symlink to a non-executable file"
+            );
+        }, qr/output is not executable/, "A relevant warning is provided for symlinks to non-executables");
+
+        like(warning {
+            is(
+                Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.symlink-directory"}),
+                0,
+                "out is a symlink to a directory"
+            );
+        }, qr/output is not a regular file or symlink/, "A relevant warning is provided for symlinks to directories");
+    };
+
+    subtest "On build status ..." => sub {
+        is(
+            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.failed"}),
+            0,
+            "Failed builds don't get run"
+        );
+    };
+
+    subtest "With dynamic runcommand disabled ..." => sub {
+        subtest "disabled on the project, enabled on the jobset" => sub {
+            $build->project->update({enable_dynamic_run_command => 0});
+            $build->jobset->update({enable_dynamic_run_command => 1});
+
+
+            like(warning {
+                is(
+                    Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}),
+                    0,
+                    "Builds don't run from a jobset with disabled dynamic runcommand"
+                );
+            }, qr/project or jobset don't have dynamic runcommand enabled./, "A relevant warning is provided for a disabled runcommand support")
+        };
+
+        subtest "enabled on the project, disabled on the jobset" => sub {
+            $build->project->update({enable_dynamic_run_command => 1});
+            $build->jobset->update({enable_dynamic_run_command => 0});
+
+            like(warning {
+                is(
+                    Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}),
+                    0,
+                    "Builds don't run from a jobset with disabled dynamic runcommand"
+                );
+            }, qr/project or jobset don't have dynamic runcommand enabled./, "A relevant warning is provided for a disabled runcommand support")
+        };
+
+        subtest "disabled on the project, disabled on the jobset" => sub {
+            $build->project->update({enable_dynamic_run_command => 0});
+            $build->jobset->update({enable_dynamic_run_command => 0});
+
+            like(warning {
+                is(
+                    Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}),
+                    0,
+                    "Builds don't run from a jobset with disabled dynamic runcommand"
+                );
+            }, qr/project or jobset don't have dynamic runcommand enabled./, "A relevant warning is provided for a disabled runcommand support")
+        };
+    };
+};
+
+
+done_testing;

t/Hydra/Plugin/RunCommand/matcher.t

@@ -7,13 +7,13 @@ use Hydra::Plugin::RunCommand;
 subtest "isEnabled" => sub {
     is(
         Hydra::Plugin::RunCommand::isEnabled({}),
-        "",
+        0,
         "Disabled by default."
     );
 
     is(
         Hydra::Plugin::RunCommand::isEnabled({ config => {}}),
-        "",
+        0,
         "Disabled by default."
     );
 
@@ -22,6 +22,121 @@ subtest "isEnabled" => sub {
         1,
         "Enabled if any runcommand blocks exist."
     );
+
+    is(
+        Hydra::Plugin::RunCommand::isEnabled({ config => { dynamicruncommand => {}}}),
+        0,
+        "Not enabled if an empty dynamicruncommand blocks exist."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::isEnabled({ config => { dynamicruncommand => { enable => 0 }}}),
+        0,
+        "Not enabled if a dynamicruncommand blocks exist without enable being set to 1."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::isEnabled({ config => { dynamicruncommand => { enable => 1 }}}),
+        1,
+        "Enabled if a dynamicruncommand blocks exist with enable being set to 1."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::isEnabled({ config => {
+            runcommand => {},
+            dynamicruncommand => { enable => 0 }
+        }}),
+        1,
+        "Enabled if a runcommand config block exists, even if a dynamicruncommand is explicitly disabled."
+    );
+};
+
+subtest "areStaticCommandsEnabled" => sub {
+    is(
+        Hydra::Plugin::RunCommand::areStaticCommandsEnabled({}),
+        0,
+        "Disabled by default."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::areStaticCommandsEnabled({}),
+        0,
+        "Disabled by default."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::areStaticCommandsEnabled({ runcommand => {}}),
+        1,
+        "Enabled if any runcommand blocks exist."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::areStaticCommandsEnabled({ dynamicruncommand => {}}),
+        0,
+        "Not enabled by dynamicruncommand blocks."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::areStaticCommandsEnabled({ dynamicruncommand => { enable => 0 }}),
+        0,
+        "Not enabled by dynamicruncommand blocks."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::areStaticCommandsEnabled({ dynamicruncommand => { enable => 1 }}),
+        0,
+        "Not enabled by dynamicruncommand blocks."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::areStaticCommandsEnabled({
+            runcommand => {},
+            dynamicruncommand => { enable => 0 }
+        }),
+        1,
+        "Enabled if a runcommand config block exists, even if a dynamicruncommand is explicitly disabled."
+    );
+};
+
+subtest "areDynamicCommandsEnabled" => sub {
+    is(
+        Hydra::Plugin::RunCommand::areDynamicCommandsEnabled({}),
+        0,
+        "Disabled by default."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::areDynamicCommandsEnabled({ runcommand => {}}),
+        0,
+        "Disabled even if any runcommand blocks exist."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::areDynamicCommandsEnabled({ dynamicruncommand => {}}),
+        0,
+        "Not enabled if an empty dynamicruncommand blocks exist."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::areDynamicCommandsEnabled({ dynamicruncommand => { enable => 0 }}),
+        0,
+        "Not enabled if a dynamicruncommand blocks exist without enable being set to 1."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::areDynamicCommandsEnabled({ dynamicruncommand => { enable => 1 }}),
+        1,
+        "Enabled if a dynamicruncommand blocks exist with enable being set to 1."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::areDynamicCommandsEnabled({
+            runcommand => {},
+            dynamicruncommand => { enable => 0 }
+        }),
+        0,
+        "Disabled if dynamicruncommand is explicitly disabled."
+    );
 };
 
 subtest "configSectionMatches" => sub {
@@ -134,44 +249,4 @@ subtest "eventMatches" => sub {
     );
 };
 
-subtest "fanoutToCommands" => sub {
-    my $config = {
-        runcommand => [
-            {
-                job => "",
-                command => "foo"
-            },
-            {
-                job => "project:*:*",
-                command => "bar"
-            },
-            {
-                job => "project:jobset:nomatch",
-                command => "baz"
-            }
-        ]
-    };
-
-    is(
-        Hydra::Plugin::RunCommand::fanoutToCommands(
-            $config,
-            "buildFinished",
-            "project",
-            "jobset",
-            "job"
-        ),
-        [
-            {
-                matcher => "",
-                command => "foo"
-            },
-            {
-                matcher => "project:*:*",
-                command => "bar"
-            }
-        ],
-        "fanoutToCommands returns a command per matching job"
-    );
-};
-
 done_testing;

t/jobs/runcommand-dynamic.nix

@@ -0,0 +1,148 @@
+with import ./config.nix;
+rec {
+  foo-bar-baz = mkDerivation {
+    name = "foo-bar-baz";
+    builder = "/bin/sh";
+    outputs = [ "out" ];
+    args = [
+      (
+        builtins.toFile "builder.sh" ''
+          #! /bin/sh
+
+          touch $out
+        ''
+      )
+    ];
+  };
+
+  runCommandHook.example = mkDerivation {
+    name = "my-build-product";
+    builder = "/bin/sh";
+    outputs = [ "out" ];
+    args = [
+      (
+        builtins.toFile "builder.sh" ''
+          #! /bin/sh
+
+          touch $out
+          chmod +x $out
+          # ... dunno ...
+        ''
+      )
+    ];
+  };
+
+  runCommandHook.symlink = mkDerivation {
+    name = "symlink-out";
+    builder = "/bin/sh";
+    outputs = [ "out" ];
+    args = [
+      (
+        builtins.toFile "builder.sh" ''
+          #! /bin/sh
+
+          ln -s $1 $out
+        ''
+      )
+
+      runCommandHook.example
+    ];
+  };
+
+  runCommandHook.no-out = mkDerivation {
+    name = "no-out";
+    builder = "/bin/sh";
+    outputs = [ "bin" ];
+    args = [
+      (
+        builtins.toFile "builder.sh" ''
+          #! /bin/sh
+          mkdir $bin
+        ''
+      )
+    ];
+  };
+
+  runCommandHook.out-is-directory = mkDerivation {
+    name = "out-is-directory";
+    builder = "/bin/sh";
+    outputs = [ "out" ];
+    args = [
+      (
+        builtins.toFile "builder.sh" ''
+          #! /bin/sh
+
+          mkdir $out
+        ''
+      )
+    ];
+  };
+
+  runCommandHook.out-is-not-executable-file = mkDerivation {
+    name = "out-is-directory";
+    builder = "/bin/sh";
+    outputs = [ "out" ];
+    args = [
+      (
+        builtins.toFile "builder.sh" ''
+          #! /bin/sh
+
+          touch $out
+        ''
+      )
+    ];
+  };
+
+  runCommandHook.symlink-non-executable = mkDerivation {
+    name = "symlink-out";
+    builder = "/bin/sh";
+    outputs = [ "out" ];
+    args = [
+      (
+        builtins.toFile "builder.sh" ''
+          #! /bin/sh
+
+          ln -s $1 $out
+        ''
+      )
+
+      runCommandHook.out-is-not-executable-file
+    ];
+  };
+
+  runCommandHook.symlink-directory = mkDerivation {
+    name = "symlink-directory";
+    builder = "/bin/sh";
+    outputs = [ "out" ];
+    args = [
+      (
+        builtins.toFile "builder.sh" ''
+          #! /bin/sh
+
+          ln -s $1 $out
+        ''
+      )
+
+      runCommandHook.out-is-directory
+    ];
+  };
+
+  runCommandHook.failed = mkDerivation {
+    name = "failed";
+    builder = "/bin/sh";
+    outputs = [ "out" ];
+    args = [
+      (
+        builtins.toFile "builder.sh" ''
+          #! /bin/sh
+
+          touch $out
+          chmod +x $out
+
+          exit 1
+        ''
+      )
+    ];
+  };
+
+}