From 4ea646130c5e637b883e9656ee5f6a71d1a728ab Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 8 Dec 2021 11:44:08 -0500
Subject: [PATCH 01/34] RunCommand: split out documentation, fixup the matcher
 syntax

---
 doc/manual/src/SUMMARY.md            |  1 +
 doc/manual/src/plugins/README.md     |  4 +++-
 doc/manual/src/plugins/RunCommand.md | 32 ++++++++++++++++++++++++++++
 3 files changed, 36 insertions(+), 1 deletion(-)
 create mode 100644 doc/manual/src/plugins/RunCommand.md

diff --git a/doc/manual/src/SUMMARY.md b/doc/manual/src/SUMMARY.md
index 80e73112..357e795a 100644
--- a/doc/manual/src/SUMMARY.md
+++ b/doc/manual/src/SUMMARY.md
@@ -7,6 +7,7 @@
 - [Hydra jobs](./jobs.md)
 - [Plugins](./plugins/README.md)
   - [Declarative Projects](./plugins/declarative-projects.md)
+  - [RunCommand](./plugins/RunCommand.md)
 - [Using the external API](api.md)
 - [Webhooks](webhooks.md)
 - [Monitoring Hydra](./monitoring/README.md)
diff --git a/doc/manual/src/plugins/README.md b/doc/manual/src/plugins/README.md
index b5486fdb..26ee2649 100644
--- a/doc/manual/src/plugins/README.md
+++ b/doc/manual/src/plugins/README.md
@@ -192,10 +192,12 @@ Writes InfluxDB events when a builds finished.
 - `influxdb.url`
 - `influxdb.db`
 
-## Run command
+## RunCommand
 
 Runs a shell command when the build is finished.
 
+See [The RunCommand Plugin](./RunCommand.md) for more information.
+
 ### Configuration options:
 
 - `runcommand.[].job`
diff --git a/doc/manual/src/plugins/RunCommand.md b/doc/manual/src/plugins/RunCommand.md
new file mode 100644
index 00000000..8b1818cc
--- /dev/null
+++ b/doc/manual/src/plugins/RunCommand.md
@@ -0,0 +1,32 @@
+## The RunCommand Plugin
+
+Hydra supports executing a program after certain builds finish.
+This behavior is disabled by default.
+
+Hydra executes these commands under the `hydra-notify` service.
+
+### Static Commands
+
+Configure specific commands to execute after the specified matching job finishes.
+
+#### Configuration
+
+- `runcommand.[].job`
+
+A matcher for jobs to match in the format `project:jobset:job`. Defaults to `*:*:*`.
+
+**Note:** This matcher format is not a regular expression.
+The `*` is a wildcard for that entire section, partial matches are not supported.
+
+- `runcommand.[].command`
+
+Command to run. Can use the `$HYDRA_JSON` environment variable to access information about the build.
+
+### Example
+
+```xml
+<runcommand>
+  job = myProject:*:*
+  command = cat $HYDRA_JSON > /tmp/hydra-output
+</runcommand>
+```

From 6ffc93c01a55c86b50d7a20dd5a474a9a5389be4 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 8 Dec 2021 12:37:13 -0500
Subject: [PATCH 02/34] RunCommand: write documentation for dynamic commands

---
 doc/manual/src/plugins/RunCommand.md | 50 ++++++++++++++++++++++++++++
 1 file changed, 50 insertions(+)

diff --git a/doc/manual/src/plugins/RunCommand.md b/doc/manual/src/plugins/RunCommand.md
index 8b1818cc..b186be80 100644
--- a/doc/manual/src/plugins/RunCommand.md
+++ b/doc/manual/src/plugins/RunCommand.md
@@ -30,3 +30,53 @@ Command to run. Can use the `$HYDRA_JSON` environment variable to access informa
   command = cat $HYDRA_JSON > /tmp/hydra-output
 </runcommand>
 ```
+
+### Dynamic Commands
+
+Hydra can optionally run RunCommand hooks defined dynamically by the jobset.
+This must be turned on explicitly in the `hydra.conf` and per jobset.
+
+#### Behavior
+
+Hydra will execute any program defined under the `runCommandHook` attribute set. These jobs must have a single output named `out`, and that output must be an executable file located directly at `$out`.
+
+#### Security Properties
+
+Safely deploying dynamic commands requires careful design of your Hydra jobs. Allowing arbitrary users to define attributes in your top level attribute set will allow that user to execute code on your Hydra.
+
+If a jobset has dynamic commands enabled, you must ensure only trusted users can define top level attributes.
+
+
+#### Configuration
+
+- `dynamicruncommand.enable`
+
+Set to 1 to enable dynamic RunCommand program execution.
+
+#### Example
+
+In your Hydra configuration, specify:
+
+```xml
+<dynamicruncommand>
+  enable = 1
+</dynamicruncommand>
+```
+
+Then create a job named `runCommandHook.example` in your jobset:
+
+```
+{ pkgs, ... }: {
+    runCommandHook = {
+        recurseForDerivations = true;
+
+        example = pkgs.writeScript "run-me" ''
+          #!${pkgs.runtimeShell}
+
+          ${pkgs.jq}/bin/jq . "$HYDRA_JSON"
+        '';
+    };
+}
+```
+
+After the `runcommandHook.example` build finishes that script will execute.

From ea311a0eb4849cca49e328719e721aa33695e0b1 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 8 Dec 2021 11:38:14 -0500
Subject: [PATCH 03/34] RunCommand: enable the plugin if dynamicruncommand is
 set

---
 src/lib/Hydra/Plugin/RunCommand.pm  |  24 +++++-
 t/Hydra/Plugin/RunCommand/matcher.t | 119 +++++++++++++++++++++++++++-
 2 files changed, 140 insertions(+), 3 deletions(-)

diff --git a/src/lib/Hydra/Plugin/RunCommand.pm b/src/lib/Hydra/Plugin/RunCommand.pm
index 401942c7..8404f280 100644
--- a/src/lib/Hydra/Plugin/RunCommand.pm
+++ b/src/lib/Hydra/Plugin/RunCommand.pm
@@ -12,7 +12,29 @@ use Try::Tiny;
 
 sub isEnabled {
     my ($self) = @_;
-    return defined $self->{config}->{runcommand};
+
+    return areStaticCommandsEnabled($self->{config}) || areDynamicCommandsEnabled($self->{config});
+}
+
+sub areStaticCommandsEnabled {
+    my ($config) = @_;
+
+    if (defined $config->{runcommand}) {
+        return 1;
+    }
+
+    return 0;
+}
+
+sub areDynamicCommandsEnabled {
+    my ($config) = @_;
+
+    if ((defined $config->{dynamicruncommand})
+        && $config->{dynamicruncommand}->{enable}) {
+        return 1;
+    }
+
+    return 0;
 }
 
 sub configSectionMatches {
diff --git a/t/Hydra/Plugin/RunCommand/matcher.t b/t/Hydra/Plugin/RunCommand/matcher.t
index bc40ba77..9797f7e1 100644
--- a/t/Hydra/Plugin/RunCommand/matcher.t
+++ b/t/Hydra/Plugin/RunCommand/matcher.t
@@ -7,13 +7,13 @@ use Hydra::Plugin::RunCommand;
 subtest "isEnabled" => sub {
     is(
         Hydra::Plugin::RunCommand::isEnabled({}),
-        "",
+        0,
         "Disabled by default."
     );
 
     is(
         Hydra::Plugin::RunCommand::isEnabled({ config => {}}),
-        "",
+        0,
         "Disabled by default."
     );
 
@@ -22,6 +22,121 @@ subtest "isEnabled" => sub {
         1,
         "Enabled if any runcommand blocks exist."
     );
+
+    is(
+        Hydra::Plugin::RunCommand::isEnabled({ config => { dynamicruncommand => {}}}),
+        0,
+        "Not enabled if an empty dynamicruncommand blocks exist."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::isEnabled({ config => { dynamicruncommand => { enable => 0 }}}),
+        0,
+        "Not enabled if a dynamicruncommand blocks exist without enable being set to 1."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::isEnabled({ config => { dynamicruncommand => { enable => 1 }}}),
+        1,
+        "Enabled if a dynamicruncommand blocks exist with enable being set to 1."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::isEnabled({ config => {
+            runcommand => {},
+            dynamicruncommand => { enable => 0 }
+        }}),
+        1,
+        "Enabled if a runcommand config block exists, even if a dynamicruncommand is explicitly disabled."
+    );
+};
+
+subtest "areStaticCommandsEnabled" => sub {
+    is(
+        Hydra::Plugin::RunCommand::areStaticCommandsEnabled({}),
+        0,
+        "Disabled by default."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::areStaticCommandsEnabled({}),
+        0,
+        "Disabled by default."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::areStaticCommandsEnabled({ runcommand => {}}),
+        1,
+        "Enabled if any runcommand blocks exist."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::areStaticCommandsEnabled({ dynamicruncommand => {}}),
+        0,
+        "Not enabled by dynamicruncommand blocks."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::areStaticCommandsEnabled({ dynamicruncommand => { enable => 0 }}),
+        0,
+        "Not enabled by dynamicruncommand blocks."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::areStaticCommandsEnabled({ dynamicruncommand => { enable => 1 }}),
+        0,
+        "Not enabled by dynamicruncommand blocks."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::areStaticCommandsEnabled({
+            runcommand => {},
+            dynamicruncommand => { enable => 0 }
+        }),
+        1,
+        "Enabled if a runcommand config block exists, even if a dynamicruncommand is explicitly disabled."
+    );
+};
+
+subtest "areDynamicCommandsEnabled" => sub {
+    is(
+        Hydra::Plugin::RunCommand::areDynamicCommandsEnabled({}),
+        0,
+        "Disabled by default."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::areDynamicCommandsEnabled({ runcommand => {}}),
+        0,
+        "Disabled even if any runcommand blocks exist."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::areDynamicCommandsEnabled({ dynamicruncommand => {}}),
+        0,
+        "Not enabled if an empty dynamicruncommand blocks exist."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::areDynamicCommandsEnabled({ dynamicruncommand => { enable => 0 }}),
+        0,
+        "Not enabled if a dynamicruncommand blocks exist without enable being set to 1."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::areDynamicCommandsEnabled({ dynamicruncommand => { enable => 1 }}),
+        1,
+        "Enabled if a dynamicruncommand blocks exist with enable being set to 1."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::areDynamicCommandsEnabled({
+            runcommand => {},
+            dynamicruncommand => { enable => 0 }
+        }),
+        0,
+        "Disabled if dynamicruncommand is explicitly disabled."
+    );
 };
 
 subtest "configSectionMatches" => sub {

From e56c49333f7d31654661f0f2f3b3d86a6ccd31cb Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 8 Dec 2021 16:03:43 -0500
Subject: [PATCH 04/34] RunCommand: Add a WIP execution of dynamic commands

This in-progress feature will run a dynamically generated set of
buildFinished hooks, which must be nested under the `runCommandHook.*`
attribute set. This implementation is not very good, with some to-dos:

1. Only run if the build succeeded
2. Verify the output is named $out and that it is an executable file
   (or a symlink to a file)
3. Require the jobset itself have a flag enabling the feature, since
   this feature can be a bit dangerous if various people of different
   trust levels can create the jobs.
---
 src/lib/Hydra/Plugin/RunCommand.pm  |  33 +++++++--
 t/Hydra/Plugin/RunCommand/fanout.t  | 108 ++++++++++++++++++++++++++++
 t/Hydra/Plugin/RunCommand/matcher.t |  40 -----------
 t/jobs/runcommand-dynamic.nix       |  27 +++++++
 4 files changed, 161 insertions(+), 47 deletions(-)
 create mode 100644 t/Hydra/Plugin/RunCommand/fanout.t
 create mode 100644 t/jobs/runcommand-dynamic.nix

diff --git a/src/lib/Hydra/Plugin/RunCommand.pm b/src/lib/Hydra/Plugin/RunCommand.pm
index 8404f280..92acc326 100644
--- a/src/lib/Hydra/Plugin/RunCommand.pm
+++ b/src/lib/Hydra/Plugin/RunCommand.pm
@@ -65,10 +65,11 @@ sub eventMatches {
 }
 
 sub fanoutToCommands {
-    my ($config, $event, $project, $jobset, $job) = @_;
+    my ($config, $event, $build) = @_;
 
     my @commands;
 
+    # Calculate all the statically defined commands to execute
     my $cfg = $config->{runcommand};
     my @config = defined $cfg ? ref $cfg eq "ARRAY" ? @$cfg : ($cfg) : ();
 
@@ -77,9 +78,10 @@ sub fanoutToCommands {
         next unless eventMatches($conf, $event);
         next unless configSectionMatches(
             $matcher,
-            $project,
-            $jobset,
-            $job);
+            $build->get_column('project'),
+            $build->get_column('jobset'),
+            $build->get_column('job')
+        );
 
         if (!defined($conf->{command})) {
             warn "<runcommand> section for '$matcher' lacks a 'command' option";
@@ -92,6 +94,25 @@ sub fanoutToCommands {
         })
     }
 
+    # Calculate all dynamically defined commands to execute
+    if (areDynamicCommandsEnabled($config)) {
+        # missing test cases:
+        #
+        # 1. is it enabled on the jobset?
+        # 2. what if the result is a directory?
+        # 3. what if the job doens't have an out?
+        # 4. what if the build failed?
+        my $job = $build->get_column('job');
+
+        if ($job =~ "^runCommandHook\.") {
+            my $out = $build->buildoutputs->find({name => "out"});
+            push(@commands, {
+                matcher => "DynamicRunCommand($job)",
+                command => $out->path
+            })
+        }
+    }
+
     return \@commands;
 }
 
@@ -160,9 +181,7 @@ sub buildFinished {
     my $commandsToRun = fanoutToCommands(
         $self->{config},
         $event,
-        $build->project->get_column('name'),
-        $build->jobset->get_column('name'),
-        $build->get_column('job')
+        $build
     );
 
     if (@$commandsToRun == 0) {
diff --git a/t/Hydra/Plugin/RunCommand/fanout.t b/t/Hydra/Plugin/RunCommand/fanout.t
new file mode 100644
index 00000000..d3a7b98a
--- /dev/null
+++ b/t/Hydra/Plugin/RunCommand/fanout.t
@@ -0,0 +1,108 @@
+use strict;
+use warnings;
+use Setup;
+
+my %ctx = test_init();
+
+use Test2::V0;
+use Hydra::Plugin::RunCommand;
+
+require Hydra::Schema;
+require Hydra::Model::DB;
+
+use Test2::V0;
+
+my $db = Hydra::Model::DB->new;
+hydra_setup($db);
+
+my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"});
+
+my $jobset = createBaseJobset("basic", "runcommand-dynamic.nix", $ctx{jobsdir});
+
+ok(evalSucceeds($jobset), "Evaluating jobs/runcommand-dynamic.nix should exit with return code 0");
+is(nrQueuedBuildsForJobset($jobset), 1, "Evaluating jobs/runcommand-dynamic.nix should result in 1 build1");
+
+(my $build) = queuedBuildsForJobset($jobset);
+
+is($build->job, "runCommandHook.example", "The only job should be runCommandHook.example");
+ok(runBuild($build), "Build should exit with return code 0");
+my $newbuild = $db->resultset('Builds')->find($build->id);
+is($newbuild->finished, 1, "Build should be finished.");
+is($newbuild->buildstatus, 0, "Build should have buildstatus 0.");
+
+subtest "fanoutToCommands" => sub {
+    my $config = {
+        runcommand => [
+            {
+                job => "",
+                command => "foo"
+            },
+            {
+                job => "tests:*:*",
+                command => "bar"
+            },
+            {
+                job => "tests:basic:nomatch",
+                command => "baz"
+            }
+        ]
+    };
+
+    is(
+        Hydra::Plugin::RunCommand::fanoutToCommands(
+            $config,
+            "buildFinished",
+            $newbuild
+        ),
+        [
+            {
+                matcher => "",
+                command => "foo"
+            },
+            {
+                matcher => "tests:*:*",
+                command => "bar"
+            }
+        ],
+        "fanoutToCommands returns a command per matching job"
+    );
+};
+
+subtest "fanoutToCommandsWithDynamicRunCommandSupport" => sub {
+    like(
+        $build->buildoutputs->find({name => "out"})->path,
+        qr/my-build-product$/,
+        "The way we find the out path is reasonable"
+    );
+
+    my $config = {
+        dynamicruncommand => { enable => 1 },
+        runcommand => [
+            {
+                job => "tests:basic:*",
+                command => "baz"
+            }
+        ]
+    };
+
+    is(
+        Hydra::Plugin::RunCommand::fanoutToCommands(
+            $config,
+            "buildFinished",
+            $build
+        ),
+        [
+            {
+                matcher => "tests:basic:*",
+                command => "baz"
+            },
+            {
+                matcher => "DynamicRunCommand(runCommandHook.example)",
+                command => $build->buildoutputs->find({name => "out"})->path
+            }
+        ],
+        "fanoutToCommands returns a command per matching job"
+    );
+};
+
+done_testing;
diff --git a/t/Hydra/Plugin/RunCommand/matcher.t b/t/Hydra/Plugin/RunCommand/matcher.t
index 9797f7e1..ca74a84c 100644
--- a/t/Hydra/Plugin/RunCommand/matcher.t
+++ b/t/Hydra/Plugin/RunCommand/matcher.t
@@ -249,44 +249,4 @@ subtest "eventMatches" => sub {
     );
 };
 
-subtest "fanoutToCommands" => sub {
-    my $config = {
-        runcommand => [
-            {
-                job => "",
-                command => "foo"
-            },
-            {
-                job => "project:*:*",
-                command => "bar"
-            },
-            {
-                job => "project:jobset:nomatch",
-                command => "baz"
-            }
-        ]
-    };
-
-    is(
-        Hydra::Plugin::RunCommand::fanoutToCommands(
-            $config,
-            "buildFinished",
-            "project",
-            "jobset",
-            "job"
-        ),
-        [
-            {
-                matcher => "",
-                command => "foo"
-            },
-            {
-                matcher => "project:*:*",
-                command => "bar"
-            }
-        ],
-        "fanoutToCommands returns a command per matching job"
-    );
-};
-
 done_testing;
diff --git a/t/jobs/runcommand-dynamic.nix b/t/jobs/runcommand-dynamic.nix
new file mode 100644
index 00000000..cf231f8f
--- /dev/null
+++ b/t/jobs/runcommand-dynamic.nix
@@ -0,0 +1,27 @@
+with import ./config.nix;
+{
+  runCommandHook.example = mkDerivation
+    {
+      name = "my-build-product";
+      builder = "/bin/sh";
+      outputs = [ "out" "bin" ];
+      args = [
+        (
+          builtins.toFile "builder.sh" ''
+            #! /bin/sh
+
+            echo "$PATH"
+
+            mkdir $bin
+            echo "foo" > $bin/bar
+
+            metrics=$out/nix-support/hydra-metrics
+            mkdir -p "$(dirname "$metrics")"
+            echo "lineCoverage 18 %" >> "$metrics"
+            echo "maxResident 27 KiB" >> "$metrics"
+          ''
+        )
+      ];
+    };
+
+}

From e7f68045f445e4c32363d2e9f6cf9fa917fa67e1 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 14 Dec 2021 16:31:19 -0500
Subject: [PATCH 05/34] DynamicRunCommand: pull out the function determining if
 a build is eligible for execution under dynamic run commands.

---
 src/lib/Hydra/Plugin/RunCommand.pm | 15 ++++++++++---
 t/Hydra/Plugin/RunCommand/fanout.t | 34 ++++++++++++++++++++++++++++++
 2 files changed, 46 insertions(+), 3 deletions(-)

diff --git a/src/lib/Hydra/Plugin/RunCommand.pm b/src/lib/Hydra/Plugin/RunCommand.pm
index 92acc326..8d3cf35a 100644
--- a/src/lib/Hydra/Plugin/RunCommand.pm
+++ b/src/lib/Hydra/Plugin/RunCommand.pm
@@ -37,6 +37,16 @@ sub areDynamicCommandsEnabled {
     return 0;
 }
 
+sub isBuildEligibleForDynamicRunCommand {
+    my ($build) = @_;
+
+    if ($build->get_column("job") =~ "^runCommandHook\..+") {
+        return 1;
+    }
+
+    return 0;
+}
+
 sub configSectionMatches {
     my ($name, $project, $jobset, $job) = @_;
 
@@ -102,9 +112,8 @@ sub fanoutToCommands {
         # 2. what if the result is a directory?
         # 3. what if the job doens't have an out?
         # 4. what if the build failed?
-        my $job = $build->get_column('job');
-
-        if ($job =~ "^runCommandHook\.") {
+        if (isBuildEligibleForDynamicRunCommand($build)) {
+            my $job = $build->get_column('job');
             my $out = $build->buildoutputs->find({name => "out"});
             push(@commands, {
                 matcher => "DynamicRunCommand($job)",
diff --git a/t/Hydra/Plugin/RunCommand/fanout.t b/t/Hydra/Plugin/RunCommand/fanout.t
index d3a7b98a..2edcb390 100644
--- a/t/Hydra/Plugin/RunCommand/fanout.t
+++ b/t/Hydra/Plugin/RunCommand/fanout.t
@@ -105,4 +105,38 @@ subtest "fanoutToCommandsWithDynamicRunCommandSupport" => sub {
     );
 };
 
+subtest "isBuildEligibleForDynamicRunCommand" => sub {
+    my $build = Hydra::Schema::Result::Builds->new({
+        "job" => "foo bar baz"
+    });
+
+    is(
+        Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($build),
+        0,
+        "The job name does not match"
+    );
+
+    $build->set_column("job", "runCommandHook");
+    is(
+        Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($build),
+        0,
+        "The job name does not match"
+    );
+
+    $build->set_column("job", "runCommandHook.");
+    is(
+        Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($build),
+        0,
+        "The job name does not match"
+    );
+
+    $build->set_column("job", "runCommandHook.a");
+    is(
+        Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($build),
+        1,
+        "The job name does match"
+    );
+};
+
+
 done_testing;

From c2be27e82b7d31ff585a5cb90b4d777d3b9c3bf2 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 14 Dec 2021 21:39:13 -0500
Subject: [PATCH 06/34] fanout.t: switch to makeAndEvaluateJobset

---
 t/Hydra/Plugin/RunCommand/fanout.t | 39 ++++++++++--------------------
 1 file changed, 13 insertions(+), 26 deletions(-)

diff --git a/t/Hydra/Plugin/RunCommand/fanout.t b/t/Hydra/Plugin/RunCommand/fanout.t
index 2edcb390..d9f67f14 100644
--- a/t/Hydra/Plugin/RunCommand/fanout.t
+++ b/t/Hydra/Plugin/RunCommand/fanout.t
@@ -1,34 +1,21 @@
 use strict;
 use warnings;
 use Setup;
-
-my %ctx = test_init();
-
 use Test2::V0;
 use Hydra::Plugin::RunCommand;
 
-require Hydra::Schema;
-require Hydra::Model::DB;
+my $ctx = test_context();
 
-use Test2::V0;
+my $builds = $ctx->makeAndEvaluateJobset(
+    expression => "runcommand-dynamic.nix",
+    build => 1
+);
 
-my $db = Hydra::Model::DB->new;
-hydra_setup($db);
-
-my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"});
-
-my $jobset = createBaseJobset("basic", "runcommand-dynamic.nix", $ctx{jobsdir});
-
-ok(evalSucceeds($jobset), "Evaluating jobs/runcommand-dynamic.nix should exit with return code 0");
-is(nrQueuedBuildsForJobset($jobset), 1, "Evaluating jobs/runcommand-dynamic.nix should result in 1 build1");
-
-(my $build) = queuedBuildsForJobset($jobset);
+my $build = $builds->{"runCommandHook.example"};
 
 is($build->job, "runCommandHook.example", "The only job should be runCommandHook.example");
-ok(runBuild($build), "Build should exit with return code 0");
-my $newbuild = $db->resultset('Builds')->find($build->id);
-is($newbuild->finished, 1, "Build should be finished.");
-is($newbuild->buildstatus, 0, "Build should have buildstatus 0.");
+is($build->finished, 1, "Build should be finished.");
+is($build->buildstatus, 0, "Build should have buildstatus 0.");
 
 subtest "fanoutToCommands" => sub {
     my $config = {
@@ -38,7 +25,7 @@ subtest "fanoutToCommands" => sub {
                 command => "foo"
             },
             {
-                job => "tests:*:*",
+                job => "*:*:*",
                 command => "bar"
             },
             {
@@ -52,7 +39,7 @@ subtest "fanoutToCommands" => sub {
         Hydra::Plugin::RunCommand::fanoutToCommands(
             $config,
             "buildFinished",
-            $newbuild
+            $build
         ),
         [
             {
@@ -60,7 +47,7 @@ subtest "fanoutToCommands" => sub {
                 command => "foo"
             },
             {
-                matcher => "tests:*:*",
+                matcher => "*:*:*",
                 command => "bar"
             }
         ],
@@ -79,7 +66,7 @@ subtest "fanoutToCommandsWithDynamicRunCommandSupport" => sub {
         dynamicruncommand => { enable => 1 },
         runcommand => [
             {
-                job => "tests:basic:*",
+                job => "*:*:*",
                 command => "baz"
             }
         ]
@@ -93,7 +80,7 @@ subtest "fanoutToCommandsWithDynamicRunCommandSupport" => sub {
         ),
         [
             {
-                matcher => "tests:basic:*",
+                matcher => "*:*:*",
                 command => "baz"
             },
             {

From 1a30a0c2f13ad9cc7a52480f918111f6278503cc Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 14 Dec 2021 22:07:15 -0500
Subject: [PATCH 07/34] Dynamic RunCommand: validate that the job's out exists,
 is a file (or points to a file) which is executable.

---
 src/lib/Hydra/Plugin/RunCommand.pm |  30 +++++-
 t/Hydra/Plugin/RunCommand/fanout.t |  88 +++++++++++------
 t/jobs/runcommand-dynamic.nix      | 145 ++++++++++++++++++++++++-----
 3 files changed, 213 insertions(+), 50 deletions(-)

diff --git a/src/lib/Hydra/Plugin/RunCommand.pm b/src/lib/Hydra/Plugin/RunCommand.pm
index 8d3cf35a..8998fc39 100644
--- a/src/lib/Hydra/Plugin/RunCommand.pm
+++ b/src/lib/Hydra/Plugin/RunCommand.pm
@@ -41,6 +41,32 @@ sub isBuildEligibleForDynamicRunCommand {
     my ($build) = @_;
 
     if ($build->get_column("job") =~ "^runCommandHook\..+") {
+        my $out = $build->buildoutputs->find({name => "out"});
+        if (!defined $out) {
+            warn "DynamicRunCommand hook on " . $build->job . " (" . $build->id . ") rejected: no output named 'out'.";
+            return 0;
+        }
+
+        my $path = $out->path;
+        if (-l $path) {
+            $path = readlink($path);
+        }
+
+        if (! -e $path) {
+            warn "DynamicRunCommand hook on " . $build->job . " (" . $build->id . ") rejected: The 'out' output doesn't exist locally. This is a bug.";
+            return 0;
+        }
+
+        if (! -x $path) {
+            warn "DynamicRunCommand hook on " . $build->job . " (" . $build->id . ") rejected: The 'out' output is not executable.";
+            return 0;
+        }
+
+        if (! -f $path) {
+            warn "DynamicRunCommand hook on " . $build->job . " (" . $build->id . ") rejected: The 'out' output is not a regular file or symlink.";
+            return 0;
+        }
+
         return 1;
     }
 
@@ -109,9 +135,7 @@ sub fanoutToCommands {
         # missing test cases:
         #
         # 1. is it enabled on the jobset?
-        # 2. what if the result is a directory?
-        # 3. what if the job doens't have an out?
-        # 4. what if the build failed?
+        # 2. what if the build failed?
         if (isBuildEligibleForDynamicRunCommand($build)) {
             my $job = $build->get_column('job');
             my $out = $build->buildoutputs->find({name => "out"});
diff --git a/t/Hydra/Plugin/RunCommand/fanout.t b/t/Hydra/Plugin/RunCommand/fanout.t
index d9f67f14..41236456 100644
--- a/t/Hydra/Plugin/RunCommand/fanout.t
+++ b/t/Hydra/Plugin/RunCommand/fanout.t
@@ -93,36 +93,72 @@ subtest "fanoutToCommandsWithDynamicRunCommandSupport" => sub {
 };
 
 subtest "isBuildEligibleForDynamicRunCommand" => sub {
-    my $build = Hydra::Schema::Result::Builds->new({
-        "job" => "foo bar baz"
-    });
+    subtest "Non-matches based on name alone ..." => sub {
+        my $build = $builds->{"foo-bar-baz"};
+        is(
+            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($build),
+            0,
+            "The job name does not match"
+        );
 
-    is(
-        Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($build),
-        0,
-        "The job name does not match"
-    );
+        $build->set_column("job", "runCommandHook");
+        is(
+            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($build),
+            0,
+            "The job name does not match"
+        );
 
-    $build->set_column("job", "runCommandHook");
-    is(
-        Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($build),
-        0,
-        "The job name does not match"
-    );
+        $build->set_column("job", "runCommandHook.");
+        is(
+            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($build),
+            0,
+            "The job name does not match"
+        );
+    };
 
-    $build->set_column("job", "runCommandHook.");
-    is(
-        Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($build),
-        0,
-        "The job name does not match"
-    );
+    subtest "On outputs ..." => sub {
+        is(
+            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}),
+            1,
+            "out is an executable file"
+        );
 
-    $build->set_column("job", "runCommandHook.a");
-    is(
-        Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($build),
-        1,
-        "The job name does match"
-    );
+        is(
+            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.symlink"}),
+            1,
+            "out is a symlink to an executable file"
+        );
+
+        is(
+            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.no-out"}),
+            0,
+            "No output named out"
+        );
+
+        is(
+            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.out-is-directory"}),
+            0,
+            "out is a directory"
+        );
+
+        is(
+            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.out-is-not-executable-file"}),
+            0,
+            "out is a file which is not not executable"
+        );
+
+        is(
+            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.symlink-non-executable"}),
+            0,
+            "out is a symlink to a non-executable file"
+        );
+
+        is(
+            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.symlink-directory"}),
+            0,
+            "out is a symlink to a directory"
+        );
+    };
 };
 
 
diff --git a/t/jobs/runcommand-dynamic.nix b/t/jobs/runcommand-dynamic.nix
index cf231f8f..c0b005b7 100644
--- a/t/jobs/runcommand-dynamic.nix
+++ b/t/jobs/runcommand-dynamic.nix
@@ -1,27 +1,130 @@
 with import ./config.nix;
-{
-  runCommandHook.example = mkDerivation
-    {
-      name = "my-build-product";
-      builder = "/bin/sh";
-      outputs = [ "out" "bin" ];
-      args = [
-        (
-          builtins.toFile "builder.sh" ''
-            #! /bin/sh
+rec {
+  foo-bar-baz = mkDerivation {
+    name = "foo-bar-baz";
+    builder = "/bin/sh";
+    outputs = [ "out" ];
+    args = [
+      (
+        builtins.toFile "builder.sh" ''
+          #! /bin/sh
 
-            echo "$PATH"
+          touch $out
+        ''
+      )
+    ];
+  };
 
-            mkdir $bin
-            echo "foo" > $bin/bar
+  runCommandHook.example = mkDerivation {
+    name = "my-build-product";
+    builder = "/bin/sh";
+    outputs = [ "out" ];
+    args = [
+      (
+        builtins.toFile "builder.sh" ''
+          #! /bin/sh
 
-            metrics=$out/nix-support/hydra-metrics
-            mkdir -p "$(dirname "$metrics")"
-            echo "lineCoverage 18 %" >> "$metrics"
-            echo "maxResident 27 KiB" >> "$metrics"
-          ''
-        )
-      ];
-    };
+          touch $out
+          chmod +x $out
+          # ... dunno ...
+        ''
+      )
+    ];
+  };
+
+  runCommandHook.symlink = mkDerivation {
+    name = "symlink-out";
+    builder = "/bin/sh";
+    outputs = [ "out" ];
+    args = [
+      (
+        builtins.toFile "builder.sh" ''
+          #! /bin/sh
+
+          ln -s $1 $out
+        ''
+      )
+
+      runCommandHook.example
+    ];
+  };
+
+  runCommandHook.no-out = mkDerivation {
+    name = "no-out";
+    builder = "/bin/sh";
+    outputs = [ "bin" ];
+    args = [
+      (
+        builtins.toFile "builder.sh" ''
+          #! /bin/sh
+          mkdir $bin
+        ''
+      )
+    ];
+  };
+
+  runCommandHook.out-is-directory = mkDerivation {
+    name = "out-is-directory";
+    builder = "/bin/sh";
+    outputs = [ "out" ];
+    args = [
+      (
+        builtins.toFile "builder.sh" ''
+          #! /bin/sh
+
+          mkdir $out
+        ''
+      )
+    ];
+  };
+
+  runCommandHook.out-is-not-executable-file = mkDerivation {
+    name = "out-is-directory";
+    builder = "/bin/sh";
+    outputs = [ "out" ];
+    args = [
+      (
+        builtins.toFile "builder.sh" ''
+          #! /bin/sh
+
+          touch $out
+        ''
+      )
+    ];
+  };
+
+  runCommandHook.symlink-non-executable = mkDerivation {
+    name = "symlink-out";
+    builder = "/bin/sh";
+    outputs = [ "out" ];
+    args = [
+      (
+        builtins.toFile "builder.sh" ''
+          #! /bin/sh
+
+          ln -s $1 $out
+        ''
+      )
+
+      runCommandHook.out-is-not-executable-file
+    ];
+  };
+
+  runCommandHook.symlink-directory = mkDerivation {
+    name = "symlink-directory";
+    builder = "/bin/sh";
+    outputs = [ "out" ];
+    args = [
+      (
+        builtins.toFile "builder.sh" ''
+          #! /bin/sh
+
+          ln -s $1 $out
+        ''
+      )
+
+      runCommandHook.out-is-directory
+    ];
+  };
 
 }

From 216d8bee3532d3d5a2326aff3f7c8bd8cff6c007 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 14 Dec 2021 22:10:02 -0500
Subject: [PATCH 08/34] DynamicRunCommand: don't run if the build failed

---
 src/lib/Hydra/Plugin/RunCommand.pm |  5 ++++-
 t/Hydra/Plugin/RunCommand/fanout.t |  8 ++++++++
 t/jobs/runcommand-dynamic.nix      | 18 ++++++++++++++++++
 3 files changed, 30 insertions(+), 1 deletion(-)

diff --git a/src/lib/Hydra/Plugin/RunCommand.pm b/src/lib/Hydra/Plugin/RunCommand.pm
index 8998fc39..2ab20274 100644
--- a/src/lib/Hydra/Plugin/RunCommand.pm
+++ b/src/lib/Hydra/Plugin/RunCommand.pm
@@ -40,6 +40,10 @@ sub areDynamicCommandsEnabled {
 sub isBuildEligibleForDynamicRunCommand {
     my ($build) = @_;
 
+    if ($build->get_column("buildstatus") != 0) {
+        return 0;
+    }
+
     if ($build->get_column("job") =~ "^runCommandHook\..+") {
         my $out = $build->buildoutputs->find({name => "out"});
         if (!defined $out) {
@@ -135,7 +139,6 @@ sub fanoutToCommands {
         # missing test cases:
         #
         # 1. is it enabled on the jobset?
-        # 2. what if the build failed?
         if (isBuildEligibleForDynamicRunCommand($build)) {
             my $job = $build->get_column('job');
             my $out = $build->buildoutputs->find({name => "out"});
diff --git a/t/Hydra/Plugin/RunCommand/fanout.t b/t/Hydra/Plugin/RunCommand/fanout.t
index 41236456..8d34e582 100644
--- a/t/Hydra/Plugin/RunCommand/fanout.t
+++ b/t/Hydra/Plugin/RunCommand/fanout.t
@@ -159,6 +159,14 @@ subtest "isBuildEligibleForDynamicRunCommand" => sub {
             "out is a symlink to a directory"
         );
     };
+
+    subtest "On build status ..." => sub {
+        is(
+            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.failed"}),
+            0,
+            "Failed builds don't get run"
+        );
+    };
 };
 
 
diff --git a/t/jobs/runcommand-dynamic.nix b/t/jobs/runcommand-dynamic.nix
index c0b005b7..1971bb82 100644
--- a/t/jobs/runcommand-dynamic.nix
+++ b/t/jobs/runcommand-dynamic.nix
@@ -127,4 +127,22 @@ rec {
     ];
   };
 
+  runCommandHook.failed = mkDerivation {
+    name = "failed";
+    builder = "/bin/sh";
+    outputs = [ "out" ];
+    args = [
+      (
+        builtins.toFile "builder.sh" ''
+          #! /bin/sh
+
+          touch $out
+          chmod +x $out
+
+          exit 1
+        ''
+      )
+    ];
+  };
+
 }

From 97a1d2d1d488887c2f9a65a22197e0f1c46be58e Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 14 Dec 2021 22:12:03 -0500
Subject: [PATCH 09/34] Jobsets: add enable_dynamic_run_command

---
 src/lib/Hydra/Schema/Result/Jobsets.pm | 12 ++++++++++--
 src/sql/hydra.sql                      |  1 +
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/lib/Hydra/Schema/Result/Jobsets.pm b/src/lib/Hydra/Schema/Result/Jobsets.pm
index 13ac09e4..bd4b7165 100644
--- a/src/lib/Hydra/Schema/Result/Jobsets.pm
+++ b/src/lib/Hydra/Schema/Result/Jobsets.pm
@@ -155,6 +155,12 @@ __PACKAGE__->table("jobsets");
   data_type: 'text'
   is_nullable: 1
 
+=head2 enable_dynamic_run_command
+
+  data_type: 'boolean'
+  default_value: false
+  is_nullable: 0
+
 =cut
 
 __PACKAGE__->add_columns(
@@ -207,6 +213,8 @@ __PACKAGE__->add_columns(
   { data_type => "integer", default_value => 0, is_nullable => 0 },
   "flake",
   { data_type => "text", is_nullable => 1 },
+  "enable_dynamic_run_command",
+  { data_type => "boolean", default_value => \"false", is_nullable => 0 },
 );
 
 =head1 PRIMARY KEY
@@ -354,8 +362,8 @@ __PACKAGE__->has_many(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-01-08 22:24:10
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:cQOnMitrWGMoJX6kZGNW+w
+# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-01-24 14:17:33
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:7wPE5ebeVTkenMCWG9Sgcg
 
 use JSON::MaybeXS;
 
diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql
index 26617789..73802d75 100644
--- a/src/sql/hydra.sql
+++ b/src/sql/hydra.sql
@@ -88,6 +88,7 @@ create table Jobsets (
     startTime     integer, -- if jobset is currently running
     type          integer not null default 0, -- 0 == legacy, 1 == flake
     flake         text,
+    enable_dynamic_run_command boolean not null default false,
     constraint jobsets_schedulingshares_nonzero_check check (schedulingShares > 0),
     constraint jobsets_type_known_check   check (type = 0 or type = 1),
     -- If the type is 0, then nixExprInput and nixExprPath should be non-null and other type-specific fields should be null

From 3cce0c5ef6a928c7a7f7091c27691a25482dccc1 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 14 Dec 2021 22:15:50 -0500
Subject: [PATCH 10/34] Only run dynamic runcommand hooks if the jobset enables
 them

---
 src/lib/Hydra/Plugin/RunCommand.pm |  7 +++----
 t/Hydra/Plugin/RunCommand/fanout.t | 13 +++++++++++++
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/src/lib/Hydra/Plugin/RunCommand.pm b/src/lib/Hydra/Plugin/RunCommand.pm
index 2ab20274..79bf72a8 100644
--- a/src/lib/Hydra/Plugin/RunCommand.pm
+++ b/src/lib/Hydra/Plugin/RunCommand.pm
@@ -71,7 +71,9 @@ sub isBuildEligibleForDynamicRunCommand {
             return 0;
         }
 
-        return 1;
+        if ($build->jobset->enable_dynamic_run_command) {
+            return 1;
+        }
     }
 
     return 0;
@@ -136,9 +138,6 @@ sub fanoutToCommands {
 
     # Calculate all dynamically defined commands to execute
     if (areDynamicCommandsEnabled($config)) {
-        # missing test cases:
-        #
-        # 1. is it enabled on the jobset?
         if (isBuildEligibleForDynamicRunCommand($build)) {
             my $job = $build->get_column('job');
             my $out = $build->buildoutputs->find({name => "out"});
diff --git a/t/Hydra/Plugin/RunCommand/fanout.t b/t/Hydra/Plugin/RunCommand/fanout.t
index 8d34e582..72d58b3c 100644
--- a/t/Hydra/Plugin/RunCommand/fanout.t
+++ b/t/Hydra/Plugin/RunCommand/fanout.t
@@ -13,6 +13,9 @@ my $builds = $ctx->makeAndEvaluateJobset(
 
 my $build = $builds->{"runCommandHook.example"};
 
+# Enable dynamic runcommand on the jobset
+$build->jobset->update({enable_dynamic_run_command => 1});
+
 is($build->job, "runCommandHook.example", "The only job should be runCommandHook.example");
 is($build->finished, 1, "Build should be finished.");
 is($build->buildstatus, 0, "Build should have buildstatus 0.");
@@ -167,6 +170,16 @@ subtest "isBuildEligibleForDynamicRunCommand" => sub {
             "Failed builds don't get run"
         );
     };
+
+    subtest "With dynamic runcommand disabled ..." => sub {
+        $build->jobset->update({enable_dynamic_run_command => 0});
+
+        is(
+            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}),
+            0,
+            "Builds don't run from a jobset with disabled dynamic runcommand"
+        );
+    };
 };
 
 

From a9bfabd6722bdfe8a1cd69d045a0bfabb0284e1d Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 15 Dec 2021 11:16:05 -0500
Subject: [PATCH 11/34] sql: add a migration for enable_dynamic_run_command

---
 src/sql/upgrade-82.sql | 2 ++
 1 file changed, 2 insertions(+)
 create mode 100644 src/sql/upgrade-82.sql

diff --git a/src/sql/upgrade-82.sql b/src/sql/upgrade-82.sql
new file mode 100644
index 00000000..eb012762
--- /dev/null
+++ b/src/sql/upgrade-82.sql
@@ -0,0 +1,2 @@
+ALTER TABLE Jobsets
+    ADD COLUMN enable_dynamic_run_command boolean not null default false;

From 85a53694c8a87544dc2192a1f8900dadca499a6f Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 15 Dec 2021 12:32:10 -0500
Subject: [PATCH 12/34] sql: add enable_dynamic_run_command to the Project as
 well

---
 src/sql/hydra.sql      | 1 +
 src/sql/upgrade-82.sql | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql
index 73802d75..eaae6da3 100644
--- a/src/sql/hydra.sql
+++ b/src/sql/hydra.sql
@@ -49,6 +49,7 @@ create table Projects (
     declfile      text, -- File containing declarative jobset specification
     decltype      text, -- Type of the input containing declarative jobset specification
     declvalue     text, -- Value of the input containing declarative jobset specification
+    enable_dynamic_run_command boolean not null default false,
     foreign key   (owner) references Users(userName) on update cascade
 );
 
diff --git a/src/sql/upgrade-82.sql b/src/sql/upgrade-82.sql
index eb012762..a619caf3 100644
--- a/src/sql/upgrade-82.sql
+++ b/src/sql/upgrade-82.sql
@@ -1,2 +1,4 @@
 ALTER TABLE Jobsets
     ADD COLUMN enable_dynamic_run_command boolean not null default false;
+ALTER TABLE Projects
+    ADD COLUMN enable_dynamic_run_command boolean not null default false;

From 0c96172c2890d0043a6ba41bdbc8d4765c2d1ca4 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 15 Dec 2021 12:33:16 -0500
Subject: [PATCH 13/34] RunCommand: only run dynamic runcommand hooks if the
 project AND jobset agree they should be enabled

---
 t/Hydra/Plugin/RunCommand/fanout.t | 40 ++++++++++++++++++++++++------
 1 file changed, 33 insertions(+), 7 deletions(-)

diff --git a/t/Hydra/Plugin/RunCommand/fanout.t b/t/Hydra/Plugin/RunCommand/fanout.t
index 72d58b3c..90bf4a6f 100644
--- a/t/Hydra/Plugin/RunCommand/fanout.t
+++ b/t/Hydra/Plugin/RunCommand/fanout.t
@@ -13,7 +13,8 @@ my $builds = $ctx->makeAndEvaluateJobset(
 
 my $build = $builds->{"runCommandHook.example"};
 
-# Enable dynamic runcommand on the jobset
+# Enable dynamic runcommand on the project and jobset
+$build->project->update({enable_dynamic_run_command => 1});
 $build->jobset->update({enable_dynamic_run_command => 1});
 
 is($build->job, "runCommandHook.example", "The only job should be runCommandHook.example");
@@ -172,13 +173,38 @@ subtest "isBuildEligibleForDynamicRunCommand" => sub {
     };
 
     subtest "With dynamic runcommand disabled ..." => sub {
-        $build->jobset->update({enable_dynamic_run_command => 0});
+        subtest "disabled on the project, enabled on the jobset" => {
+            $build->project->update({enable_dynamic_run_command => 0});
+            $build->jobset->update({enable_dynamic_run_command => 1});
 
-        is(
-            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}),
-            0,
-            "Builds don't run from a jobset with disabled dynamic runcommand"
-        );
+            is(
+                Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}),
+                0,
+                "Builds don't run from a jobset with disabled dynamic runcommand"
+            );
+        };
+
+        subtest "enabled on the project, disabled on the jobset" => {
+            $build->project->update({enable_dynamic_run_command => 1});
+            $build->jobset->update({enable_dynamic_run_command => 0});
+
+            is(
+                Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}),
+                0,
+                "Builds don't run from a jobset with disabled dynamic runcommand"
+            );
+        };
+
+        subtest "disabled on the project, disabled on the jobset" => {
+            $build->project->update({enable_dynamic_run_command => 0});
+            $build->jobset->update({enable_dynamic_run_command => 0});
+
+            is(
+                Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}),
+                0,
+                "Builds don't run from a jobset with disabled dynamic runcommand"
+            );
+        };
     };
 };
 

From aef11685a0c75b8e332b1c3d515e3bbb46206888 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 15 Dec 2021 12:34:29 -0500
Subject: [PATCH 14/34] regenerate schema files after adding the flag to the
 projects

---
 src/lib/Hydra/Schema/Result/Projects.pm | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/src/lib/Hydra/Schema/Result/Projects.pm b/src/lib/Hydra/Schema/Result/Projects.pm
index 35c3eeab..9e630b16 100644
--- a/src/lib/Hydra/Schema/Result/Projects.pm
+++ b/src/lib/Hydra/Schema/Result/Projects.pm
@@ -88,6 +88,12 @@ __PACKAGE__->table("projects");
   data_type: 'text'
   is_nullable: 1
 
+=head2 enable_dynamic_run_command
+
+  data_type: 'boolean'
+  default_value: false
+  is_nullable: 0
+
 =cut
 
 __PACKAGE__->add_columns(
@@ -111,6 +117,8 @@ __PACKAGE__->add_columns(
   { data_type => "text", is_nullable => 1 },
   "declvalue",
   { data_type => "text", is_nullable => 1 },
+  "enable_dynamic_run_command",
+  { data_type => "boolean", default_value => \"false", is_nullable => 0 },
 );
 
 =head1 PRIMARY KEY
@@ -228,8 +236,8 @@ Composing rels: L</projectmembers> -> username
 __PACKAGE__->many_to_many("usernames", "projectmembers", "username");
 
 
-# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-01-08 22:24:10
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:r/wbX3FAm5/OFrrwOQL5fA
+# Created by DBIx::Class::Schema::Loader v0.07049 @ 2022-01-24 14:20:32
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:PtXDyT8Pc7LYhhdEG39EKQ
 
 use JSON::MaybeXS;
 

From 0810f5debcaa4372f78b225453b578ec09db097b Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 15 Dec 2021 12:36:19 -0500
Subject: [PATCH 15/34] finish making the dynamic hooks only run on project &
 jobset agreement

---
 src/lib/Hydra/Plugin/RunCommand.pm | 10 ++++++++--
 t/Hydra/Plugin/RunCommand/fanout.t |  6 +++---
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/src/lib/Hydra/Plugin/RunCommand.pm b/src/lib/Hydra/Plugin/RunCommand.pm
index 79bf72a8..6d099142 100644
--- a/src/lib/Hydra/Plugin/RunCommand.pm
+++ b/src/lib/Hydra/Plugin/RunCommand.pm
@@ -71,9 +71,15 @@ sub isBuildEligibleForDynamicRunCommand {
             return 0;
         }
 
-        if ($build->jobset->enable_dynamic_run_command) {
-            return 1;
+        if (! $build->jobset->enable_dynamic_run_command) {
+            return 0;
         }
+
+        if (! $build->project->enable_dynamic_run_command) {
+            return 0;
+        }
+
+        return 1;
     }
 
     return 0;
diff --git a/t/Hydra/Plugin/RunCommand/fanout.t b/t/Hydra/Plugin/RunCommand/fanout.t
index 90bf4a6f..bd2502ec 100644
--- a/t/Hydra/Plugin/RunCommand/fanout.t
+++ b/t/Hydra/Plugin/RunCommand/fanout.t
@@ -173,7 +173,7 @@ subtest "isBuildEligibleForDynamicRunCommand" => sub {
     };
 
     subtest "With dynamic runcommand disabled ..." => sub {
-        subtest "disabled on the project, enabled on the jobset" => {
+        subtest "disabled on the project, enabled on the jobset" => sub {
             $build->project->update({enable_dynamic_run_command => 0});
             $build->jobset->update({enable_dynamic_run_command => 1});
 
@@ -184,7 +184,7 @@ subtest "isBuildEligibleForDynamicRunCommand" => sub {
             );
         };
 
-        subtest "enabled on the project, disabled on the jobset" => {
+        subtest "enabled on the project, disabled on the jobset" => sub {
             $build->project->update({enable_dynamic_run_command => 1});
             $build->jobset->update({enable_dynamic_run_command => 0});
 
@@ -195,7 +195,7 @@ subtest "isBuildEligibleForDynamicRunCommand" => sub {
             );
         };
 
-        subtest "disabled on the project, disabled on the jobset" => {
+        subtest "disabled on the project, disabled on the jobset" => sub {
             $build->project->update({enable_dynamic_run_command => 0});
             $build->jobset->update({enable_dynamic_run_command => 0});
 

From 1802bd011338f227b2100b5947431d9b52c4b4fa Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 15 Dec 2021 12:37:01 -0500
Subject: [PATCH 16/34] Declarative Jobs: add support for the
 enable_dynamic_run_command flag

---
 doc/manual/src/plugins/declarative-projects.md | 3 +++
 src/lib/Hydra/Helper/AddBuilds.pm              | 1 +
 2 files changed, 4 insertions(+)

diff --git a/doc/manual/src/plugins/declarative-projects.md b/doc/manual/src/plugins/declarative-projects.md
index 12dfed18..b72c6fd0 100644
--- a/doc/manual/src/plugins/declarative-projects.md
+++ b/doc/manual/src/plugins/declarative-projects.md
@@ -34,6 +34,7 @@ To configure a static declarative project, take the following steps:
         "checkinterval": 300,
         "schedulingshares": 100,
         "enableemail": false,
+        "enable_dynamic_run_command": false,
         "emailoverride": "",
         "keepnr": 3,
         "inputs": {
@@ -53,6 +54,7 @@ To configure a static declarative project, take the following steps:
         "checkinterval": 300,
         "schedulingshares": 100,
         "enableemail": false,
+        "enable_dynamic_run_command": false,
         "emailoverride": "",
         "keepnr": 3,
         "inputs": {
@@ -92,6 +94,7 @@ containing the configuration of the jobset, for example:
         "checkinterval": 300,
         "schedulingshares": 100,
         "enableemail": false,
+        "enable_dynamic_run_command": false,
         "emailoverride": "",
         "keepnr": 3,
         "inputs": {
diff --git a/src/lib/Hydra/Helper/AddBuilds.pm b/src/lib/Hydra/Helper/AddBuilds.pm
index 1e6d8944..f38737d3 100644
--- a/src/lib/Hydra/Helper/AddBuilds.pm
+++ b/src/lib/Hydra/Helper/AddBuilds.pm
@@ -39,6 +39,7 @@ sub updateDeclarativeJobset {
         checkinterval
         schedulingshares
         enableemail
+        enable_dynamic_run_command
         emailoverride
         keepnr
     );

From 726ea80e991aa540cf5ed5f1b66ca8a5c68efa00 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 15 Dec 2021 12:37:35 -0500
Subject: [PATCH 17/34] HTTP/Jobset: support setting / reading
 enable_dynamic_run_command

---
 hydra-api.yaml                     | 3 +++
 src/lib/Hydra/Controller/Jobset.pm | 1 +
 src/root/edit-jobset.tt            | 7 +++++++
 src/root/jobset.tt                 | 4 ++++
 t/Hydra/Controller/Jobset/http.t   | 1 +
 5 files changed, 16 insertions(+)

diff --git a/hydra-api.yaml b/hydra-api.yaml
index 7857162e..0d203a41 100644
--- a/hydra-api.yaml
+++ b/hydra-api.yaml
@@ -689,6 +689,9 @@ components:
         enableemail:
           description: when true the jobset sends emails when previously-successful builds fail
           type: boolean
+        enable_dynamic_run_command:
+          description: when true the jobset supports executing dynamically defined RunCommand hooks. Requires the server and project's configuration to also enable dynamic RunCommand.
+          type: boolean
         visible:
           description: when true the jobset is visible in the web frontend
           type: boolean
diff --git a/src/lib/Hydra/Controller/Jobset.pm b/src/lib/Hydra/Controller/Jobset.pm
index b952031f..a2d48597 100644
--- a/src/lib/Hydra/Controller/Jobset.pm
+++ b/src/lib/Hydra/Controller/Jobset.pm
@@ -268,6 +268,7 @@ sub updateJobset {
         , nixexprinput => $nixExprInput
         , enabled => $enabled
         , enableemail => defined $c->stash->{params}->{enableemail} ? 1 : 0
+        , enable_dynamic_run_command => defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0
         , emailoverride => trim($c->stash->{params}->{emailoverride}) || ""
         , hidden => defined $c->stash->{params}->{visible} ? 0 : 1
         , keepnr => int(trim($c->stash->{params}->{keepnr} // "0"))
diff --git a/src/root/edit-jobset.tt b/src/root/edit-jobset.tt
index dbd26dcc..40da8f61 100644
--- a/src/root/edit-jobset.tt
+++ b/src/root/edit-jobset.tt
@@ -157,6 +157,13 @@
     </div>
   </div>
 
+  <div class="form-group row">
+    <label class="col-sm-3" for="editjobsetenable_dynamic_run_command">Enable Dynamic RunCommand Hooks</label>
+    <div class="col-sm-9">
+      <input type="checkbox" id="editjobsetenable_dynamic_run_command" name="enable_dynamic_run_command" [% IF jobset.enable_dynamic_run_command %]checked[% END %]/>
+    </div>
+  </div>
+
   <div class="form-group row">
     <label class="col-sm-3" for="editjobsetenableemail">Email notification</label>
     <div class="col-sm-9">
diff --git a/src/root/jobset.tt b/src/root/jobset.tt
index 4fb52517..3d6ca6ae 100644
--- a/src/root/jobset.tt
+++ b/src/root/jobset.tt
@@ -160,6 +160,10 @@
         <th>Scheduling shares:</th>
         <td>[% jobset.schedulingshares %] [% IF totalShares %] ([% f = format("%.2f"); f(jobset.schedulingshares / totalShares * 100) %]% out of [% totalShares %] shares)[% END %]</td>
       </tr>
+      <tr>
+        <th>Enable Dynamic RunCommand Hooks:</th>
+        <td>[% jobset.enable_dynamic_run_command ? "Yes" : "No" %]</td>
+      </tr>
       [% IF emailNotification %]
       <tr>
         <th>Enable email notification:</th>
diff --git a/t/Hydra/Controller/Jobset/http.t b/t/Hydra/Controller/Jobset/http.t
index 32b3a681..4bca7c15 100644
--- a/t/Hydra/Controller/Jobset/http.t
+++ b/t/Hydra/Controller/Jobset/http.t
@@ -73,6 +73,7 @@ subtest 'Read newly-created jobset "job"' => sub {
      emailoverride => "",
      enabled => 2,
      enableemail => JSON::MaybeXS::false,
+     enable_dynamic_run_command => JSON::MaybeXS::false,
      errortime => undef,
      errormsg => "",
      fetcherrormsg => "",

From 1affb1cfb198cb07b8f161d03a9696e308eddd90 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 15 Dec 2021 13:55:54 -0500
Subject: [PATCH 18/34] jobset API: expose and check the
 enable_dynamic_run_command

---
 src/lib/Hydra/Schema/Result/Jobsets.pm | 1 +
 t/Hydra/Controller/Jobset/http.t       | 1 +
 2 files changed, 2 insertions(+)

diff --git a/src/lib/Hydra/Schema/Result/Jobsets.pm b/src/lib/Hydra/Schema/Result/Jobsets.pm
index bd4b7165..7b96c472 100644
--- a/src/lib/Hydra/Schema/Result/Jobsets.pm
+++ b/src/lib/Hydra/Schema/Result/Jobsets.pm
@@ -414,6 +414,7 @@ sub as_json {
 
         # boolean_columns
         "enableemail" => $self->get_column("enableemail") ? JSON::MaybeXS::true : JSON::MaybeXS::false,
+        "enable_dynamic_run_command" => $self->get_column("enable_dynamic_run_command") ? JSON::MaybeXS::true : JSON::MaybeXS::false,
         "visible" => $self->get_column("hidden") ? JSON::MaybeXS::false : JSON::MaybeXS::true,
 
         "inputs" => { map { $_->name => $_ } $self->jobsetinputs }
diff --git a/t/Hydra/Controller/Jobset/http.t b/t/Hydra/Controller/Jobset/http.t
index 4bca7c15..4e53949d 100644
--- a/t/Hydra/Controller/Jobset/http.t
+++ b/t/Hydra/Controller/Jobset/http.t
@@ -132,6 +132,7 @@ subtest 'Update jobset "job" to legacy type' => sub {
     emailoverride => "",
     enabled => 3,
     enableemail => JSON::MaybeXS::false,
+    enable_dynamic_run_command => JSON::MaybeXS::false,
     errortime => undef,
     errormsg => "",
     fetcherrormsg => "",

From 8a96f07f58bfd4c9e7da9c573718473e829bf104 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 15 Dec 2021 15:32:49 -0500
Subject: [PATCH 19/34] Project: enable enabling dynamic runcommand per project

---
 src/lib/Hydra/Controller/Project.pm     | 1 +
 src/lib/Hydra/Schema/Result/Projects.pm | 1 +
 src/root/edit-project.tt                | 8 ++++++++
 src/root/project.tt                     | 4 ++++
 4 files changed, 14 insertions(+)

diff --git a/src/lib/Hydra/Controller/Project.pm b/src/lib/Hydra/Controller/Project.pm
index ed3c527c..98a8a6eb 100644
--- a/src/lib/Hydra/Controller/Project.pm
+++ b/src/lib/Hydra/Controller/Project.pm
@@ -157,6 +157,7 @@ sub updateProject {
         , enabled => defined $c->stash->{params}->{enabled} ? 1 : 0
         , hidden => defined $c->stash->{params}->{visible} ? 0 : 1
         , owner => $owner
+        , enable_dynamic_run_command => defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0
         , declfile => trim($c->stash->{params}->{declarative}->{file})
         , decltype => trim($c->stash->{params}->{declarative}->{type})
         , declvalue => trim($c->stash->{params}->{declarative}->{value})
diff --git a/src/lib/Hydra/Schema/Result/Projects.pm b/src/lib/Hydra/Schema/Result/Projects.pm
index 9e630b16..42ca22a4 100644
--- a/src/lib/Hydra/Schema/Result/Projects.pm
+++ b/src/lib/Hydra/Schema/Result/Projects.pm
@@ -259,6 +259,7 @@ sub as_json {
 
         # boolean_columns
         "enabled" => $self->get_column("enabled") ? JSON::MaybeXS::true : JSON::MaybeXS::false,
+        "enable_dynamic_run_command" => $self->get_column("enable_dynamic_run_command") ? JSON::MaybeXS::true : JSON::MaybeXS::false,
         "hidden" => $self->get_column("hidden") ? JSON::MaybeXS::true : JSON::MaybeXS::false,
 
         "jobsets" => [ map { $_->name } $self->jobsets ]
diff --git a/src/root/edit-project.tt b/src/root/edit-project.tt
index 6149ec1d..4b99f4ab 100644
--- a/src/root/edit-project.tt
+++ b/src/root/edit-project.tt
@@ -52,6 +52,14 @@
     </div>
   </div>
 
+
+  <div class="form-group row">
+    <label class="col-sm-3" for="editprojectenable_dynamic_run_command">Enable Dynamic RunCommand Hooks for Jobsets</label>
+    <div class="col-sm-9">
+      <input type="checkbox" id="editprojectenable_dynamic_run_command" name="enable_dynamic_run_command" [% IF jobset.enable_dynamic_run_command %]checked[% END %]/>
+    </div>
+  </div>
+
   <div class="form-group row">
     <label class="col-sm-3" for="editprojectdeclfile">
       Declarative spec file
diff --git a/src/root/project.tt b/src/root/project.tt
index 04661c82..f5a51e96 100644
--- a/src/root/project.tt
+++ b/src/root/project.tt
@@ -92,6 +92,10 @@
         <th>Enabled:</th>
         <td>[% project.enabled ? "Yes" : "No" %]</td>
       </tr>
+      <tr>
+        <th>Enable Dynamic RunCommand Hooks:</th>
+        <td>[% project.enable_dynamic_run_command ? "Yes" : "No" %]</td>
+      </tr>
     </table>
   </div>
 

From 2635607b6e926ad6ac570763dcaaf5c4be8bbe7c Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 15 Dec 2021 15:41:55 -0500
Subject: [PATCH 20/34] whoops: add a test on the enable_dynamic_run_command
 field

---
 hydra-api.yaml                | 3 +++
 t/Hydra/Controller/projects.t | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/hydra-api.yaml b/hydra-api.yaml
index 0d203a41..0fe0a130 100644
--- a/hydra-api.yaml
+++ b/hydra-api.yaml
@@ -607,6 +607,9 @@ components:
         enabled:
           description: when set to true the project gets scheduled for evaluation
           type: boolean
+        enable_dynamic_run_command:
+          description: when true the project's jobsets support executing dynamically defined RunCommand hooks. Requires the server and project's configuration to also enable dynamic RunCommand.
+          type: boolean
         declarative:
           description: declarative input configured for this project
           type: object
diff --git a/t/Hydra/Controller/projects.t b/t/Hydra/Controller/projects.t
index df1290aa..130724cf 100644
--- a/t/Hydra/Controller/projects.t
+++ b/t/Hydra/Controller/projects.t
@@ -46,6 +46,7 @@ subtest "Read project 'tests'" => sub {
         description => "",
         displayname => "Tests",
         enabled => JSON::MaybeXS::true,
+        enable_dynamic_run_command => JSON::MaybeXS::false,
         hidden => JSON::MaybeXS::false,
         homepage => "",
         jobsets => [],
@@ -85,6 +86,7 @@ subtest "Transitioning from declarative project to normal" => sub {
             description => "",
             displayname => "Tests",
             enabled => JSON::MaybeXS::true,
+            enable_dynamic_run_command => JSON::MaybeXS::false,
             hidden => JSON::MaybeXS::false,
             homepage => "",
             jobsets => [".jobsets"],
@@ -128,6 +130,7 @@ subtest "Transitioning from declarative project to normal" => sub {
             description => "",
             displayname => "Tests",
             enabled => JSON::MaybeXS::true,
+            enable_dynamic_run_command => JSON::MaybeXS::false,
             hidden => JSON::MaybeXS::false,
             homepage => "",
             jobsets => [],

From bc1630bd27e6d489122e18b6c37a92ea18b83b0a Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Mon, 24 Jan 2022 15:55:18 -0500
Subject: [PATCH 21/34] fixup! RunCommand: Add a WIP execution of dynamic
 commands

---
 src/lib/Hydra/Plugin/RunCommand.pm | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/lib/Hydra/Plugin/RunCommand.pm b/src/lib/Hydra/Plugin/RunCommand.pm
index 6d099142..b55e96a9 100644
--- a/src/lib/Hydra/Plugin/RunCommand.pm
+++ b/src/lib/Hydra/Plugin/RunCommand.pm
@@ -126,8 +126,8 @@ sub fanoutToCommands {
         next unless eventMatches($conf, $event);
         next unless configSectionMatches(
             $matcher,
-            $build->get_column('project'),
-            $build->get_column('jobset'),
+            $build->jobset->get_column('project'),
+            $build->jobset->get_column('name'),
             $build->get_column('job')
         );
 

From 38514ae4940aaa9c0b499f0beed1ab8272dd8fa3 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Mon, 24 Jan 2022 16:07:42 -0500
Subject: [PATCH 22/34] fanout tests: capture warnings and test their relevance

---
 t/Hydra/Plugin/RunCommand/fanout.t | 84 +++++++++++++++++-------------
 1 file changed, 49 insertions(+), 35 deletions(-)

diff --git a/t/Hydra/Plugin/RunCommand/fanout.t b/t/Hydra/Plugin/RunCommand/fanout.t
index bd2502ec..808f661c 100644
--- a/t/Hydra/Plugin/RunCommand/fanout.t
+++ b/t/Hydra/Plugin/RunCommand/fanout.t
@@ -121,47 +121,61 @@ subtest "isBuildEligibleForDynamicRunCommand" => sub {
     };
 
     subtest "On outputs ..." => sub {
-        is(
-            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}),
-            1,
-            "out is an executable file"
-        );
+        ok(!warns {
+            is(
+                Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}),
+                1,
+                "out is an executable file"
+            );
+        }, "No warnings for an executable file.");
 
-        is(
-            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.symlink"}),
-            1,
-            "out is a symlink to an executable file"
-        );
+        ok(!warns {
+            is(
+                Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.symlink"}),
+                1,
+                "out is a symlink to an executable file"
+            );
+        }, "No warnings for a symlink to an executable file.");
 
-        is(
-            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.no-out"}),
-            0,
-            "No output named out"
-        );
+        like(warning {
+            is(
+                Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.no-out"}),
+                0,
+                "No output named out"
+            );
+        }, qr/rejected: no output named 'out'/, "A relevant warning is provided for a missing output");
 
-        is(
-            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.out-is-directory"}),
-            0,
-            "out is a directory"
-        );
+        like(warning {
+            is(
+                Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.out-is-directory"}),
+                0,
+                "out is a directory"
+            );
+        }, qr/output is not a regular file or symlink/, "A relevant warning is provided for a directory output");
 
-        is(
-            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.out-is-not-executable-file"}),
-            0,
-            "out is a file which is not not executable"
-        );
+        like(warning {
+            is(
+                Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.out-is-not-executable-file"}),
+                0,
+                "out is a file which is not a regular file or symlink"
+            );
+        }, qr/output is not executable/, "A relevant warning is provided if the file isn't executable");
 
-        is(
-            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.symlink-non-executable"}),
-            0,
-            "out is a symlink to a non-executable file"
-        );
+        like(warning {
+            is(
+                Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.symlink-non-executable"}),
+                0,
+                "out is a symlink to a non-executable file"
+            );
+        }, qr/output is not executable/, "A relevant warning is provided for symlinks to non-executables");
 
-        is(
-            Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.symlink-directory"}),
-            0,
-            "out is a symlink to a directory"
-        );
+        like(warning {
+            is(
+                Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.symlink-directory"}),
+                0,
+                "out is a symlink to a directory"
+            );
+        }, qr/output is not a regular file or symlink/, "A relevant warning is provided for symlinks to directories");
     };
 
     subtest "On build status ..." => sub {

From daa6864a58e0f29509c2718d6dc91dc84198808b Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Mon, 24 Jan 2022 16:09:45 -0500
Subject: [PATCH 23/34] Project result: add a supportsDynamicRunCommand helper

---
 src/lib/Hydra/Plugin/RunCommand.pm      | 2 +-
 src/lib/Hydra/Schema/Result/Projects.pm | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/lib/Hydra/Plugin/RunCommand.pm b/src/lib/Hydra/Plugin/RunCommand.pm
index b55e96a9..9e173278 100644
--- a/src/lib/Hydra/Plugin/RunCommand.pm
+++ b/src/lib/Hydra/Plugin/RunCommand.pm
@@ -75,7 +75,7 @@ sub isBuildEligibleForDynamicRunCommand {
             return 0;
         }
 
-        if (! $build->project->enable_dynamic_run_command) {
+        if (! $build->project->supportsDynamicRunCommand()) {
             return 0;
         }
 
diff --git a/src/lib/Hydra/Schema/Result/Projects.pm b/src/lib/Hydra/Schema/Result/Projects.pm
index 42ca22a4..d6e66bf7 100644
--- a/src/lib/Hydra/Schema/Result/Projects.pm
+++ b/src/lib/Hydra/Schema/Result/Projects.pm
@@ -246,6 +246,12 @@ sub builds {
   return $self->jobsets->related_resultset('builds');
 };
 
+sub supportsDynamicRunCommand {
+  my ($self) = @_;
+
+  return $self->get_column('enable_dynamic_run_command') == 1;
+}
+
 sub as_json {
     my $self = shift;
 

From 3aa239309181d04dc6831b22c8abb0bbf85cb657 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Mon, 24 Jan 2022 16:11:52 -0500
Subject: [PATCH 24/34] Jobsets: add a supportsDynamicRunCommand which also
 checks the project's dynamic runcommand support

---
 src/lib/Hydra/Plugin/RunCommand.pm     | 6 +-----
 src/lib/Hydra/Schema/Result/Jobsets.pm | 7 +++++++
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/lib/Hydra/Plugin/RunCommand.pm b/src/lib/Hydra/Plugin/RunCommand.pm
index 9e173278..2b3bb6f4 100644
--- a/src/lib/Hydra/Plugin/RunCommand.pm
+++ b/src/lib/Hydra/Plugin/RunCommand.pm
@@ -71,11 +71,7 @@ sub isBuildEligibleForDynamicRunCommand {
             return 0;
         }
 
-        if (! $build->jobset->enable_dynamic_run_command) {
-            return 0;
-        }
-
-        if (! $build->project->supportsDynamicRunCommand()) {
+        if (! $build->jobset->supportsDynamicRunCommand()) {
             return 0;
         }
 
diff --git a/src/lib/Hydra/Schema/Result/Jobsets.pm b/src/lib/Hydra/Schema/Result/Jobsets.pm
index 7b96c472..cd704ac8 100644
--- a/src/lib/Hydra/Schema/Result/Jobsets.pm
+++ b/src/lib/Hydra/Schema/Result/Jobsets.pm
@@ -386,6 +386,13 @@ __PACKAGE__->add_column(
     "+id" => { retrieve_on_insert => 1 }
 );
 
+sub supportsDynamicRunCommand {
+  my ($self) = @_;
+
+  return $self->get_column('enable_dynamic_run_command') == 1
+    && $self->project->supportsDynamicRunCommand();
+}
+
 sub as_json {
     my $self = shift;
 

From d8b56f022d1acd2b9f08106ab502c0a51393e6a7 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Mon, 24 Jan 2022 16:16:58 -0500
Subject: [PATCH 25/34] RunCommand: print a warning if the hook isn't run
 because the project / jobset doens't have it enabled

---
 src/lib/Hydra/Plugin/RunCommand.pm |  1 +
 t/Hydra/Plugin/RunCommand/fanout.t | 37 ++++++++++++++++++------------
 2 files changed, 23 insertions(+), 15 deletions(-)

diff --git a/src/lib/Hydra/Plugin/RunCommand.pm b/src/lib/Hydra/Plugin/RunCommand.pm
index 2b3bb6f4..43163764 100644
--- a/src/lib/Hydra/Plugin/RunCommand.pm
+++ b/src/lib/Hydra/Plugin/RunCommand.pm
@@ -72,6 +72,7 @@ sub isBuildEligibleForDynamicRunCommand {
         }
 
         if (! $build->jobset->supportsDynamicRunCommand()) {
+            warn "DynamicRunCommand hook on " . $build->job . " (" . $build->id . ") rejected: The project or jobset don't have dynamic runcommand enabled.";
             return 0;
         }
 
diff --git a/t/Hydra/Plugin/RunCommand/fanout.t b/t/Hydra/Plugin/RunCommand/fanout.t
index 808f661c..328824f9 100644
--- a/t/Hydra/Plugin/RunCommand/fanout.t
+++ b/t/Hydra/Plugin/RunCommand/fanout.t
@@ -191,33 +191,40 @@ subtest "isBuildEligibleForDynamicRunCommand" => sub {
             $build->project->update({enable_dynamic_run_command => 0});
             $build->jobset->update({enable_dynamic_run_command => 1});
 
-            is(
-                Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}),
-                0,
-                "Builds don't run from a jobset with disabled dynamic runcommand"
-            );
+
+            like(warning {
+                is(
+                    Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}),
+                    0,
+                    "Builds don't run from a jobset with disabled dynamic runcommand"
+                );
+            }, qr/project or jobset don't have dynamic runcommand enabled./, "A relevant warning is provided for a disabled runcommand support")
         };
 
         subtest "enabled on the project, disabled on the jobset" => sub {
             $build->project->update({enable_dynamic_run_command => 1});
             $build->jobset->update({enable_dynamic_run_command => 0});
 
-            is(
-                Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}),
-                0,
-                "Builds don't run from a jobset with disabled dynamic runcommand"
-            );
+            like(warning {
+                is(
+                    Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}),
+                    0,
+                    "Builds don't run from a jobset with disabled dynamic runcommand"
+                );
+            }, qr/project or jobset don't have dynamic runcommand enabled./, "A relevant warning is provided for a disabled runcommand support")
         };
 
         subtest "disabled on the project, disabled on the jobset" => sub {
             $build->project->update({enable_dynamic_run_command => 0});
             $build->jobset->update({enable_dynamic_run_command => 0});
 
-            is(
-                Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}),
-                0,
-                "Builds don't run from a jobset with disabled dynamic runcommand"
-            );
+            like(warning {
+                is(
+                    Hydra::Plugin::RunCommand::isBuildEligibleForDynamicRunCommand($builds->{"runCommandHook.example"}),
+                    0,
+                    "Builds don't run from a jobset with disabled dynamic runcommand"
+                );
+            }, qr/project or jobset don't have dynamic runcommand enabled./, "A relevant warning is provided for a disabled runcommand support")
         };
     };
 };

From 3b895aec54ba07d03feef7c151ccf980967d29cd Mon Sep 17 00:00:00 2001
From: Cole Helbling <cole.helbling@determinate.systems>
Date: Fri, 17 Dec 2021 10:29:47 -0800
Subject: [PATCH 26/34] DynamicRunCommand: needs to be enabled by server,
 project, and jobset

---
 doc/manual/src/plugins/RunCommand.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/doc/manual/src/plugins/RunCommand.md b/doc/manual/src/plugins/RunCommand.md
index b186be80..652a171e 100644
--- a/doc/manual/src/plugins/RunCommand.md
+++ b/doc/manual/src/plugins/RunCommand.md
@@ -33,8 +33,9 @@ Command to run. Can use the `$HYDRA_JSON` environment variable to access informa
 
 ### Dynamic Commands
 
-Hydra can optionally run RunCommand hooks defined dynamically by the jobset.
-This must be turned on explicitly in the `hydra.conf` and per jobset.
+Hydra can optionally run RunCommand hooks defined dynamically by the jobset. In
+order to enable dynamic commands, you must enable this feature in your
+`hydra.conf`, *as well as* in the parent project and jobset configuration.
 
 #### Behavior
 

From 3f4f1837928add779759da21522b617b93748a14 Mon Sep 17 00:00:00 2001
From: Cole Helbling <cole.helbling@determinate.systems>
Date: Fri, 17 Dec 2021 10:31:03 -0800
Subject: [PATCH 27/34] jobset.tt: more info on why Dynamic RunCommand is
 disabled

---
 src/root/jobset.tt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/root/jobset.tt b/src/root/jobset.tt
index 3d6ca6ae..56abdb50 100644
--- a/src/root/jobset.tt
+++ b/src/root/jobset.tt
@@ -162,7 +162,7 @@
       </tr>
       <tr>
         <th>Enable Dynamic RunCommand Hooks:</th>
-        <td>[% jobset.enable_dynamic_run_command ? "Yes" : "No" %]</td>
+        <td>[% c.config.dynamicruncommand.enable ? project.enable_dynamic_run_command ? jobset.enable_dynamic_run_command ? "Yes" : "No (not enabled by jobset)" : "No (not enabled by project)" : "No (not enabled by server)" %]</td>
       </tr>
       [% IF emailNotification %]
       <tr>

From dfd3a67424c0cd834d2a009d061336f3b61814ff Mon Sep 17 00:00:00 2001
From: Cole Helbling <cole.helbling@determinate.systems>
Date: Fri, 17 Dec 2021 10:31:46 -0800
Subject: [PATCH 28/34] project.tt: more info on why Dynamic RunCommand is
 disabled

---
 src/root/project.tt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/root/project.tt b/src/root/project.tt
index f5a51e96..5e8ec0c8 100644
--- a/src/root/project.tt
+++ b/src/root/project.tt
@@ -94,7 +94,7 @@
       </tr>
       <tr>
         <th>Enable Dynamic RunCommand Hooks:</th>
-        <td>[% project.enable_dynamic_run_command ? "Yes" : "No" %]</td>
+        <td>[% c.config.dynamicruncommand.enable ? project.enable_dynamic_run_command ? "Yes" : "No (not enabled by project)" : "No (not enabled by server)" %]</td>
       </tr>
     </table>
   </div>

From 6053e5fd4b2e23aeb69b632417a4e42ca88fe168 Mon Sep 17 00:00:00 2001
From: Cole Helbling <cole.helbling@determinate.systems>
Date: Fri, 17 Dec 2021 11:02:59 -0800
Subject: [PATCH 29/34] edit-jobset.tt: disable when disabled by project and
 server

Also add a tooltip describing why it's disabled, to make it easier to
chase down.
---
 src/root/edit-jobset.tt | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/root/edit-jobset.tt b/src/root/edit-jobset.tt
index 40da8f61..61e3636f 100644
--- a/src/root/edit-jobset.tt
+++ b/src/root/edit-jobset.tt
@@ -160,7 +160,15 @@
   <div class="form-group row">
     <label class="col-sm-3" for="editjobsetenable_dynamic_run_command">Enable Dynamic RunCommand Hooks</label>
     <div class="col-sm-9">
-      <input type="checkbox" id="editjobsetenable_dynamic_run_command" name="enable_dynamic_run_command" [% IF jobset.enable_dynamic_run_command %]checked[% END %]/>
+      <input type="checkbox" id="editjobsetenable_dynamic_run_command" name="enable_dynamic_run_command"
+        [% IF !c.config.dynamicruncommand.enable %]
+          title="The server has not enabled dynamic RunCommands" disabled
+        [% ELSIF !project.enable_dynamic_run_command %]
+          title="The parent project has not enabled dynamic RunCommands" disabled
+        [% ELSIF jobset.enable_dynamic_run_command %]
+          checked
+        [% END %]
+      />
     </div>
   </div>
 

From d680c209feffb3fe1087cd39e62545cd03cb22f8 Mon Sep 17 00:00:00 2001
From: Cole Helbling <cole.helbling@determinate.systems>
Date: Fri, 17 Dec 2021 11:03:55 -0800
Subject: [PATCH 30/34] edit-project.tt: disable when disabled by server

Also add a tooltip describing why it's disabled, to make it easier to
chase down.
---
 src/root/edit-project.tt | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/root/edit-project.tt b/src/root/edit-project.tt
index 4b99f4ab..bb850e5c 100644
--- a/src/root/edit-project.tt
+++ b/src/root/edit-project.tt
@@ -56,7 +56,13 @@
   <div class="form-group row">
     <label class="col-sm-3" for="editprojectenable_dynamic_run_command">Enable Dynamic RunCommand Hooks for Jobsets</label>
     <div class="col-sm-9">
-      <input type="checkbox" id="editprojectenable_dynamic_run_command" name="enable_dynamic_run_command" [% IF jobset.enable_dynamic_run_command %]checked[% END %]/>
+      <input type="checkbox" id="editprojectenable_dynamic_run_command" name="enable_dynamic_run_command"
+        [% IF !c.config.dynamicruncommand.enable %]
+          title="The server has not enabled dynamic RunCommands" disabled
+        [% ELSIF project.enable_dynamic_run_command %]
+          checked
+        [% END %]
+      />
     </div>
   </div>
 

From 928ba9e854e8fb7ee88f352a32848f114004e70c Mon Sep 17 00:00:00 2001
From: Cole Helbling <cole.helbling@determinate.systems>
Date: Fri, 17 Dec 2021 12:34:19 -0800
Subject: [PATCH 31/34] Controller/{Jobset,Project}: error when enabling
 dynamic runcommand but it's disabled elsewhere

---
 src/lib/Hydra/Controller/Jobset.pm           |  10 +-
 src/lib/Hydra/Controller/Project.pm          |   7 +-
 t/Hydra/Plugin/RunCommand/dynamic-disabled.t | 110 +++++++++++++++++++
 t/Hydra/Plugin/RunCommand/dynamic-enabled.t  | 106 ++++++++++++++++++
 4 files changed, 231 insertions(+), 2 deletions(-)
 create mode 100644 t/Hydra/Plugin/RunCommand/dynamic-disabled.t
 create mode 100644 t/Hydra/Plugin/RunCommand/dynamic-enabled.t

diff --git a/src/lib/Hydra/Controller/Jobset.pm b/src/lib/Hydra/Controller/Jobset.pm
index a2d48597..eeb4232a 100644
--- a/src/lib/Hydra/Controller/Jobset.pm
+++ b/src/lib/Hydra/Controller/Jobset.pm
@@ -261,6 +261,14 @@ sub updateJobset {
 
     my $checkinterval = int(trim($c->stash->{params}->{checkinterval}));
 
+    my $enable_dynamic_run_command = defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0;
+    if ($enable_dynamic_run_command
+        && !($c->config->{dynamicruncommand}->{enable}
+            && $jobset->project->enable_dynamic_run_command))
+    {
+        badRequest($c, "Dynamic RunCommand is not enabled by the server or the parent project.");
+    }
+
     $jobset->update(
         { name => $jobsetName
         , description => trim($c->stash->{params}->{"description"})
@@ -268,7 +276,7 @@ sub updateJobset {
         , nixexprinput => $nixExprInput
         , enabled => $enabled
         , enableemail => defined $c->stash->{params}->{enableemail} ? 1 : 0
-        , enable_dynamic_run_command => defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0
+        , enable_dynamic_run_command => $enable_dynamic_run_command
         , emailoverride => trim($c->stash->{params}->{emailoverride}) || ""
         , hidden => defined $c->stash->{params}->{visible} ? 0 : 1
         , keepnr => int(trim($c->stash->{params}->{keepnr} // "0"))
diff --git a/src/lib/Hydra/Controller/Project.pm b/src/lib/Hydra/Controller/Project.pm
index 98a8a6eb..1141de4a 100644
--- a/src/lib/Hydra/Controller/Project.pm
+++ b/src/lib/Hydra/Controller/Project.pm
@@ -149,6 +149,11 @@ sub updateProject {
     my $displayName = trim $c->stash->{params}->{displayname};
     error($c, "You must specify a display name.") if $displayName eq "";
 
+    my $enable_dynamic_run_command = defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0;
+    if ($enable_dynamic_run_command && !$c->config->{dynamicruncommand}->{enable}) {
+        badRequest($c, "Dynamic RunCommand is not enabled by the server.");
+    }
+
     $project->update(
         { name => $projectName
         , displayname => $displayName
@@ -157,7 +162,7 @@ sub updateProject {
         , enabled => defined $c->stash->{params}->{enabled} ? 1 : 0
         , hidden => defined $c->stash->{params}->{visible} ? 0 : 1
         , owner => $owner
-        , enable_dynamic_run_command => defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0
+        , enable_dynamic_run_command => $enable_dynamic_run_command
         , declfile => trim($c->stash->{params}->{declarative}->{file})
         , decltype => trim($c->stash->{params}->{declarative}->{type})
         , declvalue => trim($c->stash->{params}->{declarative}->{value})
diff --git a/t/Hydra/Plugin/RunCommand/dynamic-disabled.t b/t/Hydra/Plugin/RunCommand/dynamic-disabled.t
new file mode 100644
index 00000000..ad2e9a4b
--- /dev/null
+++ b/t/Hydra/Plugin/RunCommand/dynamic-disabled.t
@@ -0,0 +1,110 @@
+use strict;
+use warnings;
+use Setup;
+use Test2::V0;
+
+require Catalyst::Test;
+use HTTP::Request::Common qw(POST PUT GET DELETE);
+use JSON::MaybeXS qw(decode_json encode_json);
+
+my $ctx = test_context();
+Catalyst::Test->import('Hydra');
+
+# Create a user to log in to
+my $user = $ctx->db->resultset('Users')->create({ username => 'alice', emailaddress => 'root@invalid.org', password => '!' });
+$user->setPassword('foobar');
+$user->userroles->update_or_create({ role => 'admin' });
+
+subtest "can't enable dynamic RunCommand when disabled by server" => sub {
+    my $builds = $ctx->makeAndEvaluateJobset(
+        expression => "runcommand-dynamic.nix",
+        build => 1
+    );
+
+    my $build = $builds->{"runCommandHook.example"};
+    my $project = $build->project;
+    my $project_name = $project->name;
+    my $jobset = $build->jobset;
+    my $jobset_name = $jobset->name;
+
+    is($project->enable_dynamic_run_command, 0, "dynamic RunCommand is disabled on projects by default");
+    is($jobset->enable_dynamic_run_command, 0, "dynamic RunCommand is disabled on jobsets by default");
+
+    my $req = request(POST '/login',
+        Referer => 'http://localhost/',
+        Content => {
+            username => 'alice',
+            password => 'foobar'
+        }
+    );
+    is($req->code, 302, "logged in successfully");
+    my $cookie = $req->header("set-cookie");
+
+    subtest "can't enable dynamic RunCommand on project" => sub {
+        my $projectresponse = request(GET "/project/$project_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+        );
+
+        my $projectjson = decode_json($projectresponse->content);
+        $projectjson->{enable_dynamic_run_command} = 1;
+
+        my $projectupdate = request(PUT "/project/$project_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+            Content => encode_json($projectjson)
+        );
+
+        $projectresponse = request(GET "/project/$project_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+        );
+        $projectjson = decode_json($projectresponse->content);
+
+        is($projectupdate->code, 400);
+        like(
+            $projectupdate->content,
+            qr/Dynamic RunCommand is not/,
+            "failed to change enable_dynamic_run_command, not any other error"
+        );
+        is($projectjson->{enable_dynamic_run_command}, JSON::MaybeXS::false);
+    };
+
+    subtest "can't enable dynamic RunCommand on jobset" => sub {
+        my $jobsetresponse = request(GET "/jobset/$project_name/$jobset_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+        );
+
+        my $jobsetjson = decode_json($jobsetresponse->content);
+        $jobsetjson->{enable_dynamic_run_command} = 1;
+
+        my $jobsetupdate = request(PUT "/jobset/$project_name/$jobset_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+            Content => encode_json($jobsetjson)
+        );
+
+        $jobsetresponse = request(GET "/jobset/$project_name/$jobset_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+        );
+        $jobsetjson = decode_json($jobsetresponse->content);
+
+        is($jobsetupdate->code, 400);
+        like(
+            $jobsetupdate->content,
+            qr/Dynamic RunCommand is not/,
+            "failed to change enable_dynamic_run_command, not any other error"
+        );
+        is($jobsetjson->{enable_dynamic_run_command}, JSON::MaybeXS::false);
+    };
+};
+
+done_testing;
diff --git a/t/Hydra/Plugin/RunCommand/dynamic-enabled.t b/t/Hydra/Plugin/RunCommand/dynamic-enabled.t
new file mode 100644
index 00000000..68c6d593
--- /dev/null
+++ b/t/Hydra/Plugin/RunCommand/dynamic-enabled.t
@@ -0,0 +1,106 @@
+use strict;
+use warnings;
+use Setup;
+use Test2::V0;
+
+require Catalyst::Test;
+use HTTP::Request::Common qw(POST PUT GET DELETE);
+use JSON::MaybeXS qw(decode_json encode_json);
+
+my $ctx = test_context(
+    hydra_config => q|
+    <dynamicruncommand>
+    enable = 1
+    </dynamicruncommand>
+    |
+);
+Catalyst::Test->import('Hydra');
+
+# Create a user to log in to
+my $user = $ctx->db->resultset('Users')->create({ username => 'alice', emailaddress => 'root@invalid.org', password => '!' });
+$user->setPassword('foobar');
+$user->userroles->update_or_create({ role => 'admin' });
+
+subtest "can enable dynamic RunCommand when enabled by server" => sub {
+    my $builds = $ctx->makeAndEvaluateJobset(
+        expression => "runcommand-dynamic.nix",
+        build => 1
+    );
+
+    my $build = $builds->{"runCommandHook.example"};
+    my $project = $build->project;
+    my $project_name = $project->name;
+    my $jobset = $build->jobset;
+    my $jobset_name = $jobset->name;
+
+    is($project->enable_dynamic_run_command, 0, "dynamic RunCommand is disabled on projects by default");
+    is($jobset->enable_dynamic_run_command, 0, "dynamic RunCommand is disabled on jobsets by default");
+
+    my $req = request(POST '/login',
+        Referer => 'http://localhost/',
+        Content => {
+            username => 'alice',
+            password => 'foobar'
+        }
+    );
+    is($req->code, 302, "logged in successfully");
+    my $cookie = $req->header("set-cookie");
+
+    subtest "can enable dynamic RunCommand on project" => sub {
+        my $projectresponse = request(GET "/project/$project_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+        );
+
+        my $projectjson = decode_json($projectresponse->content);
+        $projectjson->{enable_dynamic_run_command} = 1;
+
+        my $projectupdate = request(PUT "/project/$project_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+            Content => encode_json($projectjson)
+        );
+
+        $projectresponse = request(GET "/project/$project_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+        );
+        $projectjson = decode_json($projectresponse->content);
+
+        is($projectupdate->code, 200);
+        is($projectjson->{enable_dynamic_run_command}, JSON::MaybeXS::true);
+    };
+
+    subtest "can enable dynamic RunCommand on jobset" => sub {
+        my $jobsetresponse = request(GET "/jobset/$project_name/$jobset_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+        );
+
+        my $jobsetjson = decode_json($jobsetresponse->content);
+        $jobsetjson->{enable_dynamic_run_command} = 1;
+
+        my $jobsetupdate = request(PUT "/jobset/$project_name/$jobset_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+            Content => encode_json($jobsetjson)
+        );
+
+        $jobsetresponse = request(GET "/jobset/$project_name/$jobset_name",
+            Accept => 'application/json',
+            Content_Type => 'application/json',
+            Cookie => $cookie,
+        );
+        $jobsetjson = decode_json($jobsetresponse->content);
+
+        is($jobsetupdate->code, 200);
+        is($jobsetjson->{enable_dynamic_run_command}, JSON::MaybeXS::true);
+    };
+};
+
+done_testing;

From a22a8fa62d777950dca470b9791e5ac8cda2ddbf Mon Sep 17 00:00:00 2001
From: Cole Helbling <cole.helbling@determinate.systems>
Date: Mon, 20 Dec 2021 09:37:14 -0800
Subject: [PATCH 32/34] AddBuilds: reject declarative jobsets with dynamic
 runcommand enabled if disabled elsewhere

---
 src/lib/Hydra/Helper/AddBuilds.pm     | 44 +++++++++++---
 src/script/hydra-eval-jobset          |  2 +-
 t/Helper/AddBuilds/dynamic-disabled.t | 85 ++++++++++++++++++++++++++
 t/Helper/AddBuilds/dynamic-enabled.t  | 88 +++++++++++++++++++++++++++
 4 files changed, 209 insertions(+), 10 deletions(-)
 create mode 100644 t/Helper/AddBuilds/dynamic-disabled.t
 create mode 100644 t/Helper/AddBuilds/dynamic-enabled.t

diff --git a/src/lib/Hydra/Helper/AddBuilds.pm b/src/lib/Hydra/Helper/AddBuilds.pm
index f38737d3..9e3ddfd2 100644
--- a/src/lib/Hydra/Helper/AddBuilds.pm
+++ b/src/lib/Hydra/Helper/AddBuilds.pm
@@ -19,14 +19,16 @@ use Hydra::Helper::CatalystUtils;
 
 our @ISA = qw(Exporter);
 our @EXPORT = qw(
+    validateDeclarativeJobset
+    createJobsetInputsRowAndData
     updateDeclarativeJobset
     handleDeclarativeJobsetBuild
     handleDeclarativeJobsetJson
 );
 
 
-sub updateDeclarativeJobset {
-    my ($db, $project, $jobsetName, $declSpec) = @_;
+sub validateDeclarativeJobset {
+    my ($config, $project, $jobsetName, $declSpec) = @_;
 
     my @allowed_keys = qw(
         enabled
@@ -62,16 +64,39 @@ sub updateDeclarativeJobset {
         }
     }
 
+    my $enable_dynamic_run_command = defined $update{enable_dynamic_run_command} ? 1 : 0;
+    if ($enable_dynamic_run_command
+        && !($config->{dynamicruncommand}->{enable}
+            && $project->{enable_dynamic_run_command}))
+    {
+        die "Dynamic RunCommand is not enabled by the server or the parent project.";
+    }
+
+    return %update;
+}
+
+sub createJobsetInputsRowAndData {
+    my ($name, $declSpec) = @_;
+    my $data = $declSpec->{"inputs"}->{$name};
+    my $row = {
+        name => $name,
+        type => $data->{type}
+    };
+    $row->{emailresponsible} = $data->{emailresponsible} // 0;
+
+    return ($row, $data);
+}
+
+sub updateDeclarativeJobset {
+    my ($config, $db, $project, $jobsetName, $declSpec) = @_;
+
+    my %update = validateDeclarativeJobset($config, $project, $jobsetName, $declSpec);
+
     $db->txn_do(sub {
         my $jobset = $project->jobsets->update_or_create(\%update);
         $jobset->jobsetinputs->delete;
         foreach my $name (keys %{$declSpec->{"inputs"}}) {
-            my $data = $declSpec->{"inputs"}->{$name};
-            my $row = {
-                name => $name,
-                type => $data->{type}
-            };
-            $row->{emailresponsible} = $data->{emailresponsible} // 0;
+            my ($row, $data) = createJobsetInputsRowAndData($name, $declSpec);
             my $input = $jobset->jobsetinputs->create($row);
             $input->jobsetinputalts->create({altnr => 0, value => $data->{value}});
         }
@@ -82,6 +107,7 @@ sub updateDeclarativeJobset {
 
 sub handleDeclarativeJobsetJson {
     my ($db, $project, $declSpec) = @_;
+    my $config = getHydraConfig();
     $db->txn_do(sub {
             my @kept = keys %$declSpec;
             push @kept, ".jobsets";
@@ -89,7 +115,7 @@ sub handleDeclarativeJobsetJson {
             foreach my $jobsetName (keys %$declSpec) {
                 my $spec = $declSpec->{$jobsetName};
                 eval {
-                    updateDeclarativeJobset($db, $project, $jobsetName, $spec);
+                    updateDeclarativeJobset($config, $db, $project, $jobsetName, $spec);
                     1;
                 } or do {
                     print STDERR "ERROR: failed to process declarative jobset ", $project->name, ":${jobsetName}, ", $@, "\n";
diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset
index de437ecd..a9bd7355 100755
--- a/src/script/hydra-eval-jobset
+++ b/src/script/hydra-eval-jobset
@@ -617,7 +617,7 @@ sub checkJobsetWrapped {
             } else {
                 # Update the jobset with the spec's inputs, and the continue
                 # evaluating the .jobsets jobset.
-                updateDeclarativeJobset($db, $project, ".jobsets", $declSpec);
+                updateDeclarativeJobset($config, $db, $project, ".jobsets", $declSpec);
                 $jobset->discard_changes;
                 $inputInfo->{"declInput"} = [ $declInput ];
                 $inputInfo->{"projectName"} = [ fetchInput($plugins, $db, $project, $jobset, "projectName", "string", $project->name, 0) ];
diff --git a/t/Helper/AddBuilds/dynamic-disabled.t b/t/Helper/AddBuilds/dynamic-disabled.t
new file mode 100644
index 00000000..0507b03e
--- /dev/null
+++ b/t/Helper/AddBuilds/dynamic-disabled.t
@@ -0,0 +1,85 @@
+use strict;
+use warnings;
+use Setup;
+use Test2::V0;
+
+require Catalyst::Test;
+use HTTP::Request::Common qw(POST PUT GET DELETE);
+use JSON::MaybeXS qw(decode_json encode_json);
+use Hydra::Helper::AddBuilds qw(validateDeclarativeJobset);
+use Hydra::Helper::Nix qw(getHydraConfig);
+
+my $ctx = test_context();
+
+sub makeJobsetSpec {
+    my ($dynamic) = @_;
+
+    return {
+        enabled => 2,
+        enable_dynamic_run_command => $dynamic ? JSON::MaybeXS::true : undef,
+        visible => JSON::MaybeXS::true,
+        name => "job",
+        type => 1,
+        description => "test jobset",
+        flake => "github:nixos/nix",
+        checkinterval => 0,
+        schedulingshares => 100,
+        keepnr => 3
+    };
+};
+
+subtest "validate declarative jobset with dynamic RunCommand disabled by server" => sub {
+    my $config = getHydraConfig();
+
+    subtest "project enabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub {
+        like(
+            dies {
+              validateDeclarativeJobset(
+                  $config,
+                  { enable_dynamic_run_command => 1 },
+                  "test-jobset",
+                  makeJobsetSpec(JSON::MaybeXS::true),
+              ),
+            },
+            qr/Dynamic RunCommand is not enabled/,
+        );
+    };
+
+    subtest "project enabled dynamic runcommand, declarative jobset disabled dynamic runcommand" => sub {
+        ok(
+            validateDeclarativeJobset(
+                $config,
+                { enable_dynamic_run_command => 1 },
+                "test-jobset",
+                makeJobsetSpec(JSON::MaybeXS::false)
+            ),
+        );
+    };
+
+    subtest "project disabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub {
+        like(
+            dies {
+              validateDeclarativeJobset(
+                  $config,
+                  { enable_dynamic_run_command => 0 },
+                  "test-jobset",
+                  makeJobsetSpec(JSON::MaybeXS::true),
+              ),
+            },
+            qr/Dynamic RunCommand is not enabled/,
+        );
+    };
+
+    subtest "project disabled dynamic runcommand, declarative jobset disabled dynamic runcommand" => sub {
+        ok(
+            validateDeclarativeJobset(
+                $config,
+                { enable_dynamic_run_command => 0 },
+                "test-jobset",
+                makeJobsetSpec(JSON::MaybeXS::false)
+            ),
+        );
+    };
+};
+
+done_testing;
diff --git a/t/Helper/AddBuilds/dynamic-enabled.t b/t/Helper/AddBuilds/dynamic-enabled.t
new file mode 100644
index 00000000..d2f5a386
--- /dev/null
+++ b/t/Helper/AddBuilds/dynamic-enabled.t
@@ -0,0 +1,88 @@
+use strict;
+use warnings;
+use Setup;
+use Test2::V0;
+
+require Catalyst::Test;
+use HTTP::Request::Common qw(POST PUT GET DELETE);
+use JSON::MaybeXS qw(decode_json encode_json);
+use Hydra::Helper::AddBuilds qw(validateDeclarativeJobset);
+use Hydra::Helper::Nix qw(getHydraConfig);
+
+my $ctx = test_context(
+    hydra_config => q|
+    <dynamicruncommand>
+    enable = 1
+    </dynamicruncommand>
+    |
+);
+
+sub makeJobsetSpec {
+    my ($dynamic) = @_;
+
+    return {
+        enabled => 2,
+        enable_dynamic_run_command => $dynamic ? JSON::MaybeXS::true : undef,
+        visible => JSON::MaybeXS::true,
+        name => "job",
+        type => 1,
+        description => "test jobset",
+        flake => "github:nixos/nix",
+        checkinterval => 0,
+        schedulingshares => 100,
+        keepnr => 3
+    };
+};
+
+subtest "validate declarative jobset with dynamic RunCommand enabled by server" => sub {
+    my $config = getHydraConfig();
+
+    subtest "project enabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub {
+        ok(
+            validateDeclarativeJobset(
+                $config,
+                { enable_dynamic_run_command => 1 },
+                "test-jobset",
+                makeJobsetSpec(JSON::MaybeXS::true)
+            ),
+        );
+    };
+
+    subtest "project enabled dynamic runcommand, declarative jobset disabled dynamic runcommand" => sub {
+        ok(
+            validateDeclarativeJobset(
+                $config,
+                { enable_dynamic_run_command => 1 },
+                "test-jobset",
+                makeJobsetSpec(JSON::MaybeXS::false)
+            ),
+        );
+    };
+
+    subtest "project disabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub {
+        like(
+            dies {
+              validateDeclarativeJobset(
+                  $config,
+                  { enable_dynamic_run_command => 0 },
+                  "test-jobset",
+                  makeJobsetSpec(JSON::MaybeXS::true),
+              ),
+            },
+            qr/Dynamic RunCommand is not enabled/,
+        );
+    };
+
+    subtest "project disabled dynamic runcommand, declarative jobset disabled dynamic runcommand" => sub {
+        ok(
+            validateDeclarativeJobset(
+                $config,
+                { enable_dynamic_run_command => 0 },
+                "test-jobset",
+                makeJobsetSpec(JSON::MaybeXS::false)
+            ),
+        );
+    };
+};
+
+done_testing;

From 8c3122cacd82d86467b1e69b58db14f058a7afe5 Mon Sep 17 00:00:00 2001
From: Cole Helbling <cole.helbling@determinate.systems>
Date: Mon, 20 Dec 2021 11:20:17 -0800
Subject: [PATCH 33/34] hydra-api: add enable_dynamic_run_command to Project
 PUT

---
 hydra-api.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/hydra-api.yaml b/hydra-api.yaml
index 0fe0a130..ce7e0f9a 100644
--- a/hydra-api.yaml
+++ b/hydra-api.yaml
@@ -178,6 +178,9 @@ paths:
                 enabled:
                   description: when set to true the project gets scheduled for evaluation
                   type: boolean
+                enable_dynamic_run_command:
+                  description: when true the project's jobsets support executing dynamically defined RunCommand hooks. Requires the server and project's configuration to also enable dynamic RunCommand.
+                  type: boolean
                 visible:
                   description: when set to true the project is displayed in the web interface
                   type: boolean

From 27ddde1e9eb95b694884dce5dfb24b67c02aefbd Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Fri, 11 Feb 2022 15:03:09 -0500
Subject: [PATCH 34/34] dynamic runcommand: print a notice on the build page if
 it is disabled

---
 src/lib/Hydra/Controller/Build.pm | 11 +++++++++++
 src/root/build.tt                 | 15 ++++++++++++++-
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm
index af648109..552f31af 100644
--- a/src/lib/Hydra/Controller/Build.pm
+++ b/src/lib/Hydra/Controller/Build.pm
@@ -38,6 +38,17 @@ sub buildChain :Chained('/') :PathPart('build') :CaptureArgs(1) {
     $c->stash->{jobset} = $c->stash->{build}->jobset;
     $c->stash->{job} = $c->stash->{build}->job;
     $c->stash->{runcommandlogs} = [$c->stash->{build}->runcommandlogs->search({}, {order_by => ["id DESC"]})];
+
+    $c->stash->{runcommandlogProblem} = undef;
+    if ($c->stash->{job} =~ qr/^runCommandHook\..*/) {
+        if (!$c->config->{dynamicruncommand}->{enable}) {
+            $c->stash->{runcommandlogProblem} = "disabled-server";
+        } elsif (!$c->stash->{project}->enable_dynamic_run_command) {
+            $c->stash->{runcommandlogProblem} = "disabled-project";
+        } elsif (!$c->stash->{jobset}->enable_dynamic_run_command) {
+            $c->stash->{runcommandlogProblem} = "disabled-jobset";
+        }
+    }
 }
 
 
diff --git a/src/root/build.tt b/src/root/build.tt
index 0848da4a..027ce3e4 100644
--- a/src/root/build.tt
+++ b/src/root/build.tt
@@ -149,7 +149,7 @@ END;
   [% IF build.dependents %]<li class="nav-item"><a class="nav-link" href="#tabs-usedby" data-toggle="tab">Used By</a></li>[% END%]
   [% IF drvAvailable %]<li class="nav-item"><a class="nav-link" href="#tabs-build-deps" data-toggle="tab">Build Dependencies</a></li>[% END %]
   [% IF localStore && available %]<li class="nav-item"><a class="nav-link" href="#tabs-runtime-deps" data-toggle="tab">Runtime Dependencies</a></li>[% END %]
-  [% IF runcommandlogs.size() > 0 %]<li class="nav-item"><a class="nav-link" href="#tabs-runcommandlogs" data-toggle="tab">RunCommand Logs</a></li>[% END %]
+  [% IF runcommandlogProblem || runcommandlogs.size() > 0 %]<li class="nav-item"><a class="nav-link" href="#tabs-runcommandlogs" data-toggle="tab">RunCommand Logs[% IF runcommandlogProblem %] <span class="badge badge-warning">Disabled</span>[% END %]</a></li>[% END %]
 </ul>
 
 <div id="generic-tabs" class="tab-content">
@@ -489,6 +489,19 @@ END;
   [% END %]
 
   <div id="tabs-runcommandlogs" class="tab-pane">
+    [% IF runcommandlogProblem %]
+      <div class="alert alert-warning" role="alert">
+        [% IF runcommandlogProblem == "disabled-server" %]
+          This server does not enable Dynamic RunCommand support.
+        [% ELSIF runcommandlogProblem == "disabled-project" %]
+          This project does not enable Dynamic RunCommand support.
+        [% ELSIF runcommandlogProblem == "disabled-jobset" %]
+          This jobset does not enable Dynamic RunCommand support.
+        [% ELSE %]
+          Dynamic RunCommand is not enabled: [% runcommandlogProblem %].
+        [% END %]
+      </div>
+    [% END %]
     <div class="d-flex flex-column">
     [% FOREACH runcommandlog IN runcommandlogs %]
       <div class="p-2 border-bottom">