Merge pull request #985 from kreisys/hydra-server-config-includes

Make hydra-server honor apache-style includes in hydra.conf like all the other components

.gitignore

@@ -1,3 +1,4 @@
+/.pls_cache
 *.o
 *~
 Makefile
@@ -22,6 +23,7 @@ Makefile.in
 /src/hydra-eval-jobs/hydra-eval-jobs
 /src/root/static/bootstrap
 /src/root/static/js/flot
+/tests
 /doc/manual/images
 /doc/manual/manual.html
 /doc/manual/manual.pdf
@@ -41,3 +43,4 @@ stamp-h1
 src/hydra-evaluator/hydra-evaluator
 src/hydra-queue-runner/hydra-queue-runner
 src/root/static/fontawesome/
+src/root/static/bootstrap*/

README.md

@@ -113,6 +113,8 @@ After making your changes, verify the test suite still passes. After following t
 ```
 $ nix-shell
 $ make check
+$ # Or, to run a single test, use:
+$ yath test ./t/foo/bar.t
 ```
 
 ### JSON API

doc/manual/src/SUMMARY.md

@@ -2,6 +2,7 @@
 
 - [Introduction](introduction.md)
 - [Installation](installation.md)
+- [Configuration](configuration.md)
 - [Creating and Managing Projects](projects.md)
 - [Hydra jobs](./jobs.md)
 - [Using the external API](api.md)

doc/manual/src/configuration.md

@@ -0,0 +1,128 @@
+Configuration
+=============
+
+This chapter is a collection of configuration snippets for different
+scenarios.
+
+Including files
+---------------
+
+`hydra.conf` supports Apache-style includes. This is **IMPORTANT**
+because that is how you keep your **secrets** out of the **Nix store**.
+Hopefully this got your attention 😌
+
+This:
+```
+<github_authorization>
+NixOS = Bearer gha-secret😱secret😱secret😱
+</github_authorization>
+```
+should **NOT** be in `hydra.conf`.
+
+`hydra.conf` is rendered in the Nix store and is therefore world-readable.
+
+Instead, the above should be written to a file outside the Nix store by
+other means (manually, using Nixops' secrets feature, etc) and included
+like so:
+```
+Include /run/keys/hydra/github_authorizations.conf
+```
+
+Serving behind reverse proxy
+----------------------------
+
+To serve hydra web server behind reverse proxy like *nginx* or *httpd*
+some additional configuration must be made.
+
+Edit your `hydra.conf` file in a similar way to this example:
+
+```conf
+using_frontend_proxy 1
+base_uri example.com
+```
+
+`base_uri` should be your hydra servers proxied URL. If you are using
+Hydra nixos module then setting `hydraURL` option should be enough.
+
+If you want to serve Hydra with a prefix path, for example
+[http://example.com/hydra]() then you need to configure your reverse
+proxy to pass `X-Request-Base` to hydra, with prefix path as value. For
+example if you are using nginx, then use configuration similar to
+following:
+
+    server {
+        listen 433 ssl;
+        server_name example.com;
+        .. other configuration ..
+        location /hydra/ {
+
+            proxy_pass     http://127.0.0.1:3000;
+            proxy_redirect http://127.0.0.1:3000 https://example.com/hydra;
+
+            proxy_set_header  Host              $host;
+            proxy_set_header  X-Real-IP         $remote_addr;
+            proxy_set_header  X-Forwarded-For   $proxy_add_x_forwarded_for;
+            proxy_set_header  X-Forwarded-Proto $scheme;
+            proxy_set_header  X-Request-Base    /hydra;
+        }
+    }
+
+Statsd Configuration
+--------------------
+
+By default, Hydra will send stats to statsd at `localhost:8125`. Point Hydra to a different server via:
+
+```
+<statsd>
+  host = alternative.host
+  port = 18125
+</statsd>
+```
+
+Using LDAP as authentication backend (optional)
+-----------------------------------------------
+
+Instead of using Hydra\'s built-in user management you can optionally
+use LDAP to manage roles and users.
+
+The `hydra-server` accepts the environment variable
+*HYDRA\_LDAP\_CONFIG*. The value of the variable should point to a valid
+YAML file containing the Catalyst LDAP configuration. The format of the
+configuration file is describe in the
+[*Catalyst::Authentication::Store::LDAP*
+documentation](https://metacpan.org/pod/Catalyst::Authentication::Store::LDAP#CONFIGURATION-OPTIONS).
+An example is given below.
+
+Roles can be assigned to users based on their LDAP group membership
+(*use\_roles: 1* in the below example). For a user to have the role
+*admin* assigned to them they should be in the group *hydra\_admin*. In
+general any LDAP group of the form *hydra\_some\_role* (notice the
+*hydra\_* prefix) will work.
+
+    credential:
+      class: Password
+      password_field: password
+      password_type: self_check
+    store:
+      class: LDAP
+      ldap_server: localhost
+      ldap_server_options.timeout: 30
+      binddn: "cn=root,dc=example"
+      bindpw: notapassword
+      start_tls: 0
+      start_tls_options
+        verify:  none
+      user_basedn: "ou=users,dc=example"
+      user_filter: "(&(objectClass=inetOrgPerson)(cn=%s))"
+      user_scope: one
+      user_field: cn
+      user_search_options:
+        deref: always
+      use_roles: 1
+      role_basedn: "ou=groups,dc=example"
+      role_filter: "(&(objectClass=groupOfNames)(member=%s))"
+      role_scope: one
+      role_field: cn
+      role_value: dn
+      role_search_options:
+        deref: always

doc/manual/src/installation.md

@@ -163,102 +163,3 @@ processes that come into play:
 All three processes must be running for Hydra to be fully functional,
 though it\'s possible to temporarily stop any one of them for
 maintenance purposes, for instance.
-
-Serving behind reverse proxy
-----------------------------
-
-To serve hydra web server behind reverse proxy like *nginx* or *httpd*
-some additional configuration must be made.
-
-Edit your `hydra.conf` file in a similar way to this example:
-
-```conf
-using_frontend_proxy 1
-base_uri example.com
-```
-
-`base_uri` should be your hydra servers proxied URL. If you are using
-Hydra nixos module then setting `hydraURL` option should be enough.
-
-If you want to serve Hydra with a prefix path, for example
-[http://example.com/hydra]() then you need to configure your reverse
-proxy to pass `X-Request-Base` to hydra, with prefix path as value. For
-example if you are using nginx, then use configuration similar to
-following:
-
-    server {
-        listen 433 ssl;
-        server_name example.com;
-        .. other configuration ..
-        location /hydra/ {
-
-            proxy_pass     http://127.0.0.1:3000;
-            proxy_redirect http://127.0.0.1:3000 https://example.com/hydra;
-
-            proxy_set_header  Host              $host;
-            proxy_set_header  X-Real-IP         $remote_addr;
-            proxy_set_header  X-Forwarded-For   $proxy_add_x_forwarded_for;
-            proxy_set_header  X-Forwarded-Proto $scheme;
-            proxy_set_header  X-Request-Base    /hydra;
-        }
-    }
-
-Statsd Configuration
---------------------
-
-By default, Hydra will send stats to statsd at `localhost:8125`. Point Hydra to a different server via:
-
-```
-<statsd>
-  host = alternative.host
-  port = 18125
-</statsd>
-```
-
-Using LDAP as authentication backend (optional)
------------------------------------------------
-
-Instead of using Hydra\'s built-in user management you can optionally
-use LDAP to manage roles and users.
-
-The `hydra-server` accepts the environment variable
-*HYDRA\_LDAP\_CONFIG*. The value of the variable should point to a valid
-YAML file containing the Catalyst LDAP configuration. The format of the
-configuration file is describe in the
-[*Catalyst::Authentication::Store::LDAP*
-documentation](https://metacpan.org/pod/Catalyst::Authentication::Store::LDAP#CONFIGURATION-OPTIONS).
-An example is given below.
-
-Roles can be assigned to users based on their LDAP group membership
-(*use\_roles: 1* in the below example). For a user to have the role
-*admin* assigned to them they should be in the group *hydra\_admin*. In
-general any LDAP group of the form *hydra\_some\_role* (notice the
-*hydra\_* prefix) will work.
-
-    credential:
-      class: Password
-      password_field: password
-      password_type: self_check
-    store:
-      class: LDAP
-      ldap_server: localhost
-      ldap_server_options.timeout: 30
-      binddn: "cn=root,dc=example"
-      bindpw: notapassword
-      start_tls: 0
-      start_tls_options
-        verify:  none
-      user_basedn: "ou=users,dc=example"
-      user_filter: "(&(objectClass=inetOrgPerson)(cn=%s))"
-      user_scope: one
-      user_field: cn
-      user_search_options:
-        deref: always
-      use_roles: 1
-      role_basedn: "ou=groups,dc=example"
-      role_filter: "(&(objectClass=groupOfNames)(member=%s))"
-      role_scope: one
-      role_field: cn
-      role_value: dn
-      role_search_options:
-        deref: always

src/lib/Hydra.pm

@@ -48,6 +48,11 @@ __PACKAGE__->config(
             file($ENV{'HYDRA_LDAP_CONFIG'})
         ) : undef
     },
+    'Plugin::ConfigLoader' => {
+        driver => {
+            'General' => \%Hydra::Config::configGeneralOpts
+        }
+    },
     'Plugin::PrometheusTiny' => {
         include_action_labels => 1,
     },

src/lib/Hydra/Config.pm

@@ -0,0 +1,5 @@
+package Hydra::Config;
+
+our %configGeneralOpts = (-UseApacheInclude => 1, -IncludeAgain => 1, -IncludeRelative => 1);
+
+1;

src/lib/Hydra/Helper/Nix.pm

@@ -5,6 +5,7 @@ use Exporter;
 use File::Path;
 use File::Basename;
 use Config::General;
+use Hydra::Config;
 use Hydra::Helper::CatalystUtils;
 use Hydra::Model::DB;
 use Nix::Store;
@@ -41,11 +42,9 @@ my $hydraConfig;
 sub getHydraConfig {
     return $hydraConfig if defined $hydraConfig;
     my $conf = $ENV{"HYDRA_CONFIG"} || (Hydra::Model::DB::getHydraPath . "/hydra.conf");
+    my %opts = (%Hydra::Config::configGeneralOpts, -ConfigFile => $conf);
     if (-f $conf) {
-        my %h = new Config::General( -ConfigFile => $conf
+        my %h = new Config::General(%opts)->getall;
-                                   , -UseApacheInclude => 1
-                                   , -IncludeAgain => 1
-                                   )->getall;
 
         $hydraConfig = \%h;
     } else {

t/Config/include.t

@@ -0,0 +1,26 @@
+use strict;
+use Setup;
+
+my %ctx = test_init(
+    use_external_destination_store => 0,
+    hydra_config                   => "include foo.conf"
+);
+
+write_file($ctx{'tmpdir'} . "/foo.conf", q|
+<foo>
+  include bar.conf
+</foo>
+|);
+
+write_file($ctx{'tmpdir'} . "/bar.conf", q|
+  bar = baz
+|);
+
+require Hydra::Helper::Nix;
+use Test2::V0;
+
+is(Hydra::Helper::Nix::getHydraConfig(), {
+    foo => { bar => "baz" }
+}, "Nested includes work.");
+
+done_testing;

t/lib/Setup.pm

@@ -9,7 +9,7 @@ use File::Basename;
 use Cwd qw(abs_path getcwd);
 
 our @ISA = qw(Exporter);
-our @EXPORT = qw(test_init hydra_setup nrBuildsForJobset queuedBuildsForJobset
+our @EXPORT = qw(test_init hydra_setup write_file nrBuildsForJobset queuedBuildsForJobset
                  nrQueuedBuildsForJobset createBaseJobset createJobsetWithOneInput
                  evalSucceeds runBuild sendNotifications updateRepository
                  captureStdoutStderr);
@@ -49,19 +49,16 @@ sub test_init {
     $ENV{'NIX_CONF_DIR'} = "$dir/nix/etc/nix";
     make_path($ENV{'NIX_CONF_DIR'});
     my $nixconf = "$ENV{'NIX_CONF_DIR'}/nix.conf";
-    open(my $fh, '>', $nixconf) or die "Could not open file '$nixconf' $!";
+    my $nix_config = "sandbox = false\n" . $opts{'nix_config'};
-    print $fh "sandbox = false\n";
+    write_file($nixconf, $nix_config);
-    print $fh $opts{'nix_config'} || "";
-    close $fh;
-
     $ENV{'HYDRA_CONFIG'} = "$dir/hydra.conf";
 
-    open(my $fh, '>', $ENV{'HYDRA_CONFIG'}) or die "Could not open file '" . $ENV{'HYDRA_CONFIG'}. " $!";
+    my $hydra_config = $opts{'hydra_config'} || "";
     if ($opts{'use_external_destination_store'} // 1) {
-        print $fh "store_uri = file:$dir/nix/dest-store\n"
+        $hydra_config = "store_uri = file:$dir/nix/dest-store\n" . $hydra_config;
     }
-    print $fh $opts{'hydra_config'} || "";
+
-    close $fh;
+    write_file($ENV{'HYDRA_CONFIG'}, $hydra_config);
 
     $ENV{'NIX_LOG_DIR'} = "$dir/nix/var/log/nix";
     $ENV{'NIX_REMOTE_SYSTEMS'} = '';
@@ -82,6 +79,13 @@ sub test_init {
     );
 }
 
+sub write_file {
+    my ($path, $text) = @_;
+    open(my $fh, '>', $path) or die "Could not open file '$path' $!";
+    print $fh $text || "";
+    close $fh;
+}
+
 sub captureStdoutStderr {
     # "Lazy"-load Hydra::Helper::Nix to avoid the compile-time
     # import of Hydra::Model::DB. Early loading of the DB class