From d764eabfcc8d8d2849796967cb2b4bbd8dfffaf6 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 5 Apr 2021 15:30:41 +0000 Subject: [PATCH 01/10] docs: mention foreman under hacking --- doc/manual/src/hacking.md | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/doc/manual/src/hacking.md b/doc/manual/src/hacking.md index a7904959..dabc7169 100644 --- a/doc/manual/src/hacking.md +++ b/doc/manual/src/hacking.md @@ -1,5 +1,4 @@ -Hacking -======= +# Hacking This section provides some notes on how to hack on Hydra. To get the latest version of Hydra from GitHub: @@ -7,13 +6,8 @@ latest version of Hydra from GitHub: $ git clone git://github.com/NixOS/hydra.git $ cd hydra -To build it and its dependencies: - - $ nix-build release.nix -A build.x86_64-linux - -To build all dependencies and start a shell in which all environment -variables (such as PERL5LIB) are set up so that those dependencies can -be found: +To enter a shell in which all environment variables (such as PERL5LIB) +are set up so that those dependencies can be found: $ nix-shell @@ -23,7 +17,12 @@ To build Hydra, you should then do: [nix-shell]$ configurePhase [nix-shell]$ make -You can run the Hydra web server in your source tree as follows: +You start a local database, the webserver, and other components with +foreman: + + $ foreman start + +You can run just the Hydra web server in your source tree as follows: $ ./src/script/hydra-server @@ -44,3 +43,9 @@ in the root of the repository at least once. if run with high parallelism [due to an issue in `Test::PostgreSQL`](https://github.com/TJC/Test-postgresql/issues/40) causing database ports to collide. + +## Building + +To build Hydra and its dependencies: + + $ nix-build release.nix -A build.x86_64-linux From 0ddf3d43f2db0565aa611da52a1e1dc72fe75d1b Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 5 Apr 2021 15:39:49 +0000 Subject: [PATCH 02/10] manual: document working on docs --- Procfile | 1 + doc/manual/src/hacking.md | 6 ++++++ foreman/start-manual.sh | 6 ++++++ 3 files changed, 13 insertions(+) create mode 100755 foreman/start-manual.sh diff --git a/Procfile b/Procfile index 45a1e8fe..f9da41e5 100644 --- a/Procfile +++ b/Procfile @@ -3,3 +3,4 @@ hydra-queue-runner: ./foreman/start-queue-runner.sh hydra-evaluator: ./foreman/start-evaluator.sh hydra-notify: ./foreman/start-notify.sh postgres: ./foreman/start-postgres.sh +manual: ./foreman/start-manual.sh diff --git a/doc/manual/src/hacking.md b/doc/manual/src/hacking.md index dabc7169..c22b7530 100644 --- a/doc/manual/src/hacking.md +++ b/doc/manual/src/hacking.md @@ -44,6 +44,12 @@ if run with high parallelism [due to an issue in `Test::PostgreSQL`](https://github.com/TJC/Test-postgresql/issues/40) causing database ports to collide. +## Working on the Manual + +By default, `foreman start` runs mdbook in "watch" mode. mdbook listens +at [http://localhost:63332/](http://localhost:63332/) by default, and +will reload the page every time you save. + ## Building To build Hydra and its dependencies: diff --git a/foreman/start-manual.sh b/foreman/start-manual.sh new file mode 100755 index 00000000..5ad78265 --- /dev/null +++ b/foreman/start-manual.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +mdbook serve \ + --port 63332 \ + --dest-dir $(pwd)/.hydra-data/manual \ + $(pwd)/doc/manual/ From 16902f3866c610649974f7dba1d231a8b7c04a96 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 5 Apr 2021 15:47:25 +0000 Subject: [PATCH 03/10] start-hydra: create a db for the current username Otherwise the logs are spammed with database not existing errors: 15:46:07 postgres.1 | 2021-04-05 15:46:07.631 UTC [30742] FATAL: database grahamc does not exist 15:46:08 postgres.1 | 2021-04-05 15:46:08.641 UTC [30759] FATAL: database grahamc does not exist 15:46:09 postgres.1 | 2021-04-05 15:46:09.650 UTC [30765] FATAL: database grahamc does not exist --- foreman/start-hydra.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/foreman/start-hydra.sh b/foreman/start-hydra.sh index 66773f81..4e348266 100755 --- a/foreman/start-hydra.sh +++ b/foreman/start-hydra.sh @@ -5,6 +5,12 @@ while ! pg_isready -h $(pwd)/.hydra-data/postgres -p 64444; do sleep 1; done createdb -h $(pwd)/.hydra-data/postgres -p 64444 hydra +# create a db for the default user. Not sure why, but +# the terminal is otherwise spammed with: +# +# FATAL: database "USERNAME" does not exist +createdb -h $(pwd)/.hydra-data/postgres -p 64444 "$(whoami)" || true + hydra-init hydra-create-user alice --password foobar --role admin From 2e47970a977576106542c3702052a55567e0799f Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 5 Apr 2021 16:07:00 +0000 Subject: [PATCH 04/10] doc: switch shell session code blocks to `console` type. --- doc/manual/src/hacking.md | 46 ++++++++++++++++++++++------------ doc/manual/src/installation.md | 40 ++++++++++++++++++++--------- doc/manual/src/projects.md | 12 ++++++--- 3 files changed, 67 insertions(+), 31 deletions(-) diff --git a/doc/manual/src/hacking.md b/doc/manual/src/hacking.md index c22b7530..98a41a20 100644 --- a/doc/manual/src/hacking.md +++ b/doc/manual/src/hacking.md @@ -3,38 +3,50 @@ This section provides some notes on how to hack on Hydra. To get the latest version of Hydra from GitHub: - $ git clone git://github.com/NixOS/hydra.git - $ cd hydra +```console +$ git clone git://github.com/NixOS/hydra.git +$ cd hydra +``` To enter a shell in which all environment variables (such as PERL5LIB) are set up so that those dependencies can be found: - $ nix-shell +```console +$ nix-shell +``` To build Hydra, you should then do: - [nix-shell]$ ./bootstrap - [nix-shell]$ configurePhase - [nix-shell]$ make +```console +[nix-shell]$ ./bootstrap +[nix-shell]$ configurePhase +[nix-shell]$ make +``` You start a local database, the webserver, and other components with foreman: - $ foreman start +```console +$ foreman start +``` You can run just the Hydra web server in your source tree as follows: - $ ./src/script/hydra-server +```console +$ ./src/script/hydra-server +``` You can run Hydra's test suite with the following: - [nix-shell]$ make check - [nix-shell]$ # to run as many tests as you have cores: - [nix-shell]$ make check YATH_JOB_COUNT=$NIX_BUILD_CORES - [nix-shell]$ # or run yath directly: - [nix-shell]$ yath test - [nix-shell]$ # to run as many tests as you have cores: - [nix-shell]$ yath test -j $NIX_BUILD_CORES +```console +[nix-shell]$ make check +[nix-shell]$ # to run as many tests as you have cores: +[nix-shell]$ make check YATH_JOB_COUNT=$NIX_BUILD_CORES +[nix-shell]$ # or run yath directly: +[nix-shell]$ yath test +[nix-shell]$ # to run as many tests as you have cores: +[nix-shell]$ yath test -j $NIX_BUILD_CORES +``` When using `yath` instead of `make check`, ensure you have run `make` in the root of the repository at least once. @@ -54,4 +66,6 @@ will reload the page every time you save. To build Hydra and its dependencies: - $ nix-build release.nix -A build.x86_64-linux +```console +$ nix-build release.nix -A build.x86_64-linux +``` diff --git a/doc/manual/src/installation.md b/doc/manual/src/installation.md index 94a12041..3fc71fcf 100644 --- a/doc/manual/src/installation.md +++ b/doc/manual/src/installation.md @@ -76,8 +76,10 @@ Hydra stores its results in a PostgreSQL database. To setup a PostgreSQL database with *hydra* as database name and user name, issue the following commands on the PostgreSQL server: - createuser -S -D -R -P hydra - createdb -O hydra hydra +```console +createuser -S -D -R -P hydra +createdb -O hydra hydra +``` Note that *\$prefix* is the location of Hydra in the nix store. @@ -86,12 +88,16 @@ used, and a variable which point to a location that holds some state. To set these variables for a PostgreSQL database, add the following to the file `~/.profile` of the user running the Hydra services. - export HYDRA_DBI="dbi:Pg:dbname=hydra;host=dbserver.example.org;user=hydra;" - export HYDRA_DATA=/var/lib/hydra +```console +export HYDRA_DBI="dbi:Pg:dbname=hydra;host=dbserver.example.org;user=hydra;" +export HYDRA_DATA=/var/lib/hydra +``` You can provide the username and password in the file `~/.pgpass`, e.g. - dbserver.example.org:*:hydra:hydra:password +``` +dbserver.example.org:*:hydra:hydra:password +``` Make sure that the *HYDRA\_DATA* directory exists and is writable for the user which will run the Hydra services. @@ -99,13 +105,17 @@ the user which will run the Hydra services. Having set these environment variables, you can now initialise the database by doing: - hydra-init +```console +hydra-init +``` To create projects, you need to create a user with *admin* privileges. This can be done using the command `hydra-create-user`: - $ hydra-create-user alice --full-name 'Alice Q. User' \ - --email-address 'alice@example.org' --password foobar --role admin +```console +$ hydra-create-user alice --full-name 'Alice Q. User' \ + --email-address 'alice@example.org' --password foobar --role admin +``` Additional users can be created through the web interface. @@ -115,14 +125,18 @@ Upgrading If you\'re upgrading Hydra from a previous version, you should do the following to perform any necessary database schema migrations: - hydra-init +```console +hydra-init +``` Getting Started --------------- To start the Hydra web server, execute: - hydra-server +```console +hydra-server +``` When the server is started, you can browse to [http://localhost:3000/]() to start configuring your Hydra instance. @@ -158,8 +172,10 @@ some additional configuration must be made. Edit your `hydra.conf` file in a similar way to this example: - using_frontend_proxy 1 - base_uri example.com +``` +using_frontend_proxy 1 +base_uri example.com +``` `base_uri` should be your hydra servers proxied URL. If you are using Hydra nixos module then setting `hydraURL` option should be enough. diff --git a/doc/manual/src/projects.md b/doc/manual/src/projects.md index c4ea8f55..4228c2dc 100644 --- a/doc/manual/src/projects.md +++ b/doc/manual/src/projects.md @@ -220,7 +220,9 @@ expressions, they can be evaluated using the standard Nix tools. To evaluate the `tarball` jobset of the above example, just run: - $ nix-build release.nix -A tarball +```console +$ nix-build release.nix -A tarball +``` However, doing this with the example as is will probably yield an error like this: @@ -230,11 +232,15 @@ yield an error like this: The error is self-explanatory. Assuming `$HOME/src/hello` points to a checkout of Hello, this can be fixed this way: - $ nix-build -I ~/src release.nix -A tarball +```console +$ nix-build -I ~/src release.nix -A tarball +``` Similarly, the `build` jobset can be evaluated: - $ nix-build -I ~/src release.nix -A build +```console +$ nix-build -I ~/src release.nix -A build +``` The `build` job reuses the result of the `tarball` job, rebuilding it only if it needs to. From b03fa96186607c506564d4fcc436669e7cbc7483 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 5 Apr 2021 16:07:19 +0000 Subject: [PATCH 05/10] doc: hacking: Quote perl5lib --- doc/manual/src/hacking.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/hacking.md b/doc/manual/src/hacking.md index 98a41a20..50f9339f 100644 --- a/doc/manual/src/hacking.md +++ b/doc/manual/src/hacking.md @@ -8,7 +8,7 @@ $ git clone git://github.com/NixOS/hydra.git $ cd hydra ``` -To enter a shell in which all environment variables (such as PERL5LIB) +To enter a shell in which all environment variables (such as `PERL5LIB`) are set up so that those dependencies can be found: ```console From 9788c52eff14d44fafa7950b6017a4825b323ed8 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 5 Apr 2021 16:07:30 +0000 Subject: [PATCH 06/10] doc: hacking: remove duplicate 'by default' --- doc/manual/src/hacking.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/hacking.md b/doc/manual/src/hacking.md index 50f9339f..fa4be54e 100644 --- a/doc/manual/src/hacking.md +++ b/doc/manual/src/hacking.md @@ -59,7 +59,7 @@ causing database ports to collide. ## Working on the Manual By default, `foreman start` runs mdbook in "watch" mode. mdbook listens -at [http://localhost:63332/](http://localhost:63332/) by default, and +at [http://localhost:63332/](http://localhost:63332/), and will reload the page every time you save. ## Building From 7bb9850656b68d246a7a28e09685888b3a54d0f7 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 5 Apr 2021 16:08:06 +0000 Subject: [PATCH 07/10] doc: hacking: fixup nix-shell dependency comment --- doc/manual/src/hacking.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/hacking.md b/doc/manual/src/hacking.md index fa4be54e..91e26cdb 100644 --- a/doc/manual/src/hacking.md +++ b/doc/manual/src/hacking.md @@ -9,7 +9,7 @@ $ cd hydra ``` To enter a shell in which all environment variables (such as `PERL5LIB`) -are set up so that those dependencies can be found: +and dependencies can be found: ```console $ nix-shell From c442f1aa1424e4f738e68597ce7a439e383354d1 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 5 Apr 2021 16:08:59 +0000 Subject: [PATCH 08/10] Procfile: sort alphabetically --- Procfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Procfile b/Procfile index f9da41e5..42e57a21 100644 --- a/Procfile +++ b/Procfile @@ -1,6 +1,6 @@ -hydra-server: ./foreman/start-hydra.sh -hydra-queue-runner: ./foreman/start-queue-runner.sh hydra-evaluator: ./foreman/start-evaluator.sh +hydra-queue-runner: ./foreman/start-queue-runner.sh hydra-notify: ./foreman/start-notify.sh -postgres: ./foreman/start-postgres.sh +hydra-server: ./foreman/start-hydra.sh manual: ./foreman/start-manual.sh +postgres: ./foreman/start-postgres.sh From c69912b2876218ce0eeeff7841854296a261e4db Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 5 Apr 2021 16:09:52 +0000 Subject: [PATCH 09/10] start-manual.sh: Drop explicit pwd's --- foreman/start-manual.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/foreman/start-manual.sh b/foreman/start-manual.sh index 5ad78265..b0ad1b8e 100755 --- a/foreman/start-manual.sh +++ b/foreman/start-manual.sh @@ -2,5 +2,5 @@ mdbook serve \ --port 63332 \ - --dest-dir $(pwd)/.hydra-data/manual \ - $(pwd)/doc/manual/ + --dest-dir ./.hydra-data/manual \ + ./doc/manual/ From 5f91353824ecdd016e2586462214ff33b3a0c4ad Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 5 Apr 2021 16:42:15 +0000 Subject: [PATCH 10/10] doc/manual/src/installation.md: give a conf syntax type Co-authored-by: Sandro --- doc/manual/src/installation.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/manual/src/installation.md b/doc/manual/src/installation.md index 3fc71fcf..79d261cb 100644 --- a/doc/manual/src/installation.md +++ b/doc/manual/src/installation.md @@ -172,7 +172,7 @@ some additional configuration must be made. Edit your `hydra.conf` file in a similar way to this example: -``` +```conf using_frontend_proxy 1 base_uri example.com ``` @@ -262,4 +262,3 @@ general any LDAP group of the form *hydra\_some\_role* (notice the role_value: dn role_search_options: deref: always -