From 3517acc5baed701fcd0e5deb54d8712a1889297d Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sat, 17 Aug 2024 18:40:43 +0200 Subject: [PATCH 1/4] Add direnv & PLS to the dev setup --- .envrc | 1 + package.nix | 1 + 2 files changed, 2 insertions(+) create mode 100644 .envrc diff --git a/.envrc b/.envrc new file mode 100644 index 00000000..3550a30f --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use flake diff --git a/package.nix b/package.nix index 07041423..0fbe85a8 100644 --- a/package.nix +++ b/package.nix @@ -190,6 +190,7 @@ stdenv.mkDerivation (finalAttrs: { postgresql_13 pixz nix-eval-jobs + perlPackages.PLS ]; checkInputs = [ -- 2.44.1 From 3a4e0d49171d603735f6089485083c0c21e4ecba Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sat, 17 Aug 2024 20:35:43 +0200 Subject: [PATCH 2/4] Get dev environment working again * justfile, inspired from Lix. * let foreman use the stuff from outputs, similar to what Lix does> * mess around with PERL5LIB[1] and PATH to get tests running locally. [1] I don't really know how `Setup` was found before tbh. --- .gitignore | 2 ++ doc/manual/src/hacking.md | 14 ++++---------- foreman/start-evaluator.sh | 2 +- foreman/start-hydra.sh | 2 +- foreman/start-notify.sh | 2 +- foreman/start-queue-runner.sh | 2 +- justfile | 11 +++++++++++ package.nix | 6 ++++-- 8 files changed, 25 insertions(+), 16 deletions(-) create mode 100644 justfile diff --git a/.gitignore b/.gitignore index 2050dddd..e0d2e4ec 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ /src/sql/tmp.sqlite result result-* +.hydra-data +outputs diff --git a/doc/manual/src/hacking.md b/doc/manual/src/hacking.md index de826720..62846819 100644 --- a/doc/manual/src/hacking.md +++ b/doc/manual/src/hacking.md @@ -12,15 +12,14 @@ To enter a shell in which all environment variables (such as `PERL5LIB`) and dependencies can be found: ```console -$ nix-shell +$ nix develop ``` To build Hydra, you should then do: ```console -[nix-shell]$ autoreconfPhase -[nix-shell]$ configurePhase -[nix-shell]$ make +[nix-shell]$ just setup +[nix-shell]$ just install ``` You start a local database, the webserver, and other components with @@ -41,18 +40,13 @@ $ ./src/script/hydra-server You can run Hydra's test suite with the following: ```console -[nix-shell]$ make check -[nix-shell]$ # to run as many tests as you have cores: -[nix-shell]$ make check YATH_JOB_COUNT=$NIX_BUILD_CORES +[nix-shell]$ just test [nix-shell]$ # or run yath directly: [nix-shell]$ yath test [nix-shell]$ # to run as many tests as you have cores: [nix-shell]$ yath test -j $NIX_BUILD_CORES ``` -When using `yath` instead of `make check`, ensure you have run `make` -in the root of the repository at least once. - **Warning**: Currently, the tests can fail if run with high parallelism [due to an issue in `Test::PostgreSQL`](https://github.com/TJC/Test-postgresql/issues/40) diff --git a/foreman/start-evaluator.sh b/foreman/start-evaluator.sh index a2858003..8b85b2a3 100755 --- a/foreman/start-evaluator.sh +++ b/foreman/start-evaluator.sh @@ -3,4 +3,4 @@ # wait for hydra-server to listen while ! nc -z localhost 63333; do sleep 1; done -HYDRA_CONFIG=$(pwd)/.hydra-data/hydra.conf exec hydra-evaluator +HYDRA_CONFIG=$(pwd)/.hydra-data/hydra.conf exec $(pwd)/outputs/out/bin/hydra-evaluator diff --git a/foreman/start-hydra.sh b/foreman/start-hydra.sh index bb8cc909..318b02f7 100755 --- a/foreman/start-hydra.sh +++ b/foreman/start-hydra.sh @@ -28,4 +28,4 @@ use-substitutes = true EOF fi -HYDRA_CONFIG=$(pwd)/.hydra-data/hydra.conf exec hydra-dev-server --port 63333 --restart --debug +HYDRA_CONFIG=$(pwd)/.hydra-data/hydra.conf exec $(pwd)/outputs/out/bin/hydra-dev-server --port 63333 --restart --debug diff --git a/foreman/start-notify.sh b/foreman/start-notify.sh index 6a647e54..f79fcd2f 100755 --- a/foreman/start-notify.sh +++ b/foreman/start-notify.sh @@ -3,4 +3,4 @@ # wait for hydra-server to listen while ! nc -z localhost 63333; do sleep 1; done -HYDRA_CONFIG=$(pwd)/.hydra-data/hydra.conf exec hydra-notify +HYDRA_CONFIG=$(pwd)/.hydra-data/hydra.conf exec $(pwd)/outputs/out/bin/hydra-notify diff --git a/foreman/start-queue-runner.sh b/foreman/start-queue-runner.sh index f4fd6048..3a1d2376 100755 --- a/foreman/start-queue-runner.sh +++ b/foreman/start-queue-runner.sh @@ -3,4 +3,4 @@ # wait until hydra is listening on port 63333 while ! nc -z localhost 63333; do sleep 1; done -NIX_REMOTE_SYSTEMS="" HYDRA_CONFIG=$(pwd)/.hydra-data/hydra.conf exec hydra-queue-runner +NIX_REMOTE_SYSTEMS="" HYDRA_CONFIG=$(pwd)/.hydra-data/hydra.conf exec $(pwd)/outputs/out/bin/hydra-queue-runner diff --git a/justfile b/justfile new file mode 100644 index 00000000..8c24774c --- /dev/null +++ b/justfile @@ -0,0 +1,11 @@ +setup *OPTIONS: + meson setup build --prefix="$PWD/outputs/out" $mesonFlags {{ OPTIONS }} + +build *OPTIONS: + meson compile -C build {{ OPTIONS }} + +install *OPTIONS: (build OPTIONS) + meson install -C build + +test *OPTIONS: + meson test -C build --print-errorlogs {{ OPTIONS }} diff --git a/package.nix b/package.nix index 0fbe85a8..8789e058 100644 --- a/package.nix +++ b/package.nix @@ -37,6 +37,7 @@ , cacert , foreman +, just , glibcLocales , libressl , openldap @@ -191,6 +192,7 @@ stdenv.mkDerivation (finalAttrs: { pixz nix-eval-jobs perlPackages.PLS + just ]; checkInputs = [ @@ -234,8 +236,8 @@ stdenv.mkDerivation (finalAttrs: { shellHook = '' pushd $(git rev-parse --show-toplevel) >/dev/null - PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-queue-runner:$PATH - PERL5LIB=$(pwd)/src/lib:$PERL5LIB + PATH=$(pwd)/outputs/out/bin:$PATH + PERL5LIB=$(pwd)/src/lib:$(pwd)/t/lib:$PERL5LIB export HYDRA_HOME="$(pwd)/src/" mkdir -p .hydra-data export HYDRA_DATA="$(pwd)/.hydra-data" -- 2.44.1 From 1f802c008c8995187c6ae021b473c1ebb64c89a6 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sun, 18 Aug 2024 00:24:04 +0200 Subject: [PATCH 3/4] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'lix': 'git+https://git.lix.systems/lix-project/lix?ref=refs/heads/main&rev=5137cea99044d54337e439510a647743110b2d7d' (2024-08-10) → 'git+https://git.lix.systems/lix-project/lix?ref=refs/heads/main&rev=278fddc317cf0cf4d3602d0ec0f24d1dd281fadb' (2024-08-17) • Updated input 'nix-eval-jobs': 'git+https://git.lix.systems/lix-project/nix-eval-jobs?ref=refs/heads/main&rev=c057494450f2d1420726ddb0bab145a5ff4ddfdd' (2024-07-17) → 'git+https://git.lix.systems/lix-project/nix-eval-jobs?ref=refs/heads/main&rev=42a160bce2fd9ffebc3809746bc80cc7208f9b08' (2024-08-13) • Updated input 'nix-eval-jobs/flake-parts': 'github:hercules-ci/flake-parts/9227223f6d922fee3c7b190b2cc238a99527bbb7' (2024-07-03) → 'github:hercules-ci/flake-parts/8471fe90ad337a8074e957b69ca4d0089218391d' (2024-08-01) • Updated input 'nix-eval-jobs/treefmt-nix': 'github:numtide/treefmt-nix/0fb28f237f83295b4dd05e342f333b447c097398' (2024-07-15) → 'github:numtide/treefmt-nix/349de7bc435bdff37785c2466f054ed1766173be' (2024-08-12) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/a781ff33ae258bbcfd4ed6e673860c3e923bf2cc' (2024-08-10) → 'github:NixOS/nixpkgs/c3d4ac725177c030b1e289015989da2ad9d56af0' (2024-08-15) --- flake.lock | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/flake.lock b/flake.lock index 854d695d..916b1458 100644 --- a/flake.lock +++ b/flake.lock @@ -24,11 +24,11 @@ ] }, "locked": { - "lastModified": 1719994518, - "narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=", + "lastModified": 1722555600, + "narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7", + "rev": "8471fe90ad337a8074e957b69ca4d0089218391d", "type": "github" }, "original": { @@ -48,11 +48,11 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1723331518, - "narHash": "sha256-JVnQ3OLbXQAlkOluFc3gWhZMbhared1Rg5YvNEc92m0=", + "lastModified": 1723919517, + "narHash": "sha256-D6+zmRXzr85p7riphuIrJQqangoJe70XM5jHhMWwXws=", "ref": "refs/heads/main", - "rev": "5137cea99044d54337e439510a647743110b2d7d", - "revCount": 16128, + "rev": "278fddc317cf0cf4d3602d0ec0f24d1dd281fadb", + "revCount": 16138, "type": "git", "url": "https://git.lix.systems/lix-project/lix" }, @@ -74,11 +74,11 @@ "treefmt-nix": "treefmt-nix" }, "locked": { - "lastModified": 1721195872, - "narHash": "sha256-TlvRq634MSl22BWLmpTy2vdtKntbZlsUwdMq8Mp9AWs=", + "lastModified": 1723579251, + "narHash": "sha256-xnHtfw0gRhV+2S9U7hQwvp2klTy1Iv7FlMMO0/WiMVc=", "ref": "refs/heads/main", - "rev": "c057494450f2d1420726ddb0bab145a5ff4ddfdd", - "revCount": 608, + "rev": "42a160bce2fd9ffebc3809746bc80cc7208f9b08", + "revCount": 609, "type": "git", "url": "https://git.lix.systems/lix-project/nix-eval-jobs" }, @@ -126,11 +126,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1723282977, - "narHash": "sha256-oTK91aOlA/4IsjNAZGMEBz7Sq1zBS0Ltu4/nIQdYDOg=", + "lastModified": 1723688146, + "narHash": "sha256-sqLwJcHYeWLOeP/XoLwAtYjr01TISlkOfz+NG82pbdg=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "a781ff33ae258bbcfd4ed6e673860c3e923bf2cc", + "rev": "c3d4ac725177c030b1e289015989da2ad9d56af0", "type": "github" }, "original": { @@ -187,11 +187,11 @@ ] }, "locked": { - "lastModified": 1721059077, - "narHash": "sha256-gCICMMX7VMSKKt99giDDtRLkHJ0cwSgBtDijJAqTlto=", + "lastModified": 1723454642, + "narHash": "sha256-S0Gvsenh0II7EAaoc9158ZB4vYyuycvMGKGxIbERNAM=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "0fb28f237f83295b4dd05e342f333b447c097398", + "rev": "349de7bc435bdff37785c2466f054ed1766173be", "type": "github" }, "original": { -- 2.44.1 From e987f7495480a7bd4918f16b8da4d6eff851d15a Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sun, 18 Aug 2024 14:47:09 +0200 Subject: [PATCH 4/4] doc: drop dev-notes & make update-dbix more discoverable `dev-notes` are severely outdated. I dropped everything except one note that I moved to hacking.md. The parts about creating users are also covered elsewhere. The `update-dbix` part got a just command to make it discoverable again. --- README.md | 27 ++++----- doc/dev-notes.txt | 122 -------------------------------------- doc/manual/src/hacking.md | 11 ++++ justfile | 10 +++- src/sql/hydra.sql | 2 +- 5 files changed, 33 insertions(+), 139 deletions(-) delete mode 100644 doc/dev-notes.txt diff --git a/README.md b/README.md index 2a085325..7f2abb31 100644 --- a/README.md +++ b/README.md @@ -78,11 +78,11 @@ $ nix-build ### Development Environment You can use the provided shell.nix to get a working development environment: + ``` -$ nix-shell -$ autoreconfPhase -$ configurePhase # NOTE: not ./configure -$ make +$ nix develop +[nix-shell]$ just setup +[nix-shell]$ just install ``` ### Executing Hydra During Development @@ -91,10 +91,9 @@ When working on new features or bug fixes you need to be able to run Hydra from can be done using [foreman](https://github.com/ddollar/foreman): ``` -$ nix-shell -$ # hack hack -$ make -$ foreman start +$ nix develop +[nix-shell]$ just install +[nix-shell]$ foreman start ``` Have a look at the [Procfile](./Procfile) if you want to see how the processes are being started. In order to avoid @@ -115,22 +114,22 @@ Start by following the steps in [Development Environment](#development-environme Then, you can run the tests and the perlcritic linter together with: ```console -$ nix-shell -$ make check +$ nix develop +[nix-shell]$ just test ``` You can run a single test with: ``` -$ nix-shell -$ yath test ./t/foo/bar.t +$ nix develop +[nix-shell]$ yath test ./t/foo/bar.t ``` And you can run just perlcritic with: ``` -$ nix-shell -$ make perlcritic +$ nix develop +[nix-shell]$ just perlcritic ``` ### JSON API diff --git a/doc/dev-notes.txt b/doc/dev-notes.txt deleted file mode 100644 index 4035c809..00000000 --- a/doc/dev-notes.txt +++ /dev/null @@ -1,122 +0,0 @@ -* Recreating the schema bindings: - - $ make -C src/sql update-dbix - -* Running the test server: - - $ DBIC_TRACE=1 ./script/hydra_server.pl - -* Setting the maximum number of concurrent builds per system type: - - $ psql -d hydra <<< "insert into SystemTypes(system, maxConcurrent) values('i686-linux', 3);" - -* Creating a user: - - $ hydra-create-user root --email-address 'e.dolstra@tudelft.nl' \ - --password-prompt - - (Replace "foobar" with the desired password.) - - To make the user an admin: - - $ hydra-create-user root --role admin - - To enable a non-admin user to create projects: - - $ hydra-create-user root --role create-projects - -* Changing the priority of a scheduled build: - - update buildschedulinginfo set priority = 200 where id = ; - -* Changing the priority of all builds for a jobset: - - update buildschedulinginfo set priority = 20 where id in (select id from builds where finished = 0 and project = 'nixpkgs' and jobset = 'trunk'); - - -* Steps to install: - - - Install the Hydra closure. - - - Set HYDRA_DATA to /somewhere. - - - Run hydra_init.pl - - - Start hydra_server - - - Visit http://localhost:3000/ - - - Create a user (see above) - - - Create a project, jobset etc. - - - Start hydra_evaluator and hydra_queue_runner - - -* Job selection: - - php-sat:build [system = "i686-linux"] - php-sat:build [same system] - tarball [same patchelfSrc] - --if system i686-linux --arg build {...} - - -* Restart all aborted builds in a given evaluation (e.g. 820909): - - > update builds set finished = 0 where id in (select id from builds where finished = 1 and buildstatus = 3 and exists (select 1 from jobsetevalmembers where eval = 820909 and build = id)); - - -* Restart all builds in a given evaluation that had a build step time out: - - > update builds set finished = 0 where id in (select id from builds where finished = 1 and buildstatus != 0 and exists (select 1 from jobsetevalmembers where eval = 926992 and build = id) and exists (select 1 from buildsteps where build = id and status = 7)); - - -* select * from (select project, jobset, job, system, max(timestamp) timestamp from builds where finished = 1 group by project, jobset, job, system) x join builds y on x.timestamp = y.timestamp and x.project = y.project and x.jobset = y.jobset and x.job = y.job and x.system = y.system; - - select * from (select project, jobset, job, system, max(timestamp) timestamp from builds where finished = 1 group by project, jobset, job, system) natural join builds; - - -* Delete all scheduled builds that are not already building: - - delete from builds where finished = 0 and not exists (select 1 from buildschedulinginfo s where s.id = builds.id and busy != 0); - - -* select x.project, x.jobset, x.job, x.system, x.id, x.timestamp, r.buildstatus, b.id, b.timestamp - from (select project, jobset, job, system, max(id) as id from Builds where finished = 1 group by project, jobset, job, system) as a_ - natural join Builds x - natural join BuildResultInfo r - left join Builds b on b.id = - (select max(id) from builds c - natural join buildresultinfo r2 - where x.project = c.project and x.jobset = c.jobset and x.job = c.job and x.system = c.system - and x.id > c.id and r.buildstatus != r2.buildstatus); - -* Using PostgreSQL (version 9.2 or newer is required): - - $ HYDRA_DBI="dbi:Pg:dbname=hydra;" hydra-server - - -* Find the builds with the highest number of build steps: - - select id, (select count(*) from buildsteps where build = x.id) as n from builds x order by n desc; - - -* Evaluating the NixOS Hydra jobs: - - $ ./hydra_eval_jobs ~/Dev/nixos-wc/release.nix --arg nixpkgs '{outPath = /home/eelco/Dev/nixpkgs-wc;}' --arg nixosSrc '{outPath = /home/eelco/Dev/nixos-wc; rev = 1234;}' --arg services '{outhPath = /home/eelco/services-wc;}' --argstr system i686-linux --argstr system x86_64-linux --arg officialRelease false - - -* Show all the failing jobs/systems in the nixpkgs:stdenv jobset that - succeed in the nixpkgs:trunk jobset: - - select job, system from builds b natural join buildresultinfo where project = 'nixpkgs' and jobset = 'stdenv' and iscurrent = 1 and finished = 1 and buildstatus != 0 and exists (select 1 from builds natural join buildresultinfo where project = 'nixpkgs' and jobset = 'trunk' and job = b.job and system = b.system and iscurrent = 1 and finished = 1 and buildstatus = 0) order by job, system; - - -* Get all Nixpkgs jobs that have never built succesfully: - - select project, jobset, job from builds b1 - where project = 'nixpkgs' and jobset = 'trunk' and iscurrent = 1 - group by project, jobset, job - having not exists - (select 1 from builds b2 where b1.project = b2.project and b1.jobset = b2.jobset and b1.job = b2.job and finished = 1 and buildstatus = 0) - order by project, jobset, job; diff --git a/doc/manual/src/hacking.md b/doc/manual/src/hacking.md index 62846819..77adf8f1 100644 --- a/doc/manual/src/hacking.md +++ b/doc/manual/src/hacking.md @@ -97,3 +97,14 @@ Off NixOS, change `/etc/nix/nix.conf`: ```conf trusted-users = root YOURUSERNAME ``` + +### Updating schema bindings + +``` +just update-dbix +``` + +### Find the builds with the highest number of build steps: + + select id, (select count(*) from buildsteps where build = x.id) as n from builds x order by n desc; + diff --git a/justfile b/justfile index 8c24774c..9964ae18 100644 --- a/justfile +++ b/justfile @@ -1,11 +1,17 @@ setup *OPTIONS: - meson setup build --prefix="$PWD/outputs/out" $mesonFlags {{ OPTIONS }} + meson setup build --prefix="$PWD/outputs/out" $mesonFlags {{ OPTIONS }} build *OPTIONS: - meson compile -C build {{ OPTIONS }} + meson compile -C build {{ OPTIONS }} install *OPTIONS: (build OPTIONS) meson install -C build test *OPTIONS: meson test -C build --print-errorlogs {{ OPTIONS }} + +update-dbix: + cd src/sql && ./update-dbix-harness.sh + +perlcritic: + perlcritic . diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index e9457972..e6dc6a36 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -6,7 +6,7 @@ -- add a map of the lowercase name of your table to the CamelCase -- version of your table. -- --- 3. Run `make -C src/sql update-dbix` in the root +-- 3. Run `just update-dbix` in the root -- of the project directory, and git add / git commit the changed, -- generated files. -- -- 2.44.1