From 07cb5d1b7c74fdc8423a93b14f4b56af1c49e4b3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 24 Jan 2024 21:04:14 -0500 Subject: [PATCH 01/12] Use `nix::ParsedDerivation::getRequiredSystemFeatures()` A slight dedup, and also ensures that floating CA derivations require a `ca-derivations` experimental feature. This fixes the scheduling issue that @SuperSandro2000 found. --- src/hydra-queue-runner/queue-monitor.cc | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index a3557316..513d399b 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -462,10 +462,7 @@ Step::ptr State::createStep(ref destStore, step->systemType = step->drv->platform; { - auto i = step->drv->env.find("requiredSystemFeatures"); - StringSet features; - if (i != step->drv->env.end()) - features = step->requiredSystemFeatures = tokenizeString>(i->second); + StringSet features = step->requiredSystemFeatures = step->parsedDrv->getRequiredSystemFeatures(); if (step->preferLocalBuild) features.insert("local"); if (!features.empty()) { From b1fa6b3aac4ce8225e506317119b32122cfc1edd Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 24 Jan 2024 21:37:13 -0500 Subject: [PATCH 02/12] Use `StoreConfig::getDefaultSystemFeatures` for default machine config We have to oddly make a `StoreConfig` subclass to get it, but https://github.com/NixOS/nix/pull/9848 will fix that. The purpose of this is to ensure that, absent an explicit config, `localhost` includes `ca-derivations` and `recursive-nix` if those experimental features are enabled. Very much the complement of #1342, the previous PR. --- src/hydra-queue-runner/hydra-queue-runner.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index 5fbcb641..1cbcafea 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -15,6 +15,7 @@ #include "state.hh" #include "hydra-build-result.hh" #include "store-api.hh" +#include "local-store.hh" #include "remote-store.hh" #include "globals.hh" @@ -226,7 +227,7 @@ void State::monitorMachinesFile() parseMachines("localhost " + (settings.thisSystem == "x86_64-linux" ? "x86_64-linux,i686-linux" : settings.thisSystem.get()) + " - " + std::to_string(settings.maxBuildJobs) + " 1 " - + concatStringsSep(",", settings.systemFeatures.get())); + + concatStringsSep(",", (LocalStoreConfig { {} }).getDefaultSystemFeatures())); machinesReadyLock.unlock(); return; } From a876e46894c73973d155b581283d07ef90706c01 Mon Sep 17 00:00:00 2001 From: Sandro Date: Thu, 25 Jan 2024 17:12:40 +0100 Subject: [PATCH 03/12] Remove automake, libtool Those are already part of autoreconfHook --- flake.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/flake.nix b/flake.nix index 8280e076..eabe5518 100644 --- a/flake.nix +++ b/flake.nix @@ -146,8 +146,6 @@ with final.buildPackages; [ makeWrapper autoreconfHook - automake - libtool nukeReferences pkg-config mdbook From 1471aacadc8868fff6eab06d2c330abd90ec2867 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jan 2024 11:20:27 -0500 Subject: [PATCH 04/12] Split out a `package.nix` Just like we did with Nix. --- flake.nix | 196 +---------------------------------------- package.nix | 246 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 248 insertions(+), 194 deletions(-) create mode 100644 package.nix diff --git a/flake.nix b/flake.nix index eabe5518..f7975b27 100644 --- a/flake.nix +++ b/flake.nix @@ -7,8 +7,6 @@ outputs = { self, nixpkgs, nix }: let - version = "${builtins.readFile ./version.txt}.${builtins.substring 0 8 (self.lastModifiedDate or "19700101")}.${self.shortRev or "DIRTY"}"; - systems = [ "x86_64-linux" "aarch64-linux" ]; forEachSystem = nixpkgs.lib.genAttrs systems; @@ -61,198 +59,8 @@ }; - hydra = let - inherit (final) lib stdenv; - perlDeps = final.buildEnv { - name = "hydra-perl-deps"; - paths = with final.perlPackages; lib.closePropagation - [ - AuthenSASL - CatalystActionREST - CatalystAuthenticationStoreDBIxClass - CatalystAuthenticationStoreLDAP - CatalystDevel - CatalystPluginAccessLog - CatalystPluginAuthorizationRoles - CatalystPluginCaptcha - CatalystPluginPrometheusTiny - CatalystPluginSessionStateCookie - CatalystPluginSessionStoreFastMmap - CatalystPluginStackTrace - CatalystTraitForRequestProxyBase - CatalystViewDownload - CatalystViewJSON - CatalystViewTT - CatalystXRoleApplicator - CatalystXScriptServerStarman - CryptPassphrase - CryptPassphraseArgon2 - CryptRandPasswd - DataDump - DateTime - DBDPg - DBDSQLite - DigestSHA1 - EmailMIME - EmailSender - FileLibMagic - FileSlurper - FileWhich - final.nix.perl-bindings - final.git - IOCompress - IPCRun - IPCRun3 - JSON - JSONMaybeXS - JSONXS - ListSomeUtils - LWP - LWPProtocolHttps - ModulePluggable - NetAmazonS3 - NetPrometheus - NetStatsd - PadWalker - ParallelForkManager - PerlCriticCommunity - PrometheusTinyShared - ReadonlyX - SetScalar - SQLSplitStatement - Starman - StringCompareConstantTime - SysHostnameLong - TermSizeAny - TermReadKey - Test2Harness - TestPostgreSQL - TextDiff - TextTable - UUID4Tiny - YAML - XMLSimple - ]; - }; - - in - stdenv.mkDerivation { - - name = "hydra-${version}"; - + hydra = final.callPackage ./package.nix { src = self; - - nativeBuildInputs = - with final.buildPackages; [ - makeWrapper - autoreconfHook - nukeReferences - pkg-config - mdbook - ]; - - buildInputs = - with final; [ - unzip - libpqxx - top-git - mercurial - darcs - subversion - breezy - openssl - bzip2 - libxslt - final.nix - perlDeps - perl - pixz - boost - postgresql_13 - (if lib.versionAtLeast lib.version "20.03pre" - then nlohmann_json - else nlohmann_json.override { multipleHeaders = true; }) - prometheus-cpp - ]; - - checkInputs = with final; [ - cacert - foreman - glibcLocales - libressl.nc - openldap - python3 - ]; - - hydraPath = with final; lib.makeBinPath ( - [ - subversion - openssh - final.nix - coreutils - findutils - pixz - gzip - bzip2 - xz - gnutar - unzip - git - top-git - mercurial - darcs - gnused - breezy - ] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] - ); - - OPENLDAP_ROOT = final.openldap; - - shellHook = '' - pushd $(git rev-parse --show-toplevel) >/dev/null - - PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$PATH - PERL5LIB=$(pwd)/src/lib:$PERL5LIB - export HYDRA_HOME="$(pwd)/src/" - mkdir -p .hydra-data - export HYDRA_DATA="$(pwd)/.hydra-data" - export HYDRA_DBI='dbi:Pg:dbname=hydra;host=localhost;port=64444' - - popd >/dev/null - ''; - - NIX_LDFLAGS = [ "-lpthread" ]; - - enableParallelBuilding = true; - - doCheck = true; - - preCheck = '' - patchShebangs . - export LOGNAME=''${LOGNAME:-foo} - # set $HOME for bzr so it can create its trace file - export HOME=$(mktemp -d) - ''; - - postInstall = '' - mkdir -p $out/nix-support - - for i in $out/bin/*; do - read -n 4 chars < $i - if [[ $chars =~ ELF ]]; then continue; fi - wrapProgram $i \ - --prefix PERL5LIB ':' $out/libexec/hydra/lib:$PERL5LIB \ - --prefix PATH ':' $out/bin:$hydraPath \ - --set HYDRA_RELEASE ${version} \ - --set HYDRA_HOME $out/libexec/hydra \ - --set NIX_RELEASE ${final.nix.name or "unknown"} - done - ''; - - dontStrip = true; - - meta.description = "Build of Hydra on ${final.stdenv.system}"; - passthru = { inherit perlDeps; inherit (final) nix; }; }; }; @@ -262,7 +70,7 @@ manual = forEachSystem (system: let pkgs = pkgsBySystem.${system}; in - pkgs.runCommand "hydra-manual-${version}" { } + pkgs.runCommand "hydra-manual-${pkgs.hydra.version}" { } '' mkdir -p $out/share cp -prvd ${pkgs.hydra}/share/doc $out/share/ diff --git a/package.nix b/package.nix new file mode 100644 index 00000000..35e29be1 --- /dev/null +++ b/package.nix @@ -0,0 +1,246 @@ +{ stdenv +, lib + +, src + +, buildEnv + +, perlPackages + +, nix +, git + +, makeWrapper +, autoreconfHook +, nukeReferences +, pkg-config +, mdbook + +, unzip +, libpqxx +, top-git +, mercurial +, darcs +, subversion +, breezy +, openssl +, bzip2 +, libxslt +, perl +, pixz +, boost +, postgresql_13 +, nlohmann_json +, prometheus-cpp + +, cacert +, foreman +, glibcLocales +, libressl +, openldap +, python3 + +, openssh +, coreutils +, findutils +, gzip +, xz +, gnutar +, gnused + +, rpm +, dpkg +, cdrkit +}: + +let + perlDeps = buildEnv { + name = "hydra-perl-deps"; + paths = with perlPackages; lib.closePropagation + [ + AuthenSASL + CatalystActionREST + CatalystAuthenticationStoreDBIxClass + CatalystAuthenticationStoreLDAP + CatalystDevel + CatalystPluginAccessLog + CatalystPluginAuthorizationRoles + CatalystPluginCaptcha + CatalystPluginPrometheusTiny + CatalystPluginSessionStateCookie + CatalystPluginSessionStoreFastMmap + CatalystPluginStackTrace + CatalystTraitForRequestProxyBase + CatalystViewDownload + CatalystViewJSON + CatalystViewTT + CatalystXRoleApplicator + CatalystXScriptServerStarman + CryptPassphrase + CryptPassphraseArgon2 + CryptRandPasswd + DataDump + DateTime + DBDPg + DBDSQLite + DigestSHA1 + EmailMIME + EmailSender + FileLibMagic + FileSlurper + FileWhich + # Not Perl + nix.perl-bindings + git + # Perl again + IOCompress + IPCRun + IPCRun3 + JSON + JSONMaybeXS + JSONXS + ListSomeUtils + LWP + LWPProtocolHttps + ModulePluggable + NetAmazonS3 + NetPrometheus + NetStatsd + PadWalker + ParallelForkManager + PerlCriticCommunity + PrometheusTinyShared + ReadonlyX + SetScalar + SQLSplitStatement + Starman + StringCompareConstantTime + SysHostnameLong + TermSizeAny + TermReadKey + Test2Harness + TestPostgreSQL + TextDiff + TextTable + UUID4Tiny + YAML + XMLSimple + ]; + }; + + version = "${builtins.readFile ./version.txt}.${builtins.substring 0 8 (src.lastModifiedDate or "19700101")}.${src.shortRev or "DIRTY"}"; +in +stdenv.mkDerivation { + pname = "hydra"; + inherit version; + + inherit src; + + nativeBuildInputs = [ + makeWrapper + autoreconfHook + nukeReferences + pkg-config + mdbook + ]; + + buildInputs = [ + unzip + libpqxx + top-git + mercurial + darcs + subversion + breezy + openssl + bzip2 + libxslt + nix + perlDeps + perl + pixz + boost + postgresql_13 + nlohmann_json + prometheus-cpp + ]; + + checkInputs = [ + cacert + foreman + glibcLocales + libressl.nc + openldap + python3 + ]; + + hydraPath = lib.makeBinPath ( + [ + subversion + openssh + nix + coreutils + findutils + pixz + gzip + bzip2 + xz + gnutar + unzip + git + top-git + mercurial + darcs + gnused + breezy + ] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] + ); + + OPENLDAP_ROOT = openldap; + + shellHook = '' + pushd $(git rev-parse --show-toplevel) >/dev/null + + PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$PATH + PERL5LIB=$(pwd)/src/lib:$PERL5LIB + export HYDRA_HOME="$(pwd)/src/" + mkdir -p .hydra-data + export HYDRA_DATA="$(pwd)/.hydra-data" + export HYDRA_DBI='dbi:Pg:dbname=hydra;host=localhost;port=64444' + + popd >/dev/null + ''; + + NIX_LDFLAGS = [ "-lpthread" ]; + + enableParallelBuilding = true; + + doCheck = true; + + preCheck = '' + patchShebangs . + export LOGNAME=''${LOGNAME:-foo} + # set $HOME for bzr so it can create its trace file + export HOME=$(mktemp -d) + ''; + + postInstall = '' + mkdir -p $out/nix-support + + for i in $out/bin/*; do + read -n 4 chars < $i + if [[ $chars =~ ELF ]]; then continue; fi + wrapProgram $i \ + --prefix PERL5LIB ':' $out/libexec/hydra/lib:$PERL5LIB \ + --prefix PATH ':' $out/bin:$hydraPath \ + --set HYDRA_RELEASE ${version} \ + --set HYDRA_HOME $out/libexec/hydra \ + --set NIX_RELEASE ${nix.name or "unknown"} + done + ''; + + dontStrip = true; + + meta.description = "Build of Hydra on ${stdenv.system}"; + passthru = { inherit perlDeps nix; }; +} From 1bd195a5138b3c69c52110d713c706cb4908ba16 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jan 2024 11:32:00 -0500 Subject: [PATCH 05/12] Clean up deps - `strictDeps` - Ensure it builds with and without `doCheck` --- flake.nix | 6 ++++++ package.nix | 28 ++++++++++++++++++---------- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/flake.nix b/flake.nix index f7975b27..a6b62cfa 100644 --- a/flake.nix +++ b/flake.nix @@ -68,6 +68,12 @@ build = forEachSystem (system: packages.${system}.hydra); + buildNoTests = forEachSystem (system: + packages.${system}.hydra.overrideAttrs (_: { + doCheck = false; + }) + ); + manual = forEachSystem (system: let pkgs = pkgsBySystem.${system}; in pkgs.runCommand "hydra-manual-${pkgs.hydra.version}" { } diff --git a/package.nix b/package.nix index 35e29be1..f91e7a24 100644 --- a/package.nix +++ b/package.nix @@ -136,41 +136,49 @@ stdenv.mkDerivation { inherit src; + strictDeps = true; + nativeBuildInputs = [ makeWrapper autoreconfHook nukeReferences pkg-config mdbook + nix + perlDeps + perl + unzip ]; buildInputs = [ - unzip libpqxx - top-git - mercurial - darcs - subversion - breezy openssl - bzip2 libxslt nix perlDeps perl - pixz boost - postgresql_13 nlohmann_json prometheus-cpp ]; + nativeCheckInputs = [ + bzip2 + darcs + top-git + mercurial + subversion + breezy + openldap + postgresql_13 + pixz + ]; + checkInputs = [ cacert foreman glibcLocales libressl.nc - openldap python3 ]; From d6d6d1b649ea3b83baf44267637e42c4eb1c92c0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jan 2024 12:03:15 -0500 Subject: [PATCH 06/12] flake.nix: Temporarily add a second Nixpkgs for `lib.fileset` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/b38e5a665e9d0aa7975beb0ed12e42d13a392e74' (2023-12-13) → 'github:NixOS/nix/03e96b9dc011a16a0f6db9c7cb021ff93f8dcf88' (2024-01-19) • Added input 'nixpkgs-for-fileset': 'github:NixOS/nixpkgs/a77ab169a83a4175169d78684ddd2e54486ac651' (2024-01-24) --- flake.lock | 25 +++++++++++++++++++++---- flake.nix | 5 +++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/flake.lock b/flake.lock index 66e30323..8231484c 100644 --- a/flake.lock +++ b/flake.lock @@ -42,11 +42,11 @@ "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1702510710, - "narHash": "sha256-9K+w1mQgmUxCmEsPaSFkpYsj/cxjO2PSwTCPkNZ/NiU=", + "lastModified": 1705666051, + "narHash": "sha256-C3eht7uA7gZp9O5XpA9Mkqdi1WoTQaFH4FMGmM4DrFM=", "owner": "NixOS", "repo": "nix", - "rev": "b38e5a665e9d0aa7975beb0ed12e42d13a392e74", + "rev": "03e96b9dc011a16a0f6db9c7cb021ff93f8dcf88", "type": "github" }, "original": { @@ -72,6 +72,22 @@ "type": "github" } }, + "nixpkgs-for-fileset": { + "locked": { + "lastModified": 1706098335, + "narHash": "sha256-r3dWjT8P9/Ah5m5ul4WqIWD8muj5F+/gbCdjiNVBKmU=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a77ab169a83a4175169d78684ddd2e54486ac651", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-23.11", + "repo": "nixpkgs", + "type": "github" + } + }, "nixpkgs-regression": { "locked": { "lastModified": 1643052045, @@ -91,7 +107,8 @@ "root": { "inputs": { "nix": "nix", - "nixpkgs": "nixpkgs" + "nixpkgs": "nixpkgs", + "nixpkgs-for-fileset": "nixpkgs-for-fileset" } } }, diff --git a/flake.nix b/flake.nix index a6b62cfa..0ab25018 100644 --- a/flake.nix +++ b/flake.nix @@ -5,6 +5,11 @@ inputs.nix.url = "github:NixOS/nix/2.19-maintenance"; inputs.nix.inputs.nixpkgs.follows = "nixpkgs"; + # TODO get rid of this once https://github.com/NixOS/nix/pull/9546 is + # mered and we upgrade or Nix, so the main `nixpkgs` input is at least + # 23.11 and has `lib.fileset`. + inputs.nixpkgs-for-fileset.url = "github:NixOS/nixpkgs/nixos-23.11"; + outputs = { self, nixpkgs, nix }: let systems = [ "x86_64-linux" "aarch64-linux" ]; From 4bbc7b8f75409aa8e99a0f3284bdc5f11f804cfb Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jan 2024 12:21:13 -0500 Subject: [PATCH 07/12] Use the Nixpkgs `fileset` library to filter source Now I can change Nix files without causing rebuilds. --- flake.nix | 5 +++-- package.nix | 26 +++++++++++++++++++++----- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/flake.nix b/flake.nix index 0ab25018..71fd6c6d 100644 --- a/flake.nix +++ b/flake.nix @@ -10,7 +10,7 @@ # 23.11 and has `lib.fileset`. inputs.nixpkgs-for-fileset.url = "github:NixOS/nixpkgs/nixos-23.11"; - outputs = { self, nixpkgs, nix }: + outputs = { self, nixpkgs, nix, nixpkgs-for-fileset }: let systems = [ "x86_64-linux" "aarch64-linux" ]; forEachSystem = nixpkgs.lib.genAttrs systems; @@ -65,7 +65,8 @@ }; hydra = final.callPackage ./package.nix { - src = self; + inherit (nixpkgs-for-fileset.lib) fileset; + rawSrc = self; }; }; diff --git a/package.nix b/package.nix index f91e7a24..eff9ed37 100644 --- a/package.nix +++ b/package.nix @@ -1,7 +1,8 @@ { stdenv , lib +, fileset -, src +, rawSrc , buildEnv @@ -128,13 +129,28 @@ let ]; }; - version = "${builtins.readFile ./version.txt}.${builtins.substring 0 8 (src.lastModifiedDate or "19700101")}.${src.shortRev or "DIRTY"}"; + version = "${builtins.readFile ./version.txt}.${builtins.substring 0 8 (rawSrc.lastModifiedDate or "19700101")}.${rawSrc.shortRev or "DIRTY"}"; in -stdenv.mkDerivation { +stdenv.mkDerivation (finalAttrs: { pname = "hydra"; inherit version; - inherit src; + src = fileset.toSource { + root = ./.; + fileset = fileset.unions ([ + ./version.txt + ./configure.ac + ./Makefile.am + ./src + ./doc + ./hydra-module.nix + # TODO only when `doCheck` + ./t + ] ++ lib.optionals finalAttrs.doCheck [ + ./.perlcriticrc + ./.yath.rc + ]); + }; strictDeps = true; @@ -251,4 +267,4 @@ stdenv.mkDerivation { meta.description = "Build of Hydra on ${stdenv.system}"; passthru = { inherit perlDeps nix; }; -} +}) From 73b6c1fb11b7c5679725599c552f4e163bf668f0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jan 2024 13:27:05 -0500 Subject: [PATCH 08/12] Filter out (mosts test) when `!doCheck` --- Makefile.am | 6 +++++- configure.ac | 20 ++++++++++++++------ package.nix | 7 +++++-- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/Makefile.am b/Makefile.am index e744cc33..9e06ebdb 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,4 +1,8 @@ -SUBDIRS = src t doc +SUBDIRS = src doc +if CAN_DO_CHECK + SUBDIRS += t +endif + BOOTCLEAN_SUBDIRS = $(SUBDIRS) DIST_SUBDIRS = $(SUBDIRS) EXTRA_DIST = hydra-module.nix diff --git a/configure.ac b/configure.ac index eec647c3..e5c57d14 100644 --- a/configure.ac +++ b/configure.ac @@ -53,9 +53,6 @@ PKG_CHECK_MODULES([NIX], [nix-main nix-expr nix-store]) testPath="$(dirname $(type -p expr))" AC_SUBST(testPath) -jobsPath="$(realpath ./t/jobs)" -AC_SUBST(jobsPath) - CXXFLAGS+=" -include nix/config.h" AC_CONFIG_FILES([ @@ -71,11 +68,22 @@ AC_CONFIG_FILES([ src/lib/Makefile src/root/Makefile src/script/Makefile - t/Makefile - t/jobs/config.nix - t/jobs/declarative/project.json ]) +# Tests might be filtered out +AM_CONDITIONAL([CAN_DO_CHECK], [test -f "$srcdir/t/api-test.t"]) +AM_COND_IF( + [CAN_DO_CHECK], + [ + jobsPath="$(realpath ./t/jobs)" + AC_SUBST(jobsPath) + AC_CONFIG_FILES([ + t/Makefile + t/jobs/config.nix + t/jobs/declarative/project.json + ]) + ]) + AC_CONFIG_COMMANDS([executable-scripts], []) AC_CONFIG_HEADER([hydra-config.h]) diff --git a/package.nix b/package.nix index eff9ed37..73ef9b3f 100644 --- a/package.nix +++ b/package.nix @@ -144,9 +144,12 @@ stdenv.mkDerivation (finalAttrs: { ./src ./doc ./hydra-module.nix - # TODO only when `doCheck` - ./t + # These are always needed to appease Automake + ./t/Makefile.am + ./t/jobs/config.nix.in + ./t/jobs/declarative/project.json.in ] ++ lib.optionals finalAttrs.doCheck [ + ./t ./.perlcriticrc ./.yath.rc ]); From c5f37eca91bee5bacbe53f826abcd0c0d0f38ef1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jan 2024 11:54:44 -0500 Subject: [PATCH 09/12] Reorganize hydra modules --- .gitignore | 1 + Makefile.am | 6 +-- flake.nix | 50 +++------------------ nixos-modules/default.nix | 49 ++++++++++++++++++++ hydra-module.nix => nixos-modules/hydra.nix | 0 package.nix | 2 +- 6 files changed, 59 insertions(+), 49 deletions(-) create mode 100644 nixos-modules/default.nix rename hydra-module.nix => nixos-modules/hydra.nix (100%) diff --git a/.gitignore b/.gitignore index 799db665..f8bf5718 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,7 @@ t/jobs/declarative/project.json hydra-config.h hydra-config.h.in result +result-* outputs config stamp-h1 diff --git a/Makefile.am b/Makefile.am index 9e06ebdb..a28e3f33 100644 --- a/Makefile.am +++ b/Makefile.am @@ -5,8 +5,8 @@ endif BOOTCLEAN_SUBDIRS = $(SUBDIRS) DIST_SUBDIRS = $(SUBDIRS) -EXTRA_DIST = hydra-module.nix +EXTRA_DIST = nixos-modules/hydra.nix -install-data-local: hydra-module.nix +install-data-local: nixos-modules/hydra.nix $(INSTALL) -d $(DESTDIR)$(datadir)/nix - $(INSTALL_DATA) hydra-module.nix $(DESTDIR)$(datadir)/nix/ + $(INSTALL_DATA) nixos-modules/hydra.nix $(DESTDIR)$(datadir)/nix/hydra-module.nix diff --git a/flake.nix b/flake.nix index 71fd6c6d..30aa61c0 100644 --- a/flake.nix +++ b/flake.nix @@ -15,9 +15,11 @@ systems = [ "x86_64-linux" "aarch64-linux" ]; forEachSystem = nixpkgs.lib.genAttrs systems; + overlayList = [ self.overlays.default nix.overlays.default ]; + pkgsBySystem = forEachSystem (system: import nixpkgs { inherit system; - overlays = [ self.overlays.default nix.overlays.default ]; + overlays = overlayList; }); # NixOS configuration used for VM tests. @@ -386,50 +388,8 @@ default = pkgsBySystem.${system}.hydra; }); - nixosModules.hydra = { - imports = [ ./hydra-module.nix ]; - nixpkgs.overlays = [ self.overlays.default nix.overlays.default ]; - }; - - nixosModules.hydraTest = { pkgs, ... }: { - imports = [ self.nixosModules.hydra ]; - - services.hydra-dev.enable = true; - services.hydra-dev.hydraURL = "http://hydra.example.org"; - services.hydra-dev.notificationSender = "admin@hydra.example.org"; - - systemd.services.hydra-send-stats.enable = false; - - services.postgresql.enable = true; - services.postgresql.package = pkgs.postgresql_11; - - # The following is to work around the following error from hydra-server: - # [error] Caught exception in engine "Cannot determine local time zone" - time.timeZone = "UTC"; - - nix.extraOptions = '' - allowed-uris = https://github.com/ - ''; - }; - - nixosModules.hydraProxy = { - services.httpd = { - enable = true; - adminAddr = "hydra-admin@example.org"; - extraConfig = '' - - Order deny,allow - Allow from all - - - ProxyRequests Off - ProxyPreserveHost On - ProxyPass /apache-errors ! - ErrorDocument 503 /apache-errors/503.html - ProxyPass / http://127.0.0.1:3000/ retry=5 disablereuse=on - ProxyPassReverse / http://127.0.0.1:3000/ - ''; - }; + nixosModules = import ./nixos-modules { + overlays = overlayList; }; nixosConfigurations.container = nixpkgs.lib.nixosSystem { diff --git a/nixos-modules/default.nix b/nixos-modules/default.nix new file mode 100644 index 00000000..6fc19d31 --- /dev/null +++ b/nixos-modules/default.nix @@ -0,0 +1,49 @@ +{ overlays }: + +rec { + hydra = { + imports = [ ./hydra.nix ]; + nixpkgs = { inherit overlays; }; + }; + + hydraTest = { pkgs, ... }: { + imports = [ hydra ]; + + services.hydra-dev.enable = true; + services.hydra-dev.hydraURL = "http://hydra.example.org"; + services.hydra-dev.notificationSender = "admin@hydra.example.org"; + + systemd.services.hydra-send-stats.enable = false; + + services.postgresql.enable = true; + services.postgresql.package = pkgs.postgresql_11; + + # The following is to work around the following error from hydra-server: + # [error] Caught exception in engine "Cannot determine local time zone" + time.timeZone = "UTC"; + + nix.extraOptions = '' + allowed-uris = https://github.com/ + ''; + }; + + hydraProxy = { + services.httpd = { + enable = true; + adminAddr = "hydra-admin@example.org"; + extraConfig = '' + + Order deny,allow + Allow from all + + + ProxyRequests Off + ProxyPreserveHost On + ProxyPass /apache-errors ! + ErrorDocument 503 /apache-errors/503.html + ProxyPass / http://127.0.0.1:3000/ retry=5 disablereuse=on + ProxyPassReverse / http://127.0.0.1:3000/ + ''; + }; + }; +} diff --git a/hydra-module.nix b/nixos-modules/hydra.nix similarity index 100% rename from hydra-module.nix rename to nixos-modules/hydra.nix diff --git a/package.nix b/package.nix index 73ef9b3f..d5e6bb69 100644 --- a/package.nix +++ b/package.nix @@ -143,7 +143,7 @@ stdenv.mkDerivation (finalAttrs: { ./Makefile.am ./src ./doc - ./hydra-module.nix + ./nixos-modules/hydra.nix # These are always needed to appease Automake ./t/Makefile.am ./t/jobs/config.nix.in From b5ed0787f7eba4f0a7d900736976e335f443c19e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jan 2024 14:25:12 -0500 Subject: [PATCH 10/12] Replace "not Perl" and "Perl again" with something more self-explanatory --- package.nix | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/package.nix b/package.nix index d5e6bb69..0f8f004d 100644 --- a/package.nix +++ b/package.nix @@ -57,8 +57,11 @@ let perlDeps = buildEnv { name = "hydra-perl-deps"; - paths = with perlPackages; lib.closePropagation - [ + paths = lib.closePropagation + ([ + nix.perl-bindings + git + ] ++ (with perlPackages; [ AuthenSASL CatalystActionREST CatalystAuthenticationStoreDBIxClass @@ -90,10 +93,6 @@ let FileLibMagic FileSlurper FileWhich - # Not Perl - nix.perl-bindings - git - # Perl again IOCompress IPCRun IPCRun3 @@ -126,7 +125,7 @@ let UUID4Tiny YAML XMLSimple - ]; + ])); }; version = "${builtins.readFile ./version.txt}.${builtins.substring 0 8 (rawSrc.lastModifiedDate or "19700101")}.${rawSrc.shortRev or "DIRTY"}"; From aed130cd17ae7da41d9f38f830af3826a85a5dc6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jan 2024 15:57:39 -0500 Subject: [PATCH 11/12] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/03e96b9dc011a16a0f6db9c7cb021ff93f8dcf88' (2024-01-19) → 'github:NixOS/nix/2c4bb93ba5a97e7078896ebc36385ce172960e4e' (2024-01-25) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 8231484c..2ddc7c16 100644 --- a/flake.lock +++ b/flake.lock @@ -42,11 +42,11 @@ "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1705666051, - "narHash": "sha256-C3eht7uA7gZp9O5XpA9Mkqdi1WoTQaFH4FMGmM4DrFM=", + "lastModified": 1706208340, + "narHash": "sha256-wNyHUEIiKKVs6UXrUzhP7RSJQv0A8jckgcuylzftl8k=", "owner": "NixOS", "repo": "nix", - "rev": "03e96b9dc011a16a0f6db9c7cb021ff93f8dcf88", + "rev": "2c4bb93ba5a97e7078896ebc36385ce172960e4e", "type": "github" }, "original": { From c64eed7d07ad432fa9c5ea979d1039a5207f4629 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jan 2024 14:48:53 -0500 Subject: [PATCH 12/12] Simplify `StoreConfig::getDefaultSystemFeatures` call That method is now static. --- src/hydra-queue-runner/hydra-queue-runner.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index 1cbcafea..df79e1dc 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -15,7 +15,6 @@ #include "state.hh" #include "hydra-build-result.hh" #include "store-api.hh" -#include "local-store.hh" #include "remote-store.hh" #include "globals.hh" @@ -227,7 +226,7 @@ void State::monitorMachinesFile() parseMachines("localhost " + (settings.thisSystem == "x86_64-linux" ? "x86_64-linux,i686-linux" : settings.thisSystem.get()) + " - " + std::to_string(settings.maxBuildJobs) + " 1 " - + concatStringsSep(",", (LocalStoreConfig { {} }).getDefaultSystemFeatures())); + + concatStringsSep(",", StoreConfig::getDefaultSystemFeatures())); machinesReadyLock.unlock(); return; }