diff --git a/.gitignore b/.gitignore index 799db665..f8bf5718 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,7 @@ t/jobs/declarative/project.json hydra-config.h hydra-config.h.in result +result-* outputs config stamp-h1 diff --git a/Makefile.am b/Makefile.am index e744cc33..a28e3f33 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,8 +1,12 @@ -SUBDIRS = src t doc +SUBDIRS = src doc +if CAN_DO_CHECK + SUBDIRS += t +endif + BOOTCLEAN_SUBDIRS = $(SUBDIRS) DIST_SUBDIRS = $(SUBDIRS) -EXTRA_DIST = hydra-module.nix +EXTRA_DIST = nixos-modules/hydra.nix -install-data-local: hydra-module.nix +install-data-local: nixos-modules/hydra.nix $(INSTALL) -d $(DESTDIR)$(datadir)/nix - $(INSTALL_DATA) hydra-module.nix $(DESTDIR)$(datadir)/nix/ + $(INSTALL_DATA) nixos-modules/hydra.nix $(DESTDIR)$(datadir)/nix/hydra-module.nix diff --git a/configure.ac b/configure.ac index eec647c3..e5c57d14 100644 --- a/configure.ac +++ b/configure.ac @@ -53,9 +53,6 @@ PKG_CHECK_MODULES([NIX], [nix-main nix-expr nix-store]) testPath="$(dirname $(type -p expr))" AC_SUBST(testPath) -jobsPath="$(realpath ./t/jobs)" -AC_SUBST(jobsPath) - CXXFLAGS+=" -include nix/config.h" AC_CONFIG_FILES([ @@ -71,11 +68,22 @@ AC_CONFIG_FILES([ src/lib/Makefile src/root/Makefile src/script/Makefile - t/Makefile - t/jobs/config.nix - t/jobs/declarative/project.json ]) +# Tests might be filtered out +AM_CONDITIONAL([CAN_DO_CHECK], [test -f "$srcdir/t/api-test.t"]) +AM_COND_IF( + [CAN_DO_CHECK], + [ + jobsPath="$(realpath ./t/jobs)" + AC_SUBST(jobsPath) + AC_CONFIG_FILES([ + t/Makefile + t/jobs/config.nix + t/jobs/declarative/project.json + ]) + ]) + AC_CONFIG_COMMANDS([executable-scripts], []) AC_CONFIG_HEADER([hydra-config.h]) diff --git a/flake.lock b/flake.lock index bdbc4197..e3ea8f61 100644 --- a/flake.lock +++ b/flake.lock @@ -42,11 +42,11 @@ "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1706100941, - "narHash": "sha256-YLYxZrQLQ6i22RLZG/rsB4sIfFzhep7XQYlodzCmKy0=", + "lastModified": 1706195509, + "narHash": "sha256-1kwfk7H/MWZAcTKHnnWXo/+KlQeOTIRtOIzc4FX3QnE=", "owner": "NixOS", "repo": "nix", - "rev": "212ba69e6f995992f8b4e4c0656d19c0156c8714", + "rev": "8df68a213fc52a57b02a57005b0e06cc8de40ce3", "type": "github" }, "original": { @@ -71,6 +71,22 @@ "type": "github" } }, + "nixpkgs-for-fileset": { + "locked": { + "lastModified": 1706098335, + "narHash": "sha256-r3dWjT8P9/Ah5m5ul4WqIWD8muj5F+/gbCdjiNVBKmU=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a77ab169a83a4175169d78684ddd2e54486ac651", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-23.11", + "repo": "nixpkgs", + "type": "github" + } + }, "nixpkgs-regression": { "locked": { "lastModified": 1643052045, @@ -90,7 +106,8 @@ "root": { "inputs": { "nix": "nix", - "nixpkgs": "nixpkgs" + "nixpkgs": "nixpkgs", + "nixpkgs-for-fileset": "nixpkgs-for-fileset" } } }, diff --git a/flake.nix b/flake.nix index c9af0174..4dc7d25e 100644 --- a/flake.nix +++ b/flake.nix @@ -5,16 +5,21 @@ inputs.nix.url = "github:NixOS/nix"; inputs.nix.inputs.nixpkgs.follows = "nixpkgs"; - outputs = { self, nixpkgs, nix }: - let - version = "${builtins.readFile ./version.txt}.${builtins.substring 0 8 (self.lastModifiedDate or "19700101")}.${self.shortRev or "DIRTY"}"; + # TODO get rid of this once https://github.com/NixOS/nix/pull/9546 is + # mered and we upgrade or Nix, so the main `nixpkgs` input is at least + # 23.11 and has `lib.fileset`. + inputs.nixpkgs-for-fileset.url = "github:NixOS/nixpkgs/nixos-23.11"; + outputs = { self, nixpkgs, nix, nixpkgs-for-fileset }: + let systems = [ "x86_64-linux" "aarch64-linux" ]; forEachSystem = nixpkgs.lib.genAttrs systems; + overlayList = [ self.overlays.default nix.overlays.default ]; + pkgsBySystem = forEachSystem (system: import nixpkgs { inherit system; - overlays = [ self.overlays.default nix.overlays.default ]; + overlays = overlayList; }); # NixOS configuration used for VM tests. @@ -61,200 +66,9 @@ }; - hydra = let - inherit (final) lib stdenv; - perlDeps = final.buildEnv { - name = "hydra-perl-deps"; - paths = with final.perlPackages; lib.closePropagation - [ - AuthenSASL - CatalystActionREST - CatalystAuthenticationStoreDBIxClass - CatalystAuthenticationStoreLDAP - CatalystDevel - CatalystPluginAccessLog - CatalystPluginAuthorizationRoles - CatalystPluginCaptcha - CatalystPluginPrometheusTiny - CatalystPluginSessionStateCookie - CatalystPluginSessionStoreFastMmap - CatalystPluginStackTrace - CatalystTraitForRequestProxyBase - CatalystViewDownload - CatalystViewJSON - CatalystViewTT - CatalystXRoleApplicator - CatalystXScriptServerStarman - CryptPassphrase - CryptPassphraseArgon2 - CryptRandPasswd - DataDump - DateTime - DBDPg - DBDSQLite - DigestSHA1 - EmailMIME - EmailSender - FileLibMagic - FileSlurper - FileWhich - final.nix.perl-bindings - final.git - IOCompress - IPCRun - IPCRun3 - JSON - JSONMaybeXS - JSONXS - ListSomeUtils - LWP - LWPProtocolHttps - ModulePluggable - NetAmazonS3 - NetPrometheus - NetStatsd - PadWalker - ParallelForkManager - PerlCriticCommunity - PrometheusTinyShared - ReadonlyX - SetScalar - SQLSplitStatement - Starman - StringCompareConstantTime - SysHostnameLong - TermSizeAny - TermReadKey - Test2Harness - TestPostgreSQL - TextDiff - TextTable - UUID4Tiny - YAML - XMLSimple - ]; - }; - - in - stdenv.mkDerivation { - - name = "hydra-${version}"; - - src = self; - - nativeBuildInputs = - with final.buildPackages; [ - makeWrapper - autoreconfHook - automake - libtool - nukeReferences - pkg-config - mdbook - ]; - - buildInputs = - with final; [ - unzip - libpqxx - top-git - mercurial - darcs - subversion - breezy - openssl - bzip2 - libxslt - final.nix - perlDeps - perl - pixz - boost - postgresql_13 - (if lib.versionAtLeast lib.version "20.03pre" - then nlohmann_json - else nlohmann_json.override { multipleHeaders = true; }) - prometheus-cpp - ]; - - checkInputs = with final; [ - cacert - foreman - glibcLocales - libressl.nc - openldap - python3 - ]; - - hydraPath = with final; lib.makeBinPath ( - [ - subversion - openssh - final.nix - coreutils - findutils - pixz - gzip - bzip2 - xz - gnutar - unzip - git - top-git - mercurial - darcs - gnused - breezy - ] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] - ); - - OPENLDAP_ROOT = final.openldap; - - shellHook = '' - pushd $(git rev-parse --show-toplevel) >/dev/null - - PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$PATH - PERL5LIB=$(pwd)/src/lib:$PERL5LIB - export HYDRA_HOME="$(pwd)/src/" - mkdir -p .hydra-data - export HYDRA_DATA="$(pwd)/.hydra-data" - export HYDRA_DBI='dbi:Pg:dbname=hydra;host=localhost;port=64444' - - popd >/dev/null - ''; - - NIX_LDFLAGS = [ "-lpthread" ]; - - enableParallelBuilding = true; - - doCheck = true; - - preCheck = '' - patchShebangs . - export LOGNAME=''${LOGNAME:-foo} - # set $HOME for bzr so it can create its trace file - export HOME=$(mktemp -d) - ''; - - postInstall = '' - mkdir -p $out/nix-support - - for i in $out/bin/*; do - read -n 4 chars < $i - if [[ $chars =~ ELF ]]; then continue; fi - wrapProgram $i \ - --prefix PERL5LIB ':' $out/libexec/hydra/lib:$PERL5LIB \ - --prefix PATH ':' $out/bin:$hydraPath \ - --set HYDRA_RELEASE ${version} \ - --set HYDRA_HOME $out/libexec/hydra \ - --set NIX_RELEASE ${final.nix.name or "unknown"} - done - ''; - - dontStrip = true; - - meta.description = "Build of Hydra on ${final.stdenv.system}"; - passthru = { inherit perlDeps; inherit (final) nix; }; + hydra = final.callPackage ./package.nix { + inherit (nixpkgs-for-fileset.lib) fileset; + rawSrc = self; }; }; @@ -262,9 +76,15 @@ build = forEachSystem (system: packages.${system}.hydra); + buildNoTests = forEachSystem (system: + packages.${system}.hydra.overrideAttrs (_: { + doCheck = false; + }) + ); + manual = forEachSystem (system: let pkgs = pkgsBySystem.${system}; in - pkgs.runCommand "hydra-manual-${version}" { } + pkgs.runCommand "hydra-manual-${pkgs.hydra.version}" { } '' mkdir -p $out/share cp -prvd ${pkgs.hydra}/share/doc $out/share/ @@ -568,50 +388,8 @@ default = pkgsBySystem.${system}.hydra; }); - nixosModules.hydra = { - imports = [ ./hydra-module.nix ]; - nixpkgs.overlays = [ self.overlays.default nix.overlays.default ]; - }; - - nixosModules.hydraTest = { pkgs, ... }: { - imports = [ self.nixosModules.hydra ]; - - services.hydra-dev.enable = true; - services.hydra-dev.hydraURL = "http://hydra.example.org"; - services.hydra-dev.notificationSender = "admin@hydra.example.org"; - - systemd.services.hydra-send-stats.enable = false; - - services.postgresql.enable = true; - services.postgresql.package = pkgs.postgresql_11; - - # The following is to work around the following error from hydra-server: - # [error] Caught exception in engine "Cannot determine local time zone" - time.timeZone = "UTC"; - - nix.extraOptions = '' - allowed-uris = https://github.com/ - ''; - }; - - nixosModules.hydraProxy = { - services.httpd = { - enable = true; - adminAddr = "hydra-admin@example.org"; - extraConfig = '' - - Order deny,allow - Allow from all - - - ProxyRequests Off - ProxyPreserveHost On - ProxyPass /apache-errors ! - ErrorDocument 503 /apache-errors/503.html - ProxyPass / http://127.0.0.1:3000/ retry=5 disablereuse=on - ProxyPassReverse / http://127.0.0.1:3000/ - ''; - }; + nixosModules = import ./nixos-modules { + overlays = overlayList; }; nixosConfigurations.container = nixpkgs.lib.nixosSystem { diff --git a/nixos-modules/default.nix b/nixos-modules/default.nix new file mode 100644 index 00000000..6fc19d31 --- /dev/null +++ b/nixos-modules/default.nix @@ -0,0 +1,49 @@ +{ overlays }: + +rec { + hydra = { + imports = [ ./hydra.nix ]; + nixpkgs = { inherit overlays; }; + }; + + hydraTest = { pkgs, ... }: { + imports = [ hydra ]; + + services.hydra-dev.enable = true; + services.hydra-dev.hydraURL = "http://hydra.example.org"; + services.hydra-dev.notificationSender = "admin@hydra.example.org"; + + systemd.services.hydra-send-stats.enable = false; + + services.postgresql.enable = true; + services.postgresql.package = pkgs.postgresql_11; + + # The following is to work around the following error from hydra-server: + # [error] Caught exception in engine "Cannot determine local time zone" + time.timeZone = "UTC"; + + nix.extraOptions = '' + allowed-uris = https://github.com/ + ''; + }; + + hydraProxy = { + services.httpd = { + enable = true; + adminAddr = "hydra-admin@example.org"; + extraConfig = '' + + Order deny,allow + Allow from all + + + ProxyRequests Off + ProxyPreserveHost On + ProxyPass /apache-errors ! + ErrorDocument 503 /apache-errors/503.html + ProxyPass / http://127.0.0.1:3000/ retry=5 disablereuse=on + ProxyPassReverse / http://127.0.0.1:3000/ + ''; + }; + }; +} diff --git a/hydra-module.nix b/nixos-modules/hydra.nix similarity index 100% rename from hydra-module.nix rename to nixos-modules/hydra.nix diff --git a/package.nix b/package.nix new file mode 100644 index 00000000..0f8f004d --- /dev/null +++ b/package.nix @@ -0,0 +1,272 @@ +{ stdenv +, lib +, fileset + +, rawSrc + +, buildEnv + +, perlPackages + +, nix +, git + +, makeWrapper +, autoreconfHook +, nukeReferences +, pkg-config +, mdbook + +, unzip +, libpqxx +, top-git +, mercurial +, darcs +, subversion +, breezy +, openssl +, bzip2 +, libxslt +, perl +, pixz +, boost +, postgresql_13 +, nlohmann_json +, prometheus-cpp + +, cacert +, foreman +, glibcLocales +, libressl +, openldap +, python3 + +, openssh +, coreutils +, findutils +, gzip +, xz +, gnutar +, gnused + +, rpm +, dpkg +, cdrkit +}: + +let + perlDeps = buildEnv { + name = "hydra-perl-deps"; + paths = lib.closePropagation + ([ + nix.perl-bindings + git + ] ++ (with perlPackages; [ + AuthenSASL + CatalystActionREST + CatalystAuthenticationStoreDBIxClass + CatalystAuthenticationStoreLDAP + CatalystDevel + CatalystPluginAccessLog + CatalystPluginAuthorizationRoles + CatalystPluginCaptcha + CatalystPluginPrometheusTiny + CatalystPluginSessionStateCookie + CatalystPluginSessionStoreFastMmap + CatalystPluginStackTrace + CatalystTraitForRequestProxyBase + CatalystViewDownload + CatalystViewJSON + CatalystViewTT + CatalystXRoleApplicator + CatalystXScriptServerStarman + CryptPassphrase + CryptPassphraseArgon2 + CryptRandPasswd + DataDump + DateTime + DBDPg + DBDSQLite + DigestSHA1 + EmailMIME + EmailSender + FileLibMagic + FileSlurper + FileWhich + IOCompress + IPCRun + IPCRun3 + JSON + JSONMaybeXS + JSONXS + ListSomeUtils + LWP + LWPProtocolHttps + ModulePluggable + NetAmazonS3 + NetPrometheus + NetStatsd + PadWalker + ParallelForkManager + PerlCriticCommunity + PrometheusTinyShared + ReadonlyX + SetScalar + SQLSplitStatement + Starman + StringCompareConstantTime + SysHostnameLong + TermSizeAny + TermReadKey + Test2Harness + TestPostgreSQL + TextDiff + TextTable + UUID4Tiny + YAML + XMLSimple + ])); + }; + + version = "${builtins.readFile ./version.txt}.${builtins.substring 0 8 (rawSrc.lastModifiedDate or "19700101")}.${rawSrc.shortRev or "DIRTY"}"; +in +stdenv.mkDerivation (finalAttrs: { + pname = "hydra"; + inherit version; + + src = fileset.toSource { + root = ./.; + fileset = fileset.unions ([ + ./version.txt + ./configure.ac + ./Makefile.am + ./src + ./doc + ./nixos-modules/hydra.nix + # These are always needed to appease Automake + ./t/Makefile.am + ./t/jobs/config.nix.in + ./t/jobs/declarative/project.json.in + ] ++ lib.optionals finalAttrs.doCheck [ + ./t + ./.perlcriticrc + ./.yath.rc + ]); + }; + + strictDeps = true; + + nativeBuildInputs = [ + makeWrapper + autoreconfHook + nukeReferences + pkg-config + mdbook + nix + perlDeps + perl + unzip + ]; + + buildInputs = [ + libpqxx + openssl + libxslt + nix + perlDeps + perl + boost + nlohmann_json + prometheus-cpp + ]; + + nativeCheckInputs = [ + bzip2 + darcs + top-git + mercurial + subversion + breezy + openldap + postgresql_13 + pixz + ]; + + checkInputs = [ + cacert + foreman + glibcLocales + libressl.nc + python3 + ]; + + hydraPath = lib.makeBinPath ( + [ + subversion + openssh + nix + coreutils + findutils + pixz + gzip + bzip2 + xz + gnutar + unzip + git + top-git + mercurial + darcs + gnused + breezy + ] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] + ); + + OPENLDAP_ROOT = openldap; + + shellHook = '' + pushd $(git rev-parse --show-toplevel) >/dev/null + + PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$PATH + PERL5LIB=$(pwd)/src/lib:$PERL5LIB + export HYDRA_HOME="$(pwd)/src/" + mkdir -p .hydra-data + export HYDRA_DATA="$(pwd)/.hydra-data" + export HYDRA_DBI='dbi:Pg:dbname=hydra;host=localhost;port=64444' + + popd >/dev/null + ''; + + NIX_LDFLAGS = [ "-lpthread" ]; + + enableParallelBuilding = true; + + doCheck = true; + + preCheck = '' + patchShebangs . + export LOGNAME=''${LOGNAME:-foo} + # set $HOME for bzr so it can create its trace file + export HOME=$(mktemp -d) + ''; + + postInstall = '' + mkdir -p $out/nix-support + + for i in $out/bin/*; do + read -n 4 chars < $i + if [[ $chars =~ ELF ]]; then continue; fi + wrapProgram $i \ + --prefix PERL5LIB ':' $out/libexec/hydra/lib:$PERL5LIB \ + --prefix PATH ':' $out/bin:$hydraPath \ + --set HYDRA_RELEASE ${version} \ + --set HYDRA_HOME $out/libexec/hydra \ + --set NIX_RELEASE ${nix.name or "unknown"} + done + ''; + + dontStrip = true; + + meta.description = "Build of Hydra on ${stdenv.system}"; + passthru = { inherit perlDeps nix; }; +}) diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index 3d48f3d7..bd62de6e 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -224,7 +224,7 @@ void State::monitorMachinesFile() parseMachines("localhost " + (settings.thisSystem == "x86_64-linux" ? "x86_64-linux,i686-linux" : settings.thisSystem.get()) + " - " + std::to_string(settings.maxBuildJobs) + " 1 " - + concatStringsSep(",", settings.systemFeatures.get())); + + concatStringsSep(",", StoreConfig::getDefaultSystemFeatures())); machinesReadyLock.unlock(); return; } diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index 99fb113c..ddcaa9cd 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -462,10 +462,7 @@ Step::ptr State::createStep(ref destStore, step->systemType = step->drv->platform; { - auto i = step->drv->env.find("requiredSystemFeatures"); - StringSet features; - if (i != step->drv->env.end()) - features = step->requiredSystemFeatures = tokenizeString>(i->second); + StringSet features = step->requiredSystemFeatures = step->parsedDrv->getRequiredSystemFeatures(); if (step->preferLocalBuild) features.insert("local"); if (!features.empty()) {