delroth/hydra
0
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Merge branch 'master' into nix-next

Browse source 
• Updated input 'nix' (merge):
    'github:NixOS/nix/212ba69e6f995992f8b4e4c0656d19c0156c8714'
    'github:NixOS/nix/2c4bb93ba5a97e7078896ebc36385ce172960e4e' (2024-01-25)
  → 'github:NixOS/nix/8df68a213fc52a57b02a57005b0e06cc8de40ce3' (2024-01-25)
This commit is contained in:
John Ericson 2024-01-25 16:25:44 -05:00
parent 449eb2d873 8a02bb7c36
commit 7a53b866f6
10 changed files with 388 additions and 262 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -38,6 +38,7 @@ t/jobs/declarative/project.json
hydra-config.h
hydra-config.h.in
result
result-*
outputs
config
stamp-h1

12
Makefile.am
View file

@ -1,8 +1,12 @@
SUBDIRS = src t doc
SUBDIRS = src doc
if CAN_DO_CHECK
SUBDIRS += t
endif
BOOTCLEAN_SUBDIRS = $(SUBDIRS)
DIST_SUBDIRS = $(SUBDIRS)
EXTRA_DIST = hydra-module.nix
EXTRA_DIST = nixos-modules/hydra.nix
install-data-local: hydra-module.nix
install-data-local: nixos-modules/hydra.nix
$(INSTALL) -d $(DESTDIR)$(datadir)/nix
$(INSTALL_DATA) hydra-module.nix $(DESTDIR)$(datadir)/nix/
$(INSTALL_DATA) nixos-modules/hydra.nix $(DESTDIR)$(datadir)/nix/hydra-module.nix

20
configure.ac
View file

@ -53,9 +53,6 @@ PKG_CHECK_MODULES([NIX], [nix-main nix-expr nix-store])
testPath="$(dirname $(type -p expr))"
AC_SUBST(testPath)
jobsPath="$(realpath ./t/jobs)"
AC_SUBST(jobsPath)
CXXFLAGS+=" -include nix/config.h"
AC_CONFIG_FILES([
@ -71,11 +68,22 @@ AC_CONFIG_FILES([
src/lib/Makefile
src/root/Makefile
src/script/Makefile
t/Makefile
t/jobs/config.nix
t/jobs/declarative/project.json
])
# Tests might be filtered out
AM_CONDITIONAL([CAN_DO_CHECK], [test -f "$srcdir/t/api-test.t"])
AM_COND_IF(
[CAN_DO_CHECK],
[
jobsPath="$(realpath ./t/jobs)"
AC_SUBST(jobsPath)
AC_CONFIG_FILES([
t/Makefile
t/jobs/config.nix
t/jobs/declarative/project.json
])
])
AC_CONFIG_COMMANDS([executable-scripts], [])
AC_CONFIG_HEADER([hydra-config.h])

25
flake.lock
View file

@ -42,11 +42,11 @@
"nixpkgs-regression": "nixpkgs-regression"
},
"locked": {
"lastModified": 1706100941,
"narHash": "sha256-YLYxZrQLQ6i22RLZG/rsB4sIfFzhep7XQYlodzCmKy0=",
"lastModified": 1706195509,
"narHash": "sha256-1kwfk7H/MWZAcTKHnnWXo/+KlQeOTIRtOIzc4FX3QnE=",
"owner": "NixOS",
"repo": "nix",
"rev": "212ba69e6f995992f8b4e4c0656d19c0156c8714",
"rev": "8df68a213fc52a57b02a57005b0e06cc8de40ce3",
"type": "github"
},
"original": {
@ -71,6 +71,22 @@
"type": "github"
}
},
"nixpkgs-for-fileset": {
"locked": {
"lastModified": 1706098335,
"narHash": "sha256-r3dWjT8P9/Ah5m5ul4WqIWD8muj5F+/gbCdjiNVBKmU=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "a77ab169a83a4175169d78684ddd2e54486ac651",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-23.11",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs-regression": {
"locked": {
"lastModified": 1643052045,
@ -90,7 +106,8 @@
"root": {
"inputs": {
"nix": "nix",
"nixpkgs": "nixpkgs"
"nixpkgs": "nixpkgs",
"nixpkgs-for-fileset": "nixpkgs-for-fileset"
}
}
},

264
flake.nix
View file

@ -5,16 +5,21 @@
inputs.nix.url = "github:NixOS/nix";
inputs.nix.inputs.nixpkgs.follows = "nixpkgs";
outputs = { self, nixpkgs, nix }:
let
version = "${builtins.readFile ./version.txt}.${builtins.substring 0 8 (self.lastModifiedDate or "19700101")}.${self.shortRev or "DIRTY"}";
# TODO get rid of this once https://github.com/NixOS/nix/pull/9546 is
# mered and we upgrade or Nix, so the main `nixpkgs` input is at least
# 23.11 and has `lib.fileset`.
inputs.nixpkgs-for-fileset.url = "github:NixOS/nixpkgs/nixos-23.11";
outputs = { self, nixpkgs, nix, nixpkgs-for-fileset }:
let
systems = [ "x86_64-linux" "aarch64-linux" ];
forEachSystem = nixpkgs.lib.genAttrs systems;
overlayList = [ self.overlays.default nix.overlays.default ];
pkgsBySystem = forEachSystem (system: import nixpkgs {
inherit system;
overlays = [ self.overlays.default nix.overlays.default ];
overlays = overlayList;
});
# NixOS configuration used for VM tests.
@ -61,200 +66,9 @@
};
hydra = let
inherit (final) lib stdenv;
perlDeps = final.buildEnv {
name = "hydra-perl-deps";
paths = with final.perlPackages; lib.closePropagation
[
AuthenSASL
CatalystActionREST
CatalystAuthenticationStoreDBIxClass
CatalystAuthenticationStoreLDAP
CatalystDevel
CatalystPluginAccessLog
CatalystPluginAuthorizationRoles
CatalystPluginCaptcha
CatalystPluginPrometheusTiny
CatalystPluginSessionStateCookie
CatalystPluginSessionStoreFastMmap
CatalystPluginStackTrace
CatalystTraitForRequestProxyBase
CatalystViewDownload
CatalystViewJSON
CatalystViewTT
CatalystXRoleApplicator
CatalystXScriptServerStarman
CryptPassphrase
CryptPassphraseArgon2
CryptRandPasswd
DataDump
DateTime
DBDPg
DBDSQLite
DigestSHA1
EmailMIME
EmailSender
FileLibMagic
FileSlurper
FileWhich
final.nix.perl-bindings
final.git
IOCompress
IPCRun
IPCRun3
JSON
JSONMaybeXS
JSONXS
ListSomeUtils
LWP
LWPProtocolHttps
ModulePluggable
NetAmazonS3
NetPrometheus
NetStatsd
PadWalker
ParallelForkManager
PerlCriticCommunity
PrometheusTinyShared
ReadonlyX
SetScalar
SQLSplitStatement
Starman
StringCompareConstantTime
SysHostnameLong
TermSizeAny
TermReadKey
Test2Harness
TestPostgreSQL
TextDiff
TextTable
UUID4Tiny
YAML
XMLSimple
];
};
in
stdenv.mkDerivation {
name = "hydra-${version}";
src = self;
nativeBuildInputs =
with final.buildPackages; [
makeWrapper
autoreconfHook
automake
libtool
nukeReferences
pkg-config
mdbook
];
buildInputs =
with final; [
unzip
libpqxx
top-git
mercurial
darcs
subversion
breezy
openssl
bzip2
libxslt
final.nix
perlDeps
perl
pixz
boost
postgresql_13
(if lib.versionAtLeast lib.version "20.03pre"
then nlohmann_json
else nlohmann_json.override { multipleHeaders = true; })
prometheus-cpp
];
checkInputs = with final; [
cacert
foreman
glibcLocales
libressl.nc
openldap
python3
];
hydraPath = with final; lib.makeBinPath (
[
subversion
openssh
final.nix
coreutils
findutils
pixz
gzip
bzip2
xz
gnutar
unzip
git
top-git
mercurial
darcs
gnused
breezy
] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ]
);
OPENLDAP_ROOT = final.openldap;
shellHook = ''
pushd $(git rev-parse --show-toplevel) >/dev/null
PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$PATH
PERL5LIB=$(pwd)/src/lib:$PERL5LIB
export HYDRA_HOME="$(pwd)/src/"
mkdir -p .hydra-data
export HYDRA_DATA="$(pwd)/.hydra-data"
export HYDRA_DBI='dbi:Pg:dbname=hydra;host=localhost;port=64444'
popd >/dev/null
'';
NIX_LDFLAGS = [ "-lpthread" ];
enableParallelBuilding = true;
doCheck = true;
preCheck = ''
patchShebangs .
export LOGNAME=''${LOGNAME:-foo}
# set $HOME for bzr so it can create its trace file
export HOME=$(mktemp -d)
'';
postInstall = ''
mkdir -p $out/nix-support
for i in $out/bin/*; do
read -n 4 chars < $i
if [[ $chars =~ ELF ]]; then continue; fi
wrapProgram $i \
--prefix PERL5LIB ':' $out/libexec/hydra/lib:$PERL5LIB \
--prefix PATH ':' $out/bin:$hydraPath \
--set HYDRA_RELEASE ${version} \
--set HYDRA_HOME $out/libexec/hydra \
--set NIX_RELEASE ${final.nix.name or "unknown"}
done
'';
dontStrip = true;
meta.description = "Build of Hydra on ${final.stdenv.system}";
passthru = { inherit perlDeps; inherit (final) nix; };
hydra = final.callPackage ./package.nix {
inherit (nixpkgs-for-fileset.lib) fileset;
rawSrc = self;
};
};
@ -262,9 +76,15 @@
build = forEachSystem (system: packages.${system}.hydra);
buildNoTests = forEachSystem (system:
packages.${system}.hydra.overrideAttrs (_: {
doCheck = false;
})
);
manual = forEachSystem (system:
let pkgs = pkgsBySystem.${system}; in
pkgs.runCommand "hydra-manual-${version}" { }
pkgs.runCommand "hydra-manual-${pkgs.hydra.version}" { }
''
mkdir -p $out/share
cp -prvd ${pkgs.hydra}/share/doc $out/share/
@ -568,50 +388,8 @@
default = pkgsBySystem.${system}.hydra;
});
nixosModules.hydra = {
imports = [ ./hydra-module.nix ];
nixpkgs.overlays = [ self.overlays.default nix.overlays.default ];
};
nixosModules.hydraTest = { pkgs, ... }: {
imports = [ self.nixosModules.hydra ];
services.hydra-dev.enable = true;
services.hydra-dev.hydraURL = "http://hydra.example.org";
services.hydra-dev.notificationSender = "admin@hydra.example.org";
systemd.services.hydra-send-stats.enable = false;
services.postgresql.enable = true;
services.postgresql.package = pkgs.postgresql_11;
# The following is to work around the following error from hydra-server:
# [error] Caught exception in engine "Cannot determine local time zone"
time.timeZone = "UTC";
nix.extraOptions = ''
allowed-uris = https://github.com/
'';
};
nixosModules.hydraProxy = {
services.httpd = {
enable = true;
adminAddr = "hydra-admin@example.org";
extraConfig = ''
<Proxy *>
Order deny,allow
Allow from all
</Proxy>
ProxyRequests Off
ProxyPreserveHost On
ProxyPass /apache-errors !
ErrorDocument 503 /apache-errors/503.html
ProxyPass / http://127.0.0.1:3000/ retry=5 disablereuse=on
ProxyPassReverse / http://127.0.0.1:3000/
'';
};
nixosModules = import ./nixos-modules {
overlays = overlayList;
};
nixosConfigurations.container = nixpkgs.lib.nixosSystem {

49
nixos-modules/default.nix Normal file
View file

@ -0,0 +1,49 @@
{ overlays }:
rec {
hydra = {
imports = [ ./hydra.nix ];
nixpkgs = { inherit overlays; };
};
hydraTest = { pkgs, ... }: {
imports = [ hydra ];
services.hydra-dev.enable = true;
services.hydra-dev.hydraURL = "http://hydra.example.org";
services.hydra-dev.notificationSender = "admin@hydra.example.org";
systemd.services.hydra-send-stats.enable = false;
services.postgresql.enable = true;
services.postgresql.package = pkgs.postgresql_11;
# The following is to work around the following error from hydra-server:
# [error] Caught exception in engine "Cannot determine local time zone"
time.timeZone = "UTC";
nix.extraOptions = ''
allowed-uris = https://github.com/
'';
};
hydraProxy = {
services.httpd = {
enable = true;
adminAddr = "hydra-admin@example.org";
extraConfig = ''
<Proxy *>
Order deny,allow
Allow from all
</Proxy>
ProxyRequests Off
ProxyPreserveHost On
ProxyPass /apache-errors !
ErrorDocument 503 /apache-errors/503.html
ProxyPass / http://127.0.0.1:3000/ retry=5 disablereuse=on
ProxyPassReverse / http://127.0.0.1:3000/
'';
};
};
}

0
hydra-module.nix → nixos-modules/hydra.nix
View file

272
package.nix Normal file
View file

@ -0,0 +1,272 @@
{ stdenv
, lib
, fileset
, rawSrc
, buildEnv
, perlPackages
, nix
, git
, makeWrapper
, autoreconfHook
, nukeReferences
, pkg-config
, mdbook
, unzip
, libpqxx
, top-git
, mercurial
, darcs
, subversion
, breezy
, openssl
, bzip2
, libxslt
, perl
, pixz
, boost
, postgresql_13
, nlohmann_json
, prometheus-cpp
, cacert
, foreman
, glibcLocales
, libressl
, openldap
, python3
, openssh
, coreutils
, findutils
, gzip
, xz
, gnutar
, gnused
, rpm
, dpkg
, cdrkit
}:
let
perlDeps = buildEnv {
name = "hydra-perl-deps";
paths = lib.closePropagation
([
nix.perl-bindings
git
] ++ (with perlPackages; [
AuthenSASL
CatalystActionREST
CatalystAuthenticationStoreDBIxClass
CatalystAuthenticationStoreLDAP
CatalystDevel
CatalystPluginAccessLog
CatalystPluginAuthorizationRoles
CatalystPluginCaptcha
CatalystPluginPrometheusTiny
CatalystPluginSessionStateCookie
CatalystPluginSessionStoreFastMmap
CatalystPluginStackTrace
CatalystTraitForRequestProxyBase
CatalystViewDownload
CatalystViewJSON
CatalystViewTT
CatalystXRoleApplicator
CatalystXScriptServerStarman
CryptPassphrase
CryptPassphraseArgon2
CryptRandPasswd
DataDump
DateTime
DBDPg
DBDSQLite
DigestSHA1
EmailMIME
EmailSender
FileLibMagic
FileSlurper
FileWhich
IOCompress
IPCRun
IPCRun3
JSON
JSONMaybeXS
JSONXS
ListSomeUtils
LWP
LWPProtocolHttps
ModulePluggable
NetAmazonS3
NetPrometheus
NetStatsd
PadWalker
ParallelForkManager
PerlCriticCommunity
PrometheusTinyShared
ReadonlyX
SetScalar
SQLSplitStatement
Starman
StringCompareConstantTime
SysHostnameLong
TermSizeAny
TermReadKey
Test2Harness
TestPostgreSQL
TextDiff
TextTable
UUID4Tiny
YAML
XMLSimple
]));
};
version = "${builtins.readFile ./version.txt}.${builtins.substring 0 8 (rawSrc.lastModifiedDate or "19700101")}.${rawSrc.shortRev or "DIRTY"}";
in
stdenv.mkDerivation (finalAttrs: {
pname = "hydra";
inherit version;
src = fileset.toSource {
root = ./.;
fileset = fileset.unions ([
./version.txt
./configure.ac
./Makefile.am
./src
./doc
./nixos-modules/hydra.nix
# These are always needed to appease Automake
./t/Makefile.am
./t/jobs/config.nix.in
./t/jobs/declarative/project.json.in
] ++ lib.optionals finalAttrs.doCheck [
./t
./.perlcriticrc
./.yath.rc
]);
};
strictDeps = true;
nativeBuildInputs = [
makeWrapper
autoreconfHook
nukeReferences
pkg-config
mdbook
nix
perlDeps
perl
unzip
];
buildInputs = [
libpqxx
openssl
libxslt
nix
perlDeps
perl
boost
nlohmann_json
prometheus-cpp
];
nativeCheckInputs = [
bzip2
darcs
top-git
mercurial
subversion
breezy
openldap
postgresql_13
pixz
];
checkInputs = [
cacert
foreman
glibcLocales
libressl.nc
python3
];
hydraPath = lib.makeBinPath (
[
subversion
openssh
nix
coreutils
findutils
pixz
gzip
bzip2
xz
gnutar
unzip
git
top-git
mercurial
darcs
gnused
breezy
] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ]
);
OPENLDAP_ROOT = openldap;
shellHook = ''
pushd $(git rev-parse --show-toplevel) >/dev/null
PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$PATH
PERL5LIB=$(pwd)/src/lib:$PERL5LIB
export HYDRA_HOME="$(pwd)/src/"
mkdir -p .hydra-data
export HYDRA_DATA="$(pwd)/.hydra-data"
export HYDRA_DBI='dbi:Pg:dbname=hydra;host=localhost;port=64444'
popd >/dev/null
'';
NIX_LDFLAGS = [ "-lpthread" ];
enableParallelBuilding = true;
doCheck = true;
preCheck = ''
patchShebangs .
export LOGNAME=''${LOGNAME:-foo}
# set $HOME for bzr so it can create its trace file
export HOME=$(mktemp -d)
'';
postInstall = ''
mkdir -p $out/nix-support
for i in $out/bin/*; do
read -n 4 chars < $i
if [[ $chars =~ ELF ]]; then continue; fi
wrapProgram $i \
--prefix PERL5LIB ':' $out/libexec/hydra/lib:$PERL5LIB \
--prefix PATH ':' $out/bin:$hydraPath \
--set HYDRA_RELEASE ${version} \
--set HYDRA_HOME $out/libexec/hydra \
--set NIX_RELEASE ${nix.name or "unknown"}
done
'';
dontStrip = true;
meta.description = "Build of Hydra on ${stdenv.system}";
passthru = { inherit perlDeps nix; };
})

2
src/hydra-queue-runner/hydra-queue-runner.cc
View file

@ -224,7 +224,7 @@ void State::monitorMachinesFile()
parseMachines("localhost " +
(settings.thisSystem == "x86_64-linux" ? "x86_64-linux,i686-linux" : settings.thisSystem.get())
+ " - " + std::to_string(settings.maxBuildJobs) + " 1 "
+ concatStringsSep(",", settings.systemFeatures.get()));
+ concatStringsSep(",", StoreConfig::getDefaultSystemFeatures()));
machinesReadyLock.unlock();
return;
}

5
src/hydra-queue-runner/queue-monitor.cc
View file

@ -462,10 +462,7 @@ Step::ptr State::createStep(ref<Store> destStore,
step->systemType = step->drv->platform;
{
auto i = step->drv->env.find("requiredSystemFeatures");
StringSet features;
if (i != step->drv->env.end())
features = step->requiredSystemFeatures = tokenizeString<std::set<std::string>>(i->second);
StringSet features = step->requiredSystemFeatures = step->parsedDrv->getRequiredSystemFeatures();
if (step->preferLocalBuild)
features.insert("local");
if (!features.empty()) {