Merge pull request #1346 from obsidiansystems/flake-reorg

Clean up the flake/build in a a few ways

.gitignore

@@ -38,6 +38,7 @@ t/jobs/declarative/project.json
 hydra-config.h
 hydra-config.h.in
 result
+result-*
 outputs
 config
 stamp-h1

Makefile.am

@@ -1,8 +1,12 @@
-SUBDIRS = src t doc
+SUBDIRS = src doc
+if CAN_DO_CHECK
+  SUBDIRS += t
+endif
+
 BOOTCLEAN_SUBDIRS = $(SUBDIRS)
 DIST_SUBDIRS      = $(SUBDIRS)
-EXTRA_DIST        = hydra-module.nix
+EXTRA_DIST        = nixos-modules/hydra.nix
 
-install-data-local: hydra-module.nix
+install-data-local: nixos-modules/hydra.nix
 	$(INSTALL) -d $(DESTDIR)$(datadir)/nix
-	$(INSTALL_DATA) hydra-module.nix $(DESTDIR)$(datadir)/nix/
+	$(INSTALL_DATA) nixos-modules/hydra.nix $(DESTDIR)$(datadir)/nix/hydra-module.nix

configure.ac

@@ -53,9 +53,6 @@ PKG_CHECK_MODULES([NIX], [nix-main nix-expr nix-store])
 testPath="$(dirname $(type -p expr))"
 AC_SUBST(testPath)
 
-jobsPath="$(realpath ./t/jobs)"
-AC_SUBST(jobsPath)
-
 CXXFLAGS+=" -include nix/config.h"
 
 AC_CONFIG_FILES([
@@ -71,10 +68,21 @@ AC_CONFIG_FILES([
   src/lib/Makefile
   src/root/Makefile
   src/script/Makefile
+])
+
+# Tests might be filtered out
+AM_CONDITIONAL([CAN_DO_CHECK], [test -f "$srcdir/t/api-test.t"])
+AM_COND_IF(
+  [CAN_DO_CHECK],
+  [
+    jobsPath="$(realpath ./t/jobs)"
+    AC_SUBST(jobsPath)
+    AC_CONFIG_FILES([
       t/Makefile
       t/jobs/config.nix
       t/jobs/declarative/project.json
-])
+    ])
+  ])
 
 AC_CONFIG_COMMANDS([executable-scripts], [])
 

flake.lock

@@ -42,11 +42,11 @@
         "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1702510710,
+        "lastModified": 1705666051,
-        "narHash": "sha256-9K+w1mQgmUxCmEsPaSFkpYsj/cxjO2PSwTCPkNZ/NiU=",
+        "narHash": "sha256-C3eht7uA7gZp9O5XpA9Mkqdi1WoTQaFH4FMGmM4DrFM=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "b38e5a665e9d0aa7975beb0ed12e42d13a392e74",
+        "rev": "03e96b9dc011a16a0f6db9c7cb021ff93f8dcf88",
         "type": "github"
       },
       "original": {
@@ -72,6 +72,22 @@
         "type": "github"
       }
     },
+    "nixpkgs-for-fileset": {
+      "locked": {
+        "lastModified": 1706098335,
+        "narHash": "sha256-r3dWjT8P9/Ah5m5ul4WqIWD8muj5F+/gbCdjiNVBKmU=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "a77ab169a83a4175169d78684ddd2e54486ac651",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-23.11",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
     "nixpkgs-regression": {
       "locked": {
         "lastModified": 1643052045,
@@ -91,7 +107,8 @@
     "root": {
       "inputs": {
         "nix": "nix",
-        "nixpkgs": "nixpkgs"
+        "nixpkgs": "nixpkgs",
+        "nixpkgs-for-fileset": "nixpkgs-for-fileset"
       }
     }
   },

flake.nix

@@ -5,16 +5,21 @@
   inputs.nix.url = "github:NixOS/nix/2.19-maintenance";
   inputs.nix.inputs.nixpkgs.follows = "nixpkgs";
 
-  outputs = { self, nixpkgs, nix }:
+  # TODO get rid of this once https://github.com/NixOS/nix/pull/9546 is
-    let
+  # mered and we upgrade or Nix, so the main `nixpkgs` input is at least
-      version = "${builtins.readFile ./version.txt}.${builtins.substring 0 8 (self.lastModifiedDate or "19700101")}.${self.shortRev or "DIRTY"}";
+  # 23.11 and has `lib.fileset`.
+  inputs.nixpkgs-for-fileset.url = "github:NixOS/nixpkgs/nixos-23.11";
 
+  outputs = { self, nixpkgs, nix, nixpkgs-for-fileset }:
+    let
       systems = [ "x86_64-linux" "aarch64-linux" ];
       forEachSystem = nixpkgs.lib.genAttrs systems;
 
+      overlayList = [ self.overlays.default nix.overlays.default ];
+
       pkgsBySystem = forEachSystem (system: import nixpkgs {
         inherit system;
-        overlays = [ self.overlays.default nix.overlays.default ];
+        overlays = overlayList;
       });
 
       # NixOS configuration used for VM tests.
@@ -61,198 +66,9 @@
 
         };
 
-        hydra = let
+        hydra = final.callPackage ./package.nix {
-          inherit (final) lib stdenv;
+          inherit (nixpkgs-for-fileset.lib) fileset;
-          perlDeps = final.buildEnv {
+          rawSrc = self;
-            name = "hydra-perl-deps";
-            paths = with final.perlPackages; lib.closePropagation
-              [
-                AuthenSASL
-                CatalystActionREST
-                CatalystAuthenticationStoreDBIxClass
-                CatalystAuthenticationStoreLDAP
-                CatalystDevel
-                CatalystPluginAccessLog
-                CatalystPluginAuthorizationRoles
-                CatalystPluginCaptcha
-                CatalystPluginPrometheusTiny
-                CatalystPluginSessionStateCookie
-                CatalystPluginSessionStoreFastMmap
-                CatalystPluginStackTrace
-                CatalystTraitForRequestProxyBase
-                CatalystViewDownload
-                CatalystViewJSON
-                CatalystViewTT
-                CatalystXRoleApplicator
-                CatalystXScriptServerStarman
-                CryptPassphrase
-                CryptPassphraseArgon2
-                CryptRandPasswd
-                DataDump
-                DateTime
-                DBDPg
-                DBDSQLite
-                DigestSHA1
-                EmailMIME
-                EmailSender
-                FileLibMagic
-                FileSlurper
-                FileWhich
-                final.nix.perl-bindings
-                final.git
-                IOCompress
-                IPCRun
-                IPCRun3
-                JSON
-                JSONMaybeXS
-                JSONXS
-                ListSomeUtils
-                LWP
-                LWPProtocolHttps
-                ModulePluggable
-                NetAmazonS3
-                NetPrometheus
-                NetStatsd
-                PadWalker
-                ParallelForkManager
-                PerlCriticCommunity
-                PrometheusTinyShared
-                ReadonlyX
-                SetScalar
-                SQLSplitStatement
-                Starman
-                StringCompareConstantTime
-                SysHostnameLong
-                TermSizeAny
-                TermReadKey
-                Test2Harness
-                TestPostgreSQL
-                TextDiff
-                TextTable
-                UUID4Tiny
-                YAML
-                XMLSimple
-              ];
-          };
-
-        in
-        stdenv.mkDerivation {
-
-          name = "hydra-${version}";
-
-          src = self;
-
-          nativeBuildInputs =
-            with final.buildPackages; [
-              makeWrapper
-              autoreconfHook
-              nukeReferences
-              pkg-config
-              mdbook
-            ];
-
-          buildInputs =
-            with final; [
-              unzip
-              libpqxx
-              top-git
-              mercurial
-              darcs
-              subversion
-              breezy
-              openssl
-              bzip2
-              libxslt
-              final.nix
-              perlDeps
-              perl
-              pixz
-              boost
-              postgresql_13
-              (if lib.versionAtLeast lib.version "20.03pre"
-              then nlohmann_json
-              else nlohmann_json.override { multipleHeaders = true; })
-              prometheus-cpp
-            ];
-
-          checkInputs = with final; [
-            cacert
-            foreman
-            glibcLocales
-            libressl.nc
-            openldap
-            python3
-          ];
-
-          hydraPath = with final; lib.makeBinPath (
-            [
-              subversion
-              openssh
-              final.nix
-              coreutils
-              findutils
-              pixz
-              gzip
-              bzip2
-              xz
-              gnutar
-              unzip
-              git
-              top-git
-              mercurial
-              darcs
-              gnused
-              breezy
-            ] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ]
-          );
-
-          OPENLDAP_ROOT = final.openldap;
-
-          shellHook = ''
-            pushd $(git rev-parse --show-toplevel) >/dev/null
-
-            PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$PATH
-            PERL5LIB=$(pwd)/src/lib:$PERL5LIB
-            export HYDRA_HOME="$(pwd)/src/"
-            mkdir -p .hydra-data
-            export HYDRA_DATA="$(pwd)/.hydra-data"
-            export HYDRA_DBI='dbi:Pg:dbname=hydra;host=localhost;port=64444'
-
-            popd >/dev/null
-          '';
-
-          NIX_LDFLAGS = [ "-lpthread" ];
-
-          enableParallelBuilding = true;
-
-          doCheck = true;
-
-          preCheck = ''
-            patchShebangs .
-            export LOGNAME=''${LOGNAME:-foo}
-            # set $HOME for bzr so it can create its trace file
-            export HOME=$(mktemp -d)
-          '';
-
-          postInstall = ''
-            mkdir -p $out/nix-support
-
-            for i in $out/bin/*; do
-                read -n 4 chars < $i
-                if [[ $chars =~ ELF ]]; then continue; fi
-                wrapProgram $i \
-                    --prefix PERL5LIB ':' $out/libexec/hydra/lib:$PERL5LIB \
-                    --prefix PATH ':' $out/bin:$hydraPath \
-                    --set HYDRA_RELEASE ${version} \
-                    --set HYDRA_HOME $out/libexec/hydra \
-                    --set NIX_RELEASE ${final.nix.name or "unknown"}
-            done
-          '';
-
-          dontStrip = true;
-
-          meta.description = "Build of Hydra on ${final.stdenv.system}";
-          passthru = { inherit perlDeps; inherit (final) nix; };
         };
       };
 
@@ -260,9 +76,15 @@
 
         build = forEachSystem (system: packages.${system}.hydra);
 
+        buildNoTests = forEachSystem (system:
+          packages.${system}.hydra.overrideAttrs (_: {
+            doCheck = false;
+          })
+        );
+
         manual = forEachSystem (system:
           let pkgs = pkgsBySystem.${system}; in
-          pkgs.runCommand "hydra-manual-${version}" { }
+          pkgs.runCommand "hydra-manual-${pkgs.hydra.version}" { }
             ''
               mkdir -p $out/share
               cp -prvd ${pkgs.hydra}/share/doc $out/share/
@@ -566,50 +388,8 @@
         default = pkgsBySystem.${system}.hydra;
       });
 
-      nixosModules.hydra = {
+      nixosModules = import ./nixos-modules {
-        imports = [ ./hydra-module.nix ];
+        overlays = overlayList;
-        nixpkgs.overlays = [ self.overlays.default nix.overlays.default ];
-      };
-
-      nixosModules.hydraTest = { pkgs, ... }: {
-        imports = [ self.nixosModules.hydra ];
-
-        services.hydra-dev.enable = true;
-        services.hydra-dev.hydraURL = "http://hydra.example.org";
-        services.hydra-dev.notificationSender = "admin@hydra.example.org";
-
-        systemd.services.hydra-send-stats.enable = false;
-
-        services.postgresql.enable = true;
-        services.postgresql.package = pkgs.postgresql_11;
-
-        # The following is to work around the following error from hydra-server:
-        #   [error] Caught exception in engine "Cannot determine local time zone"
-        time.timeZone = "UTC";
-
-        nix.extraOptions = ''
-          allowed-uris = https://github.com/
-        '';
-      };
-
-      nixosModules.hydraProxy = {
-        services.httpd = {
-          enable = true;
-          adminAddr = "hydra-admin@example.org";
-          extraConfig = ''
-            <Proxy *>
-              Order deny,allow
-              Allow from all
-            </Proxy>
-
-            ProxyRequests     Off
-            ProxyPreserveHost On
-            ProxyPass         /apache-errors !
-            ErrorDocument 503 /apache-errors/503.html
-            ProxyPass         /       http://127.0.0.1:3000/ retry=5 disablereuse=on
-            ProxyPassReverse  /       http://127.0.0.1:3000/
-          '';
-        };
       };
 
       nixosConfigurations.container = nixpkgs.lib.nixosSystem {

nixos-modules/default.nix

@@ -0,0 +1,49 @@
+{ overlays }:
+
+rec {
+  hydra = {
+    imports = [ ./hydra.nix ];
+    nixpkgs = { inherit overlays; };
+  };
+
+  hydraTest = { pkgs, ... }: {
+    imports = [ hydra ];
+
+    services.hydra-dev.enable = true;
+    services.hydra-dev.hydraURL = "http://hydra.example.org";
+    services.hydra-dev.notificationSender = "admin@hydra.example.org";
+
+    systemd.services.hydra-send-stats.enable = false;
+
+    services.postgresql.enable = true;
+    services.postgresql.package = pkgs.postgresql_11;
+
+    # The following is to work around the following error from hydra-server:
+    #   [error] Caught exception in engine "Cannot determine local time zone"
+    time.timeZone = "UTC";
+
+    nix.extraOptions = ''
+      allowed-uris = https://github.com/
+    '';
+  };
+
+  hydraProxy = {
+    services.httpd = {
+      enable = true;
+      adminAddr = "hydra-admin@example.org";
+      extraConfig = ''
+        <Proxy *>
+          Order deny,allow
+          Allow from all
+        </Proxy>
+
+        ProxyRequests     Off
+        ProxyPreserveHost On
+        ProxyPass         /apache-errors !
+        ErrorDocument 503 /apache-errors/503.html
+        ProxyPass         /       http://127.0.0.1:3000/ retry=5 disablereuse=on
+        ProxyPassReverse  /       http://127.0.0.1:3000/
+      '';
+    };
+  };
+}

package.nix

@@ -0,0 +1,272 @@
+{ stdenv
+, lib
+, fileset
+
+, rawSrc
+
+, buildEnv
+
+, perlPackages
+
+, nix
+, git
+
+, makeWrapper
+, autoreconfHook
+, nukeReferences
+, pkg-config
+, mdbook
+
+, unzip
+, libpqxx
+, top-git
+, mercurial
+, darcs
+, subversion
+, breezy
+, openssl
+, bzip2
+, libxslt
+, perl
+, pixz
+, boost
+, postgresql_13
+, nlohmann_json
+, prometheus-cpp
+
+, cacert
+, foreman
+, glibcLocales
+, libressl
+, openldap
+, python3
+
+, openssh
+, coreutils
+, findutils
+, gzip
+, xz
+, gnutar
+, gnused
+
+, rpm
+, dpkg
+, cdrkit
+}:
+
+let
+  perlDeps = buildEnv {
+    name = "hydra-perl-deps";
+    paths = lib.closePropagation
+      ([
+        nix.perl-bindings
+        git
+      ] ++ (with perlPackages; [
+        AuthenSASL
+        CatalystActionREST
+        CatalystAuthenticationStoreDBIxClass
+        CatalystAuthenticationStoreLDAP
+        CatalystDevel
+        CatalystPluginAccessLog
+        CatalystPluginAuthorizationRoles
+        CatalystPluginCaptcha
+        CatalystPluginPrometheusTiny
+        CatalystPluginSessionStateCookie
+        CatalystPluginSessionStoreFastMmap
+        CatalystPluginStackTrace
+        CatalystTraitForRequestProxyBase
+        CatalystViewDownload
+        CatalystViewJSON
+        CatalystViewTT
+        CatalystXRoleApplicator
+        CatalystXScriptServerStarman
+        CryptPassphrase
+        CryptPassphraseArgon2
+        CryptRandPasswd
+        DataDump
+        DateTime
+        DBDPg
+        DBDSQLite
+        DigestSHA1
+        EmailMIME
+        EmailSender
+        FileLibMagic
+        FileSlurper
+        FileWhich
+        IOCompress
+        IPCRun
+        IPCRun3
+        JSON
+        JSONMaybeXS
+        JSONXS
+        ListSomeUtils
+        LWP
+        LWPProtocolHttps
+        ModulePluggable
+        NetAmazonS3
+        NetPrometheus
+        NetStatsd
+        PadWalker
+        ParallelForkManager
+        PerlCriticCommunity
+        PrometheusTinyShared
+        ReadonlyX
+        SetScalar
+        SQLSplitStatement
+        Starman
+        StringCompareConstantTime
+        SysHostnameLong
+        TermSizeAny
+        TermReadKey
+        Test2Harness
+        TestPostgreSQL
+        TextDiff
+        TextTable
+        UUID4Tiny
+        YAML
+        XMLSimple
+      ]));
+  };
+
+  version = "${builtins.readFile ./version.txt}.${builtins.substring 0 8 (rawSrc.lastModifiedDate or "19700101")}.${rawSrc.shortRev or "DIRTY"}";
+in
+stdenv.mkDerivation (finalAttrs: {
+  pname = "hydra";
+  inherit version;
+
+  src = fileset.toSource {
+    root = ./.;
+    fileset = fileset.unions ([
+      ./version.txt
+      ./configure.ac
+      ./Makefile.am
+      ./src
+      ./doc
+      ./nixos-modules/hydra.nix
+      # These are always needed to appease Automake
+      ./t/Makefile.am
+      ./t/jobs/config.nix.in
+      ./t/jobs/declarative/project.json.in
+    ] ++ lib.optionals finalAttrs.doCheck [
+      ./t
+      ./.perlcriticrc
+      ./.yath.rc
+    ]);
+  };
+
+  strictDeps = true;
+
+  nativeBuildInputs = [
+    makeWrapper
+    autoreconfHook
+    nukeReferences
+    pkg-config
+    mdbook
+    nix
+    perlDeps
+    perl
+    unzip
+  ];
+
+  buildInputs = [
+    libpqxx
+    openssl
+    libxslt
+    nix
+    perlDeps
+    perl
+    boost
+    nlohmann_json
+    prometheus-cpp
+  ];
+
+  nativeCheckInputs = [
+    bzip2
+    darcs
+    top-git
+    mercurial
+    subversion
+    breezy
+    openldap
+    postgresql_13
+    pixz
+  ];
+
+  checkInputs = [
+    cacert
+    foreman
+    glibcLocales
+    libressl.nc
+    python3
+  ];
+
+  hydraPath = lib.makeBinPath (
+    [
+      subversion
+      openssh
+      nix
+      coreutils
+      findutils
+      pixz
+      gzip
+      bzip2
+      xz
+      gnutar
+      unzip
+      git
+      top-git
+      mercurial
+      darcs
+      gnused
+      breezy
+    ] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ]
+  );
+
+  OPENLDAP_ROOT = openldap;
+
+  shellHook = ''
+    pushd $(git rev-parse --show-toplevel) >/dev/null
+
+    PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$PATH
+    PERL5LIB=$(pwd)/src/lib:$PERL5LIB
+    export HYDRA_HOME="$(pwd)/src/"
+    mkdir -p .hydra-data
+    export HYDRA_DATA="$(pwd)/.hydra-data"
+    export HYDRA_DBI='dbi:Pg:dbname=hydra;host=localhost;port=64444'
+
+    popd >/dev/null
+  '';
+
+  NIX_LDFLAGS = [ "-lpthread" ];
+
+  enableParallelBuilding = true;
+
+  doCheck = true;
+
+  preCheck = ''
+    patchShebangs .
+    export LOGNAME=''${LOGNAME:-foo}
+    # set $HOME for bzr so it can create its trace file
+    export HOME=$(mktemp -d)
+  '';
+
+  postInstall = ''
+    mkdir -p $out/nix-support
+
+    for i in $out/bin/*; do
+        read -n 4 chars < $i
+        if [[ $chars =~ ELF ]]; then continue; fi
+        wrapProgram $i \
+            --prefix PERL5LIB ':' $out/libexec/hydra/lib:$PERL5LIB \
+            --prefix PATH ':' $out/bin:$hydraPath \
+            --set HYDRA_RELEASE ${version} \
+            --set HYDRA_HOME $out/libexec/hydra \
+            --set NIX_RELEASE ${nix.name or "unknown"}
+    done
+  '';
+
+  dontStrip = true;
+
+  meta.description = "Build of Hydra on ${stdenv.system}";
+  passthru = { inherit perlDeps nix; };
+})