flake.lock: Update

Flake lock file updates:

• Updated input 'flake-parts':
    'github:hercules-ci/flake-parts/9227223f6d922fee3c7b190b2cc238a99527bbb7' (2024-07-03)
  → 'github:hercules-ci/flake-parts/8471fe90ad337a8074e957b69ca4d0089218391d' (2024-08-01)
• Updated input 'lix':
    'ef0de7c79f.tar.gz?narHash=sha256-GUH5%2BB1JztzDNSN1D7KbndrYSq0LWvVIJnuWKHlpN3Q%3D' (2024-07-16)
  → 'b016eb0895.tar.gz?narHash=sha256-kOpGI9WPmte1L4QWHviuXsr8jxmGn27zwi82jtzYObM%3D&rev=b016eb0895bb6714a4f6530d9a2bb6577ac6c3cf' (2024-08-13)
• Updated input 'lix/nix2container':
    'github:nlewo/nix2container/20aad300c925639d5d6cbe30013c8357ce9f2a2e' (2024-04-13)
  → 'github:nlewo/nix2container/3853e5caf9ad24103b13aa6e0e8bcebb47649fe4' (2024-07-10)
• Updated input 'lix/pre-commit-hooks':
    'github:cachix/git-hooks.nix/e35aed5fda3cc79f88ed7f1795021e559582093a' (2024-04-02)
  → 'github:cachix/git-hooks.nix/f451c19376071a90d8c58ab1a953c6e9840527fd' (2024-07-15)
• Updated input 'nixpkgs':
    'github:NixOS/nixpkgs/732b4f3a3afdfe6a6c4fcb2511e529588d4e5ccd' (2024-07-15)
  → 'github:NixOS/nixpkgs/fb81cec9eda2a6b5365ad723995f0329d9e356fd' (2024-08-13)
• Updated input 'treefmt-nix':
    'github:numtide/treefmt-nix/0fb28f237f83295b4dd05e342f333b447c097398' (2024-07-15)
  → 'github:numtide/treefmt-nix/349de7bc435bdff37785c2466f054ed1766173be' (2024-08-12)

.nix-version

@@ -1 +0,0 @@
-unstable

README.md

@@ -1,5 +1,8 @@
 # nix-eval-jobs
 
+> [!NOTE]
+> This is a fork of nix-eval-jobs that works with Lix.
+
 This project evaluates nix attribute sets in parallel with streamable json
 output. This is useful for time and memory intensive evaluations such as NixOS
 machines, i.e. in a CI context. The evaluation is done with a controllable
@@ -89,11 +92,12 @@ we collect example ci configuration for various CIs.
 
 ## Organisation of this repository
 
-On the `main` branch we target nixUnstable. When a release of nix happens, we
+`main` follows Lix HEAD, and is updated alongside the Lix NixOS module. When we
-fork for a release branch i.e. `release-2.8` and change the nix version in
+release we will make a `release-2.90` etc branch, which receives backports.
-`.nix-version`. Changes and improvements made in `main` also may be backported
+
-to these release branches. At the time of writing we only intent to support the
+The version of nix-eval-jobs follows the major version of Lix and minor
-latest release branch.
+versions of nix-eval-jobs are released as necessary when changes are made in
+n-e-j itself.
 
 ## Projects using nix-eval-jobs
 

default.nix

@@ -11,7 +11,7 @@ let
 in
 stdenv.mkDerivation {
   pname = "nix-eval-jobs";
-  version = "2.19.0";
+  version = "2.90.0-unstable";
   src = if srcDir == null then filterMesonBuild ./. else srcDir;
   buildInputs = with pkgs; [
     nlohmann_json

flake.lock

@@ -23,11 +23,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1701473968,
+        "lastModified": 1722555600,
-        "narHash": "sha256-YcVE5emp1qQ8ieHUnxt1wCZCC3ZfAS+SRRWZ2TMda7E=",
+        "narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=",
         "owner": "hercules-ci",
         "repo": "flake-parts",
-        "rev": "34fed993f1674c8d06d58b37ce1e0fe5eebcb9f5",
+        "rev": "8471fe90ad337a8074e957b69ca4d0089218391d",
         "type": "github"
       },
       "original": {
@@ -39,6 +39,7 @@
     "lix": {
       "inputs": {
         "flake-compat": "flake-compat",
+        "nix2container": "nix2container",
         "nixpkgs": [
           "nixpkgs"
         ],
@@ -46,18 +47,15 @@
         "pre-commit-hooks": "pre-commit-hooks"
       },
       "locked": {
-        "lastModified": 1714955862,
+        "lastModified": 1723577950,
-        "narHash": "sha256-REWlo2RYHfJkxnmZTEJu3Cd/2VM+wjjpPy7Xi4BdDTQ=",
+        "narHash": "sha256-kOpGI9WPmte1L4QWHviuXsr8jxmGn27zwi82jtzYObM=",
-        "ref": "refs/tags/2.90-beta.1",
+        "rev": "b016eb0895bb6714a4f6530d9a2bb6577ac6c3cf",
-        "rev": "b6799ab0374a8e1907a48915d3187e07da41d88c",
+        "type": "tarball",
-        "revCount": 15501,
+        "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/b016eb0895bb6714a4f6530d9a2bb6577ac6c3cf.tar.gz?rev=b016eb0895bb6714a4f6530d9a2bb6577ac6c3cf"
-        "type": "git",
-        "url": "https://git@git.lix.systems/lix-project/lix"
       },
       "original": {
-        "ref": "refs/tags/2.90-beta.1",
+        "type": "tarball",
-        "type": "git",
+        "url": "https://git.lix.systems/lix-project/lix/archive/main.tar.gz"
-        "url": "https://git@git.lix.systems/lix-project/lix"
       }
     },
     "nix-github-actions": {
@@ -67,11 +65,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1701208414,
+        "lastModified": 1720066371,
-        "narHash": "sha256-xrQ0FyhwTZK6BwKhahIkUVZhMNk21IEI1nUcWSONtpo=",
+        "narHash": "sha256-uPlLYH2S0ACj0IcgaK9Lsf4spmJoGejR9DotXiXSBZQ=",
         "owner": "nix-community",
         "repo": "nix-github-actions",
-        "rev": "93e39cc1a087d65bcf7a132e75a650c44dd2b734",
+        "rev": "622f829f5fe69310a866c8a6cd07e747c44ef820",
         "type": "github"
       },
       "original": {
@@ -80,18 +78,34 @@
         "type": "github"
       }
     },
+    "nix2container": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1720642556,
+        "narHash": "sha256-qsnqk13UmREKmRT7c8hEnz26X3GFFyIQrqx4EaRc1Is=",
+        "owner": "nlewo",
+        "repo": "nix2container",
+        "rev": "3853e5caf9ad24103b13aa6e0e8bcebb47649fe4",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nlewo",
+        "repo": "nix2container",
+        "type": "github"
+      }
+    },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1703134684,
+        "lastModified": 1723540975,
-        "narHash": "sha256-SQmng1EnBFLzS7WSRyPM9HgmZP2kLJcPAz+Ug/nug6o=",
+        "narHash": "sha256-rxpxOz2VSqgmwI7g7FGVAoye5bxwO1MSpnELY5bsITw=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "d6863cbcbbb80e71cecfc03356db1cda38919523",
+        "rev": "fb81cec9eda2a6b5365ad723995f0329d9e356fd",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixpkgs-unstable",
+        "ref": "nixos-24.05-small",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -115,11 +129,11 @@
     "pre-commit-hooks": {
       "flake": false,
       "locked": {
-        "lastModified": 1712055707,
+        "lastModified": 1721042469,
-        "narHash": "sha256-4XLvuSIDZJGS17xEwSrNuJLL7UjDYKGJSbK1WWX2AK8=",
+        "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=",
         "owner": "cachix",
         "repo": "git-hooks.nix",
-        "rev": "e35aed5fda3cc79f88ed7f1795021e559582093a",
+        "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd",
         "type": "github"
       },
       "original": {
@@ -144,11 +158,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1702979157,
+        "lastModified": 1723454642,
-        "narHash": "sha256-RnFBbLbpqtn4AoJGXKevQMCGhra4h6G2MPcuTSZZQ+g=",
+        "narHash": "sha256-S0Gvsenh0II7EAaoc9158ZB4vYyuycvMGKGxIbERNAM=",
         "owner": "numtide",
         "repo": "treefmt-nix",
-        "rev": "2961375283668d867e64129c22af532de8e77734",
+        "rev": "349de7bc435bdff37785c2466f054ed1766173be",
         "type": "github"
       },
       "original": {

flake.nix

@@ -1,7 +1,7 @@
 {
   description = "Hydra's builtin hydra-eval-jobs as a standalone";
 
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05-small";
   inputs.flake-parts.url = "github:hercules-ci/flake-parts";
   inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs";
   inputs.treefmt-nix.url = "github:numtide/treefmt-nix";
@@ -9,7 +9,7 @@
   inputs.nix-github-actions.url = "github:nix-community/nix-github-actions";
   inputs.nix-github-actions.inputs.nixpkgs.follows = "nixpkgs";
   inputs.lix = {
-    url = "git+https://git@git.lix.systems/lix-project/lix?ref=refs/tags/2.90-beta.1";
+    url = "https://git.lix.systems/lix-project/lix/archive/main.tar.gz";
     inputs.nixpkgs.follows = "nixpkgs";
   };
 
@@ -17,7 +17,6 @@
     let
       inherit (inputs.nixpkgs) lib;
       inherit (inputs) self;
-      nixVersion = lib.fileContents ./.nix-version;
     in
     flake-parts.lib.mkFlake { inherit inputs; }
       {

meson.build

@@ -1,6 +1,11 @@
 project('nix-eval-jobs', 'cpp',
   version : '0.1.6',
   license : 'GPL-3.0',
+  default_options : [
+    'debug=true',
+    'optimization=2',
+    'cpp_std=c++20',
+  ],
 )
 
 nix_main_dep = dependency('lix-main', required: true)

shell.nix

@@ -23,6 +23,7 @@ in
 pkgs.mkShell {
   inherit (nix-eval-jobs) buildInputs;
   nativeBuildInputs = nix-eval-jobs.nativeBuildInputs ++ [
+    pkgs.clang-tools
     (pkgs.python3.withPackages (ps: [
       ps.pytest
     ]))

src/drv.cc

@@ -23,14 +23,17 @@
 #include "drv.hh"
 #include "eval-args.hh"
 
-static bool queryIsCached(nix::Store &store,
+static bool
-                          std::map<std::string, std::string> &outputs) {
+queryIsCached(nix::Store &store,
+              std::map<std::string, std::optional<std::string>> &outputs) {
     uint64_t downloadSize, narSize;
     nix::StorePathSet willBuild, willSubstitute, unknown;
 
     std::vector<nix::StorePathWithOutputs> paths;
     for (auto const &[key, val] : outputs) {
-        paths.push_back(followLinksToStorePathWithOutputs(store, val));
+        if (val) {
+            paths.push_back(followLinksToStorePathWithOutputs(store, *val));
+        }
     }
 
     store.queryMissing(toDerivedPaths(paths), willBuild, willSubstitute,
@@ -45,9 +48,19 @@ Drv::Drv(std::string &attrPath, nix::EvalState &state, nix::DrvInfo &drvInfo,
     auto localStore = state.store.dynamic_pointer_cast<nix::LocalFSStore>();
 
     try {
-        for (auto out : drvInfo.queryOutputs(true)) {
+        // CA derivations do not have static output paths, so we have to
-            if (out.second)
+        // defensively not query output paths in case we encounter one.
-                outputs[out.first] = localStore->printStorePath(*out.second);
+        for (auto &[outputName, optOutputPath] :
+             drvInfo.queryOutputs(!nix::experimentalFeatureSettings.isEnabled(
+                 nix::Xp::CaDerivations))) {
+            if (optOutputPath) {
+                outputs[outputName] =
+                    localStore->printStorePath(*optOutputPath);
+            } else {
+                assert(nix::experimentalFeatureSettings.isEnabled(
+                    nix::Xp::CaDerivations));
+                outputs[outputName] = std::nullopt;
+            }
         }
     } catch (const std::exception &e) {
         throw nix::EvalError(state,
@@ -98,10 +111,16 @@ Drv::Drv(std::string &attrPath, nix::EvalState &state, nix::DrvInfo &drvInfo,
 }
 
 void to_json(nlohmann::json &json, const Drv &drv) {
+    std::map<std::string, nlohmann::json> outputsJson;
+    for (auto &[name, optPath] : drv.outputs) {
+        outputsJson[name] =
+            optPath ? nlohmann::json(*optPath) : nlohmann::json(nullptr);
+    }
+
     json = nlohmann::json{{"name", drv.name},
                           {"system", drv.system},
                           {"drvPath", drv.drvPath},
-                          {"outputs", drv.outputs},
+                          {"outputs", outputsJson},
                           {"inputDrvs", drv.inputDrvs}};
 
     if (drv.meta.has_value()) {

src/drv.hh

@@ -24,7 +24,7 @@ struct Drv {
     std::string drvPath;
 
     enum class CacheStatus { Cached, Uncached, Unknown } cacheStatus;
-    std::map<std::string, std::string> outputs;
+    std::map<std::string, std::optional<std::string>> outputs;
     std::map<std::string, std::set<std::string>> inputDrvs;
     std::optional<nlohmann::json> meta;
 

src/eval-args.hh

@@ -12,7 +12,7 @@
 
 class MyArgs : virtual public nix::MixEvalArgs,
                virtual public nix::MixCommonArgs,
-               virtual nix::RootArgs {
+               virtual public nix::RootArgs {
   public:
     std::string releaseExpr;
     nix::Path gcRootsDir;

src/meson.build

@@ -1,10 +1,10 @@
-src = [
+src = files(
   'nix-eval-jobs.cc',
   'eval-args.cc',
   'drv.cc',
   'buffered-io.cc',
-  'worker.cc'
+  'worker.cc',
-]
+)
 
 cc = meson.get_compiler('cpp')
 
@@ -31,4 +31,4 @@ executable('nix-eval-jobs', src,
              threads_dep
            ],
            install: true,
-           cpp_args: ['-std=c++2a', '-fvisibility=hidden', '--include', 'autotools-config.h'])
+           cpp_args: ['--include', 'autotools-config.h'])

src/nix-eval-jobs.cc

@@ -8,6 +8,7 @@
 #include <sys/wait.h>
 #include <nlohmann/json.hpp>
 #include <errno.h>
+#include <pthread.h>
 #include <signal.h>
 #include <stdlib.h>
 #include <string.h>
@@ -17,13 +18,13 @@
 #include <lix/libutil/error.hh>
 #include <lix/libstore/globals.hh>
 #include <lix/libutil/logging.hh>
+#include <lix/libutil/terminal.hh>
 #include <nlohmann/detail/iterators/iter_impl.hpp>
 #include <nlohmann/detail/json_ref.hpp>
 #include <nlohmann/json_fwd.hpp>
 #include <lix/libutil/ref.hh>
 #include <lix/libstore/store-api.hh>
 #include <map>
-#include <thread>
 #include <condition_variable>
 #include <filesystem>
 #include <exception>
@@ -92,12 +93,60 @@ struct Proc {
 
         to = std::move(toPipe.writeSide);
         from = std::move(fromPipe.readSide);
-        pid = p;
+        pid = std::move(p);
     }
 
     ~Proc() {}
 };
 
+// We'd highly prefer using std::thread here; but this won't let us configure the stack
+// size. macOS uses 512KiB size stacks for non-main threads, and musl defaults to 128k.
+// While Nix configures a 64MiB size for the main thread, this doesn't propagate to the
+// threads we launch here. It turns out, running the evaluator under an anemic stack of
+// 0.5MiB has it overflow way too quickly. Hence, we have our own custom Thread struct.
+struct Thread {
+    pthread_t thread;
+
+    Thread(const Thread &) = delete;
+    Thread(Thread &&) noexcept = default;
+
+    Thread(std::function<void(void)> f) {
+        int s;
+        pthread_attr_t attr;
+
+        auto func = std::make_unique<std::function<void(void)>>(std::move(f));
+
+        if ((s = pthread_attr_init(&attr)) != 0) {
+            throw SysError(s, "calling pthread_attr_init");
+        }
+        if ((s = pthread_attr_setstacksize(&attr, 64 * 1024 * 1024)) != 0) {
+            throw SysError(s, "calling pthread_attr_setstacksize");
+        }
+        if ((s = pthread_create(&thread, &attr, Thread::init, func.release())) != 0) {
+            throw SysError(s, "calling pthread_launch");
+        }
+        if ((s = pthread_attr_destroy(&attr)) != 0) {
+            throw SysError(s, "calling pthread_attr_destroy");
+        }
+    }
+
+    void join() {
+        int s;
+        s = pthread_join(thread, nullptr);
+        if (s != 0) {
+            throw SysError(s, "calling pthread_join");
+        }
+    }
+private:
+    static void *init(void *ptr) {
+        std::unique_ptr<std::function<void(void)>> func;
+        func.reset(static_cast<std::function<void(void)> *>(ptr));
+
+        (*func)();
+        return 0;
+    }
+};
+
 struct State {
     std::set<json> todo = json::array({json::array()});
     std::set<json> active;
@@ -331,10 +380,10 @@ int main(int argc, char **argv) {
         Sync<State> state_;
 
         /* Start a collector thread per worker process. */
-        std::vector<std::thread> threads;
+        std::vector<Thread> threads;
         std::condition_variable wakeup;
         for (size_t i = 0; i < myArgs.nrWorkers; i++) {
-            threads.emplace_back(collector, std::ref(state_), std::ref(wakeup));
+            threads.emplace_back(std::bind(collector, std::ref(state_), std::ref(wakeup)));
         }
 
         for (auto &thread : threads)

src/worker.cc

@@ -27,18 +27,15 @@
 #include <lix/libstore/store-api.hh>
 #include <lix/libexpr/symbol-table.hh>
 #include <lix/libutil/types.hh>
-#include <lix/libutil/util.hh>
 #include <lix/libexpr/value.hh>
+#include <lix/libutil/terminal.hh>
 #include <exception>
-#include <map>
-#include <memory>
 #include <numeric>
 #include <optional>
 #include <sstream>
 #include <string>
 #include <string_view>
 #include <utility>
-#include <vector>
 
 #include "worker.hh"
 #include "drv.hh"
@@ -51,7 +48,7 @@ static nix::Value *releaseExprTopLevelValue(nix::EvalState &state,
     nix::Value vTop;
 
     if (args.fromArgs) {
-        nix::Expr *e = state.parseExprFromString(
+        nix::Expr &e = state.parseExprFromString(
             args.releaseExpr, state.rootPath(nix::CanonPath::fromCwd()));
         state.eval(e, vTop);
     } else {