lix-project/nix-eval-jobs
2
0
Fork 0
Code Issues 1 Pull requests Packages Projects Releases Wiki Activity

Compare commits

base: lix-project:main
Branches Tags
lix-project:main
lix-project:release-2.90
lix-project:own-thread
lix-project:jade/fixup-readme-and-version
lix-project:jade/lixexpr
..
compare: lix-project:jade/lixexpr
Branches Tags
lix-project:main
lix-project:release-2.90
lix-project:own-thread
lix-project:jade/fixup-readme-and-version
lix-project:jade/lixexpr

No commits in common. "main" and "jade/lixexpr" have entirely different histories.
main ... jade/lixex

11 changed files with 47 additions and 115 deletions
Show stats Expand all files Collapse all files

1
.nix-version Normal file
View file

@ -0,0 +1 @@
unstable

14
README.md
View file

@ -1,8 +1,5 @@
# nix-eval-jobs
> [!NOTE]
> This is a fork of nix-eval-jobs that works with Lix.
This project evaluates nix attribute sets in parallel with streamable json
output. This is useful for time and memory intensive evaluations such as NixOS
machines, i.e. in a CI context. The evaluation is done with a controllable
@ -92,12 +89,11 @@ we collect example ci configuration for various CIs.
## Organisation of this repository
`main` follows Lix HEAD, and is updated alongside the Lix NixOS module. When we
release we will make a `release-2.90` etc branch, which receives backports.
The version of nix-eval-jobs follows the major version of Lix and minor
versions of nix-eval-jobs are released as necessary when changes are made in
n-e-j itself.
On the `main` branch we target nixUnstable. When a release of nix happens, we
fork for a release branch i.e. `release-2.8` and change the nix version in
`.nix-version`. Changes and improvements made in `main` also may be backported
to these release branches. At the time of writing we only intent to support the
latest release branch.
## Projects using nix-eval-jobs

2
default.nix
View file

@ -11,7 +11,7 @@ let
in
stdenv.mkDerivation {
pname = "nix-eval-jobs";
version = "2.90.0-unstable";
version = "2.19.0";
src = if srcDir == null then filterMesonBuild ./. else srcDir;
buildInputs = with pkgs; [
nlohmann_json

60
flake.lock
View file

@ -23,11 +23,11 @@
]
},
"locked": {
"lastModified": 1717285511,
"narHash": "sha256-iKzJcpdXih14qYVcZ9QC9XuZYnPc6T8YImb6dX166kw=",
"lastModified": 1701473968,
"narHash": "sha256-YcVE5emp1qQ8ieHUnxt1wCZCC3ZfAS+SRRWZ2TMda7E=",
"owner": "hercules-ci",
"repo": "flake-parts",
"rev": "2a55567fcf15b1b1c7ed712a2c6fadaec7412ea8",
"rev": "34fed993f1674c8d06d58b37ce1e0fe5eebcb9f5",
"type": "github"
},
"original": {
@ -39,7 +39,6 @@
"lix": {
"inputs": {
"flake-compat": "flake-compat",
"nix2container": "nix2container",
"nixpkgs": [
"nixpkgs"
],
@ -47,15 +46,18 @@
"pre-commit-hooks": "pre-commit-hooks"
},
"locked": {
"lastModified": 1719348166,
"narHash": "sha256-GK6PusfbMgkg+qdgChmrw78KTNQkm7SDoJ6+lJKY6vg=",
"rev": "f170870ae7b18c8ee13fb42fe19b5aa05ddf56c0",
"type": "tarball",
"url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/f170870ae7b18c8ee13fb42fe19b5aa05ddf56c0.tar.gz"
"lastModified": 1714955862,
"narHash": "sha256-REWlo2RYHfJkxnmZTEJu3Cd/2VM+wjjpPy7Xi4BdDTQ=",
"ref": "refs/tags/2.90-beta.1",
"rev": "b6799ab0374a8e1907a48915d3187e07da41d88c",
"revCount": 15501,
"type": "git",
"url": "https://git@git.lix.systems/lix-project/lix"
},
"original": {
"type": "tarball",
"url": "https://git.lix.systems/lix-project/lix/archive/main.tar.gz"
"ref": "refs/tags/2.90-beta.1",
"type": "git",
"url": "https://git@git.lix.systems/lix-project/lix"
}
},
"nix-github-actions": {
@ -65,11 +67,11 @@
]
},
"locked": {
"lastModified": 1703863825,
"narHash": "sha256-rXwqjtwiGKJheXB43ybM8NwWB8rO2dSRrEqes0S7F5Y=",
"lastModified": 1701208414,
"narHash": "sha256-xrQ0FyhwTZK6BwKhahIkUVZhMNk21IEI1nUcWSONtpo=",
"owner": "nix-community",
"repo": "nix-github-actions",
"rev": "5163432afc817cf8bd1f031418d1869e4c9d5547",
"rev": "93e39cc1a087d65bcf7a132e75a650c44dd2b734",
"type": "github"
},
"original": {
@ -78,34 +80,18 @@
"type": "github"
}
},
"nix2container": {
"flake": false,
"locked": {
"lastModified": 1712990762,
"narHash": "sha256-hO9W3w7NcnYeX8u8cleHiSpK2YJo7ecarFTUlbybl7k=",
"owner": "nlewo",
"repo": "nix2container",
"rev": "20aad300c925639d5d6cbe30013c8357ce9f2a2e",
"type": "github"
},
"original": {
"owner": "nlewo",
"repo": "nix2container",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1718676691,
"narHash": "sha256-DWKbARWtRpT1yiLLm+5vMijx65YB7NwGZMV6NXOtrJ8=",
"lastModified": 1703134684,
"narHash": "sha256-SQmng1EnBFLzS7WSRyPM9HgmZP2kLJcPAz+Ug/nug6o=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "d9e18354acbf59c625505b7315c85508e9831bf4",
"rev": "d6863cbcbbb80e71cecfc03356db1cda38919523",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-24.05-small",
"ref": "nixpkgs-unstable",
"repo": "nixpkgs",
"type": "github"
}
@ -158,11 +144,11 @@
]
},
"locked": {
"lastModified": 1718139168,
"narHash": "sha256-1TZQcdETNdJMcfwwoshVeCjwWfrPtkSQ8y8wFX3it7k=",
"lastModified": 1702979157,
"narHash": "sha256-RnFBbLbpqtn4AoJGXKevQMCGhra4h6G2MPcuTSZZQ+g=",
"owner": "numtide",
"repo": "treefmt-nix",
"rev": "1cb529bffa880746a1d0ec4e0f5076876af931f1",
"rev": "2961375283668d867e64129c22af532de8e77734",
"type": "github"
},
"original": {

5
flake.nix
View file

@ -1,7 +1,7 @@
{
description = "Hydra's builtin hydra-eval-jobs as a standalone";
inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05-small";
inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
inputs.flake-parts.url = "github:hercules-ci/flake-parts";
inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs";
inputs.treefmt-nix.url = "github:numtide/treefmt-nix";
@ -9,7 +9,7 @@
inputs.nix-github-actions.url = "github:nix-community/nix-github-actions";
inputs.nix-github-actions.inputs.nixpkgs.follows = "nixpkgs";
inputs.lix = {
url = "https://git.lix.systems/lix-project/lix/archive/main.tar.gz";
url = "git+https://git@git.lix.systems/lix-project/lix?ref=refs/tags/2.90-beta.1";
inputs.nixpkgs.follows = "nixpkgs";
};
@ -17,6 +17,7 @@
let
inherit (inputs.nixpkgs) lib;
inherit (inputs) self;
nixVersion = lib.fileContents ./.nix-version;
in
flake-parts.lib.mkFlake { inherit inputs; }
{

5
meson.build
View file

@ -1,11 +1,6 @@
project('nix-eval-jobs', 'cpp',
version : '0.1.6',
license : 'GPL-3.0',
default_options : [
'debug=true',
'optimization=2',
'cpp_std=c++20',
],
)
nix_main_dep = dependency('lix-main', required: true)

1
shell.nix
View file

@ -23,7 +23,6 @@ in
pkgs.mkShell {
inherit (nix-eval-jobs) buildInputs;
nativeBuildInputs = nix-eval-jobs.nativeBuildInputs ++ [
pkgs.clang-tools
(pkgs.python3.withPackages (ps: [
ps.pytest
]))

2
src/eval-args.hh
View file

@ -12,7 +12,7 @@
class MyArgs : virtual public nix::MixEvalArgs,
virtual public nix::MixCommonArgs,
virtual public nix::RootArgs {
virtual nix::RootArgs {
public:
std::string releaseExpr;
nix::Path gcRootsDir;

8
src/meson.build
View file

@ -1,10 +1,10 @@
src = files(
src = [
'nix-eval-jobs.cc',
'eval-args.cc',
'drv.cc',
'buffered-io.cc',
'worker.cc',
)
'worker.cc'
]
cc = meson.get_compiler('cpp')
@ -31,4 +31,4 @@ executable('nix-eval-jobs', src,
threads_dep
],
install: true,
cpp_args: ['--include', 'autotools-config.h'])
cpp_args: ['-std=c++2a', '-fvisibility=hidden', '--include', 'autotools-config.h'])

57
src/nix-eval-jobs.cc
View file

@ -8,7 +8,6 @@
#include <sys/wait.h>
#include <nlohmann/json.hpp>
#include <errno.h>
#include <pthread.h>
#include <signal.h>
#include <stdlib.h>
#include <string.h>
@ -18,13 +17,13 @@
#include <lix/libutil/error.hh>
#include <lix/libstore/globals.hh>
#include <lix/libutil/logging.hh>
#include <lix/libutil/terminal.hh>
#include <nlohmann/detail/iterators/iter_impl.hpp>
#include <nlohmann/detail/json_ref.hpp>
#include <nlohmann/json_fwd.hpp>
#include <lix/libutil/ref.hh>
#include <lix/libstore/store-api.hh>
#include <map>
#include <thread>
#include <condition_variable>
#include <filesystem>
#include <exception>
@ -93,60 +92,12 @@ struct Proc {
to = std::move(toPipe.writeSide);
from = std::move(fromPipe.readSide);
pid = std::move(p);
pid = p;
}
~Proc() {}
};
// We'd highly prefer using std::thread here; but this won't let us configure the stack
// size. macOS uses 512KiB size stacks for non-main threads, and musl defaults to 128k.
// While Nix configures a 64MiB size for the main thread, this doesn't propagate to the
// threads we launch here. It turns out, running the evaluator under an anemic stack of
// 0.5MiB has it overflow way too quickly. Hence, we have our own custom Thread struct.
struct Thread {
pthread_t thread;
Thread(const Thread &) = delete;
Thread(Thread &&) noexcept = default;
Thread(std::function<void(void)> f) {
int s;
pthread_attr_t attr;
auto func = std::make_unique<std::function<void(void)>>(std::move(f));
if ((s = pthread_attr_init(&attr)) != 0) {
throw SysError(s, "calling pthread_attr_init");
}
if ((s = pthread_attr_setstacksize(&attr, 64 * 1024 * 1024)) != 0) {
throw SysError(s, "calling pthread_attr_setstacksize");
}
if ((s = pthread_create(&thread, &attr, Thread::init, func.release())) != 0) {
throw SysError(s, "calling pthread_launch");
}
if ((s = pthread_attr_destroy(&attr)) != 0) {
throw SysError(s, "calling pthread_attr_destroy");
}
}
void join() {
int s;
s = pthread_join(thread, nullptr);
if (s != 0) {
throw SysError(s, "calling pthread_join");
}
}
private:
static void *init(void *ptr) {
std::unique_ptr<std::function<void(void)>> func;
func.reset(static_cast<std::function<void(void)> *>(ptr));
(*func)();
return 0;
}
};
struct State {
std::set<json> todo = json::array({json::array()});
std::set<json> active;
@ -380,10 +331,10 @@ int main(int argc, char **argv) {
Sync<State> state_;
/* Start a collector thread per worker process. */
std::vector<Thread> threads;
std::vector<std::thread> threads;
std::condition_variable wakeup;
for (size_t i = 0; i < myArgs.nrWorkers; i++) {
threads.emplace_back(std::bind(collector, std::ref(state_), std::ref(wakeup)));
threads.emplace_back(collector, std::ref(state_), std::ref(wakeup));
}
for (auto &thread : threads)

7
src/worker.cc
View file

@ -27,15 +27,18 @@
#include <lix/libstore/store-api.hh>
#include <lix/libexpr/symbol-table.hh>
#include <lix/libutil/types.hh>
#include <lix/libutil/util.hh>
#include <lix/libexpr/value.hh>
#include <lix/libutil/terminal.hh>
#include <exception>
#include <map>
#include <memory>
#include <numeric>
#include <optional>
#include <sstream>
#include <string>
#include <string_view>
#include <utility>
#include <vector>
#include "worker.hh"
#include "drv.hh"
@ -48,7 +51,7 @@ static nix::Value *releaseExprTopLevelValue(nix::EvalState &state,
nix::Value vTop;
if (args.fromArgs) {
nix::Expr &e = state.parseExprFromString(
nix::Expr *e = state.parseExprFromString(
args.releaseExpr, state.rootPath(nix::CanonPath::fromCwd()));
state.eval(e, vTop);
} else {