Compare commits
No commits in common. "main" and "jade/lixexpr" have entirely different histories.
main
...
jade/lixex
1
.nix-version
Normal file
1
.nix-version
Normal file
|
|
@ -0,0 +1 @@
|
|||
unstable
|
||||
14
README.md
14
README.md
|
|
@ -1,8 +1,5 @@
|
|||
# nix-eval-jobs
|
||||
|
||||
> [!NOTE]
|
||||
> This is a fork of nix-eval-jobs that works with Lix.
|
||||
|
||||
This project evaluates nix attribute sets in parallel with streamable json
|
||||
output. This is useful for time and memory intensive evaluations such as NixOS
|
||||
machines, i.e. in a CI context. The evaluation is done with a controllable
|
||||
|
|
@ -92,12 +89,11 @@ we collect example ci configuration for various CIs.
|
|||
|
||||
## Organisation of this repository
|
||||
|
||||
`main` follows Lix HEAD, and is updated alongside the Lix NixOS module. When we
|
||||
release we will make a `release-2.90` etc branch, which receives backports.
|
||||
|
||||
The version of nix-eval-jobs follows the major version of Lix and minor
|
||||
versions of nix-eval-jobs are released as necessary when changes are made in
|
||||
n-e-j itself.
|
||||
On the `main` branch we target nixUnstable. When a release of nix happens, we
|
||||
fork for a release branch i.e. `release-2.8` and change the nix version in
|
||||
`.nix-version`. Changes and improvements made in `main` also may be backported
|
||||
to these release branches. At the time of writing we only intent to support the
|
||||
latest release branch.
|
||||
|
||||
## Projects using nix-eval-jobs
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ let
|
|||
in
|
||||
stdenv.mkDerivation {
|
||||
pname = "nix-eval-jobs";
|
||||
version = "2.90.0-unstable";
|
||||
version = "2.19.0";
|
||||
src = if srcDir == null then filterMesonBuild ./. else srcDir;
|
||||
buildInputs = with pkgs; [
|
||||
nlohmann_json
|
||||
|
|
|
|||
66
flake.lock
66
flake.lock
|
|
@ -23,11 +23,11 @@
|
|||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1730504689,
|
||||
"narHash": "sha256-hgmguH29K2fvs9szpq2r3pz2/8cJd2LPS+b4tfNFCwE=",
|
||||
"lastModified": 1701473968,
|
||||
"narHash": "sha256-YcVE5emp1qQ8ieHUnxt1wCZCC3ZfAS+SRRWZ2TMda7E=",
|
||||
"owner": "hercules-ci",
|
||||
"repo": "flake-parts",
|
||||
"rev": "506278e768c2a08bec68eb62932193e341f55c90",
|
||||
"rev": "34fed993f1674c8d06d58b37ce1e0fe5eebcb9f5",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
|
@ -39,7 +39,6 @@
|
|||
"lix": {
|
||||
"inputs": {
|
||||
"flake-compat": "flake-compat",
|
||||
"nix2container": "nix2container",
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
],
|
||||
|
|
@ -47,15 +46,18 @@
|
|||
"pre-commit-hooks": "pre-commit-hooks"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1731683711,
|
||||
"narHash": "sha256-bq21I1EjXJa/s5Rra9J9ot2NkPCnI0F5uNPurwYLdpE=",
|
||||
"rev": "c859d03013712b349d82ee6223948d6d03e63a8d",
|
||||
"type": "tarball",
|
||||
"url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/c859d03013712b349d82ee6223948d6d03e63a8d.tar.gz?rev=c859d03013712b349d82ee6223948d6d03e63a8d"
|
||||
"lastModified": 1714955862,
|
||||
"narHash": "sha256-REWlo2RYHfJkxnmZTEJu3Cd/2VM+wjjpPy7Xi4BdDTQ=",
|
||||
"ref": "refs/tags/2.90-beta.1",
|
||||
"rev": "b6799ab0374a8e1907a48915d3187e07da41d88c",
|
||||
"revCount": 15501,
|
||||
"type": "git",
|
||||
"url": "https://git@git.lix.systems/lix-project/lix"
|
||||
},
|
||||
"original": {
|
||||
"type": "tarball",
|
||||
"url": "https://git.lix.systems/lix-project/lix/archive/main.tar.gz"
|
||||
"ref": "refs/tags/2.90-beta.1",
|
||||
"type": "git",
|
||||
"url": "https://git@git.lix.systems/lix-project/lix"
|
||||
}
|
||||
},
|
||||
"nix-github-actions": {
|
||||
|
|
@ -65,11 +67,11 @@
|
|||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1729742964,
|
||||
"narHash": "sha256-B4mzTcQ0FZHdpeWcpDYPERtyjJd/NIuaQ9+BV1h+MpA=",
|
||||
"lastModified": 1701208414,
|
||||
"narHash": "sha256-xrQ0FyhwTZK6BwKhahIkUVZhMNk21IEI1nUcWSONtpo=",
|
||||
"owner": "nix-community",
|
||||
"repo": "nix-github-actions",
|
||||
"rev": "e04df33f62cdcf93d73e9a04142464753a16db67",
|
||||
"rev": "93e39cc1a087d65bcf7a132e75a650c44dd2b734",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
|
@ -78,34 +80,18 @@
|
|||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nix2container": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
"lastModified": 1724996935,
|
||||
"narHash": "sha256-njRK9vvZ1JJsP8oV2OgkBrpJhgQezI03S7gzskCcHos=",
|
||||
"owner": "nlewo",
|
||||
"repo": "nix2container",
|
||||
"rev": "fa6bb0a1159f55d071ba99331355955ae30b3401",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nlewo",
|
||||
"repo": "nix2container",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1731663789,
|
||||
"narHash": "sha256-x07g4NcqGP6mQn6AISXJaks9sQYDjZmTMBlKIvajvyc=",
|
||||
"lastModified": 1703134684,
|
||||
"narHash": "sha256-SQmng1EnBFLzS7WSRyPM9HgmZP2kLJcPAz+Ug/nug6o=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "035d434d48f4375ac5d3a620954cf5fda7dd7c36",
|
||||
"rev": "d6863cbcbbb80e71cecfc03356db1cda38919523",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"ref": "nixos-24.05-small",
|
||||
"ref": "nixpkgs-unstable",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
|
|
@ -129,11 +115,11 @@
|
|||
"pre-commit-hooks": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
"lastModified": 1726745158,
|
||||
"narHash": "sha256-D5AegvGoEjt4rkKedmxlSEmC+nNLMBPWFxvmYnVLhjk=",
|
||||
"lastModified": 1712055707,
|
||||
"narHash": "sha256-4XLvuSIDZJGS17xEwSrNuJLL7UjDYKGJSbK1WWX2AK8=",
|
||||
"owner": "cachix",
|
||||
"repo": "git-hooks.nix",
|
||||
"rev": "4e743a6920eab45e8ba0fbe49dc459f1423a4b74",
|
||||
"rev": "e35aed5fda3cc79f88ed7f1795021e559582093a",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
|
@ -158,11 +144,11 @@
|
|||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1730321837,
|
||||
"narHash": "sha256-vK+a09qq19QNu2MlLcvN4qcRctJbqWkX7ahgPZ/+maI=",
|
||||
"lastModified": 1702979157,
|
||||
"narHash": "sha256-RnFBbLbpqtn4AoJGXKevQMCGhra4h6G2MPcuTSZZQ+g=",
|
||||
"owner": "numtide",
|
||||
"repo": "treefmt-nix",
|
||||
"rev": "746901bb8dba96d154b66492a29f5db0693dbfcc",
|
||||
"rev": "2961375283668d867e64129c22af532de8e77734",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
description = "Hydra's builtin hydra-eval-jobs as a standalone";
|
||||
|
||||
inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05-small";
|
||||
inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
|
||||
inputs.flake-parts.url = "github:hercules-ci/flake-parts";
|
||||
inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs";
|
||||
inputs.treefmt-nix.url = "github:numtide/treefmt-nix";
|
||||
|
|
@ -9,7 +9,7 @@
|
|||
inputs.nix-github-actions.url = "github:nix-community/nix-github-actions";
|
||||
inputs.nix-github-actions.inputs.nixpkgs.follows = "nixpkgs";
|
||||
inputs.lix = {
|
||||
url = "https://git.lix.systems/lix-project/lix/archive/main.tar.gz";
|
||||
url = "git+https://git@git.lix.systems/lix-project/lix?ref=refs/tags/2.90-beta.1";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
|
||||
|
|
@ -17,6 +17,7 @@
|
|||
let
|
||||
inherit (inputs.nixpkgs) lib;
|
||||
inherit (inputs) self;
|
||||
nixVersion = lib.fileContents ./.nix-version;
|
||||
in
|
||||
flake-parts.lib.mkFlake { inherit inputs; }
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1,11 +1,6 @@
|
|||
project('nix-eval-jobs', 'cpp',
|
||||
version : '0.1.6',
|
||||
license : 'GPL-3.0',
|
||||
default_options : [
|
||||
'debug=true',
|
||||
'optimization=2',
|
||||
'cpp_std=c++20',
|
||||
],
|
||||
)
|
||||
|
||||
nix_main_dep = dependency('lix-main', required: true)
|
||||
|
|
|
|||
|
|
@ -23,7 +23,6 @@ in
|
|||
pkgs.mkShell {
|
||||
inherit (nix-eval-jobs) buildInputs;
|
||||
nativeBuildInputs = nix-eval-jobs.nativeBuildInputs ++ [
|
||||
pkgs.clang-tools
|
||||
(pkgs.python3.withPackages (ps: [
|
||||
ps.pytest
|
||||
]))
|
||||
|
|
|
|||
33
src/drv.cc
33
src/drv.cc
|
|
@ -23,17 +23,14 @@
|
|||
#include "drv.hh"
|
||||
#include "eval-args.hh"
|
||||
|
||||
static bool
|
||||
queryIsCached(nix::Store &store,
|
||||
std::map<std::string, std::optional<std::string>> &outputs) {
|
||||
static bool queryIsCached(nix::Store &store,
|
||||
std::map<std::string, std::string> &outputs) {
|
||||
uint64_t downloadSize, narSize;
|
||||
nix::StorePathSet willBuild, willSubstitute, unknown;
|
||||
|
||||
std::vector<nix::StorePathWithOutputs> paths;
|
||||
for (auto const &[key, val] : outputs) {
|
||||
if (val) {
|
||||
paths.push_back(followLinksToStorePathWithOutputs(store, *val));
|
||||
}
|
||||
paths.push_back(followLinksToStorePathWithOutputs(store, val));
|
||||
}
|
||||
|
||||
store.queryMissing(toDerivedPaths(paths), willBuild, willSubstitute,
|
||||
|
|
@ -48,19 +45,9 @@ Drv::Drv(std::string &attrPath, nix::EvalState &state, nix::DrvInfo &drvInfo,
|
|||
auto localStore = state.store.dynamic_pointer_cast<nix::LocalFSStore>();
|
||||
|
||||
try {
|
||||
// CA derivations do not have static output paths, so we have to
|
||||
// defensively not query output paths in case we encounter one.
|
||||
for (auto &[outputName, optOutputPath] :
|
||||
drvInfo.queryOutputs(!nix::experimentalFeatureSettings.isEnabled(
|
||||
nix::Xp::CaDerivations))) {
|
||||
if (optOutputPath) {
|
||||
outputs[outputName] =
|
||||
localStore->printStorePath(*optOutputPath);
|
||||
} else {
|
||||
assert(nix::experimentalFeatureSettings.isEnabled(
|
||||
nix::Xp::CaDerivations));
|
||||
outputs[outputName] = std::nullopt;
|
||||
}
|
||||
for (auto out : drvInfo.queryOutputs(true)) {
|
||||
if (out.second)
|
||||
outputs[out.first] = localStore->printStorePath(*out.second);
|
||||
}
|
||||
} catch (const std::exception &e) {
|
||||
throw nix::EvalError(state,
|
||||
|
|
@ -111,16 +98,10 @@ Drv::Drv(std::string &attrPath, nix::EvalState &state, nix::DrvInfo &drvInfo,
|
|||
}
|
||||
|
||||
void to_json(nlohmann::json &json, const Drv &drv) {
|
||||
std::map<std::string, nlohmann::json> outputsJson;
|
||||
for (auto &[name, optPath] : drv.outputs) {
|
||||
outputsJson[name] =
|
||||
optPath ? nlohmann::json(*optPath) : nlohmann::json(nullptr);
|
||||
}
|
||||
|
||||
json = nlohmann::json{{"name", drv.name},
|
||||
{"system", drv.system},
|
||||
{"drvPath", drv.drvPath},
|
||||
{"outputs", outputsJson},
|
||||
{"outputs", drv.outputs},
|
||||
{"inputDrvs", drv.inputDrvs}};
|
||||
|
||||
if (drv.meta.has_value()) {
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ struct Drv {
|
|||
std::string drvPath;
|
||||
|
||||
enum class CacheStatus { Cached, Uncached, Unknown } cacheStatus;
|
||||
std::map<std::string, std::optional<std::string>> outputs;
|
||||
std::map<std::string, std::string> outputs;
|
||||
std::map<std::string, std::set<std::string>> inputDrvs;
|
||||
std::optional<nlohmann::json> meta;
|
||||
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@
|
|||
|
||||
class MyArgs : virtual public nix::MixEvalArgs,
|
||||
virtual public nix::MixCommonArgs,
|
||||
virtual public nix::RootArgs {
|
||||
virtual nix::RootArgs {
|
||||
public:
|
||||
std::string releaseExpr;
|
||||
nix::Path gcRootsDir;
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
src = files(
|
||||
src = [
|
||||
'nix-eval-jobs.cc',
|
||||
'eval-args.cc',
|
||||
'drv.cc',
|
||||
'buffered-io.cc',
|
||||
'worker.cc',
|
||||
)
|
||||
'worker.cc'
|
||||
]
|
||||
|
||||
cc = meson.get_compiler('cpp')
|
||||
|
||||
|
|
@ -31,4 +31,4 @@ executable('nix-eval-jobs', src,
|
|||
threads_dep
|
||||
],
|
||||
install: true,
|
||||
cpp_args: ['--include', 'autotools-config.h'])
|
||||
cpp_args: ['-std=c++2a', '-fvisibility=hidden', '--include', 'autotools-config.h'])
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@
|
|||
#include <sys/wait.h>
|
||||
#include <nlohmann/json.hpp>
|
||||
#include <errno.h>
|
||||
#include <pthread.h>
|
||||
#include <signal.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
|
@ -18,13 +17,13 @@
|
|||
#include <lix/libutil/error.hh>
|
||||
#include <lix/libstore/globals.hh>
|
||||
#include <lix/libutil/logging.hh>
|
||||
#include <lix/libutil/terminal.hh>
|
||||
#include <nlohmann/detail/iterators/iter_impl.hpp>
|
||||
#include <nlohmann/detail/json_ref.hpp>
|
||||
#include <nlohmann/json_fwd.hpp>
|
||||
#include <lix/libutil/ref.hh>
|
||||
#include <lix/libstore/store-api.hh>
|
||||
#include <map>
|
||||
#include <thread>
|
||||
#include <condition_variable>
|
||||
#include <filesystem>
|
||||
#include <exception>
|
||||
|
|
@ -93,60 +92,12 @@ struct Proc {
|
|||
|
||||
to = std::move(toPipe.writeSide);
|
||||
from = std::move(fromPipe.readSide);
|
||||
pid = std::move(p);
|
||||
pid = p;
|
||||
}
|
||||
|
||||
~Proc() {}
|
||||
};
|
||||
|
||||
// We'd highly prefer using std::thread here; but this won't let us configure the stack
|
||||
// size. macOS uses 512KiB size stacks for non-main threads, and musl defaults to 128k.
|
||||
// While Nix configures a 64MiB size for the main thread, this doesn't propagate to the
|
||||
// threads we launch here. It turns out, running the evaluator under an anemic stack of
|
||||
// 0.5MiB has it overflow way too quickly. Hence, we have our own custom Thread struct.
|
||||
struct Thread {
|
||||
pthread_t thread;
|
||||
|
||||
Thread(const Thread &) = delete;
|
||||
Thread(Thread &&) noexcept = default;
|
||||
|
||||
Thread(std::function<void(void)> f) {
|
||||
int s;
|
||||
pthread_attr_t attr;
|
||||
|
||||
auto func = std::make_unique<std::function<void(void)>>(std::move(f));
|
||||
|
||||
if ((s = pthread_attr_init(&attr)) != 0) {
|
||||
throw SysError(s, "calling pthread_attr_init");
|
||||
}
|
||||
if ((s = pthread_attr_setstacksize(&attr, 64 * 1024 * 1024)) != 0) {
|
||||
throw SysError(s, "calling pthread_attr_setstacksize");
|
||||
}
|
||||
if ((s = pthread_create(&thread, &attr, Thread::init, func.release())) != 0) {
|
||||
throw SysError(s, "calling pthread_launch");
|
||||
}
|
||||
if ((s = pthread_attr_destroy(&attr)) != 0) {
|
||||
throw SysError(s, "calling pthread_attr_destroy");
|
||||
}
|
||||
}
|
||||
|
||||
void join() {
|
||||
int s;
|
||||
s = pthread_join(thread, nullptr);
|
||||
if (s != 0) {
|
||||
throw SysError(s, "calling pthread_join");
|
||||
}
|
||||
}
|
||||
private:
|
||||
static void *init(void *ptr) {
|
||||
std::unique_ptr<std::function<void(void)>> func;
|
||||
func.reset(static_cast<std::function<void(void)> *>(ptr));
|
||||
|
||||
(*func)();
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
|
||||
struct State {
|
||||
std::set<json> todo = json::array({json::array()});
|
||||
std::set<json> active;
|
||||
|
|
@ -344,16 +295,24 @@ int main(int argc, char **argv) {
|
|||
|
||||
return handleExceptions(argv[0], [&]() {
|
||||
initNix();
|
||||
initLibExpr();
|
||||
initGC();
|
||||
|
||||
myArgs.parseArgs(argv, argc);
|
||||
|
||||
/* FIXME: The build hook in conjunction with import-from-derivation is
|
||||
* causing "unexpected EOF" during eval */
|
||||
settings.builders = "";
|
||||
|
||||
/* Prevent access to paths outside of the Nix search path and
|
||||
to the environment. */
|
||||
evalSettings.restrictEval = false;
|
||||
|
||||
/* When building a flake, use pure evaluation (no access to
|
||||
'getEnv', 'currentSystem' etc. */
|
||||
if (myArgs.impure) {
|
||||
evalSettings.pureEval.override(false);
|
||||
evalSettings.pureEval = false;
|
||||
} else if (myArgs.flake) {
|
||||
evalSettings.pureEval.override(true);
|
||||
evalSettings.pureEval = true;
|
||||
}
|
||||
|
||||
if (myArgs.releaseExpr == "")
|
||||
|
|
@ -366,16 +325,16 @@ int main(int argc, char **argv) {
|
|||
}
|
||||
|
||||
if (myArgs.showTrace) {
|
||||
loggerSettings.showTrace.override(true);
|
||||
loggerSettings.showTrace.assign(true);
|
||||
}
|
||||
|
||||
Sync<State> state_;
|
||||
|
||||
/* Start a collector thread per worker process. */
|
||||
std::vector<Thread> threads;
|
||||
std::vector<std::thread> threads;
|
||||
std::condition_variable wakeup;
|
||||
for (size_t i = 0; i < myArgs.nrWorkers; i++) {
|
||||
threads.emplace_back(std::bind(collector, std::ref(state_), std::ref(wakeup)));
|
||||
threads.emplace_back(collector, std::ref(state_), std::ref(wakeup));
|
||||
}
|
||||
|
||||
for (auto &thread : threads)
|
||||
|
|
|
|||
|
|
@ -27,15 +27,18 @@
|
|||
#include <lix/libstore/store-api.hh>
|
||||
#include <lix/libexpr/symbol-table.hh>
|
||||
#include <lix/libutil/types.hh>
|
||||
#include <lix/libutil/util.hh>
|
||||
#include <lix/libexpr/value.hh>
|
||||
#include <lix/libutil/terminal.hh>
|
||||
#include <exception>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <numeric>
|
||||
#include <optional>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "worker.hh"
|
||||
#include "drv.hh"
|
||||
|
|
@ -48,7 +51,7 @@ static nix::Value *releaseExprTopLevelValue(nix::EvalState &state,
|
|||
nix::Value vTop;
|
||||
|
||||
if (args.fromArgs) {
|
||||
nix::Expr &e = state.parseExprFromString(
|
||||
nix::Expr *e = state.parseExprFromString(
|
||||
args.releaseExpr, state.rootPath(nix::CanonPath::fromCwd()));
|
||||
state.eval(e, vTop);
|
||||
} else {
|
||||
|
|
|
|||
Loading…
Reference in a new issue