forked from lix-project/nix-eval-jobs
Compare commits
23 commits
jade/fixup
...
main
Author | SHA1 | Date | |
---|---|---|---|
df3edf3730 | |||
bdbe3e1304 | |||
dfc286ca3d | |||
912a9d6331 | |||
bbbebaf380 | |||
b9e0abe9e6 | |||
57ddb99e78 | |||
9160d40f80 | |||
43aaa943bf | |||
50a1455953 | |||
|
42a160bce2 | ||
c057494450 | |||
d9a46559a4 | |||
f8869bdcca | |||
fd86a1a068 | |||
9c23772cf2 | |||
11d467fecd | |||
040db2fe26 | |||
ee79f8a236 | |||
ed7a959ae1 | |||
d05f8e86a9 | |||
b67c46d320 | |||
30cf61fd26 |
12 changed files with 169 additions and 89 deletions
66
flake.lock
66
flake.lock
|
@ -23,11 +23,11 @@
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1701473968,
|
"lastModified": 1730504689,
|
||||||
"narHash": "sha256-YcVE5emp1qQ8ieHUnxt1wCZCC3ZfAS+SRRWZ2TMda7E=",
|
"narHash": "sha256-hgmguH29K2fvs9szpq2r3pz2/8cJd2LPS+b4tfNFCwE=",
|
||||||
"owner": "hercules-ci",
|
"owner": "hercules-ci",
|
||||||
"repo": "flake-parts",
|
"repo": "flake-parts",
|
||||||
"rev": "34fed993f1674c8d06d58b37ce1e0fe5eebcb9f5",
|
"rev": "506278e768c2a08bec68eb62932193e341f55c90",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
|
@ -39,6 +39,7 @@
|
||||||
"lix": {
|
"lix": {
|
||||||
"inputs": {
|
"inputs": {
|
||||||
"flake-compat": "flake-compat",
|
"flake-compat": "flake-compat",
|
||||||
|
"nix2container": "nix2container",
|
||||||
"nixpkgs": [
|
"nixpkgs": [
|
||||||
"nixpkgs"
|
"nixpkgs"
|
||||||
],
|
],
|
||||||
|
@ -46,18 +47,15 @@
|
||||||
"pre-commit-hooks": "pre-commit-hooks"
|
"pre-commit-hooks": "pre-commit-hooks"
|
||||||
},
|
},
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1714955862,
|
"lastModified": 1733448312,
|
||||||
"narHash": "sha256-REWlo2RYHfJkxnmZTEJu3Cd/2VM+wjjpPy7Xi4BdDTQ=",
|
"narHash": "sha256-id5U81bzXk/Lg900nGLM4CQb0wmTdzIvQz7CZk2OcTM=",
|
||||||
"ref": "refs/tags/2.90-beta.1",
|
"rev": "2a9e560570982a91937d199af3e7b7a8f3cbe14b",
|
||||||
"rev": "b6799ab0374a8e1907a48915d3187e07da41d88c",
|
"type": "tarball",
|
||||||
"revCount": 15501,
|
"url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/2a9e560570982a91937d199af3e7b7a8f3cbe14b.tar.gz?rev=2a9e560570982a91937d199af3e7b7a8f3cbe14b"
|
||||||
"type": "git",
|
|
||||||
"url": "https://git@git.lix.systems/lix-project/lix"
|
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
"ref": "refs/tags/2.90-beta.1",
|
"type": "tarball",
|
||||||
"type": "git",
|
"url": "https://git.lix.systems/lix-project/lix/archive/main.tar.gz"
|
||||||
"url": "https://git@git.lix.systems/lix-project/lix"
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"nix-github-actions": {
|
"nix-github-actions": {
|
||||||
|
@ -67,11 +65,11 @@
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1701208414,
|
"lastModified": 1731952509,
|
||||||
"narHash": "sha256-xrQ0FyhwTZK6BwKhahIkUVZhMNk21IEI1nUcWSONtpo=",
|
"narHash": "sha256-p4gB3Rhw8R6Ak4eMl8pqjCPOLCZRqaehZxdZ/mbFClM=",
|
||||||
"owner": "nix-community",
|
"owner": "nix-community",
|
||||||
"repo": "nix-github-actions",
|
"repo": "nix-github-actions",
|
||||||
"rev": "93e39cc1a087d65bcf7a132e75a650c44dd2b734",
|
"rev": "7b5f051df789b6b20d259924d349a9ba3319b226",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
|
@ -80,18 +78,34 @@
|
||||||
"type": "github"
|
"type": "github"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"nix2container": {
|
||||||
|
"flake": false,
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1724996935,
|
||||||
|
"narHash": "sha256-njRK9vvZ1JJsP8oV2OgkBrpJhgQezI03S7gzskCcHos=",
|
||||||
|
"owner": "nlewo",
|
||||||
|
"repo": "nix2container",
|
||||||
|
"rev": "fa6bb0a1159f55d071ba99331355955ae30b3401",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "nlewo",
|
||||||
|
"repo": "nix2container",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
"nixpkgs": {
|
"nixpkgs": {
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1703134684,
|
"lastModified": 1733408989,
|
||||||
"narHash": "sha256-SQmng1EnBFLzS7WSRyPM9HgmZP2kLJcPAz+Ug/nug6o=",
|
"narHash": "sha256-VCQpCQy+6ik+oYKWUCvq0WM2V7UtEKldqdsEzCNEOLc=",
|
||||||
"owner": "NixOS",
|
"owner": "NixOS",
|
||||||
"repo": "nixpkgs",
|
"repo": "nixpkgs",
|
||||||
"rev": "d6863cbcbbb80e71cecfc03356db1cda38919523",
|
"rev": "d916387b68a74040a3873ad2a08a559c60cedb5e",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
"owner": "NixOS",
|
"owner": "NixOS",
|
||||||
"ref": "nixpkgs-unstable",
|
"ref": "nixos-24.05-small",
|
||||||
"repo": "nixpkgs",
|
"repo": "nixpkgs",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
}
|
}
|
||||||
|
@ -115,11 +129,11 @@
|
||||||
"pre-commit-hooks": {
|
"pre-commit-hooks": {
|
||||||
"flake": false,
|
"flake": false,
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1712055707,
|
"lastModified": 1726745158,
|
||||||
"narHash": "sha256-4XLvuSIDZJGS17xEwSrNuJLL7UjDYKGJSbK1WWX2AK8=",
|
"narHash": "sha256-D5AegvGoEjt4rkKedmxlSEmC+nNLMBPWFxvmYnVLhjk=",
|
||||||
"owner": "cachix",
|
"owner": "cachix",
|
||||||
"repo": "git-hooks.nix",
|
"repo": "git-hooks.nix",
|
||||||
"rev": "e35aed5fda3cc79f88ed7f1795021e559582093a",
|
"rev": "4e743a6920eab45e8ba0fbe49dc459f1423a4b74",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
|
@ -144,11 +158,11 @@
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1702979157,
|
"lastModified": 1732292307,
|
||||||
"narHash": "sha256-RnFBbLbpqtn4AoJGXKevQMCGhra4h6G2MPcuTSZZQ+g=",
|
"narHash": "sha256-5WSng844vXt8uytT5djmqBCkopyle6ciFgteuA9bJpw=",
|
||||||
"owner": "numtide",
|
"owner": "numtide",
|
||||||
"repo": "treefmt-nix",
|
"repo": "treefmt-nix",
|
||||||
"rev": "2961375283668d867e64129c22af532de8e77734",
|
"rev": "705df92694af7093dfbb27109ce16d828a79155f",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
{
|
{
|
||||||
description = "Hydra's builtin hydra-eval-jobs as a standalone";
|
description = "Hydra's builtin hydra-eval-jobs as a standalone";
|
||||||
|
|
||||||
inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
|
inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05-small";
|
||||||
inputs.flake-parts.url = "github:hercules-ci/flake-parts";
|
inputs.flake-parts.url = "github:hercules-ci/flake-parts";
|
||||||
inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs";
|
inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs";
|
||||||
inputs.treefmt-nix.url = "github:numtide/treefmt-nix";
|
inputs.treefmt-nix.url = "github:numtide/treefmt-nix";
|
||||||
|
@ -9,7 +9,7 @@
|
||||||
inputs.nix-github-actions.url = "github:nix-community/nix-github-actions";
|
inputs.nix-github-actions.url = "github:nix-community/nix-github-actions";
|
||||||
inputs.nix-github-actions.inputs.nixpkgs.follows = "nixpkgs";
|
inputs.nix-github-actions.inputs.nixpkgs.follows = "nixpkgs";
|
||||||
inputs.lix = {
|
inputs.lix = {
|
||||||
url = "git+https://git@git.lix.systems/lix-project/lix?ref=refs/tags/2.90-beta.1";
|
url = "https://git.lix.systems/lix-project/lix/archive/main.tar.gz";
|
||||||
inputs.nixpkgs.follows = "nixpkgs";
|
inputs.nixpkgs.follows = "nixpkgs";
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,11 @@
|
||||||
project('nix-eval-jobs', 'cpp',
|
project('nix-eval-jobs', 'cpp',
|
||||||
version : '0.1.6',
|
version : '0.1.6',
|
||||||
license : 'GPL-3.0',
|
license : 'GPL-3.0',
|
||||||
|
default_options : [
|
||||||
|
'debug=true',
|
||||||
|
'optimization=2',
|
||||||
|
'cpp_std=c++20',
|
||||||
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
nix_main_dep = dependency('lix-main', required: true)
|
nix_main_dep = dependency('lix-main', required: true)
|
||||||
|
|
|
@ -23,6 +23,7 @@ in
|
||||||
pkgs.mkShell {
|
pkgs.mkShell {
|
||||||
inherit (nix-eval-jobs) buildInputs;
|
inherit (nix-eval-jobs) buildInputs;
|
||||||
nativeBuildInputs = nix-eval-jobs.nativeBuildInputs ++ [
|
nativeBuildInputs = nix-eval-jobs.nativeBuildInputs ++ [
|
||||||
|
pkgs.clang-tools
|
||||||
(pkgs.python3.withPackages (ps: [
|
(pkgs.python3.withPackages (ps: [
|
||||||
ps.pytest
|
ps.pytest
|
||||||
]))
|
]))
|
||||||
|
|
50
src/drv.cc
50
src/drv.cc
|
@ -23,14 +23,17 @@
|
||||||
#include "drv.hh"
|
#include "drv.hh"
|
||||||
#include "eval-args.hh"
|
#include "eval-args.hh"
|
||||||
|
|
||||||
static bool queryIsCached(nix::Store &store,
|
static bool
|
||||||
std::map<std::string, std::string> &outputs) {
|
queryIsCached(nix::Store &store,
|
||||||
|
std::map<std::string, std::optional<std::string>> &outputs) {
|
||||||
uint64_t downloadSize, narSize;
|
uint64_t downloadSize, narSize;
|
||||||
nix::StorePathSet willBuild, willSubstitute, unknown;
|
nix::StorePathSet willBuild, willSubstitute, unknown;
|
||||||
|
|
||||||
std::vector<nix::StorePathWithOutputs> paths;
|
std::vector<nix::StorePathWithOutputs> paths;
|
||||||
for (auto const &[key, val] : outputs) {
|
for (auto const &[key, val] : outputs) {
|
||||||
paths.push_back(followLinksToStorePathWithOutputs(store, val));
|
if (val) {
|
||||||
|
paths.push_back(followLinksToStorePathWithOutputs(store, *val));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
store.queryMissing(toDerivedPaths(paths), willBuild, willSubstitute,
|
store.queryMissing(toDerivedPaths(paths), willBuild, willSubstitute,
|
||||||
|
@ -42,26 +45,37 @@ static bool queryIsCached(nix::Store &store,
|
||||||
Drv::Drv(std::string &attrPath, nix::EvalState &state, nix::DrvInfo &drvInfo,
|
Drv::Drv(std::string &attrPath, nix::EvalState &state, nix::DrvInfo &drvInfo,
|
||||||
MyArgs &args) {
|
MyArgs &args) {
|
||||||
|
|
||||||
auto localStore = state.store.dynamic_pointer_cast<nix::LocalFSStore>();
|
auto localStore = state.ctx.store.dynamic_pointer_cast<nix::LocalFSStore>();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
for (auto out : drvInfo.queryOutputs(true)) {
|
// CA derivations do not have static output paths, so we have to
|
||||||
if (out.second)
|
// defensively not query output paths in case we encounter one.
|
||||||
outputs[out.first] = localStore->printStorePath(*out.second);
|
for (auto &[outputName, optOutputPath] :
|
||||||
|
drvInfo.queryOutputs(state, !nix::experimentalFeatureSettings.isEnabled(
|
||||||
|
nix::Xp::CaDerivations))) {
|
||||||
|
if (optOutputPath) {
|
||||||
|
outputs[outputName] =
|
||||||
|
localStore->printStorePath(*optOutputPath);
|
||||||
|
} else {
|
||||||
|
assert(nix::experimentalFeatureSettings.isEnabled(
|
||||||
|
nix::Xp::CaDerivations));
|
||||||
|
outputs[outputName] = std::nullopt;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} catch (const std::exception &e) {
|
} catch (const std::exception &e) {
|
||||||
throw nix::EvalError(state,
|
state.ctx.errors.make<nix::EvalError>(
|
||||||
"derivation '%s' does not have valid outputs: %s",
|
"derivation '%s' does not have valid outputs: %s",
|
||||||
attrPath, e.what());
|
attrPath, e.what()
|
||||||
|
).debugThrow();
|
||||||
}
|
}
|
||||||
|
|
||||||
if (args.meta) {
|
if (args.meta) {
|
||||||
nlohmann::json meta_;
|
nlohmann::json meta_;
|
||||||
for (auto &metaName : drvInfo.queryMetaNames()) {
|
for (auto &metaName : drvInfo.queryMetaNames(state)) {
|
||||||
nix::NixStringContext context;
|
nix::NixStringContext context;
|
||||||
std::stringstream ss;
|
std::stringstream ss;
|
||||||
|
|
||||||
auto metaValue = drvInfo.queryMeta(metaName);
|
auto metaValue = drvInfo.queryMeta(state, metaName);
|
||||||
// Skip non-serialisable types
|
// Skip non-serialisable types
|
||||||
// TODO: Fix serialisation of derivations to store paths
|
// TODO: Fix serialisation of derivations to store paths
|
||||||
if (metaValue == 0) {
|
if (metaValue == 0) {
|
||||||
|
@ -83,9 +97,9 @@ Drv::Drv(std::string &attrPath, nix::EvalState &state, nix::DrvInfo &drvInfo,
|
||||||
cacheStatus = Drv::CacheStatus::Unknown;
|
cacheStatus = Drv::CacheStatus::Unknown;
|
||||||
}
|
}
|
||||||
|
|
||||||
drvPath = localStore->printStorePath(drvInfo.requireDrvPath());
|
drvPath = localStore->printStorePath(drvInfo.requireDrvPath(state));
|
||||||
|
|
||||||
auto drv = localStore->readDerivation(drvInfo.requireDrvPath());
|
auto drv = localStore->readDerivation(drvInfo.requireDrvPath(state));
|
||||||
for (const auto &[inputDrvPath, inputNode] : drv.inputDrvs.map) {
|
for (const auto &[inputDrvPath, inputNode] : drv.inputDrvs.map) {
|
||||||
std::set<std::string> inputDrvOutputs;
|
std::set<std::string> inputDrvOutputs;
|
||||||
for (auto &outputName : inputNode.value) {
|
for (auto &outputName : inputNode.value) {
|
||||||
|
@ -93,15 +107,21 @@ Drv::Drv(std::string &attrPath, nix::EvalState &state, nix::DrvInfo &drvInfo,
|
||||||
}
|
}
|
||||||
inputDrvs[localStore->printStorePath(inputDrvPath)] = inputDrvOutputs;
|
inputDrvs[localStore->printStorePath(inputDrvPath)] = inputDrvOutputs;
|
||||||
}
|
}
|
||||||
name = drvInfo.queryName();
|
name = drvInfo.queryName(state);
|
||||||
system = drv.platform;
|
system = drv.platform;
|
||||||
}
|
}
|
||||||
|
|
||||||
void to_json(nlohmann::json &json, const Drv &drv) {
|
void to_json(nlohmann::json &json, const Drv &drv) {
|
||||||
|
std::map<std::string, nlohmann::json> outputsJson;
|
||||||
|
for (auto &[name, optPath] : drv.outputs) {
|
||||||
|
outputsJson[name] =
|
||||||
|
optPath ? nlohmann::json(*optPath) : nlohmann::json(nullptr);
|
||||||
|
}
|
||||||
|
|
||||||
json = nlohmann::json{{"name", drv.name},
|
json = nlohmann::json{{"name", drv.name},
|
||||||
{"system", drv.system},
|
{"system", drv.system},
|
||||||
{"drvPath", drv.drvPath},
|
{"drvPath", drv.drvPath},
|
||||||
{"outputs", drv.outputs},
|
{"outputs", outputsJson},
|
||||||
{"inputDrvs", drv.inputDrvs}};
|
{"inputDrvs", drv.inputDrvs}};
|
||||||
|
|
||||||
if (drv.meta.has_value()) {
|
if (drv.meta.has_value()) {
|
||||||
|
|
|
@ -24,7 +24,7 @@ struct Drv {
|
||||||
std::string drvPath;
|
std::string drvPath;
|
||||||
|
|
||||||
enum class CacheStatus { Cached, Uncached, Unknown } cacheStatus;
|
enum class CacheStatus { Cached, Uncached, Unknown } cacheStatus;
|
||||||
std::map<std::string, std::string> outputs;
|
std::map<std::string, std::optional<std::string>> outputs;
|
||||||
std::map<std::string, std::set<std::string>> inputDrvs;
|
std::map<std::string, std::set<std::string>> inputDrvs;
|
||||||
std::optional<nlohmann::json> meta;
|
std::optional<nlohmann::json> meta;
|
||||||
|
|
||||||
|
|
|
@ -100,5 +100,5 @@ MyArgs::MyArgs() : MixCommonArgs("nix-eval-jobs") {
|
||||||
}
|
}
|
||||||
|
|
||||||
void MyArgs::parseArgs(char **argv, int argc) {
|
void MyArgs::parseArgs(char **argv, int argc) {
|
||||||
parseCmdline(nix::argvToStrings(argc, argv));
|
parseCmdline(nix::Strings(argv + 1, argv + argc));
|
||||||
}
|
}
|
||||||
|
|
|
@ -12,7 +12,7 @@
|
||||||
|
|
||||||
class MyArgs : virtual public nix::MixEvalArgs,
|
class MyArgs : virtual public nix::MixEvalArgs,
|
||||||
virtual public nix::MixCommonArgs,
|
virtual public nix::MixCommonArgs,
|
||||||
virtual nix::RootArgs {
|
virtual public nix::RootArgs {
|
||||||
public:
|
public:
|
||||||
std::string releaseExpr;
|
std::string releaseExpr;
|
||||||
nix::Path gcRootsDir;
|
nix::Path gcRootsDir;
|
||||||
|
|
|
@ -1,10 +1,10 @@
|
||||||
src = [
|
src = files(
|
||||||
'nix-eval-jobs.cc',
|
'nix-eval-jobs.cc',
|
||||||
'eval-args.cc',
|
'eval-args.cc',
|
||||||
'drv.cc',
|
'drv.cc',
|
||||||
'buffered-io.cc',
|
'buffered-io.cc',
|
||||||
'worker.cc'
|
'worker.cc',
|
||||||
]
|
)
|
||||||
|
|
||||||
cc = meson.get_compiler('cpp')
|
cc = meson.get_compiler('cpp')
|
||||||
|
|
||||||
|
@ -31,4 +31,4 @@ executable('nix-eval-jobs', src,
|
||||||
threads_dep
|
threads_dep
|
||||||
],
|
],
|
||||||
install: true,
|
install: true,
|
||||||
cpp_args: ['-std=c++2a', '-fvisibility=hidden', '--include', 'autotools-config.h'])
|
cpp_args: ['--include', 'autotools-config.h'])
|
||||||
|
|
|
@ -8,6 +8,7 @@
|
||||||
#include <sys/wait.h>
|
#include <sys/wait.h>
|
||||||
#include <nlohmann/json.hpp>
|
#include <nlohmann/json.hpp>
|
||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
|
#include <pthread.h>
|
||||||
#include <signal.h>
|
#include <signal.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
|
@ -17,13 +18,12 @@
|
||||||
#include <lix/libutil/error.hh>
|
#include <lix/libutil/error.hh>
|
||||||
#include <lix/libstore/globals.hh>
|
#include <lix/libstore/globals.hh>
|
||||||
#include <lix/libutil/logging.hh>
|
#include <lix/libutil/logging.hh>
|
||||||
|
#include <lix/libutil/terminal.hh>
|
||||||
#include <nlohmann/detail/iterators/iter_impl.hpp>
|
#include <nlohmann/detail/iterators/iter_impl.hpp>
|
||||||
#include <nlohmann/detail/json_ref.hpp>
|
#include <nlohmann/detail/json_ref.hpp>
|
||||||
#include <nlohmann/json_fwd.hpp>
|
#include <nlohmann/json_fwd.hpp>
|
||||||
#include <lix/libutil/ref.hh>
|
#include <lix/libutil/ref.hh>
|
||||||
#include <lix/libstore/store-api.hh>
|
#include <lix/libstore/store-api.hh>
|
||||||
#include <map>
|
|
||||||
#include <thread>
|
|
||||||
#include <condition_variable>
|
#include <condition_variable>
|
||||||
#include <filesystem>
|
#include <filesystem>
|
||||||
#include <exception>
|
#include <exception>
|
||||||
|
@ -46,9 +46,8 @@ using namespace nlohmann;
|
||||||
|
|
||||||
static MyArgs myArgs;
|
static MyArgs myArgs;
|
||||||
|
|
||||||
typedef std::function<void(ref<EvalState> state, Bindings &autoArgs,
|
using Processor = std::function<void(ref<nix::eval_cache::CachingEvaluator> state, Bindings &autoArgs,
|
||||||
AutoCloseFD &to, AutoCloseFD &from, MyArgs &args)>
|
AutoCloseFD &to, AutoCloseFD &from, MyArgs &args)>;
|
||||||
Processor;
|
|
||||||
|
|
||||||
/* Auto-cleanup of fork's process and fds. */
|
/* Auto-cleanup of fork's process and fds. */
|
||||||
struct Proc {
|
struct Proc {
|
||||||
|
@ -69,10 +68,10 @@ struct Proc {
|
||||||
auto evalStore = myArgs.evalStoreUrl
|
auto evalStore = myArgs.evalStoreUrl
|
||||||
? openStore(*myArgs.evalStoreUrl)
|
? openStore(*myArgs.evalStoreUrl)
|
||||||
: openStore();
|
: openStore();
|
||||||
auto state = std::make_shared<EvalState>(myArgs.searchPath,
|
auto evaluator = nix::make_ref<nix::eval_cache::CachingEvaluator>(myArgs.searchPath,
|
||||||
evalStore);
|
evalStore);
|
||||||
Bindings &autoArgs = *myArgs.getAutoArgs(*state);
|
Bindings &autoArgs = *myArgs.getAutoArgs(*evaluator);
|
||||||
proc(ref<EvalState>(state), autoArgs, *to, *from, myArgs);
|
proc(evaluator, autoArgs, *to, *from, myArgs);
|
||||||
} catch (Error &e) {
|
} catch (Error &e) {
|
||||||
nlohmann::json err;
|
nlohmann::json err;
|
||||||
auto msg = e.msg();
|
auto msg = e.msg();
|
||||||
|
@ -92,12 +91,60 @@ struct Proc {
|
||||||
|
|
||||||
to = std::move(toPipe.writeSide);
|
to = std::move(toPipe.writeSide);
|
||||||
from = std::move(fromPipe.readSide);
|
from = std::move(fromPipe.readSide);
|
||||||
pid = p;
|
pid = std::move(p);
|
||||||
}
|
}
|
||||||
|
|
||||||
~Proc() {}
|
~Proc() {}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// We'd highly prefer using std::thread here; but this won't let us configure the stack
|
||||||
|
// size. macOS uses 512KiB size stacks for non-main threads, and musl defaults to 128k.
|
||||||
|
// While Nix configures a 64MiB size for the main thread, this doesn't propagate to the
|
||||||
|
// threads we launch here. It turns out, running the evaluator under an anemic stack of
|
||||||
|
// 0.5MiB has it overflow way too quickly. Hence, we have our own custom Thread struct.
|
||||||
|
struct Thread {
|
||||||
|
pthread_t thread;
|
||||||
|
|
||||||
|
Thread(const Thread &) = delete;
|
||||||
|
Thread(Thread &&) noexcept = default;
|
||||||
|
|
||||||
|
Thread(std::function<void(void)> f) {
|
||||||
|
int s;
|
||||||
|
pthread_attr_t attr;
|
||||||
|
|
||||||
|
auto func = std::make_unique<std::function<void(void)>>(std::move(f));
|
||||||
|
|
||||||
|
if ((s = pthread_attr_init(&attr)) != 0) {
|
||||||
|
throw SysError(s, "calling pthread_attr_init");
|
||||||
|
}
|
||||||
|
if ((s = pthread_attr_setstacksize(&attr, 64 * 1024 * 1024)) != 0) {
|
||||||
|
throw SysError(s, "calling pthread_attr_setstacksize");
|
||||||
|
}
|
||||||
|
if ((s = pthread_create(&thread, &attr, Thread::init, func.release())) != 0) {
|
||||||
|
throw SysError(s, "calling pthread_launch");
|
||||||
|
}
|
||||||
|
if ((s = pthread_attr_destroy(&attr)) != 0) {
|
||||||
|
throw SysError(s, "calling pthread_attr_destroy");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void join() {
|
||||||
|
int s;
|
||||||
|
s = pthread_join(thread, nullptr);
|
||||||
|
if (s != 0) {
|
||||||
|
throw SysError(s, "calling pthread_join");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
private:
|
||||||
|
static void *init(void *ptr) {
|
||||||
|
std::unique_ptr<std::function<void(void)>> func;
|
||||||
|
func.reset(static_cast<std::function<void(void)> *>(ptr));
|
||||||
|
|
||||||
|
(*func)();
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
struct State {
|
struct State {
|
||||||
std::set<json> todo = json::array({json::array()});
|
std::set<json> todo = json::array({json::array()});
|
||||||
std::set<json> active;
|
std::set<json> active;
|
||||||
|
@ -295,24 +342,16 @@ int main(int argc, char **argv) {
|
||||||
|
|
||||||
return handleExceptions(argv[0], [&]() {
|
return handleExceptions(argv[0], [&]() {
|
||||||
initNix();
|
initNix();
|
||||||
initGC();
|
initLibExpr();
|
||||||
|
|
||||||
myArgs.parseArgs(argv, argc);
|
myArgs.parseArgs(argv, argc);
|
||||||
|
|
||||||
/* FIXME: The build hook in conjunction with import-from-derivation is
|
|
||||||
* causing "unexpected EOF" during eval */
|
|
||||||
settings.builders = "";
|
|
||||||
|
|
||||||
/* Prevent access to paths outside of the Nix search path and
|
|
||||||
to the environment. */
|
|
||||||
evalSettings.restrictEval = false;
|
|
||||||
|
|
||||||
/* When building a flake, use pure evaluation (no access to
|
/* When building a flake, use pure evaluation (no access to
|
||||||
'getEnv', 'currentSystem' etc. */
|
'getEnv', 'currentSystem' etc. */
|
||||||
if (myArgs.impure) {
|
if (myArgs.impure) {
|
||||||
evalSettings.pureEval = false;
|
evalSettings.pureEval.override(false);
|
||||||
} else if (myArgs.flake) {
|
} else if (myArgs.flake) {
|
||||||
evalSettings.pureEval = true;
|
evalSettings.pureEval.override(true);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (myArgs.releaseExpr == "")
|
if (myArgs.releaseExpr == "")
|
||||||
|
@ -325,16 +364,16 @@ int main(int argc, char **argv) {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (myArgs.showTrace) {
|
if (myArgs.showTrace) {
|
||||||
loggerSettings.showTrace.assign(true);
|
loggerSettings.showTrace.override(true);
|
||||||
}
|
}
|
||||||
|
|
||||||
Sync<State> state_;
|
Sync<State> state_;
|
||||||
|
|
||||||
/* Start a collector thread per worker process. */
|
/* Start a collector thread per worker process. */
|
||||||
std::vector<std::thread> threads;
|
std::vector<Thread> threads;
|
||||||
std::condition_variable wakeup;
|
std::condition_variable wakeup;
|
||||||
for (size_t i = 0; i < myArgs.nrWorkers; i++) {
|
for (size_t i = 0; i < myArgs.nrWorkers; i++) {
|
||||||
threads.emplace_back(collector, std::ref(state_), std::ref(wakeup));
|
threads.emplace_back(std::bind(collector, std::ref(state_), std::ref(wakeup)));
|
||||||
}
|
}
|
||||||
|
|
||||||
for (auto &thread : threads)
|
for (auto &thread : threads)
|
||||||
|
|
|
@ -14,6 +14,7 @@
|
||||||
#include <lix/libutil/canon-path.hh>
|
#include <lix/libutil/canon-path.hh>
|
||||||
#include <lix/libcmd/common-eval-args.hh>
|
#include <lix/libcmd/common-eval-args.hh>
|
||||||
#include <lix/libutil/error.hh>
|
#include <lix/libutil/error.hh>
|
||||||
|
#include <lix/libexpr/eval-cache.hh>
|
||||||
#include <lix/libexpr/eval-inline.hh>
|
#include <lix/libexpr/eval-inline.hh>
|
||||||
#include <lix/libexpr/eval.hh>
|
#include <lix/libexpr/eval.hh>
|
||||||
#include <lix/libexpr/flake/flakeref.hh>
|
#include <lix/libexpr/flake/flakeref.hh>
|
||||||
|
@ -27,18 +28,15 @@
|
||||||
#include <lix/libstore/store-api.hh>
|
#include <lix/libstore/store-api.hh>
|
||||||
#include <lix/libexpr/symbol-table.hh>
|
#include <lix/libexpr/symbol-table.hh>
|
||||||
#include <lix/libutil/types.hh>
|
#include <lix/libutil/types.hh>
|
||||||
#include <lix/libutil/util.hh>
|
|
||||||
#include <lix/libexpr/value.hh>
|
#include <lix/libexpr/value.hh>
|
||||||
|
#include <lix/libutil/terminal.hh>
|
||||||
#include <exception>
|
#include <exception>
|
||||||
#include <map>
|
|
||||||
#include <memory>
|
|
||||||
#include <numeric>
|
#include <numeric>
|
||||||
#include <optional>
|
#include <optional>
|
||||||
#include <sstream>
|
#include <sstream>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <string_view>
|
#include <string_view>
|
||||||
#include <utility>
|
#include <utility>
|
||||||
#include <vector>
|
|
||||||
|
|
||||||
#include "worker.hh"
|
#include "worker.hh"
|
||||||
#include "drv.hh"
|
#include "drv.hh"
|
||||||
|
@ -51,14 +49,14 @@ static nix::Value *releaseExprTopLevelValue(nix::EvalState &state,
|
||||||
nix::Value vTop;
|
nix::Value vTop;
|
||||||
|
|
||||||
if (args.fromArgs) {
|
if (args.fromArgs) {
|
||||||
nix::Expr *e = state.parseExprFromString(
|
nix::Expr &e = state.ctx.parseExprFromString(
|
||||||
args.releaseExpr, state.rootPath(nix::CanonPath::fromCwd()));
|
args.releaseExpr, nix::CanonPath::fromCwd());
|
||||||
state.eval(e, vTop);
|
state.eval(e, vTop);
|
||||||
} else {
|
} else {
|
||||||
state.evalFile(lookupFileArg(state, args.releaseExpr), vTop);
|
state.evalFile(nix::lookupFileArg(state.ctx, args.releaseExpr), vTop);
|
||||||
}
|
}
|
||||||
|
|
||||||
auto vRoot = state.allocValue();
|
auto vRoot = state.ctx.mem.allocValue();
|
||||||
|
|
||||||
state.autoCallFunction(autoArgs, vTop, *vRoot);
|
state.autoCallFunction(autoArgs, vTop, *vRoot);
|
||||||
|
|
||||||
|
@ -76,16 +74,17 @@ static std::string attrPathJoin(nlohmann::json input) {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
void worker(nix::ref<nix::EvalState> state, nix::Bindings &autoArgs,
|
void worker(nix::ref<nix::eval_cache::CachingEvaluator> evaluator, nix::Bindings &autoArgs,
|
||||||
nix::AutoCloseFD &to, nix::AutoCloseFD &from, MyArgs &args) {
|
nix::AutoCloseFD &to, nix::AutoCloseFD &from, MyArgs &args) {
|
||||||
|
|
||||||
nix::Value *vRoot = [&]() {
|
nix::Value *vRoot = [&]() {
|
||||||
|
auto state = evaluator->begin();
|
||||||
if (args.flake) {
|
if (args.flake) {
|
||||||
auto [flakeRef, fragment, outputSpec] =
|
auto [flakeRef, fragment, outputSpec] =
|
||||||
nix::parseFlakeRefWithFragmentAndExtendedOutputsSpec(
|
nix::parseFlakeRefWithFragmentAndExtendedOutputsSpec(
|
||||||
args.releaseExpr, nix::absPath("."));
|
args.releaseExpr, nix::absPath("."));
|
||||||
nix::InstallableFlake flake{
|
nix::InstallableFlake flake{
|
||||||
{}, state, std::move(flakeRef), fragment, outputSpec,
|
{}, evaluator, std::move(flakeRef), fragment, outputSpec,
|
||||||
{}, {}, args.lockFlags};
|
{}, {}, args.lockFlags};
|
||||||
|
|
||||||
return flake.toValue(*state).first;
|
return flake.toValue(*state).first;
|
||||||
|
@ -95,6 +94,7 @@ void worker(nix::ref<nix::EvalState> state, nix::Bindings &autoArgs,
|
||||||
}();
|
}();
|
||||||
|
|
||||||
LineReader fromReader(from.release());
|
LineReader fromReader(from.release());
|
||||||
|
auto state = evaluator->begin();
|
||||||
|
|
||||||
while (true) {
|
while (true) {
|
||||||
/* Wait for the collector to send us a job name. */
|
/* Wait for the collector to send us a job name. */
|
||||||
|
@ -122,7 +122,7 @@ void worker(nix::ref<nix::EvalState> state, nix::Bindings &autoArgs,
|
||||||
nix::findAlongAttrPath(*state, attrPathS, autoArgs, *vRoot)
|
nix::findAlongAttrPath(*state, attrPathS, autoArgs, *vRoot)
|
||||||
.first;
|
.first;
|
||||||
|
|
||||||
auto v = state->allocValue();
|
auto v = evaluator->mem.allocValue();
|
||||||
state->autoCallFunction(autoArgs, *vTmp, *v);
|
state->autoCallFunction(autoArgs, *vTmp, *v);
|
||||||
|
|
||||||
if (v->type() == nix::nAttrs) {
|
if (v->type() == nix::nAttrs) {
|
||||||
|
@ -139,7 +139,7 @@ void worker(nix::ref<nix::EvalState> state, nix::Bindings &autoArgs,
|
||||||
std::string(nix::baseNameOf(drv.drvPath));
|
std::string(nix::baseNameOf(drv.drvPath));
|
||||||
if (!nix::pathExists(root)) {
|
if (!nix::pathExists(root)) {
|
||||||
auto localStore =
|
auto localStore =
|
||||||
state->store
|
evaluator->store
|
||||||
.dynamic_pointer_cast<nix::LocalFSStore>();
|
.dynamic_pointer_cast<nix::LocalFSStore>();
|
||||||
auto storePath =
|
auto storePath =
|
||||||
localStore->parseStorePath(drv.drvPath);
|
localStore->parseStorePath(drv.drvPath);
|
||||||
|
@ -154,14 +154,14 @@ void worker(nix::ref<nix::EvalState> state, nix::Bindings &autoArgs,
|
||||||
// = true;` for top-level attrset
|
// = true;` for top-level attrset
|
||||||
|
|
||||||
for (auto &i :
|
for (auto &i :
|
||||||
v->attrs->lexicographicOrder(state->symbols)) {
|
v->attrs->lexicographicOrder(evaluator->symbols)) {
|
||||||
const std::string &name = state->symbols[i->name];
|
const std::string &name = evaluator->symbols[i->name];
|
||||||
attrs.push_back(name);
|
attrs.push_back(name);
|
||||||
|
|
||||||
if (name == "recurseForDerivations" &&
|
if (name == "recurseForDerivations" &&
|
||||||
!args.forceRecurse) {
|
!args.forceRecurse) {
|
||||||
auto attrv =
|
auto attrv =
|
||||||
v->attrs->get(state->sRecurseForDerivations);
|
v->attrs->get(evaluator->s.recurseForDerivations);
|
||||||
recurse = state->forceBool(
|
recurse = state->forceBool(
|
||||||
*attrv->value, attrv->pos,
|
*attrv->value, attrv->pos,
|
||||||
"while evaluating recurseForDerivations");
|
"while evaluating recurseForDerivations");
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
#include <lix/libmain/shared.hh>
|
#include <lix/libmain/shared.hh>
|
||||||
#include <lix/libexpr/eval.hh>
|
#include <lix/libexpr/eval.hh>
|
||||||
|
#include <lix/libexpr/eval-cache.hh>
|
||||||
|
|
||||||
#include "eval-args.hh"
|
#include "eval-args.hh"
|
||||||
|
|
||||||
|
@ -13,5 +14,5 @@ class EvalState;
|
||||||
template <typename T> class ref;
|
template <typename T> class ref;
|
||||||
} // namespace nix
|
} // namespace nix
|
||||||
|
|
||||||
void worker(nix::ref<nix::EvalState> state, nix::Bindings &autoArgs,
|
void worker(nix::ref<nix::eval_cache::CachingEvaluator> evaluator, nix::Bindings &autoArgs,
|
||||||
nix::AutoCloseFD &to, nix::AutoCloseFD &from, MyArgs &args);
|
nix::AutoCloseFD &to, nix::AutoCloseFD &from, MyArgs &args);
|
||||||
|
|
Loading…
Reference in a new issue