alois31/nix-eval-jobs
0
0
Fork 0
forked from lix-project/nix-eval-jobs
Code Pull requests Activity

Merge pull request #65 from nix-community/treefmt-clang-format

Browse source 
Add treefmt with clang-format, nixpkgs-fmt, prettier (toml) & black (python)
This commit is contained in:
adisbladis 2022-04-26 23:13:49 +07:00 committed by GitHub
parent 9482b5ab3e efca7105c7
commit 796f752cb8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 181 additions and 182 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
.editorconfig
View file

@ -1,24 +0,0 @@
# top-most EditorConfig file
root = true
# Unix-style newlines with a newline ending every file
[*]
charset = utf-8
end_of_line = lf
insert_final_newline = true
trim_trailing_whitespace = true
[*.{cc,hh,hpp,pl,pm,sh,t}]
indent_style = space
intend_size = 4
[Makefile]
indent_style = tab
[*.nix]
indent_style = space
indent_size = 2
# Match diffs, avoid to trim trailing whitespace
[*.{diff,patch}]
trim_trailing_whitespace = false

3
.prettierrc.js Normal file
View file

@ -0,0 +1,3 @@
module.exports = {
"plugins": ["prettier-plugin-toml"],
};

42
flake.nix
View file

@ -8,6 +8,7 @@
flake-utils.lib.eachDefaultSystem (system: flake-utils.lib.eachDefaultSystem (system:
let let
pkgs = nixpkgs.legacyPackages.${system}; pkgs = nixpkgs.legacyPackages.${system};
inherit (pkgs) stdenv;
drvArgs = { srcDir = self; }; drvArgs = { srcDir = self; };
in in
rec { rec {
@ -16,9 +17,7 @@
checks = checks =
let let
mkVariant = nix: (packages.nix-eval-jobs.override { mkVariant = nix: (packages.nix-eval-jobs.override {
# TODO: fix to stable after next nix release inherit nix;
nix = pkgs.nix;
#inherit nix;
}).overrideAttrs (_: { }).overrideAttrs (_: {
name = "nix-eval-jobs-${nix.version}"; name = "nix-eval-jobs-${nix.version}";
inherit (nix) version; inherit (nix) version;
@ -26,32 +25,31 @@
in in
{ {
editorconfig = pkgs.runCommand "editorconfig-check" treefmt =
{ let
nativeBuildInputs = [ devShell = devShells.default;
pkgs.editorconfig-checker in
]; stdenv.mkDerivation {
} '' name = "treefmt-check";
editorconfig-checker ${self} src = self;
touch $out nativeBuildInputs = devShell.nativeBuildInputs;
dontConfigure = true;
inherit (devShell) NODE_PATH;
buildPhase = ''
env HOME=$(mktemp -d) treefmt --fail-on-change
''; '';
nixpkgs-fmt = pkgs.runCommand "fmt-check" installPhase = "touch $out";
{ };
nativeBuildInputs = [
pkgs.nixpkgs-fmt
];
} ''
nixpkgs-fmt --check .
touch $out
'';
build = mkVariant pkgs.nix; build = mkVariant pkgs.nix;
build-unstable = mkVariant pkgs.nixUnstable; build-unstable = mkVariant pkgs.nixUnstable;
}; };
defaultPackage = self.packages.${system}.nix-eval-jobs; packages.default = self.packages.${system}.nix-eval-jobs;
devShell = pkgs.callPackage ./shell.nix drvArgs; devShells.default = pkgs.callPackage ./shell.nix drvArgs;
} }
); );

8
shell.nix
View file

@ -26,16 +26,20 @@ in
nativeBuildInputs = old.nativeBuildInputs ++ [ nativeBuildInputs = old.nativeBuildInputs ++ [
pkgs.editorconfig-checker pkgs.treefmt
pkgs.llvmPackages.clang # clang-format
pkgs.nixpkgs-fmt pkgs.nixpkgs-fmt
pkgs.nodePackages.prettier
(pkgs.python3.withPackages (ps: [ (pkgs.python3.withPackages (ps: [
ps.pytest ps.pytest
ps.black
])) ]))
]; ];
NODE_PATH = "${pkgs.nodePackages.prettier-plugin-toml}/lib/node_modules";
shellHook = lib.optionalString stdenv.isLinux '' shellHook = lib.optionalString stdenv.isLinux ''
export NIX_DEBUG_INFO_DIRS="${pkgs.curl.debug}/lib/debug:${nix.debug}/lib/debug''${NIX_DEBUG_INFO_DIRS:+:$NIX_DEBUG_INFO_DIRS}" export NIX_DEBUG_INFO_DIRS="${pkgs.curl.debug}/lib/debug:${nix.debug}/lib/debug''${NIX_DEBUG_INFO_DIRS:+:$NIX_DEBUG_INFO_DIRS}"
''; '';

156
src/nix-eval-jobs.cc
View file

@ -33,10 +33,12 @@ using namespace nlohmann;
typedef enum { evalAuto, evalImpure, evalPure } pureEval; typedef enum { evalAuto, evalImpure, evalPure } pureEval;
// Safe to ignore - the args will be static. // Safe to ignore - the args will be static.
#ifdef __GNUC__
#pragma GCC diagnostic ignored "-Wnon-virtual-dtor" #pragma GCC diagnostic ignored "-Wnon-virtual-dtor"
#elif __clang__
#pragma clang diagnostic ignored "-Wnon-virtual-dtor" #pragma clang diagnostic ignored "-Wnon-virtual-dtor"
struct MyArgs : MixEvalArgs, MixCommonArgs #endif
{ struct MyArgs : MixEvalArgs, MixCommonArgs {
Path releaseExpr; Path releaseExpr;
Path gcRootsDir; Path gcRootsDir;
bool flake = false; bool flake = false;
@ -46,8 +48,7 @@ struct MyArgs : MixEvalArgs, MixCommonArgs
size_t maxMemorySize = 4096; size_t maxMemorySize = 4096;
pureEval evalMode = evalAuto; pureEval evalMode = evalAuto;
MyArgs() : MixCommonArgs("nix-eval-jobs") MyArgs() : MixCommonArgs("nix-eval-jobs") {
{
addFlag({ addFlag({
.longName = "help", .longName = "help",
.description = "show usage information", .description = "show usage information",
@ -57,7 +58,8 @@ struct MyArgs : MixEvalArgs, MixCommonArgs
if (hiddenCategories.count(flag->category)) { if (hiddenCategories.count(flag->category)) {
continue; continue;
} }
printf(" --%-20s %s\n", name.c_str(), flag->description.c_str()); printf(" --%-20s %s\n", name.c_str(),
flag->description.c_str());
} }
::exit(0); ::exit(0);
}}, }},
@ -66,59 +68,47 @@ struct MyArgs : MixEvalArgs, MixCommonArgs
addFlag({ addFlag({
.longName = "impure", .longName = "impure",
.description = "set evaluation mode", .description = "set evaluation mode",
.handler = {[&]() { .handler = {[&]() { evalMode = evalImpure; }},
evalMode = evalImpure;
}},
}); });
addFlag({ addFlag({.longName = "gc-roots-dir",
.longName = "gc-roots-dir",
.description = "garbage collector roots directory", .description = "garbage collector roots directory",
.labels = {"path"}, .labels = {"path"},
.handler = {&gcRootsDir} .handler = {&gcRootsDir}});
});
addFlag({ addFlag(
.longName = "workers", {.longName = "workers",
.description = "number of evaluate workers", .description = "number of evaluate workers",
.labels = {"workers"}, .labels = {"workers"},
.handler = {[=](std::string s) { .handler = {[=](std::string s) { nrWorkers = std::stoi(s); }}});
nrWorkers = std::stoi(s);
}}
});
addFlag({ addFlag({.longName = "max-memory-size",
.longName = "max-memory-size",
.description = "maximum evaluation memory size", .description = "maximum evaluation memory size",
.labels = {"size"}, .labels = {"size"},
.handler = {[=](std::string s) { .handler = {
maxMemorySize = std::stoi(s); [=](std::string s) { maxMemorySize = std::stoi(s); }}});
}}
});
addFlag({ addFlag({.longName = "flake",
.longName = "flake",
.description = "build a flake", .description = "build a flake",
.handler = {&flake, true} .handler = {&flake, true}});
});
addFlag({ addFlag({.longName = "meta",
.longName = "meta",
.description = "include derivation meta field in output", .description = "include derivation meta field in output",
.handler = {&meta, true} .handler = {&meta, true}});
});
addFlag({ addFlag({.longName = "show-trace",
.longName = "show-trace", .description =
.description = "print out a stack trace in case of evaluation errors", "print out a stack trace in case of evaluation errors",
.handler = {&showTrace, true} .handler = {&showTrace, true}});
});
expectArg("expr", &releaseExpr); expectArg("expr", &releaseExpr);
} }
}; };
#pragma GCC diagnostic warning "-Wnon-virtual-dtor" #ifdef __GNUC__
#pragma clang diagnostic warning "-Wnon-virtual-dtor" #pragma GCC diagnostic ignored "-Wnon-virtual-dtor"
#elif __clang__
#pragma clang diagnostic ignored "-Wnon-virtual-dtor"
#endif
static MyArgs myArgs; static MyArgs myArgs;
@ -137,7 +127,8 @@ static Value* releaseExprTopLevelValue(EvalState & state, Bindings & autoArgs) {
static Value *flakeTopLevelValue(EvalState &state, Bindings &autoArgs) { static Value *flakeTopLevelValue(EvalState &state, Bindings &autoArgs) {
using namespace flake; using namespace flake;
auto [flakeRef, fragment] = parseFlakeRefWithFragment(myArgs.releaseExpr, absPath(".")); auto [flakeRef, fragment] =
parseFlakeRefWithFragment(myArgs.releaseExpr, absPath("."));
auto vFlake = state.allocValue(); auto vFlake = state.allocValue();
@ -169,8 +160,7 @@ static Value* flakeTopLevelValue(EvalState & state, Bindings & autoArgs) {
} }
Value *topLevelValue(EvalState &state, Bindings &autoArgs) { Value *topLevelValue(EvalState &state, Bindings &autoArgs) {
return myArgs.flake return myArgs.flake ? flakeTopLevelValue(state, autoArgs)
? flakeTopLevelValue(state, autoArgs)
: releaseExprTopLevelValue(state, autoArgs); : releaseExprTopLevelValue(state, autoArgs);
} }
@ -191,7 +181,6 @@ struct Drv {
for (auto out : drvInfo.queryOutputs(true)) { for (auto out : drvInfo.queryOutputs(true)) {
if (out.second) if (out.second)
outputs[out.first] = localStore->printStorePath(*out.second); outputs[out.first] = localStore->printStorePath(*out.second);
} }
if (myArgs.meta) { if (myArgs.meta) {
@ -230,7 +219,6 @@ static void to_json(nlohmann::json & json, const Drv & drv) {
if (drv.meta.has_value()) if (drv.meta.has_value())
json["meta"] = drv.meta.value(); json["meta"] = drv.meta.value();
} }
std::string attrPathJoin(json input) { std::string attrPathJoin(json input) {
@ -244,12 +232,8 @@ std::string attrPathJoin(json input) {
}); });
} }
static void worker( static void worker(EvalState &state, Bindings &autoArgs, AutoCloseFD &to,
EvalState & state, AutoCloseFD &from) {
Bindings & autoArgs,
AutoCloseFD & to,
AutoCloseFD & from)
{
auto vRoot = topLevelValue(state, autoArgs); auto vRoot = topLevelValue(state, autoArgs);
while (true) { while (true) {
@ -257,8 +241,10 @@ static void worker(
writeLine(to.get(), "next"); writeLine(to.get(), "next");
auto s = readLine(from.get()); auto s = readLine(from.get());
if (s == "exit") break; if (s == "exit")
if (!hasPrefix(s, "do ")) abort(); break;
if (!hasPrefix(s, "do "))
abort();
auto path = json::parse(s.substr(3)); auto path = json::parse(s.substr(3));
auto attrPathS = attrPathJoin(path); auto attrPathS = attrPathJoin(path);
@ -267,7 +253,8 @@ static void worker(
/* Evaluate it and send info back to the collector. */ /* Evaluate it and send info back to the collector. */
json reply = json{{"attr", attrPathS}, {"attrPath", path}}; json reply = json{{"attr", attrPathS}, {"attrPath", path}};
try { try {
auto vTmp = findAlongAttrPath(state, attrPathS, autoArgs, *vRoot).first; auto vTmp =
findAlongAttrPath(state, attrPathS, autoArgs, *vRoot).first;
auto v = state.allocValue(); auto v = state.allocValue();
state.autoCallFunction(autoArgs, *vTmp, *v); state.autoCallFunction(autoArgs, *vTmp, *v);
@ -281,27 +268,32 @@ static void worker(
registers roots for jobs that we may have already registers roots for jobs that we may have already
done. */ done. */
if (myArgs.gcRootsDir != "") { if (myArgs.gcRootsDir != "") {
Path root = myArgs.gcRootsDir + "/" + std::string(baseNameOf(drv.drvPath)); Path root = myArgs.gcRootsDir + "/" +
std::string(baseNameOf(drv.drvPath));
if (!pathExists(root)) { if (!pathExists(root)) {
auto localStore = state.store.dynamic_pointer_cast<LocalFSStore>(); auto localStore =
auto storePath = localStore->parseStorePath(drv.drvPath); state.store.dynamic_pointer_cast<LocalFSStore>();
auto storePath =
localStore->parseStorePath(drv.drvPath);
localStore->addPermRoot(storePath, root); localStore->addPermRoot(storePath, root);
} }
} }
} }
else if (v->type() == nAttrs) else if (v->type() == nAttrs) {
{
auto attrs = nlohmann::json::array(); auto attrs = nlohmann::json::array();
bool recurse = path.size() == 0; // Dont require `recurseForDerivations = true;` for top-level attrset bool recurse =
path.size() == 0; // Dont require `recurseForDerivations =
// true;` for top-level attrset
for (auto &i : v->attrs->lexicographicOrder()) { for (auto &i : v->attrs->lexicographicOrder()) {
std::string name(i->name); std::string name(i->name);
attrs.push_back(name); attrs.push_back(name);
if (name == "recurseForDerivations") { if (name == "recurseForDerivations") {
auto attrv = v->attrs->get(state.sRecurseForDerivations); auto attrv =
v->attrs->get(state.sRecurseForDerivations);
recurse = state.forceBool(*attrv->value, *attrv->pos); recurse = state.forceBool(*attrv->value, *attrv->pos);
} }
} }
@ -314,7 +306,9 @@ static void worker(
else if (v->type() == nNull) else if (v->type() == nNull)
; ;
else throw TypeError("attribute '%s' is %s, which is not supported", path, showType(*v)); else
throw TypeError("attribute '%s' is %s, which is not supported",
path, showType(*v));
} catch (EvalError &e) { } catch (EvalError &e) {
auto err = e.info(); auto err = e.info();
@ -337,13 +331,15 @@ static void worker(
start a new process. */ start a new process. */
struct rusage r; struct rusage r;
getrusage(RUSAGE_SELF, &r); getrusage(RUSAGE_SELF, &r);
if ((size_t) r.ru_maxrss > myArgs.maxMemorySize * 1024) break; if ((size_t)r.ru_maxrss > myArgs.maxMemorySize * 1024)
break;
} }
writeLine(to.get(), "restart"); writeLine(to.get(), "restart");
} }
typedef std::function<void(EvalState & state, Bindings & autoArgs, AutoCloseFD & to, AutoCloseFD & from)> typedef std::function<void(EvalState &state, Bindings &autoArgs,
AutoCloseFD &to, AutoCloseFD &from)>
Processor; Processor;
/* Auto-cleanup of fork's process and fds. */ /* Auto-cleanup of fork's process and fds. */
@ -358,9 +354,8 @@ struct Proc {
auto p = startProcess( auto p = startProcess(
[&, [&,
to{std::make_shared<AutoCloseFD>(std::move(fromPipe.writeSide))}, to{std::make_shared<AutoCloseFD>(std::move(fromPipe.writeSide))},
from{std::make_shared<AutoCloseFD>(std::move(toPipe.readSide))} from{
]() std::make_shared<AutoCloseFD>(std::move(toPipe.readSide))}]() {
{
debug("created worker process %d", getpid()); debug("created worker process %d", getpid());
try { try {
EvalState state(myArgs.searchPath, openStore()); EvalState state(myArgs.searchPath, openStore());
@ -387,22 +382,21 @@ struct Proc {
~Proc() {} ~Proc() {}
}; };
struct State struct State {
{
std::set<json> todo = json::array({json::array()}); std::set<json> todo = json::array({json::array()});
std::set<json> active; std::set<json> active;
std::exception_ptr exc; std::exception_ptr exc;
}; };
std::function<void()> collector(Sync<State> & state_, std::condition_variable & wakeup) { std::function<void()> collector(Sync<State> &state_,
std::condition_variable &wakeup) {
return [&]() { return [&]() {
try { try {
std::optional<std::unique_ptr<Proc>> proc_; std::optional<std::unique_ptr<Proc>> proc_;
while (true) { while (true) {
auto proc = proc_.has_value() auto proc = proc_.has_value() ? std::move(proc_.value())
? std::move(proc_.value())
: std::make_unique<Proc>(worker); : std::make_unique<Proc>(worker);
/* Check whether the existing worker process is still there. */ /* Check whether the existing worker process is still there. */
@ -421,7 +415,8 @@ std::function<void()> collector(Sync<State> & state_, std::condition_variable &
while (true) { while (true) {
checkInterrupt(); checkInterrupt();
auto state(state_.lock()); auto state(state_.lock());
if ((state->todo.empty() && state->active.empty()) || state->exc) { if ((state->todo.empty() && state->active.empty()) ||
state->exc) {
writeLine(proc->to.get(), "exit"); writeLine(proc->to.get(), "exit");
return; return;
} }
@ -474,8 +469,7 @@ std::function<void()> collector(Sync<State> & state_, std::condition_variable &
}; };
} }
int main(int argc, char * * argv) int main(int argc, char **argv) {
{
/* Prevent undeclared dependencies in the evaluation via /* Prevent undeclared dependencies in the evaluation via
$NIX_PATH. */ $NIX_PATH. */
unsetenv("NIX_PATH"); unsetenv("NIX_PATH");
@ -489,7 +483,8 @@ int main(int argc, char * * argv)
myArgs.parseCmdline(argvToStrings(argc, argv)); myArgs.parseCmdline(argvToStrings(argc, argv));
/* FIXME: The build hook in conjunction with import-from-derivation is causing "unexpected EOF" during eval */ /* FIXME: The build hook in conjunction with import-from-derivation is
* causing "unexpected EOF" during eval */
settings.builders = ""; settings.builders = "";
/* Prevent access to paths outside of the Nix search path and /* Prevent access to paths outside of the Nix search path and
@ -498,11 +493,15 @@ int main(int argc, char * * argv)
/* When building a flake, use pure evaluation (no access to /* When building a flake, use pure evaluation (no access to
'getEnv', 'currentSystem' etc. */ 'getEnv', 'currentSystem' etc. */
evalSettings.pureEval = myArgs.evalMode == evalAuto ? myArgs.flake : myArgs.evalMode == evalPure; evalSettings.pureEval = myArgs.evalMode == evalAuto
? myArgs.flake
: myArgs.evalMode == evalPure;
if (myArgs.releaseExpr == "") throw UsageError("no expression specified"); if (myArgs.releaseExpr == "")
throw UsageError("no expression specified");
if (myArgs.gcRootsDir == "") printMsg(lvlError, "warning: `--gc-roots-dir' not specified"); if (myArgs.gcRootsDir == "")
printMsg(lvlError, "warning: `--gc-roots-dir' not specified");
if (myArgs.showTrace) { if (myArgs.showTrace) {
loggerSettings.showTrace.assign(true); loggerSettings.showTrace.assign(true);
@ -523,6 +522,5 @@ int main(int argc, char * * argv)
if (state->exc) if (state->exc)
std::rethrow_exception(state->exc); std::rethrow_exception(state->exc);
}); });
} }

3
tests/assets/ci.nix
View file

@ -1,5 +1,4 @@
{ { pkgs ? import (builtins.getFlake (toString ./.)).inputs.nixpkgs { }
pkgs ? import (builtins.getFlake (toString ./.)).inputs.nixpkgs { }
, system ? pkgs.system , system ? pkgs.system
}: }:

6
tests/test_eval.py
View file

@ -30,10 +30,10 @@ def common_test(extra_args: List[str]) -> None:
assert built_job["name"] == "job1" assert built_job["name"] == "job1"
assert built_job["outputs"]["out"].startswith("/nix/store") assert built_job["outputs"]["out"].startswith("/nix/store")
assert built_job["drvPath"].endswith(".drv") assert built_job["drvPath"].endswith(".drv")
assert built_job["meta"]['broken'] is False assert built_job["meta"]["broken"] is False
dotted_job = results[1] dotted_job = results[1]
assert dotted_job["attr"] == "\"dotted.attr\"" assert dotted_job["attr"] == '"dotted.attr"'
assert dotted_job["attrPath"] == ["dotted.attr"] assert dotted_job["attrPath"] == ["dotted.attr"]
recurse_drv = results[2] recurse_drv = results[2]
@ -46,7 +46,7 @@ def common_test(extra_args: List[str]) -> None:
substituted_job = results[4] substituted_job = results[4]
assert substituted_job["attr"] == "substitutedJob" assert substituted_job["attr"] == "substitutedJob"
assert substituted_job["name"].startswith("hello-") assert substituted_job["name"].startswith("hello-")
assert substituted_job["meta"]['broken'] is False assert substituted_job["meta"]["broken"] is False
def test_flake() -> None: def test_flake() -> None:

21
treefmt.toml Normal file
View file

@ -0,0 +1,21 @@
[formatter."c++"]
command = "clang-format"
options = [
"-i",
"-style",
"{BasedOnStyle: llvm, IndentWidth: 4, SortIncludes: false}"
]
includes = ["*.c", "*.cpp", "*.cc", "*.h", "*.hpp"]
[formatter.nix]
command = "nixpkgs-fmt"
includes = ["*.nix"]
[formatter.toml]
command = "prettier"
options = ["--write"]
includes = ["*.toml"]
[formatter.python]
command = "black"
includes = ["*.py"]