Compare commits

...

23 commits

Author SHA1 Message Date
jade df3edf3730 Merge pull request 'Fix compilation for latest Lix main' (#19) from jade/fix-compilation into main
Reviewed-on: lix-project/nix-eval-jobs#19
2024-12-06 21:54:01 +00:00
jade bdbe3e1304 Fix compilation for latest Lix main 2024-12-06 13:50:43 -08:00
Maximilian Bosch dfc286ca3d
flake.lock: Update
Flake lock file updates:

• Updated input 'lix':
    'c859d03013.tar.gz?narHash=sha256-bq21I1EjXJa/s5Rra9J9ot2NkPCnI0F5uNPurwYLdpE%3D&rev=c859d03013712b349d82ee6223948d6d03e63a8d' (2024-11-15)
  → '66f6dbda32.tar.gz?narHash=sha256-H7GN4%2B%2Ba4vE49SUNojZx%2BFSk4mmpb2ifJUtJMJHProI%3D&rev=66f6dbda32959dd5cf3a9aaba15af72d037ab7ff' (2024-11-20)
• Updated input 'nix-github-actions':
    'github:nix-community/nix-github-actions/e04df33f62cdcf93d73e9a04142464753a16db67' (2024-10-24)
  → 'github:nix-community/nix-github-actions/7b5f051df789b6b20d259924d349a9ba3319b226' (2024-11-18)
• Updated input 'nixpkgs':
    'github:NixOS/nixpkgs/035d434d48f4375ac5d3a620954cf5fda7dd7c36' (2024-11-15)
  → 'github:NixOS/nixpkgs/df94f897ffe1af1bcd60cb68697c5d8e6431346e' (2024-11-22)
• Updated input 'treefmt-nix':
    'github:numtide/treefmt-nix/746901bb8dba96d154b66492a29f5db0693dbfcc' (2024-10-30)
  → 'github:numtide/treefmt-nix/705df92694af7093dfbb27109ce16d828a79155f' (2024-11-22)
2024-11-23 09:47:15 +01:00
leo60228 912a9d6331
Merge branch 'alois31-fix-build' 2024-11-17 19:49:28 -05:00
alois31 bbbebaf380
fix build with latest Lix
Commit 8088927b90ff84dd37c342e1ef0a91bc8feca6ec renamed initGC to initLibExpr.
Use the new name so that the build works again.
2024-11-16 20:14:39 +01:00
alois31 b9e0abe9e6
flake.lock: Update
Flake lock file updates:

• Updated input 'flake-parts':
    'github:hercules-ci/flake-parts/3d04084d54bedc3d6b8b736c70ef449225c361b1' (2024-10-01)
  → 'github:hercules-ci/flake-parts/506278e768c2a08bec68eb62932193e341f55c90' (2024-11-01)
• Updated input 'lix':
    '2734a9cf94.tar.gz?narHash=sha256-XME7TzBvjK6GEmZqPLK%2B2%2BWk0qnwc7DCwYH434hMcOM%3D&rev=2734a9cf94debc6baef4e7d4d9fa28cc28f5b31d' (2024-10-23)
  → 'c859d03013.tar.gz?narHash=sha256-bq21I1EjXJa/s5Rra9J9ot2NkPCnI0F5uNPurwYLdpE%3D&rev=c859d03013712b349d82ee6223948d6d03e63a8d' (2024-11-15)
• Updated input 'nixpkgs':
    'github:NixOS/nixpkgs/45e5197248e59e92e88956c5aa12553a7f62337f' (2024-10-25)
  → 'github:NixOS/nixpkgs/035d434d48f4375ac5d3a620954cf5fda7dd7c36' (2024-11-15)
• Updated input 'treefmt-nix':
    'github:numtide/treefmt-nix/aac86347fb5063960eccb19493e0cadcdb4205ca' (2024-10-22)
  → 'github:numtide/treefmt-nix/746901bb8dba96d154b66492a29f5db0693dbfcc' (2024-10-30)
2024-11-16 19:56:09 +01:00
jade 57ddb99e78 Merge pull request 'update and fix build with latest Lix' (#14) from fixes/build-settings-opeq into main
Reviewed-on: lix-project/nix-eval-jobs#14
2024-11-09 20:50:43 +00:00
jade 9160d40f80 Address feedback from alois31
- Remove restrict-eval stuff that did nothing
- Remove builders stuff that appears unnecessary:

/* FIXME: The build hook in conjunction with import-from-derivation is
 * causing "unexpected EOF" during eval */
settings.builders.setDefault("");

We removed that line and then observed that it works, so idk:

ifdtest.nix:

let
  ifd = builtins.derivation {
    name = "wat2";
    builder = "/bin/sh";
    args = [ "-c" "echo meow > $out" ];
    system = "aarch64-linux";
  };
in
  builtins.readFile ifd

 » NIX_CONFIG="builders = @/etc/nix/machines" build/src/nix-eval-jobs ifdtest.nix
warning: unknown setting 'trusted-users'
warning: `--gc-roots-dir' not specified
building '/nix/store/xxnd5rb49n3anyla5v71lgdk0wmhmijp-wat2.drv' on 'ssh-ng://root@voracle.jade.fyi'...
copying 0 paths...
building '/nix/store/xxnd5rb49n3anyla5v71lgdk0wmhmijp-wat2.drv'...
copying 1 paths...
copying path '/nix/store/h2yxq9lb7l0nd9plgqrcgf7nvsg67gl7-wat2' from 'ssh-ng://root@voracle.jade.fyi'...

- Changed the impure/flake code to override the pureEval setting, which
  it was definitely *supposed* to be doing in the first place.
2024-11-09 12:46:21 -08:00
Qyriad 43aaa943bf fix build with latest Lix
Lix commit 4dbbd721e[1] changed the way settings are changed, removing
operator= in the process. This commit changes the places where we use
operator= to using either setDefault(), or override(). I *believe* I
have used the correct ones for each changed setting.

Fixes #13.

[1]: 4dbbd721eb9db75d4968a624b8cb9e75e979a144
2024-10-25 14:29:57 -06:00
Qyriad 50a1455953 flake.lock: Update
Flake lock file updates:

• Updated input 'flake-parts':
    'github:hercules-ci/flake-parts/8471fe90ad337a8074e957b69ca4d0089218391d' (2024-08-01)
  → 'github:hercules-ci/flake-parts/3d04084d54bedc3d6b8b736c70ef449225c361b1' (2024-10-01)
• Updated input 'lix':
    'b016eb0895.tar.gz?narHash=sha256-kOpGI9WPmte1L4QWHviuXsr8jxmGn27zwi82jtzYObM%3D&rev=b016eb0895bb6714a4f6530d9a2bb6577ac6c3cf' (2024-08-13)
  → '2734a9cf94.tar.gz?narHash=sha256-XME7TzBvjK6GEmZqPLK%2B2%2BWk0qnwc7DCwYH434hMcOM%3D&rev=2734a9cf94debc6baef4e7d4d9fa28cc28f5b31d' (2024-10-23)
• Updated input 'lix/nix2container':
    'github:nlewo/nix2container/3853e5caf9ad24103b13aa6e0e8bcebb47649fe4' (2024-07-10)
  → 'github:nlewo/nix2container/fa6bb0a1159f55d071ba99331355955ae30b3401' (2024-08-30)
• Updated input 'lix/pre-commit-hooks':
    'github:cachix/git-hooks.nix/f451c19376071a90d8c58ab1a953c6e9840527fd' (2024-07-15)
  → 'github:cachix/git-hooks.nix/4e743a6920eab45e8ba0fbe49dc459f1423a4b74' (2024-09-19)
• Updated input 'nix-github-actions':
    'github:nix-community/nix-github-actions/622f829f5fe69310a866c8a6cd07e747c44ef820' (2024-07-04)
  → 'github:nix-community/nix-github-actions/e04df33f62cdcf93d73e9a04142464753a16db67' (2024-10-24)
• Updated input 'nixpkgs':
    'github:NixOS/nixpkgs/fb81cec9eda2a6b5365ad723995f0329d9e356fd' (2024-08-13)
  → 'github:NixOS/nixpkgs/45e5197248e59e92e88956c5aa12553a7f62337f' (2024-10-25)
• Updated input 'treefmt-nix':
    'github:numtide/treefmt-nix/349de7bc435bdff37785c2466f054ed1766173be' (2024-08-12)
  → 'github:numtide/treefmt-nix/aac86347fb5063960eccb19493e0cadcdb4205ca' (2024-10-22)
2024-10-25 12:48:08 -06:00
Yureka 42a160bce2 flake.lock: Update
Flake lock file updates:

• Updated input 'flake-parts':
    'github:hercules-ci/flake-parts/9227223f6d922fee3c7b190b2cc238a99527bbb7' (2024-07-03)
  → 'github:hercules-ci/flake-parts/8471fe90ad337a8074e957b69ca4d0089218391d' (2024-08-01)
• Updated input 'lix':
    'ef0de7c79f.tar.gz?narHash=sha256-GUH5%2BB1JztzDNSN1D7KbndrYSq0LWvVIJnuWKHlpN3Q%3D' (2024-07-16)
  → 'b016eb0895.tar.gz?narHash=sha256-kOpGI9WPmte1L4QWHviuXsr8jxmGn27zwi82jtzYObM%3D&rev=b016eb0895bb6714a4f6530d9a2bb6577ac6c3cf' (2024-08-13)
• Updated input 'lix/nix2container':
    'github:nlewo/nix2container/20aad300c925639d5d6cbe30013c8357ce9f2a2e' (2024-04-13)
  → 'github:nlewo/nix2container/3853e5caf9ad24103b13aa6e0e8bcebb47649fe4' (2024-07-10)
• Updated input 'lix/pre-commit-hooks':
    'github:cachix/git-hooks.nix/e35aed5fda3cc79f88ed7f1795021e559582093a' (2024-04-02)
  → 'github:cachix/git-hooks.nix/f451c19376071a90d8c58ab1a953c6e9840527fd' (2024-07-15)
• Updated input 'nixpkgs':
    'github:NixOS/nixpkgs/732b4f3a3afdfe6a6c4fcb2511e529588d4e5ccd' (2024-07-15)
  → 'github:NixOS/nixpkgs/fb81cec9eda2a6b5365ad723995f0329d9e356fd' (2024-08-13)
• Updated input 'treefmt-nix':
    'github:numtide/treefmt-nix/0fb28f237f83295b4dd05e342f333b447c097398' (2024-07-15)
  → 'github:numtide/treefmt-nix/349de7bc435bdff37785c2466f054ed1766173be' (2024-08-12)
2024-08-13 22:00:51 +02:00
Pierre Bourdon c057494450
flake.lock: Update
Flake lock file updates:

• Updated input 'flake-parts':
    'github:hercules-ci/flake-parts/2a55567fcf15b1b1c7ed712a2c6fadaec7412ea8' (2024-06-01)
  → 'github:hercules-ci/flake-parts/9227223f6d922fee3c7b190b2cc238a99527bbb7' (2024-07-03)
• Updated input 'lix':
    'f170870ae7.tar.gz?narHash=sha256-GK6PusfbMgkg%2BqdgChmrw78KTNQkm7SDoJ6%2BlJKY6vg%3D' (2024-06-25)
  → 'ef0de7c79f.tar.gz?narHash=sha256-GUH5%2BB1JztzDNSN1D7KbndrYSq0LWvVIJnuWKHlpN3Q%3D' (2024-07-16)
• Updated input 'nix-github-actions':
    'github:nix-community/nix-github-actions/5163432afc817cf8bd1f031418d1869e4c9d5547' (2023-12-29)
  → 'github:nix-community/nix-github-actions/622f829f5fe69310a866c8a6cd07e747c44ef820' (2024-07-04)
• Updated input 'nixpkgs':
    'github:NixOS/nixpkgs/d9e18354acbf59c625505b7315c85508e9831bf4' (2024-06-18)
  → 'github:NixOS/nixpkgs/732b4f3a3afdfe6a6c4fcb2511e529588d4e5ccd' (2024-07-15)
• Updated input 'treefmt-nix':
    'github:numtide/treefmt-nix/1cb529bffa880746a1d0ec4e0f5076876af931f1' (2024-06-11)
  → 'github:numtide/treefmt-nix/0fb28f237f83295b4dd05e342f333b447c097398' (2024-07-15)
2024-07-17 07:57:52 +02:00
Pierre Bourdon d9a46559a4
drv: backport CA derivations support changes from hydra-eval-jobs
It is not possible to query output paths for CA derivations since
they're not static / known at eval time. Instead, return JSON nulls for
outputs paths.

This is a partial port of the following Hydra commits:

- 9ba4417940ffdd0fadea43f68c61ef948a4b8d39
- 069b7775c565f5999fe33e8c3f28c7b9306039ca
- fcde5908d8e51f975b883329b34d24a9f30ea4b3

By the following authors:

Co-Authored-By: John Ericson <John.Ericson@Obsidian.Systems>
Co-Authored-By: Théophane Hufschmitt <theophane.hufschmitt@tweag.io>
Co-Authored-By: Alexander Sosedkin <monk@unboiled.info>
Co-Authored-By: Andrea Ciceri <andrea.ciceri@autistici.org>
Co-Authored-By: Charlotte 🦝 Delenk Mlotte@chir.rs>
Co-Authored-By: Sandro Jäckel <sandro.jaeckel@gmail.com>
2024-07-16 09:15:21 +02:00
eldritch horrors f8869bdcca update for lix 2.91-dev
the api of nix::Pid changed, causing a build failure.
2024-06-25 23:57:51 +02:00
jade fd86a1a068 Fix on latest lix
Fixes: lix-project/nix-eval-jobs#9
2024-06-18 23:08:04 -07:00
jade 9c23772cf2 Tidy and make it work on release-2.90 2024-06-18 22:31:42 -07:00
puck 11d467fecd Use our own Thread struct instead of std::thread
We'd highly prefer using std::thread here; but this won't let us configure the stack
size. macOS uses 512KiB size stacks for non-main threads, and musl defaults to 128k.
While Nix configures a 64MiB size for the main thread, this doesn't propagate to the
threads we launch here. It turns out, running the evaluator under an anemic stack of
0.5MiB has it overflow way too quickly. Hence, we have our own custom Thread struct.
2024-06-12 22:39:53 +00:00
puck 040db2fe26 flake.nix: nixpkgs-unstable -> nixos-23.11-unstable; flake update
This makes nix-eval-jobs build again.
2024-06-12 22:39:29 +00:00
jade ee79f8a236 Merge pull request 'Fixup readme and version to say what this is' (#6) from jade/fixup-readme-and-version into main
Reviewed-on: lix-project/nix-eval-jobs#6
2024-06-12 21:06:56 +00:00
jade ed7a959ae1 update lix pin to actually work 2024-05-30 12:50:13 -07:00
jade d05f8e86a9 Merge pull request 'lix: deal with util.hh removal' (#7) from jade/util-hh-removal into main
Reviewed-on: lix-project/nix-eval-jobs#7
2024-05-30 02:33:13 +00:00
jade b67c46d320 lix: deal with util.hh removal 2024-05-29 19:17:54 -07:00
jade 30cf61fd26 fix: don't crash on startup on macOS
This is caused, through several layers of absurdity, by runtime type
information of Nix things being invisible due to -fvisibility=hidden
inside n-e-j.

We don't have any idea why n-e-j has -fvisibility=hidden, since blame
says it's from the initial commit. It is plausible that it was some
ill-advised optimization but it's not sound.

The crash is caused by dynamic_cast<RootArgs *>(MyArgs *) failing, which
is in turn caused by the RTTI being invisible.

See: https://www.qt.io/blog/quality-assurance/one-way-dynamic_cast-across-library-boundaries-can-fail-and-how-to-fix-it
Fixes: lix-project/nix-eval-jobs#2
2024-05-27 18:23:13 -06:00
12 changed files with 169 additions and 89 deletions

View file

@ -23,11 +23,11 @@
]
},
"locked": {
"lastModified": 1701473968,
"narHash": "sha256-YcVE5emp1qQ8ieHUnxt1wCZCC3ZfAS+SRRWZ2TMda7E=",
"lastModified": 1730504689,
"narHash": "sha256-hgmguH29K2fvs9szpq2r3pz2/8cJd2LPS+b4tfNFCwE=",
"owner": "hercules-ci",
"repo": "flake-parts",
"rev": "34fed993f1674c8d06d58b37ce1e0fe5eebcb9f5",
"rev": "506278e768c2a08bec68eb62932193e341f55c90",
"type": "github"
},
"original": {
@ -39,6 +39,7 @@
"lix": {
"inputs": {
"flake-compat": "flake-compat",
"nix2container": "nix2container",
"nixpkgs": [
"nixpkgs"
],
@ -46,18 +47,15 @@
"pre-commit-hooks": "pre-commit-hooks"
},
"locked": {
"lastModified": 1714955862,
"narHash": "sha256-REWlo2RYHfJkxnmZTEJu3Cd/2VM+wjjpPy7Xi4BdDTQ=",
"ref": "refs/tags/2.90-beta.1",
"rev": "b6799ab0374a8e1907a48915d3187e07da41d88c",
"revCount": 15501,
"type": "git",
"url": "https://git@git.lix.systems/lix-project/lix"
"lastModified": 1733448312,
"narHash": "sha256-id5U81bzXk/Lg900nGLM4CQb0wmTdzIvQz7CZk2OcTM=",
"rev": "2a9e560570982a91937d199af3e7b7a8f3cbe14b",
"type": "tarball",
"url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/2a9e560570982a91937d199af3e7b7a8f3cbe14b.tar.gz?rev=2a9e560570982a91937d199af3e7b7a8f3cbe14b"
},
"original": {
"ref": "refs/tags/2.90-beta.1",
"type": "git",
"url": "https://git@git.lix.systems/lix-project/lix"
"type": "tarball",
"url": "https://git.lix.systems/lix-project/lix/archive/main.tar.gz"
}
},
"nix-github-actions": {
@ -67,11 +65,11 @@
]
},
"locked": {
"lastModified": 1701208414,
"narHash": "sha256-xrQ0FyhwTZK6BwKhahIkUVZhMNk21IEI1nUcWSONtpo=",
"lastModified": 1731952509,
"narHash": "sha256-p4gB3Rhw8R6Ak4eMl8pqjCPOLCZRqaehZxdZ/mbFClM=",
"owner": "nix-community",
"repo": "nix-github-actions",
"rev": "93e39cc1a087d65bcf7a132e75a650c44dd2b734",
"rev": "7b5f051df789b6b20d259924d349a9ba3319b226",
"type": "github"
},
"original": {
@ -80,18 +78,34 @@
"type": "github"
}
},
"nix2container": {
"flake": false,
"locked": {
"lastModified": 1724996935,
"narHash": "sha256-njRK9vvZ1JJsP8oV2OgkBrpJhgQezI03S7gzskCcHos=",
"owner": "nlewo",
"repo": "nix2container",
"rev": "fa6bb0a1159f55d071ba99331355955ae30b3401",
"type": "github"
},
"original": {
"owner": "nlewo",
"repo": "nix2container",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1703134684,
"narHash": "sha256-SQmng1EnBFLzS7WSRyPM9HgmZP2kLJcPAz+Ug/nug6o=",
"lastModified": 1733408989,
"narHash": "sha256-VCQpCQy+6ik+oYKWUCvq0WM2V7UtEKldqdsEzCNEOLc=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "d6863cbcbbb80e71cecfc03356db1cda38919523",
"rev": "d916387b68a74040a3873ad2a08a559c60cedb5e",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixpkgs-unstable",
"ref": "nixos-24.05-small",
"repo": "nixpkgs",
"type": "github"
}
@ -115,11 +129,11 @@
"pre-commit-hooks": {
"flake": false,
"locked": {
"lastModified": 1712055707,
"narHash": "sha256-4XLvuSIDZJGS17xEwSrNuJLL7UjDYKGJSbK1WWX2AK8=",
"lastModified": 1726745158,
"narHash": "sha256-D5AegvGoEjt4rkKedmxlSEmC+nNLMBPWFxvmYnVLhjk=",
"owner": "cachix",
"repo": "git-hooks.nix",
"rev": "e35aed5fda3cc79f88ed7f1795021e559582093a",
"rev": "4e743a6920eab45e8ba0fbe49dc459f1423a4b74",
"type": "github"
},
"original": {
@ -144,11 +158,11 @@
]
},
"locked": {
"lastModified": 1702979157,
"narHash": "sha256-RnFBbLbpqtn4AoJGXKevQMCGhra4h6G2MPcuTSZZQ+g=",
"lastModified": 1732292307,
"narHash": "sha256-5WSng844vXt8uytT5djmqBCkopyle6ciFgteuA9bJpw=",
"owner": "numtide",
"repo": "treefmt-nix",
"rev": "2961375283668d867e64129c22af532de8e77734",
"rev": "705df92694af7093dfbb27109ce16d828a79155f",
"type": "github"
},
"original": {

View file

@ -1,7 +1,7 @@
{
description = "Hydra's builtin hydra-eval-jobs as a standalone";
inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05-small";
inputs.flake-parts.url = "github:hercules-ci/flake-parts";
inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs";
inputs.treefmt-nix.url = "github:numtide/treefmt-nix";
@ -9,7 +9,7 @@
inputs.nix-github-actions.url = "github:nix-community/nix-github-actions";
inputs.nix-github-actions.inputs.nixpkgs.follows = "nixpkgs";
inputs.lix = {
url = "git+https://git@git.lix.systems/lix-project/lix?ref=refs/tags/2.90-beta.1";
url = "https://git.lix.systems/lix-project/lix/archive/main.tar.gz";
inputs.nixpkgs.follows = "nixpkgs";
};

View file

@ -1,6 +1,11 @@
project('nix-eval-jobs', 'cpp',
version : '0.1.6',
license : 'GPL-3.0',
default_options : [
'debug=true',
'optimization=2',
'cpp_std=c++20',
],
)
nix_main_dep = dependency('lix-main', required: true)

View file

@ -23,6 +23,7 @@ in
pkgs.mkShell {
inherit (nix-eval-jobs) buildInputs;
nativeBuildInputs = nix-eval-jobs.nativeBuildInputs ++ [
pkgs.clang-tools
(pkgs.python3.withPackages (ps: [
ps.pytest
]))

View file

@ -23,14 +23,17 @@
#include "drv.hh"
#include "eval-args.hh"
static bool queryIsCached(nix::Store &store,
std::map<std::string, std::string> &outputs) {
static bool
queryIsCached(nix::Store &store,
std::map<std::string, std::optional<std::string>> &outputs) {
uint64_t downloadSize, narSize;
nix::StorePathSet willBuild, willSubstitute, unknown;
std::vector<nix::StorePathWithOutputs> paths;
for (auto const &[key, val] : outputs) {
paths.push_back(followLinksToStorePathWithOutputs(store, val));
if (val) {
paths.push_back(followLinksToStorePathWithOutputs(store, *val));
}
}
store.queryMissing(toDerivedPaths(paths), willBuild, willSubstitute,
@ -42,26 +45,37 @@ static bool queryIsCached(nix::Store &store,
Drv::Drv(std::string &attrPath, nix::EvalState &state, nix::DrvInfo &drvInfo,
MyArgs &args) {
auto localStore = state.store.dynamic_pointer_cast<nix::LocalFSStore>();
auto localStore = state.ctx.store.dynamic_pointer_cast<nix::LocalFSStore>();
try {
for (auto out : drvInfo.queryOutputs(true)) {
if (out.second)
outputs[out.first] = localStore->printStorePath(*out.second);
// CA derivations do not have static output paths, so we have to
// defensively not query output paths in case we encounter one.
for (auto &[outputName, optOutputPath] :
drvInfo.queryOutputs(state, !nix::experimentalFeatureSettings.isEnabled(
nix::Xp::CaDerivations))) {
if (optOutputPath) {
outputs[outputName] =
localStore->printStorePath(*optOutputPath);
} else {
assert(nix::experimentalFeatureSettings.isEnabled(
nix::Xp::CaDerivations));
outputs[outputName] = std::nullopt;
}
}
} catch (const std::exception &e) {
throw nix::EvalError(state,
state.ctx.errors.make<nix::EvalError>(
"derivation '%s' does not have valid outputs: %s",
attrPath, e.what());
attrPath, e.what()
).debugThrow();
}
if (args.meta) {
nlohmann::json meta_;
for (auto &metaName : drvInfo.queryMetaNames()) {
for (auto &metaName : drvInfo.queryMetaNames(state)) {
nix::NixStringContext context;
std::stringstream ss;
auto metaValue = drvInfo.queryMeta(metaName);
auto metaValue = drvInfo.queryMeta(state, metaName);
// Skip non-serialisable types
// TODO: Fix serialisation of derivations to store paths
if (metaValue == 0) {
@ -83,9 +97,9 @@ Drv::Drv(std::string &attrPath, nix::EvalState &state, nix::DrvInfo &drvInfo,
cacheStatus = Drv::CacheStatus::Unknown;
}
drvPath = localStore->printStorePath(drvInfo.requireDrvPath());
drvPath = localStore->printStorePath(drvInfo.requireDrvPath(state));
auto drv = localStore->readDerivation(drvInfo.requireDrvPath());
auto drv = localStore->readDerivation(drvInfo.requireDrvPath(state));
for (const auto &[inputDrvPath, inputNode] : drv.inputDrvs.map) {
std::set<std::string> inputDrvOutputs;
for (auto &outputName : inputNode.value) {
@ -93,15 +107,21 @@ Drv::Drv(std::string &attrPath, nix::EvalState &state, nix::DrvInfo &drvInfo,
}
inputDrvs[localStore->printStorePath(inputDrvPath)] = inputDrvOutputs;
}
name = drvInfo.queryName();
name = drvInfo.queryName(state);
system = drv.platform;
}
void to_json(nlohmann::json &json, const Drv &drv) {
std::map<std::string, nlohmann::json> outputsJson;
for (auto &[name, optPath] : drv.outputs) {
outputsJson[name] =
optPath ? nlohmann::json(*optPath) : nlohmann::json(nullptr);
}
json = nlohmann::json{{"name", drv.name},
{"system", drv.system},
{"drvPath", drv.drvPath},
{"outputs", drv.outputs},
{"outputs", outputsJson},
{"inputDrvs", drv.inputDrvs}};
if (drv.meta.has_value()) {

View file

@ -24,7 +24,7 @@ struct Drv {
std::string drvPath;
enum class CacheStatus { Cached, Uncached, Unknown } cacheStatus;
std::map<std::string, std::string> outputs;
std::map<std::string, std::optional<std::string>> outputs;
std::map<std::string, std::set<std::string>> inputDrvs;
std::optional<nlohmann::json> meta;

View file

@ -100,5 +100,5 @@ MyArgs::MyArgs() : MixCommonArgs("nix-eval-jobs") {
}
void MyArgs::parseArgs(char **argv, int argc) {
parseCmdline(nix::argvToStrings(argc, argv));
parseCmdline(nix::Strings(argv + 1, argv + argc));
}

View file

@ -12,7 +12,7 @@
class MyArgs : virtual public nix::MixEvalArgs,
virtual public nix::MixCommonArgs,
virtual nix::RootArgs {
virtual public nix::RootArgs {
public:
std::string releaseExpr;
nix::Path gcRootsDir;

View file

@ -1,10 +1,10 @@
src = [
src = files(
'nix-eval-jobs.cc',
'eval-args.cc',
'drv.cc',
'buffered-io.cc',
'worker.cc'
]
'worker.cc',
)
cc = meson.get_compiler('cpp')
@ -31,4 +31,4 @@ executable('nix-eval-jobs', src,
threads_dep
],
install: true,
cpp_args: ['-std=c++2a', '-fvisibility=hidden', '--include', 'autotools-config.h'])
cpp_args: ['--include', 'autotools-config.h'])

View file

@ -8,6 +8,7 @@
#include <sys/wait.h>
#include <nlohmann/json.hpp>
#include <errno.h>
#include <pthread.h>
#include <signal.h>
#include <stdlib.h>
#include <string.h>
@ -17,13 +18,12 @@
#include <lix/libutil/error.hh>
#include <lix/libstore/globals.hh>
#include <lix/libutil/logging.hh>
#include <lix/libutil/terminal.hh>
#include <nlohmann/detail/iterators/iter_impl.hpp>
#include <nlohmann/detail/json_ref.hpp>
#include <nlohmann/json_fwd.hpp>
#include <lix/libutil/ref.hh>
#include <lix/libstore/store-api.hh>
#include <map>
#include <thread>
#include <condition_variable>
#include <filesystem>
#include <exception>
@ -46,9 +46,8 @@ using namespace nlohmann;
static MyArgs myArgs;
typedef std::function<void(ref<EvalState> state, Bindings &autoArgs,
AutoCloseFD &to, AutoCloseFD &from, MyArgs &args)>
Processor;
using Processor = std::function<void(ref<nix::eval_cache::CachingEvaluator> state, Bindings &autoArgs,
AutoCloseFD &to, AutoCloseFD &from, MyArgs &args)>;
/* Auto-cleanup of fork's process and fds. */
struct Proc {
@ -69,10 +68,10 @@ struct Proc {
auto evalStore = myArgs.evalStoreUrl
? openStore(*myArgs.evalStoreUrl)
: openStore();
auto state = std::make_shared<EvalState>(myArgs.searchPath,
auto evaluator = nix::make_ref<nix::eval_cache::CachingEvaluator>(myArgs.searchPath,
evalStore);
Bindings &autoArgs = *myArgs.getAutoArgs(*state);
proc(ref<EvalState>(state), autoArgs, *to, *from, myArgs);
Bindings &autoArgs = *myArgs.getAutoArgs(*evaluator);
proc(evaluator, autoArgs, *to, *from, myArgs);
} catch (Error &e) {
nlohmann::json err;
auto msg = e.msg();
@ -92,12 +91,60 @@ struct Proc {
to = std::move(toPipe.writeSide);
from = std::move(fromPipe.readSide);
pid = p;
pid = std::move(p);
}
~Proc() {}
};
// We'd highly prefer using std::thread here; but this won't let us configure the stack
// size. macOS uses 512KiB size stacks for non-main threads, and musl defaults to 128k.
// While Nix configures a 64MiB size for the main thread, this doesn't propagate to the
// threads we launch here. It turns out, running the evaluator under an anemic stack of
// 0.5MiB has it overflow way too quickly. Hence, we have our own custom Thread struct.
struct Thread {
pthread_t thread;
Thread(const Thread &) = delete;
Thread(Thread &&) noexcept = default;
Thread(std::function<void(void)> f) {
int s;
pthread_attr_t attr;
auto func = std::make_unique<std::function<void(void)>>(std::move(f));
if ((s = pthread_attr_init(&attr)) != 0) {
throw SysError(s, "calling pthread_attr_init");
}
if ((s = pthread_attr_setstacksize(&attr, 64 * 1024 * 1024)) != 0) {
throw SysError(s, "calling pthread_attr_setstacksize");
}
if ((s = pthread_create(&thread, &attr, Thread::init, func.release())) != 0) {
throw SysError(s, "calling pthread_launch");
}
if ((s = pthread_attr_destroy(&attr)) != 0) {
throw SysError(s, "calling pthread_attr_destroy");
}
}
void join() {
int s;
s = pthread_join(thread, nullptr);
if (s != 0) {
throw SysError(s, "calling pthread_join");
}
}
private:
static void *init(void *ptr) {
std::unique_ptr<std::function<void(void)>> func;
func.reset(static_cast<std::function<void(void)> *>(ptr));
(*func)();
return 0;
}
};
struct State {
std::set<json> todo = json::array({json::array()});
std::set<json> active;
@ -295,24 +342,16 @@ int main(int argc, char **argv) {
return handleExceptions(argv[0], [&]() {
initNix();
initGC();
initLibExpr();
myArgs.parseArgs(argv, argc);
/* FIXME: The build hook in conjunction with import-from-derivation is
* causing "unexpected EOF" during eval */
settings.builders = "";
/* Prevent access to paths outside of the Nix search path and
to the environment. */
evalSettings.restrictEval = false;
/* When building a flake, use pure evaluation (no access to
'getEnv', 'currentSystem' etc. */
if (myArgs.impure) {
evalSettings.pureEval = false;
evalSettings.pureEval.override(false);
} else if (myArgs.flake) {
evalSettings.pureEval = true;
evalSettings.pureEval.override(true);
}
if (myArgs.releaseExpr == "")
@ -325,16 +364,16 @@ int main(int argc, char **argv) {
}
if (myArgs.showTrace) {
loggerSettings.showTrace.assign(true);
loggerSettings.showTrace.override(true);
}
Sync<State> state_;
/* Start a collector thread per worker process. */
std::vector<std::thread> threads;
std::vector<Thread> threads;
std::condition_variable wakeup;
for (size_t i = 0; i < myArgs.nrWorkers; i++) {
threads.emplace_back(collector, std::ref(state_), std::ref(wakeup));
threads.emplace_back(std::bind(collector, std::ref(state_), std::ref(wakeup)));
}
for (auto &thread : threads)

View file

@ -14,6 +14,7 @@
#include <lix/libutil/canon-path.hh>
#include <lix/libcmd/common-eval-args.hh>
#include <lix/libutil/error.hh>
#include <lix/libexpr/eval-cache.hh>
#include <lix/libexpr/eval-inline.hh>
#include <lix/libexpr/eval.hh>
#include <lix/libexpr/flake/flakeref.hh>
@ -27,18 +28,15 @@
#include <lix/libstore/store-api.hh>
#include <lix/libexpr/symbol-table.hh>
#include <lix/libutil/types.hh>
#include <lix/libutil/util.hh>
#include <lix/libexpr/value.hh>
#include <lix/libutil/terminal.hh>
#include <exception>
#include <map>
#include <memory>
#include <numeric>
#include <optional>
#include <sstream>
#include <string>
#include <string_view>
#include <utility>
#include <vector>
#include "worker.hh"
#include "drv.hh"
@ -51,14 +49,14 @@ static nix::Value *releaseExprTopLevelValue(nix::EvalState &state,
nix::Value vTop;
if (args.fromArgs) {
nix::Expr *e = state.parseExprFromString(
args.releaseExpr, state.rootPath(nix::CanonPath::fromCwd()));
nix::Expr &e = state.ctx.parseExprFromString(
args.releaseExpr, nix::CanonPath::fromCwd());
state.eval(e, vTop);
} else {
state.evalFile(lookupFileArg(state, args.releaseExpr), vTop);
state.evalFile(nix::lookupFileArg(state.ctx, args.releaseExpr), vTop);
}
auto vRoot = state.allocValue();
auto vRoot = state.ctx.mem.allocValue();
state.autoCallFunction(autoArgs, vTop, *vRoot);
@ -76,16 +74,17 @@ static std::string attrPathJoin(nlohmann::json input) {
});
}
void worker(nix::ref<nix::EvalState> state, nix::Bindings &autoArgs,
void worker(nix::ref<nix::eval_cache::CachingEvaluator> evaluator, nix::Bindings &autoArgs,
nix::AutoCloseFD &to, nix::AutoCloseFD &from, MyArgs &args) {
nix::Value *vRoot = [&]() {
auto state = evaluator->begin();
if (args.flake) {
auto [flakeRef, fragment, outputSpec] =
nix::parseFlakeRefWithFragmentAndExtendedOutputsSpec(
args.releaseExpr, nix::absPath("."));
nix::InstallableFlake flake{
{}, state, std::move(flakeRef), fragment, outputSpec,
{}, evaluator, std::move(flakeRef), fragment, outputSpec,
{}, {}, args.lockFlags};
return flake.toValue(*state).first;
@ -95,6 +94,7 @@ void worker(nix::ref<nix::EvalState> state, nix::Bindings &autoArgs,
}();
LineReader fromReader(from.release());
auto state = evaluator->begin();
while (true) {
/* Wait for the collector to send us a job name. */
@ -122,7 +122,7 @@ void worker(nix::ref<nix::EvalState> state, nix::Bindings &autoArgs,
nix::findAlongAttrPath(*state, attrPathS, autoArgs, *vRoot)
.first;
auto v = state->allocValue();
auto v = evaluator->mem.allocValue();
state->autoCallFunction(autoArgs, *vTmp, *v);
if (v->type() == nix::nAttrs) {
@ -139,7 +139,7 @@ void worker(nix::ref<nix::EvalState> state, nix::Bindings &autoArgs,
std::string(nix::baseNameOf(drv.drvPath));
if (!nix::pathExists(root)) {
auto localStore =
state->store
evaluator->store
.dynamic_pointer_cast<nix::LocalFSStore>();
auto storePath =
localStore->parseStorePath(drv.drvPath);
@ -154,14 +154,14 @@ void worker(nix::ref<nix::EvalState> state, nix::Bindings &autoArgs,
// = true;` for top-level attrset
for (auto &i :
v->attrs->lexicographicOrder(state->symbols)) {
const std::string &name = state->symbols[i->name];
v->attrs->lexicographicOrder(evaluator->symbols)) {
const std::string &name = evaluator->symbols[i->name];
attrs.push_back(name);
if (name == "recurseForDerivations" &&
!args.forceRecurse) {
auto attrv =
v->attrs->get(state->sRecurseForDerivations);
v->attrs->get(evaluator->s.recurseForDerivations);
recurse = state->forceBool(
*attrv->value, attrv->pos,
"while evaluating recurseForDerivations");

View file

@ -1,6 +1,7 @@
#pragma once
#include <lix/libmain/shared.hh>
#include <lix/libexpr/eval.hh>
#include <lix/libexpr/eval-cache.hh>
#include "eval-args.hh"
@ -13,5 +14,5 @@ class EvalState;
template <typename T> class ref;
} // namespace nix
void worker(nix::ref<nix::EvalState> state, nix::Bindings &autoArgs,
void worker(nix::ref<nix::eval_cache::CachingEvaluator> evaluator, nix::Bindings &autoArgs,
nix::AutoCloseFD &to, nix::AutoCloseFD &from, MyArgs &args);