Merge branch '2.18-maintenance' into ifd-buildStore-2.18

This commit is contained in:
Shea Levy 2024-01-11 07:21:16 -05:00
commit ba48ab4b95
No known key found for this signature in database
GPG key ID: 5C0BD6957D86FE27
685 changed files with 680 additions and 841 deletions

2
.github/labeler.yml vendored
View file

@ -20,4 +20,4 @@
# Unit tests # Unit tests
- src/*/tests/**/* - src/*/tests/**/*
# Functional and integration tests # Functional and integration tests
- tests/**/* - tests/functional/**/*

40
.gitignore vendored
View file

@ -41,14 +41,14 @@ perl/Makefile.config
/src/libexpr/parser-tab.hh /src/libexpr/parser-tab.hh
/src/libexpr/parser-tab.output /src/libexpr/parser-tab.output
/src/libexpr/nix.tbl /src/libexpr/nix.tbl
/src/libexpr/tests/libnixexpr-tests /tests/unit/libexpr/libnixexpr-tests
# /src/libstore/ # /src/libstore/
*.gen.* *.gen.*
/src/libstore/tests/libnixstore-tests /tests/unit/libstore/libnixstore-tests
# /src/libutil/ # /src/libutil/
/src/libutil/tests/libnixutil-tests /tests/unit/libutil/libnixutil-tests
/src/nix/nix /src/nix/nix
@ -79,24 +79,24 @@ perl/Makefile.config
/src/build-remote/build-remote /src/build-remote/build-remote
# /tests/ # /tests/functional/
/tests/test-tmp /tests/functional/test-tmp
/tests/common/vars-and-functions.sh /tests/functional/common/vars-and-functions.sh
/tests/result* /tests/functional/result*
/tests/restricted-innocent /tests/functional/restricted-innocent
/tests/shell /tests/functional/shell
/tests/shell.drv /tests/functional/shell.drv
/tests/config.nix /tests/functional/config.nix
/tests/ca/config.nix /tests/functional/ca/config.nix
/tests/dyn-drv/config.nix /tests/functional/dyn-drv/config.nix
/tests/repl-result-out /tests/functional/repl-result-out
/tests/test-libstoreconsumer/test-libstoreconsumer /tests/functional/test-libstoreconsumer/test-libstoreconsumer
# /tests/lang/ # /tests/functional/lang/
/tests/lang/*.out /tests/functional/lang/*.out
/tests/lang/*.out.xml /tests/functional/lang/*.out.xml
/tests/lang/*.err /tests/functional/lang/*.err
/tests/lang/*.ast /tests/functional/lang/*.ast
/perl/lib/Nix/Config.pm /perl/lib/Nix/Config.pm
/perl/lib/Nix/Store.cc /perl/lib/Nix/Store.cc

View file

@ -1 +1 @@
2.18.0 2.18.1

View file

@ -52,7 +52,7 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy).
- [ ] Fixes an [idea approved](https://github.com/NixOS/nix/labels/idea%20approved) issue - [ ] Fixes an [idea approved](https://github.com/NixOS/nix/labels/idea%20approved) issue
- [ ] Tests, as appropriate: - [ ] Tests, as appropriate:
- Functional tests [`tests/**.sh`](./tests) - Functional tests [`tests/functional/**.sh`](./tests/functional)
- Unit tests [`src/*/tests`](./src/) - Unit tests [`src/*/tests`](./src/)
- Integration tests [`tests/nixos/*`](./tests/nixos) - Integration tests [`tests/nixos/*`](./tests/nixos)
- [ ] User documentation in the [manual](..doc/manual/src) - [ ] User documentation in the [manual](..doc/manual/src)

View file

@ -23,14 +23,17 @@ makefiles = \
ifeq ($(tests), yes) ifeq ($(tests), yes)
makefiles += \ makefiles += \
src/libutil/tests/local.mk \ tests/unit/libutil/local.mk \
src/libstore/tests/local.mk \ tests/unit/libutil-support/local.mk \
src/libexpr/tests/local.mk \ tests/unit/libstore/local.mk \
tests/local.mk \ tests/unit/libstore-support/local.mk \
tests/ca/local.mk \ tests/unit/libexpr/local.mk \
tests/dyn-drv/local.mk \ tests/unit/libexpr-support/local.mk \
tests/test-libstoreconsumer/local.mk \ tests/functional/local.mk \
tests/plugins/local.mk tests/functional/ca/local.mk \
tests/functional/dyn-drv/local.mk \
tests/functional/test-libstoreconsumer/local.mk \
tests/functional/plugins/local.mk
else else
makefiles += \ makefiles += \
mk/disable-tests.mk mk/disable-tests.mk

View file

@ -39,17 +39,21 @@ INPUT = \
src/libcmd \ src/libcmd \
src/libexpr \ src/libexpr \
src/libexpr/flake \ src/libexpr/flake \
src/libexpr/tests \ tests/unit/libexpr \
src/libexpr/tests/value \ tests/unit/libexpr/value \
tests/unit/libexpr/test \
tests/unit/libexpr/test/value \
src/libexpr/value \ src/libexpr/value \
src/libfetchers \ src/libfetchers \
src/libmain \ src/libmain \
src/libstore \ src/libstore \
src/libstore/build \ src/libstore/build \
src/libstore/builtins \ src/libstore/builtins \
src/libstore/tests \ tests/unit/libstore \
tests/unit/libstore/test \
src/libutil \ src/libutil \
src/libutil/tests \ tests/unit/libutil \
tests/unit/libutil/test \
src/nix \ src/nix \
src/nix-env \ src/nix-env \
src/nix-store src/nix-store

30
doc/manual/_redirects Normal file
View file

@ -0,0 +1,30 @@
# redirect rules for paths (server-side) to prevent link rot.
# see ./redirects.js for redirects based on URL fragments (client-side)
#
# concrete user story this supports:
# - user finds URL to the manual for Nix x.y
# - Nix x.z (z > y) is the most recent release
# - updating the version in the URL will show the right thing
#
# format documentation:
# - https://docs.netlify.com/routing/redirects/#syntax-for-the-redirects-file
# - https://docs.netlify.com/routing/redirects/redirect-options/
#
# conventions:
# - always force (<CODE>!) since this allows re-using file names
# - group related paths to ease readability
# - always append new redirects to the end of the file
# - redirects that should have been there but are missing can be inserted where they belong
/expressions/expression-language /language/ 301!
/expressions/language-values /language/values 301!
/expressions/language-constructs /language/constructs 301!
/expressions/language-operators /language/operators 301!
/expressions/* /language/:splat 301!
/package-management/basic-package-mgmt /command-ref/nix-env 301!
/package-management/channels* /command-ref/nix-channel 301!
/package-management/s3-substituter* /command-ref/new-cli/nix3-help-stores#s3-binary-cache-store 301!

View file

@ -5,7 +5,7 @@ let
inherit (import ./utils.nix) concatStrings optionalString filterAttrs trim squash unique showSettings; inherit (import ./utils.nix) concatStrings optionalString filterAttrs trim squash unique showSettings;
in in
commandDump: inlineHTML: commandDump:
let let
@ -30,7 +30,7 @@ let
${maybeSubcommands} ${maybeSubcommands}
${maybeDocumentation} ${maybeStoreDocs}
${maybeOptions} ${maybeOptions}
''; '';
@ -70,7 +70,7 @@ let
* [`${command} ${name}`](./${appendName filename name}.md) - ${subcmd.description} * [`${command} ${name}`](./${appendName filename name}.md) - ${subcmd.description}
''; '';
maybeDocumentation = optionalString maybeStoreDocs = optionalString
(details ? doc) (details ? doc)
(replaceStrings ["@stores@"] [storeDocs] details.doc); (replaceStrings ["@stores@"] [storeDocs] details.doc);
@ -91,17 +91,20 @@ let
listOptions = opts: concatStringsSep "\n" (attrValues (mapAttrs showOption opts)); listOptions = opts: concatStringsSep "\n" (attrValues (mapAttrs showOption opts));
showOption = name: option: showOption = name: option:
let let
result = trim ''
- ${item}
${option.description}
'';
item = if inlineHTML
then ''<span id="opt-${name}">[`--${name}`](#opt-${name})</span> ${shortName} ${labels}''
else "`--${name}` ${shortName} ${labels}";
shortName = optionalString shortName = optionalString
(option ? shortName) (option ? shortName)
("/ `-${option.shortName}`"); ("/ `-${option.shortName}`");
labels = optionalString labels = optionalString
(option ? labels) (option ? labels)
(concatStringsSep " " (map (s: "*${s}*") option.labels)); (concatStringsSep " " (map (s: "*${s}*") option.labels));
in trim '' in result;
- <span id="opt-${name}">[`--${name}`](#opt-${name})</span> ${shortName} ${labels}
${option.description}
'';
categories = sort lessThan (unique (map (cmd: cmd.category) (attrValues allOptions))); categories = sort lessThan (unique (map (cmd: cmd.category) (attrValues allOptions)));
in concatStrings (map showCategory categories); in concatStrings (map showCategory categories);
in squash result; in squash result;
@ -162,7 +165,7 @@ let
**Settings**: **Settings**:
${showSettings { useAnchors = false; } settings} ${showSettings { inherit inlineHTML; } settings}
''; '';
in concatStrings (attrValues (mapAttrs showStore commandInfo.stores)); in concatStrings (attrValues (mapAttrs showStore commandInfo.stores));

View file

@ -98,12 +98,12 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/sr
$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(bindir)/nix $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(bindir)/nix
@rm -rf $@ $@.tmp @rm -rf $@ $@.tmp
$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix (builtins.readFile $<)' $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix true (builtins.readFile $<)'
@mv $@.tmp $@ @mv $@.tmp $@
$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix
@cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp
$(trace-gen) $(nix-eval) --expr '(import doc/manual/utils.nix).showSettings { useAnchors = true; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; $(trace-gen) $(nix-eval) --expr '(import doc/manual/utils.nix).showSettings { inlineHTML = true; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
@mv $@.tmp $@ @mv $@.tmp $@
$(d)/nix.json: $(bindir)/nix $(d)/nix.json: $(bindir)/nix

View file

@ -1,7 +1,9 @@
// redirect rules for anchors ensure backwards compatibility of URLs. // redirect rules for URL fragments (client-side) to prevent link rot.
// this must be done on the client side, as web servers do not see the anchor part of the URL. // this must be done on the client side, as web servers do not see the fragment part of the URL.
// it will only work with JavaScript enabled in the browser, but this is the best we can do here.
// see ./_redirects for path redirects (client-side)
// redirections are declared as follows: // redirects are declared as follows:
// each entry has as its key a path matching the requested URL path, relative to the mdBook document root. // each entry has as its key a path matching the requested URL path, relative to the mdBook document root.
// //
// IMPORTANT: it must specify the full path with file name and suffix // IMPORTANT: it must specify the full path with file name and suffix
@ -19,6 +21,7 @@ const redirects = {
"chap-distributed-builds": "advanced-topics/distributed-builds.html", "chap-distributed-builds": "advanced-topics/distributed-builds.html",
"chap-post-build-hook": "advanced-topics/post-build-hook.html", "chap-post-build-hook": "advanced-topics/post-build-hook.html",
"chap-post-build-hook-caveats": "advanced-topics/post-build-hook.html#implementation-caveats", "chap-post-build-hook-caveats": "advanced-topics/post-build-hook.html#implementation-caveats",
"chap-writing-nix-expressions": "language/index.html",
"part-command-ref": "command-ref/command-ref.html", "part-command-ref": "command-ref/command-ref.html",
"conf-allow-import-from-derivation": "command-ref/conf-file.html#conf-allow-import-from-derivation", "conf-allow-import-from-derivation": "command-ref/conf-file.html#conf-allow-import-from-derivation",
"conf-allow-new-privileges": "command-ref/conf-file.html#conf-allow-new-privileges", "conf-allow-new-privileges": "command-ref/conf-file.html#conf-allow-new-privileges",

View file

@ -109,7 +109,6 @@
- [CLI guideline](contributing/cli-guideline.md) - [CLI guideline](contributing/cli-guideline.md)
- [C++ style guide](contributing/cxx.md) - [C++ style guide](contributing/cxx.md)
- [Release Notes](release-notes/release-notes.md) - [Release Notes](release-notes/release-notes.md)
- [Release X.Y (202?-??-??)](release-notes/rl-next.md)
- [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md) - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md)
- [Release 2.17 (2023-07-24)](release-notes/rl-2.17.md) - [Release 2.17 (2023-07-24)](release-notes/rl-2.17.md)
- [Release 2.16 (2023-05-31)](release-notes/rl-2.16.md) - [Release 2.16 (2023-05-31)](release-notes/rl-2.16.md)

View file

@ -3,16 +3,28 @@
## Unit-tests ## Unit-tests
The unit-tests for each Nix library (`libexpr`, `libstore`, etc..) are defined The unit-tests for each Nix library (`libexpr`, `libstore`, etc..) are defined
under `src/{library_name}/tests` using the under `tests/unit/{library_name}/tests` using the
[googletest](https://google.github.io/googletest/) and [googletest](https://google.github.io/googletest/) and
[rapidcheck](https://github.com/emil-e/rapidcheck) frameworks. [rapidcheck](https://github.com/emil-e/rapidcheck) frameworks.
You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`.
Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option, or the `GTEST_FILTER` environment variable. Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option, or the `GTEST_FILTER` environment variable.
### Unit test support libraries
There are headers and code which are not just used to test the library in question, but also downstream libraries.
For example, we do [property testing] with the [rapidcheck] library.
This requires writing `Arbitrary` "instances", which are used to describe how to generate values of a given type for the sake of running property tests.
Because types contain other types, `Arbitrary` "instances" for some type are not just useful for testing that type, but also any other type that contains it.
Downstream types frequently contain upstream types, so it is very important that we share arbitrary instances so that downstream libraries' property tests can also use them.
It is important that these testing libraries don't contain any actual tests themselves.
On some platforms they would be run as part of every test executable that uses them, which is redundant.
On other platforms they wouldn't be run at all.
## Functional tests ## Functional tests
The functional tests reside under the `tests` directory and are listed in `tests/local.mk`. The functional tests reside under the `tests/functional` directory and are listed in `tests/functional/local.mk`.
Each test is a bash script. Each test is a bash script.
### Running the whole test suite ### Running the whole test suite
@ -21,8 +33,8 @@ The whole test suite can be run with:
```shell-session ```shell-session
$ make install && make installcheck $ make install && make installcheck
ran test tests/foo.sh... [PASS] ran test tests/functional/foo.sh... [PASS]
ran test tests/bar.sh... [PASS] ran test tests/functional/bar.sh... [PASS]
... ...
``` ```
@ -30,14 +42,14 @@ ran test tests/bar.sh... [PASS]
Sometimes it is useful to group related tests so they can be easily run together without running the entire test suite. Sometimes it is useful to group related tests so they can be easily run together without running the entire test suite.
Each test group is in a subdirectory of `tests`. Each test group is in a subdirectory of `tests`.
For example, `tests/ca/local.mk` defines a `ca` test group for content-addressed derivation outputs. For example, `tests/functional/ca/local.mk` defines a `ca` test group for content-addressed derivation outputs.
That test group can be run like this: That test group can be run like this:
```shell-session ```shell-session
$ make ca.test-group -j50 $ make ca.test-group -j50
ran test tests/ca/nix-run.sh... [PASS] ran test tests/functional/ca/nix-run.sh... [PASS]
ran test tests/ca/import-derivation.sh... [PASS] ran test tests/functional/ca/import-derivation.sh... [PASS]
... ...
``` ```
@ -56,21 +68,21 @@ install-tests-groups += $(test-group-name)
Individual tests can be run with `make`: Individual tests can be run with `make`:
```shell-session ```shell-session
$ make tests/${testName}.sh.test $ make tests/functional/${testName}.sh.test
ran test tests/${testName}.sh... [PASS] ran test tests/functional/${testName}.sh... [PASS]
``` ```
or without `make`: or without `make`:
```shell-session ```shell-session
$ ./mk/run-test.sh tests/${testName}.sh $ ./mk/run-test.sh tests/functional/${testName}.sh
ran test tests/${testName}.sh... [PASS] ran test tests/functional/${testName}.sh... [PASS]
``` ```
To see the complete output, one can also run: To see the complete output, one can also run:
```shell-session ```shell-session
$ ./mk/debug-test.sh tests/${testName}.sh $ ./mk/debug-test.sh tests/functional/${testName}.sh
+ foo + foo
output from foo output from foo
+ bar + bar
@ -105,7 +117,7 @@ edit it like so:
Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point: Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point:
```shell-session ```shell-session
$ ./mk/debug-test.sh tests/${testName}.sh $ ./mk/debug-test.sh tests/functional/${testName}.sh
... ...
+ gdb blash blub + gdb blash blub
GNU gdb (GDB) 12.1 GNU gdb (GDB) 12.1
@ -124,9 +136,11 @@ This technique is to include the exact output/behavior of a former version of Ni
For example, this technique is used for the language tests, to check both the printed final value if evaluation was successful, and any errors and warnings encountered. For example, this technique is used for the language tests, to check both the printed final value if evaluation was successful, and any errors and warnings encountered.
It is frequently useful to regenerate the expected output. It is frequently useful to regenerate the expected output.
To do that, rerun the failed test with `_NIX_TEST_ACCEPT=1`. To do that, rerun the failed test(s) with `_NIX_TEST_ACCEPT=1`.
(At least, this is the convention we've used for `tests/lang.sh`. For example:
If we add more characterization testing we should always strive to be consistent.) ```bash
_NIX_TEST_ACCEPT=1 make tests/functional/lang.sh.test
```
An interesting situation to document is the case when these tests are "overfitted". An interesting situation to document is the case when these tests are "overfitted".
The language tests are, again, an example of this. The language tests are, again, an example of this.

View file

@ -44,10 +44,10 @@ rec {
optionalString = cond: string: if cond then string else ""; optionalString = cond: string: if cond then string else "";
showSetting = { useAnchors }: name: { description, documentDefault, defaultValue, aliases, value, experimentalFeature }: showSetting = { inlineHTML }: name: { description, documentDefault, defaultValue, aliases, value, experimentalFeature }:
let let
result = squash '' result = squash ''
- ${if useAnchors - ${if inlineHTML
then ''<span id="conf-${name}">[`${name}`](#conf-${name})</span>'' then ''<span id="conf-${name}">[`${name}`](#conf-${name})</span>''
else ''`${name}`''} else ''`${name}`''}

View file

@ -34,16 +34,16 @@
}, },
"nixpkgs": { "nixpkgs": {
"locked": { "locked": {
"lastModified": 1695124524, "lastModified": 1700748986,
"narHash": "sha256-trXDytVCqf3KryQQQrHOZKUabu1/lB8/ndOAuZKQrOE=", "narHash": "sha256-/nqLrNU297h3PCw4QyDpZKZEUHmialJdZW2ceYFobds=",
"owner": "edolstra", "owner": "NixOS",
"repo": "nixpkgs", "repo": "nixpkgs",
"rev": "a3d30b525535e3158221abc1a957ce798ab159fe", "rev": "9ba29e2346bc542e9909d1021e8fd7d4b3f64db0",
"type": "github" "type": "github"
}, },
"original": { "original": {
"owner": "edolstra", "owner": "NixOS",
"ref": "fix-aws-sdk-cpp", "ref": "nixos-23.05-small",
"repo": "nixpkgs", "repo": "nixpkgs",
"type": "github" "type": "github"
} }

View file

@ -1,8 +1,7 @@
{ {
description = "The purely functional package manager"; description = "The purely functional package manager";
#inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small";
inputs.nixpkgs.url = "github:edolstra/nixpkgs/fix-aws-sdk-cpp";
inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; };
inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; };
@ -12,7 +11,7 @@
let let
inherit (nixpkgs) lib; inherit (nixpkgs) lib;
officialRelease = false; officialRelease = true;
version = lib.fileContents ./.version + versionSuffix; version = lib.fileContents ./.version + versionSuffix;
versionSuffix = versionSuffix =
@ -59,35 +58,28 @@
nixSrc = fileset.toSource { nixSrc = fileset.toSource {
root = ./.; root = ./.;
fileset = fileset.intersect baseFiles ( fileset = fileset.intersect baseFiles (fileset.unions [
fileset.difference ./.version
(fileset.unions [ ./boehmgc-coroutine-sp-fallback.diff
./.version ./bootstrap.sh
./boehmgc-coroutine-sp-fallback.diff ./configure.ac
./bootstrap.sh ./doc
./configure.ac ./local.mk
./doc ./m4
./local.mk ./Makefile
./m4 ./Makefile.config.in
./Makefile ./misc
./Makefile.config.in ./mk
./misc ./precompiled-headers.h
./mk ./src
./precompiled-headers.h ./tests/functional
./src ./tests/unit
./tests ./COPYING
./COPYING ./scripts/local.mk
./scripts/local.mk (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts)
(fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) # TODO: do we really need README.md? It doesn't seem used in the build.
# TODO: do we really need README.md? It doesn't seem used in the build. ./README.md
./README.md ]);
])
(fileset.unions [
# Removed file sets
./tests/nixos
./tests/installer
])
);
}; };
# Memoize nixpkgs for different platforms for efficiency. # Memoize nixpkgs for different platforms for efficiency.

View file

@ -1,11 +1,15 @@
test_dir=tests/functional
test=$(echo -n "$test" | sed -e "s|^$test_dir/||")
TESTS_ENVIRONMENT=("TEST_NAME=${test%.*}" 'NIX_REMOTE=') TESTS_ENVIRONMENT=("TEST_NAME=${test%.*}" 'NIX_REMOTE=')
: ${BASH:=/usr/bin/env bash} : ${BASH:=/usr/bin/env bash}
init_test () { init_test () {
cd tests && env "${TESTS_ENVIRONMENT[@]}" $BASH -e init.sh 2>/dev/null > /dev/null cd "$test_dir" && env "${TESTS_ENVIRONMENT[@]}" $BASH -e init.sh 2>/dev/null > /dev/null
} }
run_test_proper () { run_test_proper () {
cd $(dirname $test) && env "${TESTS_ENVIRONMENT[@]}" $BASH -e $(basename $test) cd "$test_dir/$(dirname $test)" && env "${TESTS_ENVIRONMENT[@]}" $BASH -e $(basename $test)
} }

View file

@ -1536,7 +1536,9 @@ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args,
auto path = realisePath(state, pos, arg, { .checkForPureEval = false }); auto path = realisePath(state, pos, arg, { .checkForPureEval = false });
/* SourcePath doesn't know about trailing slash. */ /* SourcePath doesn't know about trailing slash. */
auto mustBeDir = arg.type() == nString && arg.str().ends_with("/"); auto mustBeDir = arg.type() == nString
&& (arg.str().ends_with("/")
|| arg.str().ends_with("/."));
try { try {
auto checked = state.checkSourcePath(path); auto checked = state.checkSourcePath(path);

View file

@ -395,7 +395,8 @@ static RegisterPrimOp primop_fetchGit({
- `shallow` (default: `false`) - `shallow` (default: `false`)
A Boolean parameter that specifies whether fetching a shallow clone is allowed. A Boolean parameter that specifies whether fetching from a shallow remote repository is allowed.
This still performs a full clone of what is available on the remote.
- `allRefs` - `allRefs`

View file

@ -1,19 +0,0 @@
check: libexpr-tests_RUN
programs += libexpr-tests
libexpr-tests_NAME := libnixexpr-tests
libexpr-tests_DIR := $(d)
libexpr-tests_INSTALL_DIR :=
libexpr-tests_SOURCES := \
$(wildcard $(d)/*.cc) \
$(wildcard $(d)/value/*.cc)
libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests -I src/libfetchers
libexpr-tests_LIBS = libstore-tests libutils-tests libexpr libutil libstore libfetchers
libexpr-tests_LDFLAGS := $(GTEST_LIBS) -lgmock

View file

@ -1,157 +0,0 @@
#include "create-derivation-and-realise-goal.hh"
#include "worker.hh"
namespace nix {
CreateDerivationAndRealiseGoal::CreateDerivationAndRealiseGoal(ref<SingleDerivedPath> drvReq,
const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode)
: Goal(worker, DerivedPath::Built { .drvPath = drvReq, .outputs = wantedOutputs })
, drvReq(drvReq)
, wantedOutputs(wantedOutputs)
, buildMode(buildMode)
{
state = &CreateDerivationAndRealiseGoal::getDerivation;
name = fmt(
"outer obtaining drv from '%s' and then building outputs %s",
drvReq->to_string(worker.store),
std::visit(overloaded {
[&](const OutputsSpec::All) -> std::string {
return "* (all of them)";
},
[&](const OutputsSpec::Names os) {
return concatStringsSep(", ", quoteStrings(os));
},
}, wantedOutputs.raw));
trace("created outer");
worker.updateProgress();
}
CreateDerivationAndRealiseGoal::~CreateDerivationAndRealiseGoal()
{
}
static StorePath pathPartOfReq(const SingleDerivedPath & req)
{
return std::visit(overloaded {
[&](const SingleDerivedPath::Opaque & bo) {
return bo.path;
},
[&](const SingleDerivedPath::Built & bfd) {
return pathPartOfReq(*bfd.drvPath);
},
}, req.raw());
}
std::string CreateDerivationAndRealiseGoal::key()
{
/* Ensure that derivations get built in order of their name,
i.e. a derivation named "aardvark" always comes before "baboon". And
substitution goals and inner derivation goals always happen before
derivation goals (due to "b$"). */
return "c$" + std::string(pathPartOfReq(*drvReq).name()) + "$" + drvReq->to_string(worker.store);
}
void CreateDerivationAndRealiseGoal::timedOut(Error && ex)
{
}
void CreateDerivationAndRealiseGoal::work()
{
(this->*state)();
}
void CreateDerivationAndRealiseGoal::addWantedOutputs(const OutputsSpec & outputs)
{
/* If we already want all outputs, there is nothing to do. */
auto newWanted = wantedOutputs.union_(outputs);
bool needRestart = !newWanted.isSubsetOf(wantedOutputs);
wantedOutputs = newWanted;
if (!needRestart) return;
if (!optDrvPath)
// haven't started steps where the outputs matter yet
return;
worker.makeDerivationGoal(*optDrvPath, outputs, buildMode);
}
void CreateDerivationAndRealiseGoal::getDerivation()
{
trace("outer init");
/* The first thing to do is to make sure that the derivation
exists. If it doesn't, it may be created through a
substitute. */
if (auto optDrvPath = [this]() -> std::optional<StorePath> {
if (buildMode != bmNormal) return std::nullopt;
auto drvPath = StorePath::dummy;
try {
drvPath = resolveDerivedPath(worker.store, *drvReq);
} catch (MissingRealisation &) {
return std::nullopt;
}
return worker.evalStore.isValidPath(drvPath) || worker.store.isValidPath(drvPath)
? std::optional { drvPath }
: std::nullopt;
}()) {
trace(fmt("already have drv '%s' for '%s', can go straight to building",
worker.store.printStorePath(*optDrvPath),
drvReq->to_string(worker.store)));
loadAndBuildDerivation();
} else {
trace("need to obtain drv we want to build");
addWaitee(worker.makeGoal(DerivedPath::fromSingle(*drvReq)));
state = &CreateDerivationAndRealiseGoal::loadAndBuildDerivation;
if (waitees.empty()) work();
}
}
void CreateDerivationAndRealiseGoal::loadAndBuildDerivation()
{
trace("outer load and build derivation");
if (nrFailed != 0) {
amDone(ecFailed, Error("cannot build missing derivation '%s'", drvReq->to_string(worker.store)));
return;
}
StorePath drvPath = resolveDerivedPath(worker.store, *drvReq);
/* Build this step! */
concreteDrvGoal = worker.makeDerivationGoal(drvPath, wantedOutputs, buildMode);
addWaitee(upcast_goal(concreteDrvGoal));
state = &CreateDerivationAndRealiseGoal::buildDone;
optDrvPath = std::move(drvPath);
if (waitees.empty()) work();
}
void CreateDerivationAndRealiseGoal::buildDone()
{
trace("outer build done");
buildResult = upcast_goal(concreteDrvGoal)->getBuildResult(DerivedPath::Built {
.drvPath = drvReq,
.outputs = wantedOutputs,
});
if (buildResult.success())
amDone(ecSuccess);
else
amDone(ecFailed, Error("building '%s' failed", drvReq->to_string(worker.store)));
}
}

View file

@ -1,96 +0,0 @@
#pragma once
#include "parsed-derivations.hh"
#include "lock.hh"
#include "store-api.hh"
#include "pathlocks.hh"
#include "goal.hh"
namespace nix {
struct DerivationGoal;
/**
* This goal type is essentially the serial composition (like function
* composition) of a goal for getting a derivation, and then a
* `DerivationGoal` using the newly-obtained derivation.
*
* In the (currently experimental) general inductive case of derivations
* that are themselves build outputs, that first goal will be *another*
* `CreateDerivationAndRealiseGoal`. In the (much more common) base-case
* where the derivation has no provence and is just referred to by
* (content-addressed) store path, that first goal is a
* `SubstitutionGoal`.
*
* If we already have the derivation (e.g. if the evalutator has created
* the derivation locally and then instructured the store to build it),
* we can skip the first goal entirely as a small optimization.
*/
struct CreateDerivationAndRealiseGoal : public Goal
{
/**
* How to obtain a store path of the derivation to build.
*/
ref<SingleDerivedPath> drvReq;
/**
* The path of the derivation, once obtained.
**/
std::optional<StorePath> optDrvPath;
/**
* The goal for the corresponding concrete derivation.
**/
std::shared_ptr<DerivationGoal> concreteDrvGoal;
/**
* The specific outputs that we need to build.
*/
OutputsSpec wantedOutputs;
typedef void (CreateDerivationAndRealiseGoal::*GoalState)();
GoalState state;
/**
* The final output paths of the build.
*
* - For input-addressed derivations, always the precomputed paths
*
* - For content-addressed derivations, calcuated from whatever the
* hash ends up being. (Note that fixed outputs derivations that
* produce the "wrong" output still install that data under its
* true content-address.)
*/
OutputPathMap finalOutputs;
BuildMode buildMode;
CreateDerivationAndRealiseGoal(ref<SingleDerivedPath> drvReq,
const OutputsSpec & wantedOutputs, Worker & worker,
BuildMode buildMode = bmNormal);
virtual ~CreateDerivationAndRealiseGoal();
void timedOut(Error && ex) override;
std::string key() override;
void work() override;
/**
* Add wanted outputs to an already existing derivation goal.
*/
void addWantedOutputs(const OutputsSpec & outputs);
/**
* The states.
*/
void getDerivation();
void loadAndBuildDerivation();
void buildDone();
JobCategory jobCategory() const override {
return JobCategory::Administration;
};
};
}

View file

@ -71,7 +71,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath,
, wantedOutputs(wantedOutputs) , wantedOutputs(wantedOutputs)
, buildMode(buildMode) , buildMode(buildMode)
{ {
state = &DerivationGoal::loadDerivation; state = &DerivationGoal::getDerivation;
name = fmt( name = fmt(
"building of '%s' from .drv file", "building of '%s' from .drv file",
DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store)); DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store));
@ -164,6 +164,24 @@ void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs)
} }
void DerivationGoal::getDerivation()
{
trace("init");
/* The first thing to do is to make sure that the derivation
exists. If it doesn't, it may be created through a
substitute. */
if (buildMode == bmNormal && worker.evalStore.isValidPath(drvPath)) {
loadDerivation();
return;
}
addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath)));
state = &DerivationGoal::loadDerivation;
}
void DerivationGoal::loadDerivation() void DerivationGoal::loadDerivation()
{ {
trace("loading derivation"); trace("loading derivation");
@ -1498,24 +1516,23 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result)
if (!useDerivation) return; if (!useDerivation) return;
auto & fullDrv = *dynamic_cast<Derivation *>(drv.get()); auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
std::optional info = tryGetConcreteDrvGoal(waitee); auto * dg = dynamic_cast<DerivationGoal *>(&*waitee);
if (!info) return; if (!dg) return;
const auto & [dg, drvReq] = *info;
auto * nodeP = fullDrv.inputDrvs.findSlot(drvReq.get()); auto * nodeP = fullDrv.inputDrvs.findSlot(DerivedPath::Opaque { .path = dg->drvPath });
if (!nodeP) return; if (!nodeP) return;
auto & outputs = nodeP->value; auto & outputs = nodeP->value;
for (auto & outputName : outputs) { for (auto & outputName : outputs) {
auto buildResult = dg.get().getBuildResult(DerivedPath::Built { auto buildResult = dg->getBuildResult(DerivedPath::Built {
.drvPath = makeConstantStorePathRef(dg.get().drvPath), .drvPath = makeConstantStorePathRef(dg->drvPath),
.outputs = OutputsSpec::Names { outputName }, .outputs = OutputsSpec::Names { outputName },
}); });
if (buildResult.success()) { if (buildResult.success()) {
auto i = buildResult.builtOutputs.find(outputName); auto i = buildResult.builtOutputs.find(outputName);
if (i != buildResult.builtOutputs.end()) if (i != buildResult.builtOutputs.end())
inputDrvOutputs.insert_or_assign( inputDrvOutputs.insert_or_assign(
{ dg.get().drvPath, outputName }, { dg->drvPath, outputName },
i->second.outPath); i->second.outPath);
} }
} }

View file

@ -52,10 +52,6 @@ struct InitialOutput {
/** /**
* A goal for building some or all of the outputs of a derivation. * A goal for building some or all of the outputs of a derivation.
*
* The derivation must already be present, either in the store in a drv
* or in memory. If the derivation itself needs to be gotten first, a
* `CreateDerivationAndRealiseGoal` goal must be used instead.
*/ */
struct DerivationGoal : public Goal struct DerivationGoal : public Goal
{ {
@ -235,6 +231,7 @@ struct DerivationGoal : public Goal
/** /**
* The states. * The states.
*/ */
void getDerivation();
void loadDerivation(); void loadDerivation();
void haveDerivation(); void haveDerivation();
void outputsSubstitutionTried(); void outputsSubstitutionTried();

View file

@ -1,6 +1,5 @@
#include "worker.hh" #include "worker.hh"
#include "substitution-goal.hh" #include "substitution-goal.hh"
#include "create-derivation-and-realise-goal.hh"
#include "derivation-goal.hh" #include "derivation-goal.hh"
#include "local-store.hh" #include "local-store.hh"
@ -16,7 +15,7 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
worker.run(goals); worker.run(goals);
StringSet failed; StorePathSet failed;
std::optional<Error> ex; std::optional<Error> ex;
for (auto & i : goals) { for (auto & i : goals) {
if (i->ex) { if (i->ex) {
@ -26,10 +25,10 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
ex = std::move(i->ex); ex = std::move(i->ex);
} }
if (i->exitCode != Goal::ecSuccess) { if (i->exitCode != Goal::ecSuccess) {
if (auto i2 = dynamic_cast<CreateDerivationAndRealiseGoal *>(i.get())) if (auto i2 = dynamic_cast<DerivationGoal *>(i.get()))
failed.insert(i2->drvReq->to_string(*this)); failed.insert(i2->drvPath);
else if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get())) else if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get()))
failed.insert(printStorePath(i2->storePath)); failed.insert(i2->storePath);
} }
} }
@ -38,7 +37,7 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
throw std::move(*ex); throw std::move(*ex);
} else if (!failed.empty()) { } else if (!failed.empty()) {
if (ex) logError(ex->info()); if (ex) logError(ex->info());
throw Error(worker.failingExitStatus(), "build of %s failed", concatStringsSep(", ", quoteStrings(failed))); throw Error(worker.failingExitStatus(), "build of %s failed", showPaths(failed));
} }
} }

View file

@ -49,16 +49,6 @@ enum struct JobCategory {
* A substitution an arbitrary store object; it will use network resources. * A substitution an arbitrary store object; it will use network resources.
*/ */
Substitution, Substitution,
/**
* A goal that does no "real" work by itself, and just exists to depend on
* other goals which *do* do real work. These goals therefore are not
* limited.
*
* These goals cannot infinitely create themselves, so there is no risk of
* a "fork bomb" type situation (which would be a problem even though the
* goal do no real work) either.
*/
Administration,
}; };
struct Goal : public std::enable_shared_from_this<Goal> struct Goal : public std::enable_shared_from_this<Goal>

View file

@ -2,7 +2,6 @@
#include "worker.hh" #include "worker.hh"
#include "substitution-goal.hh" #include "substitution-goal.hh"
#include "drv-output-substitution-goal.hh" #include "drv-output-substitution-goal.hh"
#include "create-derivation-and-realise-goal.hh"
#include "local-derivation-goal.hh" #include "local-derivation-goal.hh"
#include "hook-instance.hh" #include "hook-instance.hh"
@ -42,24 +41,6 @@ Worker::~Worker()
} }
std::shared_ptr<CreateDerivationAndRealiseGoal> Worker::makeCreateDerivationAndRealiseGoal(
ref<SingleDerivedPath> drvReq,
const OutputsSpec & wantedOutputs,
BuildMode buildMode)
{
std::weak_ptr<CreateDerivationAndRealiseGoal> & goal_weak = outerDerivationGoals.ensureSlot(*drvReq).value;
std::shared_ptr<CreateDerivationAndRealiseGoal> goal = goal_weak.lock();
if (!goal) {
goal = std::make_shared<CreateDerivationAndRealiseGoal>(drvReq, wantedOutputs, *this, buildMode);
goal_weak = goal;
wakeUp(goal);
} else {
goal->addWantedOutputs(wantedOutputs);
}
return goal;
}
std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon( std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
const StorePath & drvPath, const StorePath & drvPath,
const OutputsSpec & wantedOutputs, const OutputsSpec & wantedOutputs,
@ -130,7 +111,10 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
{ {
return std::visit(overloaded { return std::visit(overloaded {
[&](const DerivedPath::Built & bfd) -> GoalPtr { [&](const DerivedPath::Built & bfd) -> GoalPtr {
return makeCreateDerivationAndRealiseGoal(bfd.drvPath, bfd.outputs, buildMode); if (auto bop = std::get_if<DerivedPath::Opaque>(&*bfd.drvPath))
return makeDerivationGoal(bop->path, bfd.outputs, buildMode);
else
throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented.");
}, },
[&](const DerivedPath::Opaque & bo) -> GoalPtr { [&](const DerivedPath::Opaque & bo) -> GoalPtr {
return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair); return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair);
@ -139,46 +123,24 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
} }
template<typename K, typename V, typename F>
static void cullMap(std::map<K, V> & goalMap, F f)
{
for (auto i = goalMap.begin(); i != goalMap.end();)
if (!f(i->second))
i = goalMap.erase(i);
else ++i;
}
template<typename K, typename G> template<typename K, typename G>
static void removeGoal(std::shared_ptr<G> goal, std::map<K, std::weak_ptr<G>> & goalMap) static void removeGoal(std::shared_ptr<G> goal, std::map<K, std::weak_ptr<G>> & goalMap)
{ {
/* !!! inefficient */ /* !!! inefficient */
cullMap(goalMap, [&](const std::weak_ptr<G> & gp) -> bool { for (auto i = goalMap.begin();
return gp.lock() != goal; i != goalMap.end(); )
}); if (i->second.lock() == goal) {
} auto j = i; ++j;
goalMap.erase(i);
template<typename K> i = j;
static void removeGoal(std::shared_ptr<CreateDerivationAndRealiseGoal> goal, std::map<K, DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>::ChildNode> & goalMap); }
else ++i;
template<typename K>
static void removeGoal(std::shared_ptr<CreateDerivationAndRealiseGoal> goal, std::map<K, DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>::ChildNode> & goalMap)
{
/* !!! inefficient */
cullMap(goalMap, [&](DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>::ChildNode & node) -> bool {
if (node.value.lock() == goal)
node.value.reset();
removeGoal(goal, node.childMap);
return !node.value.expired() || !node.childMap.empty();
});
} }
void Worker::removeGoal(GoalPtr goal) void Worker::removeGoal(GoalPtr goal)
{ {
if (auto drvGoal = std::dynamic_pointer_cast<CreateDerivationAndRealiseGoal>(goal)) if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
nix::removeGoal(drvGoal, outerDerivationGoals.map);
else if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
nix::removeGoal(drvGoal, derivationGoals); nix::removeGoal(drvGoal, derivationGoals);
else if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal)) else if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal))
nix::removeGoal(subGoal, substitutionGoals); nix::removeGoal(subGoal, substitutionGoals);
@ -236,19 +198,8 @@ void Worker::childStarted(GoalPtr goal, const std::set<int> & fds,
child.respectTimeouts = respectTimeouts; child.respectTimeouts = respectTimeouts;
children.emplace_back(child); children.emplace_back(child);
if (inBuildSlot) { if (inBuildSlot) {
switch (goal->jobCategory()) { if (goal->jobCategory() == JobCategory::Substitution) nrSubstitutions++;
case JobCategory::Substitution: else nrLocalBuilds++;
nrSubstitutions++;
break;
case JobCategory::Build:
nrLocalBuilds++;
break;
case JobCategory::Administration:
/* Intentionally not limited, see docs */
break;
default:
abort();
}
} }
} }
@ -260,20 +211,12 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers)
if (i == children.end()) return; if (i == children.end()) return;
if (i->inBuildSlot) { if (i->inBuildSlot) {
switch (goal->jobCategory()) { if (goal->jobCategory() == JobCategory::Substitution) {
case JobCategory::Substitution:
assert(nrSubstitutions > 0); assert(nrSubstitutions > 0);
nrSubstitutions--; nrSubstitutions--;
break; } else {
case JobCategory::Build:
assert(nrLocalBuilds > 0); assert(nrLocalBuilds > 0);
nrLocalBuilds--; nrLocalBuilds--;
break;
case JobCategory::Administration:
/* Intentionally not limited, see docs */
break;
default:
abort();
} }
} }
@ -324,9 +267,9 @@ void Worker::run(const Goals & _topGoals)
for (auto & i : _topGoals) { for (auto & i : _topGoals) {
topGoals.insert(i); topGoals.insert(i);
if (auto goal = dynamic_cast<CreateDerivationAndRealiseGoal *>(i.get())) { if (auto goal = dynamic_cast<DerivationGoal *>(i.get())) {
topPaths.push_back(DerivedPath::Built { topPaths.push_back(DerivedPath::Built {
.drvPath = goal->drvReq, .drvPath = makeConstantStorePathRef(goal->drvPath),
.outputs = goal->wantedOutputs, .outputs = goal->wantedOutputs,
}); });
} else if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) { } else if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) {
@ -589,19 +532,4 @@ GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal)
return subGoal; return subGoal;
} }
GoalPtr upcast_goal(std::shared_ptr<DerivationGoal> subGoal)
{
return subGoal;
}
std::optional<std::pair<std::reference_wrapper<const DerivationGoal>, std::reference_wrapper<const SingleDerivedPath>>> tryGetConcreteDrvGoal(GoalPtr waitee)
{
auto * odg = dynamic_cast<CreateDerivationAndRealiseGoal *>(&*waitee);
if (!odg) return std::nullopt;
return {{
std::cref(*odg->concreteDrvGoal),
std::cref(*odg->drvReq),
}};
}
} }

View file

@ -4,7 +4,6 @@
#include "types.hh" #include "types.hh"
#include "lock.hh" #include "lock.hh"
#include "store-api.hh" #include "store-api.hh"
#include "derived-path-map.hh"
#include "goal.hh" #include "goal.hh"
#include "realisation.hh" #include "realisation.hh"
@ -14,7 +13,6 @@
namespace nix { namespace nix {
/* Forward definition. */ /* Forward definition. */
struct CreateDerivationAndRealiseGoal;
struct DerivationGoal; struct DerivationGoal;
struct PathSubstitutionGoal; struct PathSubstitutionGoal;
class DrvOutputSubstitutionGoal; class DrvOutputSubstitutionGoal;
@ -33,25 +31,9 @@ class DrvOutputSubstitutionGoal;
*/ */
GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal); GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal);
GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal); GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal);
GoalPtr upcast_goal(std::shared_ptr<DerivationGoal> subGoal);
typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point; typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
/**
* The current implementation of impure derivations has
* `DerivationGoal`s accumulate realisations from their waitees.
* Unfortunately, `DerivationGoal`s don't directly depend on other
* goals, but instead depend on `CreateDerivationAndRealiseGoal`s.
*
* We try not to share any of the details of any goal type with any
* other, for sake of modularity and quicker rebuilds. This means we
* cannot "just" downcast and fish out the field. So as an escape hatch,
* we have made the function, written in `worker.cc` where all the goal
* types are visible, and use it instead.
*/
std::optional<std::pair<std::reference_wrapper<const DerivationGoal>, std::reference_wrapper<const SingleDerivedPath>>> tryGetConcreteDrvGoal(GoalPtr waitee);
/** /**
* A mapping used to remember for each child process to what goal it * A mapping used to remember for each child process to what goal it
* belongs, and file descriptors for receiving log data and output * belongs, and file descriptors for receiving log data and output
@ -119,9 +101,6 @@ private:
* Maps used to prevent multiple instantiations of a goal for the * Maps used to prevent multiple instantiations of a goal for the
* same derivation / path. * same derivation / path.
*/ */
DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>> outerDerivationGoals;
std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals; std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals;
std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals; std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals;
std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals; std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals;
@ -209,9 +188,6 @@ public:
* @ref DerivationGoal "derivation goal" * @ref DerivationGoal "derivation goal"
*/ */
private: private:
std::shared_ptr<CreateDerivationAndRealiseGoal> makeCreateDerivationAndRealiseGoal(
ref<SingleDerivedPath> drvPath,
const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal);
std::shared_ptr<DerivationGoal> makeDerivationGoalCommon( std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
const StorePath & drvPath, const OutputsSpec & wantedOutputs, const StorePath & drvPath, const OutputsSpec & wantedOutputs,
std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal); std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);

View file

@ -51,11 +51,8 @@ typename DerivedPathMap<V>::ChildNode * DerivedPathMap<V>::findSlot(const Single
// instantiations // instantiations
#include "create-derivation-and-realise-goal.hh"
namespace nix { namespace nix {
template struct DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>;
GENERATE_CMP_EXT( GENERATE_CMP_EXT(
template<>, template<>,
DerivedPathMap<std::set<std::string>>::ChildNode, DerivedPathMap<std::set<std::string>>::ChildNode,

View file

@ -20,11 +20,8 @@ namespace nix {
* *
* @param V A type to instantiate for each output. It should probably * @param V A type to instantiate for each output. It should probably
* should be an "optional" type so not every interior node has to have a * should be an "optional" type so not every interior node has to have a
* value. For example, the scheduler uses * value. `* const Something` or `std::optional<Something>` would be
* `DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>` to * good choices for "optional" types.
* remember which goals correspond to which outputs. `* const Something`
* or `std::optional<Something>` would also be good choices for
* "optional" types.
*/ */
template<typename V> template<typename V>
struct DerivedPathMap { struct DerivedPathMap {

View file

@ -776,7 +776,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
} }
}; };
/* Synchronisation point for testing, see tests/gc-concurrent.sh. */ /* Synchronisation point for testing, see tests/functional/gc-concurrent.sh. */
if (auto p = getEnv("_NIX_TEST_GC_SYNC")) if (auto p = getEnv("_NIX_TEST_GC_SYNC"))
readFile(*p); readFile(*p);

View file

@ -24,6 +24,9 @@
#include "config-impl.hh" #include "config-impl.hh"
#ifdef __APPLE__
#include <sys/sysctl.h>
#endif
namespace nix { namespace nix {
@ -154,6 +157,29 @@ unsigned int Settings::getDefaultCores()
return concurrency; return concurrency;
} }
#if __APPLE__
static bool hasVirt() {
int hasVMM;
int hvSupport;
size_t size;
size = sizeof(hasVMM);
if (sysctlbyname("kern.hv_vmm_present", &hasVMM, &size, NULL, 0) == 0) {
if (hasVMM)
return false;
}
// whether the kernel and hardware supports virt
size = sizeof(hvSupport);
if (sysctlbyname("kern.hv_support", &hvSupport, &size, NULL, 0) == 0) {
return hvSupport == 1;
} else {
return false;
}
}
#endif
StringSet Settings::getDefaultSystemFeatures() StringSet Settings::getDefaultSystemFeatures()
{ {
/* For backwards compatibility, accept some "features" that are /* For backwards compatibility, accept some "features" that are
@ -170,6 +196,11 @@ StringSet Settings::getDefaultSystemFeatures()
features.insert("kvm"); features.insert("kvm");
#endif #endif
#if __APPLE__
if (hasVirt())
features.insert("apple-virt");
#endif
return features; return features;
} }

View file

@ -708,6 +708,7 @@ public:
`kvm` feature. `kvm` feature.
This setting by default includes `kvm` if `/dev/kvm` is accessible, This setting by default includes `kvm` if `/dev/kvm` is accessible,
`apple-virt` if hardware virtualization is available on macOS,
and the pseudo-features `nixos-test`, `benchmark` and `big-parallel` and the pseudo-features `nixos-test`, `benchmark` and `big-parallel`
that are used in Nixpkgs to route builds to specific machines. that are used in Nixpkgs to route builds to specific machines.
)", {}, false}; )", {}, false};

View file

@ -3,6 +3,6 @@
namespace nix { namespace nix {
static constexpr std::string_view nameRegexStr = R"([0-9a-zA-Z\+\-\._\?=]+)"; static constexpr std::string_view nameRegexStr = R"([0-9a-zA-Z\+\-_\?=][0-9a-zA-Z\+\-\._\?=]*)";
} }

View file

@ -11,6 +11,8 @@ static void checkName(std::string_view path, std::string_view name)
if (name.size() > StorePath::MaxPathLen) if (name.size() > StorePath::MaxPathLen)
throw BadStorePath("store path '%s' has a name longer than %d characters", throw BadStorePath("store path '%s' has a name longer than %d characters",
path, StorePath::MaxPathLen); path, StorePath::MaxPathLen);
if (name[0] == '.')
throw BadStorePath("store path '%s' starts with illegal character '.'", path);
// See nameRegexStr for the definition // See nameRegexStr for the definition
for (auto c : name) for (auto c : name)
if (!((c >= '0' && c <= '9') if (!((c >= '0' && c <= '9')

View file

@ -1,29 +0,0 @@
check: libstore-tests-exe_RUN
programs += libstore-tests-exe
libstore-tests-exe_NAME = libnixstore-tests
libstore-tests-exe_DIR := $(d)
libstore-tests-exe_INSTALL_DIR :=
libstore-tests-exe_LIBS = libstore-tests
libstore-tests-exe_LDFLAGS := $(GTEST_LIBS)
libraries += libstore-tests
libstore-tests_NAME = libnixstore-tests
libstore-tests_DIR := $(d)
libstore-tests_INSTALL_DIR :=
libstore-tests_SOURCES := $(wildcard $(d)/*.cc)
libstore-tests_CXXFLAGS += -I src/libstore -I src/libutil
libstore-tests_LIBS = libutil-tests libstore libutil
libstore-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS)

View file

@ -1,29 +0,0 @@
check: libutil-tests_RUN
programs += libutil-tests
libutil-tests-exe_NAME = libnixutil-tests
libutil-tests-exe_DIR := $(d)
libutil-tests-exe_INSTALL_DIR :=
libutil-tests-exe_LIBS = libutil-tests
libutil-tests-exe_LDFLAGS := $(GTEST_LIBS)
libraries += libutil-tests
libutil-tests_NAME = libnixutil-tests
libutil-tests_DIR := $(d)
libutil-tests_INSTALL_DIR :=
libutil-tests_SOURCES := $(wildcard $(d)/*.cc)
libutil-tests_CXXFLAGS += -I src/libutil
libutil-tests_LIBS = libutil
libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS)

View file

@ -30,7 +30,7 @@ extern std::regex refRegex;
/// Instead of defining what a good Git Ref is, we define what a bad Git Ref is /// Instead of defining what a good Git Ref is, we define what a bad Git Ref is
/// This is because of the definition of a ref in refs.c in https://github.com/git/git /// This is because of the definition of a ref in refs.c in https://github.com/git/git
/// See tests/fetchGitRefs.sh for the full definition /// See tests/functional/fetchGitRefs.sh for the full definition
const static std::string badGitRefRegexS = "//|^[./]|/\\.|\\.\\.|[[:cntrl:][:space:]:?^~\[]|\\\\|\\*|\\.lock$|\\.lock/|@\\{|[/.]$|^@$|^$"; const static std::string badGitRefRegexS = "//|^[./]|/\\.|\\.\\.|[[:cntrl:][:space:]:?^~\[]|\\\\|\\*|\\.lock$|\\.lock/|@\\{|[/.]$|^@$|^$";
extern std::regex badGitRefRegex; extern std::regex badGitRefRegex;

View file

@ -218,7 +218,8 @@ static void showHelp(std::vector<std::string> subcommand, NixArgs & toplevel)
vDump->mkString(toplevel.dumpCli()); vDump->mkString(toplevel.dumpCli());
auto vRes = state.allocValue(); auto vRes = state.allocValue();
state.callFunction(*vGenerateManpage, *vDump, *vRes, noPos); state.callFunction(*vGenerateManpage, state.getBuiltin("false"), *vRes, noPos);
state.callFunction(*vRes, *vDump, *vRes, noPos);
auto attr = vRes->attrs->get(state.symbols.create(mdName + ".md")); auto attr = vRes->attrs->get(state.symbols.create(mdName + ".md"));
if (!attr) if (!attr)

View file

@ -1,11 +0,0 @@
#!/usr/bin/env bash
source common.sh
out1=$(nix-build ./text-hashed-output.nix -A hello --no-out-link)
clearStore
out2=$(nix-build ./text-hashed-output.nix -A wrapper --no-out-link)
diff -r $out1 $out2

View file

@ -1,6 +1,6 @@
source common.sh source common.sh
sed -e "s|@localstatedir@|$TEST_ROOT/profile-var|g" -e "s|@coreutils@|$coreutils|g" < ../scripts/nix-profile.sh.in > $TEST_ROOT/nix-profile.sh sed -e "s|@localstatedir@|$TEST_ROOT/profile-var|g" -e "s|@coreutils@|$coreutils|g" < ../../scripts/nix-profile.sh.in > $TEST_ROOT/nix-profile.sh
user=$(whoami) user=$(whoami)
rm -rf $TEST_HOME $TEST_ROOT/profile-var rm -rf $TEST_HOME $TEST_ROOT/profile-var

View file

@ -6,7 +6,7 @@ unset NIX_STATE_DIR
remoteDir=$TEST_ROOT/remote remoteDir=$TEST_ROOT/remote
# Note: ssh{-ng}://localhost bypasses ssh. See tests/build-remote.sh for # Note: ssh{-ng}://localhost bypasses ssh. See tests/functional/build-remote.sh for
# more details. # more details.
nix-build $file -o $TEST_ROOT/result --max-jobs 0 \ nix-build $file -o $TEST_ROOT/result --max-jobs 0 \
--arg busybox $busybox \ --arg busybox $busybox \

View file

@ -25,4 +25,4 @@ clean-files += \
$(d)/config.nix $(d)/config.nix
test-deps += \ test-deps += \
tests/ca/config.nix tests/functional/ca/config.nix

Some files were not shown because too many files have changed in this diff Show more