zebreus/lix
0
0
Fork 0
forked from lix-project/lix
Code Pull requests 1 Activity

Merge remote-tracking branch 'upstream/master' into source-path

Browse source
This commit is contained in:
Robert Hensing 2023-04-17 11:41:50 +02:00
parent a9759407e5 9af9c260fc
commit cb2615cf47
148 changed files with 3908 additions and 1449 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
.github/labeler.yml vendored
View file

@ -2,5 +2,22 @@
- doc/manual/*
- src/nix/**/*.md
"store":
- src/libstore/store-api.*
- src/libstore/*-store.*
"fetching":
- src/libfetchers/**/*
"repl":
- src/libcmd/repl.*
- src/nix/repl.*
"new-cli":
- src/nix/**/*
"tests":
# Unit tests
- src/*/tests/**/*
# Functional and integration tests
- tests/**/*

3
.gitignore vendored
View file

@ -19,9 +19,12 @@ perl/Makefile.config
/doc/manual/nix.json
/doc/manual/conf-file.json
/doc/manual/builtins.json
/doc/manual/xp-features.json
/doc/manual/src/SUMMARY.md
/doc/manual/src/command-ref/new-cli
/doc/manual/src/command-ref/conf-file.md
/doc/manual/src/command-ref/experimental-features-shortlist.md
/doc/manual/src/contributing/experimental-feature-descriptions.md
/doc/manual/src/language/builtins.md
# /scripts/

2
.version
View file

@ -1 +1 @@
2.15.0
2.16.0

22
boehmgc-coroutine-sp-fallback.diff
View file

@ -1,8 +1,8 @@
diff --git a/darwin_stop_world.c b/darwin_stop_world.c
index 3dbaa3fb..36a1d1f7 100644
index 0468aaec..b348d869 100644
--- a/darwin_stop_world.c
+++ b/darwin_stop_world.c
@@ -352,6 +352,7 @@ GC_INNER void GC_push_all_stacks(void)
@@ -356,6 +356,7 @@ GC_INNER void GC_push_all_stacks(void)
int nthreads = 0;
word total_size = 0;
mach_msg_type_number_t listcount = (mach_msg_type_number_t)THREAD_TABLE_SZ;
@ -10,7 +10,7 @@ index 3dbaa3fb..36a1d1f7 100644
if (!EXPECT(GC_thr_initialized, TRUE))
GC_thr_init();
@@ -407,6 +408,19 @@ GC_INNER void GC_push_all_stacks(void)
@@ -411,6 +412,19 @@ GC_INNER void GC_push_all_stacks(void)
GC_push_all_stack_sections(lo, hi, p->traced_stack_sect);
}
if (altstack_lo) {
@ -30,6 +30,22 @@ index 3dbaa3fb..36a1d1f7 100644
total_size += altstack_hi - altstack_lo;
GC_push_all_stack(altstack_lo, altstack_hi);
}
diff --git a/include/gc.h b/include/gc.h
index edab6c22..f2c61282 100644
--- a/include/gc.h
+++ b/include/gc.h
@@ -2172,6 +2172,11 @@ GC_API void GC_CALL GC_win32_free_heap(void);
(*GC_amiga_allocwrapper_do)(a,GC_malloc_atomic_ignore_off_page)
#endif /* _AMIGA && !GC_AMIGA_MAKINGLIB */
+#if !__APPLE__
+/* Patch doesn't work on apple */
+#define NIX_BOEHM_PATCH_VERSION 1
+#endif
+
#ifdef __cplusplus
} /* extern "C" */
#endif
diff --git a/pthread_stop_world.c b/pthread_stop_world.c
index b5d71e62..aed7b0bf 100644
--- a/pthread_stop_world.c

15
configure.ac
View file

@ -289,13 +289,24 @@ PKG_CHECK_MODULES([GTEST], [gtest_main])
# Look for rapidcheck.
AC_ARG_VAR([RAPIDCHECK_HEADERS], [include path of gtest headers shipped by RAPIDCHECK])
# No pkg-config yet, https://github.com/emil-e/rapidcheck/issues/302
AC_LANG_PUSH(C++)
AC_SUBST(RAPIDCHECK_HEADERS)
[CXXFLAGS="-I $RAPIDCHECK_HEADERS $CXXFLAGS"]
[LIBS="-lrapidcheck -lgtest $LIBS"]
AC_CHECK_HEADERS([rapidcheck/gtest.h], [], [], [#include <gtest/gtest.h>])
dnl No good for C++ libs with mangled symbols
dnl AC_CHECK_LIB([rapidcheck], [])
dnl AC_CHECK_LIB doesn't work for C++ libs with mangled symbols
AC_LINK_IFELSE([
AC_LANG_PROGRAM([[
#include <gtest/gtest.h>
#include <rapidcheck/gtest.h>
]], [[
return RUN_ALL_TESTS();
]])
],
[],
[AC_MSG_ERROR([librapidcheck is not found.])])
AC_LANG_POP(C++)
fi

32
doc/manual/generate-manpage.nix
View file

@ -10,7 +10,9 @@ let
result = ''
> **Warning** \
> This program is **experimental** and its interface is subject to change.
> This program is
> [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command)
> and its interface is subject to change.
# Name
@ -29,19 +31,18 @@ let
showSynopsis = command: args:
let
showArgument = arg: "*${arg.label}*" + (if arg ? arity then "" else "...");
showArgument = arg: "*${arg.label}*" + optionalString (! arg ? arity) "...";
arguments = concatStringsSep " " (map showArgument args);
in ''
`${command}` [*option*...] ${arguments}
'';
maybeSubcommands = if details ? commands && details.commands != {}
then ''
maybeSubcommands = optionalString (details ? commands && details.commands != {})
''
where *subcommand* is one of the following:
${subcommands}
''
else "";
'';
subcommands = if length categories > 1
then listCategories
@ -63,12 +64,11 @@ let
* [`${command} ${name}`](./${appendName filename name}.md) - ${subcmd.description}
'';
maybeDocumentation =
if details ? doc
then replaceStrings ["@stores@"] [storeDocs] details.doc
else "";
maybeDocumentation = optionalString
(details ? doc)
(replaceStrings ["@stores@"] [storeDocs] details.doc);
maybeOptions = if details.flags == {} then "" else ''
maybeOptions = optionalString (details.flags != {}) ''
# Options
${showOptions details.flags toplevel.flags}
@ -78,15 +78,19 @@ let
let
allOptions = options // commonOptions;
showCategory = cat: ''
${if cat != "" then "**${cat}:**" else ""}
${optionalString (cat != "") "**${cat}:**"}
${listOptions (filterAttrs (n: v: v.category == cat) allOptions)}
'';
listOptions = opts: concatStringsSep "\n" (attrValues (mapAttrs showOption opts));
showOption = name: option:
let
shortName = if option ? shortName then "/ `-${option.shortName}`" else "";
labels = if option ? labels then (concatStringsSep " " (map (s: "*${s}*") option.labels)) else "";
shortName = optionalString
(option ? shortName)
("/ `-${option.shortName}`");
labels = optionalString
(option ? labels)
(concatStringsSep " " (map (s: "*${s}*") option.labels));
in trim ''
- `--${name}` ${shortName} ${labels}

9
doc/manual/generate-xp-features-shortlist.nix Normal file
View file

@ -0,0 +1,9 @@
with builtins;
with import ./utils.nix;
let
showExperimentalFeature = name: doc:
''
- [`${name}`](@docroot@/contributing/experimental-features.md#xp-feature-${name})
'';
in xps: indent " " (concatStrings (attrValues (mapAttrs showExperimentalFeature xps)))

11
doc/manual/generate-xp-features.nix Normal file
View file

@ -0,0 +1,11 @@
with builtins;
with import ./utils.nix;
let
showExperimentalFeature = name: doc:
squash ''
## [`${name}`]{#xp-feature-${name}}
${doc}
'';
in xps: (concatStringsSep "\n" (attrValues (mapAttrs showExperimentalFeature xps)))

23
doc/manual/local.mk
View file

@ -81,19 +81,20 @@ $(d)/%.8: $(d)/src/command-ref/%.md
$(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md
@printf "Title: %s\n\n" "$$(basename $@ .5)" > $^.tmp
@cat $^ >> $^.tmp
@$(call process-includes,$^,$^.tmp)
$(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@
@rm $^.tmp
$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli
$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md
@cp $< $@
@$(call process-includes,$@,$@)
$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix
$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(bindir)/nix
@rm -rf $@ $@.tmp
$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix (builtins.readFile $<)'
@mv $@.tmp $@
$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix
$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix
@cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp
$(trace-gen) $(nix-eval) --expr '(import doc/manual/utils.nix).showSettings { useAnchors = true; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
@mv $@.tmp $@
@ -106,6 +107,20 @@ $(d)/conf-file.json: $(bindir)/nix
$(trace-gen) $(dummy-env) $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp
@mv $@.tmp $@
$(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(bindir)/nix
@rm -rf $@ $@.tmp
$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features.nix (builtins.fromJSON (builtins.readFile $<))'
@mv $@.tmp $@
$(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(bindir)/nix
@rm -rf $@ $@.tmp
$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features-shortlist.nix (builtins.fromJSON (builtins.readFile $<))'
@mv $@.tmp $@
$(d)/xp-features.json: $(bindir)/nix
$(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-xp-features > $@.tmp
@mv $@.tmp $@
$(d)/src/language/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix
@cat doc/manual/src/language/builtins-prefix.md > $@.tmp
$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
@ -145,7 +160,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli
done
@touch $@
$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md
$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md
$(trace-gen) \
tmp="$$(mktemp -d)"; \
cp -r doc/manual "$$tmp"; \

2
doc/manual/src/SUMMARY.md.in
View file

@ -95,9 +95,11 @@
- [Glossary](glossary.md)
- [Contributing](contributing/contributing.md)
- [Hacking](contributing/hacking.md)
- [Experimental Features](contributing/experimental-features.md)
- [CLI guideline](contributing/cli-guideline.md)
- [Release Notes](release-notes/release-notes.md)
- [Release X.Y (202?-??-??)](release-notes/rl-next.md)
- [Release 2.15 (2023-04-11)](release-notes/rl-2.15.md)
- [Release 2.14 (2023-02-28)](release-notes/rl-2.14.md)
- [Release 2.13 (2023-01-17)](release-notes/rl-2.13.md)
- [Release 2.12 (2022-12-06)](release-notes/rl-2.12.md)

2
doc/manual/src/command-ref/experimental-commands.md
View file

@ -1,6 +1,6 @@
# Experimental Commands
This section lists experimental commands.
This section lists [experimental commands](@docroot@/contributing/experimental-features.md#xp-feature-nix-command).
> **Warning**
>

3
doc/manual/src/command-ref/nix-shell.md
View file

@ -120,7 +120,8 @@ shell in which to build it:
```console
$ nix-shell '<nixpkgs>' -A pan
[nix-shell]$ eval ${unpackPhase:-unpackPhase}
[nix-shell]$ cd pan-*
[nix-shell]$ cd $sourceRoot
[nix-shell]$ eval ${patchPhase:-patchPhase}
[nix-shell]$ eval ${configurePhase:-configurePhase}
[nix-shell]$ eval ${buildPhase:-buildPhase}
[nix-shell]$ ./pan/gui/pan

95
doc/manual/src/contributing/experimental-features.md Normal file
View file

@ -0,0 +1,95 @@
This section describes the notion of *experimental features*, and how it fits into the big picture of the development of Nix.
# What are experimental features?
Experimental features are considered unstable, which means that they can be changed or removed at any time.
Users must explicitly enable them by toggling the associated [experimental feature flags](@docroot@/command-ref/conf-file.md#conf-experimental-features).
This allows accessing unstable functionality without unwittingly relying on it.
Experimental feature flags were first introduced in [Nix 2.4](@docroot@/release-notes/rl-2.4.md).
Before that, Nix did have experimental features, but they were not guarded by flags and were merely documented as unstable.
This was a source of confusion and controversy.
# When should a new feature be marked experimental?
A change in the Nix codebase should be guarded by an experimental feature flag if it is considered likely to be reverted or adapted in a backwards-incompatible manner after gathering more experience with it in practice.
Examples:
- Changes to the Nix language, such as new built-ins, syntactic or semantic changes, etc.
- Changes to the command-line interface
# Lifecycle of an experimental feature
Experimental features have to be treated on a case-by-case basis.
However, the standard workflow for an experimental feature is as follows:
- A new feature is implemented in a *pull request*
- It is guarded by an experimental feature flag that is disabled by default
- The pull request is merged, the *experimental* feature ends up in a release
- Using the feature requires explicitly enabling it, signifying awareness of the potential risks
- Being experimental, the feature can still be changed arbitrarily
- The feature can be *removed*
- The associated experimental feature flag is also removed
- The feature can be declared *stable*
- The associated experimental feature flag is removed
- There should be enough evidence of users having tried the feature, such as feedback, fixed bugs, demonstrations of how it is put to use
- Maintainers must feel confident that:
- The feature is designed and implemented sensibly, that it is fit for purpose
- Potential interactions are well-understood
- Stabilising the feature will not incur an outsized maintenance burden in the future
The following diagram illustrates the process:
```
.------.
| idea |
'------'
|
discussion, design, implementation
|
| .-------.
| | |
v v |
.--------------. review
| pull request | |
'--------------' |
| ^ | |
| | '-------'
.---' '----.
| |
merge user feedback,
| (breaking) changes
| |
'---. .----'
| |
v |
+--------------+
.---| experimental |----.
| +--------------+ |
| |
decision to stabilise decision against
| keeping the feature
| |
v v
+--------+ +---------+
| stable | | removed |
+--------+ +---------+
```
# Relation to the RFC process
Experimental features and [RFCs](https://github.com/NixOS/rfcs/) both allow approaching substantial changes while minimizing the risk.
However they serve different purposes:
- An experimental feature enables developers to iterate on and deliver a new idea without committing to it or requiring a costly long-running fork.
It is primarily an issue of *implementation*, targeting Nix developers and early testers.
- The goal of an RFC is to make explicit all the implications of a change:
Explain why it is wanted, which new use-cases it enables, which interface changes it requires, etc.
It is primarily an issue of *design* and *communication*, targeting the broader community.
This means that experimental features and RFCs are orthogonal mechanisms, and can be used independently or together as needed.
# Currently available experimental features
{{#include ./experimental-feature-descriptions.md}}

32
doc/manual/src/glossary.md
View file

@ -15,7 +15,7 @@
Example: `/nix/store/g946hcz4c8mdvq2g8vxx42z51qb71rvp-git-2.38.1.drv`
See [`nix show-derivation`](./command-ref/new-cli/nix3-show-derivation.md) (experimental) for displaying the contents of store derivations.
See [`nix derivation show`](./command-ref/new-cli/nix3-derivation-show.md) (experimental) for displaying the contents of store derivations.
[store derivation]: #gloss-store-derivation
@ -54,7 +54,7 @@
invoked, the Nix store can be referred to
as a "_local_" or a "_remote_" one:
+ A *local store* exists on the filesystem of
+ A [local store]{#gloss-local-store} exists on the filesystem of
the machine where Nix is invoked. You can use other
local stores by passing the `--store` flag to the
`nix` command. Local stores can be used for building derivations.
@ -65,17 +65,17 @@
served by the `nix-serve` Perl script.
[store]: #gloss-store
[local store]: #gloss-local-store
- [chroot store]{#gloss-chroot-store}\
A local store whose canonical path is anything other than `/nix/store`.
A [local store] whose canonical path is anything other than `/nix/store`.
- [binary cache]{#gloss-binary-cache}\
A *binary cache* is a Nix store which uses a different format: its
metadata and signatures are kept in `.narinfo` files rather than in a
Nix database. This different format simplifies serving store objects
[Nix database]. This different format simplifies serving store objects
over the network, but cannot host builds. Examples of binary caches
include S3 buckets and the [NixOS binary
cache](https://cache.nixos.org).
include S3 buckets and the [NixOS binary cache](https://cache.nixos.org).
- [store path]{#gloss-store-path}\
The location of a [store object] in the file system, i.e., an
@ -108,7 +108,7 @@
[fixed-output derivations](#gloss-fixed-output-derivation).
- [substitute]{#gloss-substitute}\
A substitute is a command invocation stored in the Nix database that
A substitute is a command invocation stored in the [Nix database] that
describes how to build a store object, bypassing the normal build
mechanism (i.e., derivations). Typically, the substitute builds the
store object by downloading a pre-built version of the store object
@ -127,6 +127,14 @@
builder can rely on external inputs such as the network or the
system time) but the Nix model assumes it.
- Nix database{#gloss-nix-database}\
An SQlite database to track [reference]s between [store object]s.
This is an implementation detail of the [local store].
Default location: `/nix/var/nix/db`.
[Nix database]: #gloss-nix-database
- [Nix expression]{#gloss-nix-expression}\
A high-level description of software packages and compositions
thereof. Deploying software using Nix entails writing Nix
@ -175,9 +183,9 @@
- [validity]{#gloss-validity}\
A store path is valid if all [store object]s in its [closure] can be read from the [store].
For a local store, this means:
For a [local store], this means:
- The store path leads to an existing [store object] in that [store].
- The store path is listed in the Nix database as being valid.
- The store path is listed in the [Nix database] as being valid.
- All paths in the store path's [closure] are valid.
[validity]: #gloss-validity
@ -217,3 +225,9 @@
[string]: ./language/values.md#type-string
[path]: ./language/values.md#type-path
[attribute name]: ./language/values.md#attribute-set
- [experimental feature]{#gloss-experimental-feature}\
Not yet stabilized functionality guarded by named experimental feature flags.
These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting.
See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/contributing/experimental-features.md).

26
doc/manual/src/language/advanced-attributes.md
View file

@ -208,12 +208,26 @@ Derivations can declare some infrequently used optional attributes.
about converting to and from base-32 notation.)
- [`__contentAddressed`]{#adv-attr-__contentAddressed}
If this **experimental** attribute is set to true, then the derivation
> **Warning**
> This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md).
>
> To use this attribute, you must enable the
> [`ca-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) experimental feature.
> For example, in [nix.conf](../command-ref/conf-file.md) you could add:
>
> ```
> extra-experimental-features = ca-derivations
> ```
If this attribute is set to `true`, then the derivation
outputs will be stored in a content-addressed location rather than the
traditional input-addressed one.
This only has an effect if the `ca-derivations` experimental feature is enabled.
Setting this attribute also requires setting `outputHashMode` and `outputHashAlgo` like for *fixed-output derivations* (see above).
Setting this attribute also requires setting
[`outputHashMode`](#adv-attr-outputHashMode)
and
[`outputHashAlgo`](#adv-attr-outputHashAlgo)
like for *fixed-output derivations* (see above).
- [`passAsFile`]{#adv-attr-passAsFile}\
A list of names of attributes that should be passed via files rather
@ -307,9 +321,11 @@ Derivations can declare some infrequently used optional attributes.
- [`unsafeDiscardReferences`]{#adv-attr-unsafeDiscardReferences}\
> **Warning**
> This is an experimental feature.
> This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md).
>
> To enable it, add the following to [nix.conf](../command-ref/conf-file.md):
> To use this attribute, you must enable the
> [`discard-references`](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) experimental feature.
> For example, in [nix.conf](../command-ref/conf-file.md) you could add:
>
> ```
> extra-experimental-features = discard-references

12
doc/manual/src/quick-start.md
View file

@ -19,7 +19,7 @@ to subsequent chapters.
channel:
```console
$ nix-env -qaP
$ nix-env --query --available --attr-path
nixpkgs.docbook_xml_dtd_43 docbook-xml-4.3
nixpkgs.docbook_xml_dtd_45 docbook-xml-4.5
nixpkgs.firefox firefox-33.0.2
@ -31,7 +31,7 @@ to subsequent chapters.
1. Install some packages from the channel:
```console
$ nix-env -iA nixpkgs.hello
$ nix-env --install --attr nixpkgs.hello
```
This should download pre-built packages; it should not build them
@ -49,13 +49,13 @@ to subsequent chapters.
1. Uninstall a package:
```console
$ nix-env -e hello
$ nix-env --uninstall hello
```
1. You can also test a package without installing it:
```console
$ nix-shell -p hello
$ nix-shell --packages hello
```
This builds or downloads GNU Hello and its dependencies, then drops
@ -76,7 +76,7 @@ to subsequent chapters.
```console
$ nix-channel --update nixpkgs
$ nix-env -u '*'
$ nix-env --upgrade '*'
```
The latter command will upgrade each installed package for which
@ -95,5 +95,5 @@ to subsequent chapters.
them:
```console
$ nix-collect-garbage -d
$ nix-collect-garbage --delete-old
```

58
doc/manual/src/release-notes/rl-2.15.md Normal file
View file

@ -0,0 +1,58 @@
# Release 2.15 (2023-04-11)
* Commands which take installables on the command line can now read them from the standard input if
passed the `--stdin` flag. This is primarily useful when you have a large amount of paths which
exceed the OS argument limit.
* The `nix-hash` command now supports Base64 and SRI. Use the flags `--base64`
or `--sri` to specify the format of output hash as Base64 or SRI, and `--to-base64`
or `--to-sri` to convert a hash to Base64 or SRI format, respectively.
As the choice of hash formats is no longer binary, the `--base16` flag is also added
to explicitly specify the Base16 format, which is still the default.
* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](../glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents.
The new `^` syntax for store paths introduced in Nix 2.13 allows explicitly referencing output paths of a derivation.
Using this is better and more clear than relying on the now-removed `.drv` special handling.
For example,
```shell-session
$ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv
```
now gives info about the derivation itself, while
```shell-session
$ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^*
```
provides information about each of its outputs.
* The experimental command `nix describe-stores` has been removed.
* Nix stores and their settings are now documented in [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md).
* Documentation for operations of `nix-store` and `nix-env` are now available on separate pages of the manual.
They include all common options that can be specified and common environment variables that affect these commands.
These pages can be viewed offline with `man` using
* `man nix-store-<operation>` and `man nix-env-<operation>`
* `nix-store --help --<operation>` and `nix-env --help --<operation>`.
* Nix when used as a client now checks whether the store (the server) trusts the client.
(The store always had to check whether it trusts the client, but now the client is informed of the store's decision.)
This is useful for scripting interactions with (non-legacy-ssh) remote Nix stores.
`nix store ping` and `nix doctor` now display this information.
* The new command `nix derivation add` allows adding derivations to the store without involving the Nix language.
It exists to round out our collection of basic utility/plumbing commands, and allow for a low barrier-to-entry way of experimenting with alternative front-ends to the Nix Store.
It uses the same JSON layout as `nix derivation show`, and is its inverse.
* `nix show-derivation` has been renamed to `nix derivation show`.
This matches `nix derivation add`, and avoids bloating the top-level namespace.
The old name is still kept as an alias for compatibility, however.
* The `nix derivation {add,show}` JSON format now includes the derivation name as a top-level field.
This is useful in general, but especially necessary for the `add` direction, as otherwise we would need to pass in the name out of band for certain cases.

39
doc/manual/src/release-notes/rl-next.md
View file

@ -1,41 +1,2 @@
# Release X.Y (202?-??-??)
* Commands which take installables on the command line can now read them from the standard input if
passed the `--stdin` flag. This is primarily useful when you have a large amount of paths which
exceed the OS arg limit.
* The `nix-hash` command now supports Base64 and SRI. Use the flags `--base64`
or `--sri` to specify the format of output hash as Base64 or SRI, and `--to-base64`
or `--to-sri` to convert a hash to Base64 or SRI format, respectively.
As the choice of hash formats is no longer binary, the `--base16` flag is also added
to explicitly specify the Base16 format, which is still the default.
* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](../glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents.
The new `^` syntax for store paths introduced in Nix 2.13 allows explicitly referencing output paths of a derivation.
Using this is better and more clear than relying on the now-removed `.drv` special handling.
For example,
```shell-session
$ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv
```
now gives info about the derivation itself, while
```shell-session
$ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^*
```
provides information about each of its outputs.
* The experimental command `nix describe-stores` has been removed.
* Nix stores and their settings are now documented in [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md).
* Documentation for operations of `nix-store` and `nix-env` are now available on separate pages of the manual.
They include all common options that can be specified and common environment variables that affect these commands.
These pages can be viewed offline with `man` using
* `man nix-store-<operation>` and `man nix-env-<operation>`
* `nix-store --help --<operation>` and `nix-env --help --<operation>`.

31
doc/manual/utils.nix
View file

@ -5,6 +5,9 @@ rec {
concatStrings = concatStringsSep "";
attrsToList = a:
map (name: { inherit name; value = a.${name}; }) (builtins.attrNames a);
replaceStringsRec = from: to: string:
# recursively replace occurrences of `from` with `to` within `string`
# example:
@ -39,7 +42,9 @@ rec {
filterAttrs = pred: set:
listToAttrs (concatMap (name: let v = set.${name}; in if pred name v then [(nameValuePair name v)] else []) (attrNames set));
showSetting = { useAnchors }: name: { description, documentDefault, defaultValue, aliases, value }:
optionalString = cond: string: if cond then string else "";
showSetting = { useAnchors }: name: { description, documentDefault, defaultValue, aliases, value, experimentalFeature }:
let
result = squash ''
- ${if useAnchors
@ -49,10 +54,28 @@ rec {
${indent " " body}
'';
experimentalFeatureNote = optionalString (experimentalFeature != null) ''
> **Warning**
> This setting is part of an
> [experimental feature](@docroot@/contributing/experimental-features.md).
To change this setting, you need to make sure the corresponding experimental feature,
[`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}),
is enabled.
For example, include the following in [`nix.conf`](#):
```
extra-experimental-features = ${experimentalFeature}
${name} = ...
```
'';
# separate body to cleanly handle indentation
body = ''
${description}
${experimentalFeatureNote}
**Default:** ${showDefault documentDefault defaultValue}
${showAliases aliases}
@ -71,13 +94,13 @@ rec {
else "*machine-specific*";
showAliases = aliases:
if aliases == [] then "" else
optionalString (aliases != [])
"**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}";
in result;
indent = prefix: s:
concatStringsSep "\n" (map (x: if x == "" then x else "${prefix}${x}") (splitLines s));
in result;
showSettings = args: settingsInfo: concatStrings (attrValues (mapAttrs (showSetting args) settingsInfo));
}

1
flake.nix
View file

@ -219,6 +219,7 @@
enableParallelBuilding = true;
configureFlags = testConfigureFlags; # otherwise configure fails
dontBuild = true;
doInstallCheck = true;

4
local.mk
View file

@ -1,6 +1,8 @@
clean-files += Makefile.config
GLOBAL_CXXFLAGS += -Wno-deprecated-declarations
GLOBAL_CXXFLAGS += -Wno-deprecated-declarations -Werror=switch
# Allow switch-enum to be overridden for files that do not support it, usually because of dependency headers.
ERROR_SWITCH_ENUM = -Werror=switch-enum
$(foreach i, config.h $(wildcard src/lib*/*.hh), \
$(eval $(call install-file-in, $(i), $(includedir)/nix, 0644)))

8
maintainers/README.md
View file

@ -56,7 +56,7 @@ Meeting notes are collected on a [collaborative scratchpad](https://pad.lassul.u
The team uses a [GitHub project board](https://github.com/orgs/NixOS/projects/19/views/1) for tracking its work.
Issues on the board progress through the following states:
Items on the board progress through the following states:
- No Status
@ -69,6 +69,7 @@ Issues on the board progress through the following states:
2. [security](https://github.com/NixOS/nix/labels/security)
3. [regression](https://github.com/NixOS/nix/labels/regression)
4. [bug](https://github.com/NixOS/nix/issues?q=is%3Aopen+label%3Abug+sort%3Areactions-%2B1-desc)
5. [tests of existing functionality](https://github.com/NixOS/nix/issues?q=is%3Aopen+label%3Atests+-label%3Afeature+sort%3Areactions-%2B1-desc)
- [oldest pull requests](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+sort%3Acreated-asc)
- [most popular pull requests](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+sort%3Areactions-%2B1-desc)
@ -79,6 +80,9 @@ Issues on the board progress through the following states:
If there is disagreement on the general idea behind an issue or pull request, it is moved to _To discuss_, otherwise to _In review_.
To ensure process quality and reliability, all non-trivial pull requests must be triaged before merging.
What constitutes a trivial pull request is up to maintainers' judgement.
- To discuss
Pull requests and issues that are deemed important and controversial are discussed by the team during discussion meetings.
@ -91,7 +95,7 @@ Issues on the board progress through the following states:
Contributors who took the time to implement concrete change proposals should not wait indefinitely.
- Prioritise fixing bugs over documentation, improvements or new features
- Prioritise fixing bugs and testing over documentation, improvements or new features
The team values stability and accessibility higher than raw functionality.

4
mk/patterns.mk
View file

@ -1,10 +1,10 @@
$(buildprefix)%.o: %.cc
@mkdir -p "$(dir $@)"
$(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP
$(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep, $@) -MP
$(buildprefix)%.o: %.cpp
@mkdir -p "$(dir $@)"
$(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP
$(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep, $@) -MP
$(buildprefix)%.o: %.c
@mkdir -p "$(dir $@)"

6
src/libcmd/editor-for.hh
View file

@ -6,8 +6,10 @@
namespace nix {
/* Helper function to generate args that invoke $EDITOR on
filename:lineno. */
/**
* Helper function to generate args that invoke $EDITOR on
* filename:lineno.
*/
Strings editorFor(const SourcePath & file, uint32_t line);
}

22
src/libcmd/installables.cc
View file

@ -102,6 +102,28 @@ MixFlakeOptions::MixFlakeOptions()
}}
});
addFlag({
.longName = "reference-lock-file",
.description = "Read the given lock file instead of `flake.lock` within the top-level flake.",
.category = category,
.labels = {"flake-lock-path"},
.handler = {[&](std::string lockFilePath) {
lockFlags.referenceLockFilePath = lockFilePath;
}},
.completer = completePath
});
addFlag({
.longName = "output-lock-file",
.description = "Write the given lock file instead of `flake.lock` within the top-level flake.",
.category = category,
.labels = {"flake-lock-path"},
.handler = {[&](std::string lockFilePath) {
lockFlags.outputLockFilePath = lockFilePath;
}},
.completer = completePath
});
addFlag({
.longName = "inputs-from",
.description = "Use the inputs of the specified flake as registry entries.",

28
src/libcmd/repl.cc
View file

@ -40,6 +40,7 @@ extern "C" {
#include "markdown.hh"
#include "local-fs-store.hh"
#include "progress-bar.hh"
#include "print.hh"
#if HAVE_BOEHMGC
#define GC_INCLUDE_NEW
@ -249,7 +250,9 @@ void NixRepl::mainLoop()
el_hist_size = 1000;
#endif
read_history(historyFile.c_str());
auto oldRepl = curRepl;
curRepl = this;
Finally restoreRepl([&] { curRepl = oldRepl; });
#ifndef READLINE
rl_set_complete_func(completionCallback);
rl_set_list_possib_func(listPossibleCallback);
@ -420,6 +423,7 @@ StringSet NixRepl::completePrefix(const std::string & prefix)
}
// FIXME: DRY and match or use the parser
static bool isVarName(std::string_view s)
{
if (s.size() == 0) return false;
@ -888,17 +892,6 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
}
std::ostream & printStringValue(std::ostream & str, const char * string) {
str << "\"";
for (const char * i = string; *i; i++)
if (*i == '\"' || *i == '\\') str << "\\" << *i;
else if (*i == '\n') str << "\\n";
else if (*i == '\r') str << "\\r";
else if (*i == '\t') str << "\\t";
else str << *i;
str << "\"";
return str;
}
// FIXME: lot of cut&paste from Nix's eval.cc.
@ -916,12 +909,14 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
break;
case nBool:
str << ANSI_CYAN << (v.boolean ? "true" : "false") << ANSI_NORMAL;
str << ANSI_CYAN;
printLiteralBool(str, v.boolean);
str << ANSI_NORMAL;
break;
case nString:
str << ANSI_WARNING;
printStringValue(str, v.string.s);
printLiteralString(str, v.string.s);
str << ANSI_NORMAL;
break;
@ -958,10 +953,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
sorted.emplace(state->symbols[i.name], i.value);
for (auto & i : sorted) {
if (isVarName(i.first))
str << i.first;
else
printStringValue(str, i.first.c_str());
printAttributeName(str, i.first);
str << " = ";
if (seen.count(i.second))
str << "«repeated»";
@ -1020,6 +1012,8 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
str << v.fpoint;
break;
case nThunk:
case nExternal:
default:
str << ANSI_RED "«unknown»" ANSI_NORMAL;
break;

4
src/libexpr/attr-path.hh
View file

@ -17,7 +17,9 @@ std::pair<Value *, PosIdx> findAlongAttrPath(
Bindings & autoArgs,
Value & vIn);
/* Heuristic to find the filename and lineno or a nix value. */
/**
* Heuristic to find the filename and lineno or a nix value.
*/
std::pair<SourcePath, uint32_t> findPackageFilename(EvalState & state, Value & v, std::string what);
std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s);

26
src/libexpr/attr-set.hh
View file

@ -13,7 +13,9 @@ namespace nix {
class EvalState;
struct Value;
/* Map one attribute name to its value. */
/**
* Map one attribute name to its value.
*/
struct Attr
{
/* the placement of `name` and `pos` in this struct is important.
@ -37,10 +39,12 @@ static_assert(sizeof(Attr) == 2 * sizeof(uint32_t) + sizeof(Value *),
"avoid introducing any padding into Attr if at all possible, and do not "
"introduce new fields that need not be present for almost every instance.");
/* Bindings contains all the attributes of an attribute set. It is defined
by its size and its capacity, the capacity being the number of Attr
elements allocated after this structure, while the size corresponds to
the number of elements already inserted in this structure. */
/**
* Bindings contains all the attributes of an attribute set. It is defined
* by its size and its capacity, the capacity being the number of Attr
* elements allocated after this structure, while the size corresponds to
* the number of elements already inserted in this structure.
*/
class Bindings
{
public:
@ -95,7 +99,9 @@ public:
size_t capacity() { return capacity_; }
/* Returns the attributes in lexicographically sorted order. */
/**
* Returns the attributes in lexicographically sorted order.
*/
std::vector<const Attr *> lexicographicOrder(const SymbolTable & symbols) const
{
std::vector<const Attr *> res;
@ -112,9 +118,11 @@ public:
friend class EvalState;
};
/* A wrapper around Bindings that ensures that its always in sorted
order at the end. The only way to consume a BindingsBuilder is to
call finish(), which sorts the bindings. */
/**
* A wrapper around Bindings that ensures that its always in sorted
* order at the end. The only way to consume a BindingsBuilder is to
* call finish(), which sorts the bindings.
*/
class BindingsBuilder
{
Bindings * bindings;

10
src/libexpr/eval-cache.hh
View file

@ -110,8 +110,10 @@ public:
ref<AttrCursor> getAttr(std::string_view name);
/* Get an attribute along a chain of attrsets. Note that this does
not auto-call functors or functions. */
/**
* Get an attribute along a chain of attrsets. Note that this does
* not auto-call functors or functions.
*/
OrSuggestions<ref<AttrCursor>> findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force = false);
std::string getString();
@ -130,7 +132,9 @@ public:
Value & forceValue();
/* Force creation of the .drv file in the Nix store. */
/**
* Force creation of the .drv file in the Nix store.
*/
StorePath forceDerivation();
};

4
src/libexpr/eval-inline.hh
View file

@ -5,7 +5,9 @@
namespace nix {
/* Note: Various places expect the allocated memory to be zeroed. */
/**
* Note: Various places expect the allocated memory to be zeroed.
*/
[[gnu::always_inline]]
inline void * allocBytes(size_t n)
{

57
src/libexpr/eval.cc
View file

@ -9,6 +9,7 @@
#include "filetransfer.hh"
#include "function-trace.hh"
#include "profiles.hh"
#include "print.hh"
#include <algorithm>
#include <chrono>
@ -104,18 +105,10 @@ void Value::print(const SymbolTable & symbols, std::ostream & str,
str << integer;
break;
case tBool:
str << (boolean ? "true" : "false");
printLiteralBool(str, boolean);
break;
case tString:
str << "\"";
for (const char * i = string.s; *i; i++)
if (*i == '\"' || *i == '\\') str << "\\" << *i;
else if (*i == '\n') str << "\\n";
else if (*i == '\r') str << "\\r";
else if (*i == '\t') str << "\\t";
else if (*i == '$' && *(i+1) == '{') str << "\\" << *i;
else str << *i;
str << "\"";
printLiteralString(str, string.s);
break;
case tPath:
str << path().to_string(); // !!! escaping?
@ -173,7 +166,17 @@ void Value::print(const SymbolTable & symbols, std::ostream & str,
case tFloat:
str << fpoint;
break;
case tBlackhole:
// Although we know for sure that it's going to be an infinite recursion
// when this value is accessed _in the current context_, it's likely
// that the user will misinterpret a simpler «infinite recursion» output
// as a definitive statement about the value, while in fact it may be
// a valid value after `builtins.trace` and perhaps some other steps
// have completed.
str << "«potential infinite recursion»";
break;
default:
printError("Nix evaluator internal error: Value::print(): invalid value type %1%", internalType);
abort();
}
}
@ -229,6 +232,9 @@ std::string_view showType(ValueType type)
std::string showType(const Value & v)
{
// Allow selecting a subset of enum values
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wswitch-enum"
switch (v.internalType) {
case tString: return v.string.context ? "a string with context" : "a string";
case tPrimOp:
@ -242,16 +248,21 @@ std::string showType(const Value & v)
default:
return std::string(showType(v.type()));
}
#pragma GCC diagnostic pop
}
PosIdx Value::determinePos(const PosIdx pos) const
{
// Allow selecting a subset of enum values
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wswitch-enum"
switch (internalType) {
case tAttrs: return attrs->pos;
case tLambda: return lambda.fun->pos;
case tApp: return app.left->determinePos(pos);
default: return pos;
}
#pragma GCC diagnostic pop
}
bool Value::isTrivial() const
@ -326,6 +337,22 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env)
}
}
#if HAVE_BOEHMGC
/* Disable GC while this object lives. Used by CoroutineContext.
*
* Boehm keeps a count of GC_disable() and GC_enable() calls,
* and only enables GC when the count matches.
*/
class BoehmDisableGC {
public:
BoehmDisableGC() {
GC_disable();
};
~BoehmDisableGC() {
GC_enable();
};
};
#endif
static bool gcInitialised = false;
@ -350,6 +377,15 @@ void initGC()
StackAllocator::defaultAllocator = &boehmGCStackAllocator;
#if NIX_BOEHM_PATCH_VERSION != 1
printTalkative("Unpatched BoehmGC, disabling GC inside coroutines");
/* Used to disable GC when entering coroutines on macOS */
create_coro_gc_hook = []() -> std::shared_ptr<void> {
return std::make_shared<BoehmDisableGC>();
};
#endif
/* Set the initial heap size to something fairly big (25% of
physical RAM, up to a maximum of 384 MiB) so that in most cases
we don't need to garbage collect at all. (Collection has a
@ -2334,6 +2370,7 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v
case nFloat:
return v1.fpoint == v2.fpoint;
case nThunk: // Must not be left by forceValue
default:
error("cannot compare %1% with %2%", showType(v1), showType(v2)).withTrace(pos, errorCtx).debugThrow<EvalError>();
}

232
src/libexpr/eval.hh
View file

@ -44,7 +44,10 @@ struct PrimOp
struct Env
{
Env * up;
unsigned short prevWith:14; // nr of levels up to next `with' environment
/**
* Number of of levels up to next `with` environment
*/
unsigned short prevWith:14;
enum { Plain = 0, HasWithExpr, HasWithAttrs } type:2;
Value * values[0];
};
@ -66,7 +69,9 @@ typedef std::pair<std::string, std::string> SearchPathElem;
typedef std::list<SearchPathElem> SearchPath;
/* Initialise the Boehm GC, if applicable. */
/**
* Initialise the Boehm GC, if applicable.
*/
void initGC();
@ -138,28 +143,38 @@ public:
sPrefix,
sOutputSpecified;
/* If set, force copying files to the Nix store even if they
already exist there. */
/**
* If set, force copying files to the Nix store even if they
* already exist there.
*/
RepairFlag repair;
/* The allowed filesystem paths in restricted or pure evaluation
mode. */
/**
* The allowed filesystem paths in restricted or pure evaluation
* mode.
*/
std::optional<PathSet> allowedPaths;
Bindings emptyBindings;
const SourcePath derivationInternal;
/* Store used to materialise .drv files. */
/**
* Store used to materialise .drv files.
*/
const ref<Store> store;
/* Store used to build stuff. */
/**
* Store used to build stuff.
*/
const ref<Store> buildStore;
RootValue vCallFlake = nullptr;
RootValue vImportedDrvToDerivation = nullptr;
/* Debugger */
/**
* Debugger
*/
void (* debugRepl)(ref<EvalState> es, const ValMap & extraEnv);
bool debugStop;
bool debugQuit;
@ -218,7 +233,9 @@ private:
paths. */
std::map<SourcePath, StorePath> srcToStore;
/* A cache from path names to parse trees. */
/**
* A cache from path names to parse trees.
*/
#if HAVE_BOEHMGC
typedef std::map<SourcePath, Expr *, std::less<SourcePath>, traceable_allocator<std::pair<const SourcePath, Expr *>>> FileParseCache;
#else
@ -226,7 +243,9 @@ private:
#endif
FileParseCache fileParseCache;
/* A cache from path names to values. */
/**
* A cache from path names to values.
*/
#if HAVE_BOEHMGC
typedef std::map<SourcePath, Value, std::less<SourcePath>, traceable_allocator<std::pair<const SourcePath, Value>>> FileEvalCache;
#else
@ -238,17 +257,25 @@ private:
std::map<std::string, std::pair<bool, std::string>> searchPathResolved;
/* Cache used by checkSourcePath(). */
/**
* Cache used by checkSourcePath().
*/
std::unordered_map<Path, SourcePath> resolvedPaths;
/* Cache used by prim_match(). */
/**
* Cache used by prim_match().
*/
std::shared_ptr<RegexCache> regexCache;
#if HAVE_BOEHMGC
/* Allocation cache for GC'd Value objects. */
/**
* Allocation cache for GC'd Value objects.
*/
std::shared_ptr<void *> valueAllocCache;
/* Allocation cache for size-1 Env objects. */
/**
* Allocation cache for size-1 Env objects.
*/
std::shared_ptr<void *> env1AllocCache;
#endif
@ -270,47 +297,65 @@ public:
*/
SourcePath rootPath(CanonPath path);
/* Allow access to a path. */
/**
* Allow access to a path.
*/
void allowPath(const Path & path);
/* Allow access to a store path. Note that this gets remapped to
the real store path if `store` is a chroot store. */
/**
* Allow access to a store path. Note that this gets remapped to
* the real store path if `store` is a chroot store.
*/
void allowPath(const StorePath & storePath);
/* Allow access to a store path and return it as a string. */
/**
* Allow access to a store path and return it as a string.
*/
void allowAndSetStorePathString(const StorePath & storePath, Value & v);
/* Check whether access to a path is allowed and throw an error if
not. Otherwise return the canonicalised path. */
/**
* Check whether access to a path is allowed and throw an error if
* not. Otherwise return the canonicalised path.
*/
SourcePath checkSourcePath(const SourcePath & path);
void checkURI(const std::string & uri);
/* When using a diverted store and 'path' is in the Nix store, map
'path' to the diverted location (e.g. /nix/store/foo is mapped
to /home/alice/my-nix/nix/store/foo). However, this is only
done if the context is not empty, since otherwise we're
probably trying to read from the actual /nix/store. This is
intended to distinguish between import-from-derivation and
sources stored in the actual /nix/store. */
/**
* When using a diverted store and 'path' is in the Nix store, map
* 'path' to the diverted location (e.g. /nix/store/foo is mapped
* to /home/alice/my-nix/nix/store/foo). However, this is only
* done if the context is not empty, since otherwise we're
* probably trying to read from the actual /nix/store. This is
* intended to distinguish between import-from-derivation and
* sources stored in the actual /nix/store.
*/
Path toRealPath(const Path & path, const PathSet & context);
/* Parse a Nix expression from the specified file. */
/**
* Parse a Nix expression from the specified file.
*/
Expr * parseExprFromFile(const SourcePath & path);
Expr * parseExprFromFile(const SourcePath & path, std::shared_ptr<StaticEnv> & staticEnv);
/* Parse a Nix expression from the specified string. */
/**
* Parse a Nix expression from the specified string.
*/
Expr * parseExprFromString(std::string s, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv);
Expr * parseExprFromString(std::string s, const SourcePath & basePath);
Expr * parseStdin();
/* Evaluate an expression read from the given file to normal
form. Optionally enforce that the top-level expression is
trivial (i.e. doesn't require arbitrary computation). */
/**
* Evaluate an expression read from the given file to normal
* form. Optionally enforce that the top-level expression is
* trivial (i.e. doesn't require arbitrary computation).
*/
void evalFile(const SourcePath & path, Value & v, bool mustBeTrivial = false);
/* Like `evalFile`, but with an already parsed expression. */
/**
* Like `evalFile`, but with an already parsed expression.
*/
void cacheFile(
const SourcePath & path,
const SourcePath & resolvedPath,
@ -320,37 +365,52 @@ public:
void resetFileCache();
/* Look up a file in the search path. */
/**
* Look up a file in the search path.
*/
SourcePath findFile(const std::string_view path);
SourcePath findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos);
/* If the specified search path element is a URI, download it. */
/**
* If the specified search path element is a URI, download it.
*/
std::pair<bool, std::string> resolveSearchPathElem(const SearchPathElem & elem);
/* Evaluate an expression to normal form, storing the result in
value `v'. */
/**
* Evaluate an expression to normal form
*
* @param [out] v The resulting is stored here.
*/
void eval(Expr * e, Value & v);
/* Evaluation the expression, then verify that it has the expected
type. */
/**
* Evaluation the expression, then verify that it has the expected
* type.
*/
inline bool evalBool(Env & env, Expr * e);
inline bool evalBool(Env & env, Expr * e, const PosIdx pos, std::string_view errorCtx);
inline void evalAttrs(Env & env, Expr * e, Value & v, const PosIdx pos, std::string_view errorCtx);
/* If `v' is a thunk, enter it and overwrite `v' with the result
of the evaluation of the thunk. If `v' is a delayed function
application, call the function and overwrite `v' with the
result. Otherwise, this is a no-op. */
/**
* If `v` is a thunk, enter it and overwrite `v` with the result
* of the evaluation of the thunk. If `v` is a delayed function
* application, call the function and overwrite `v` with the
* result. Otherwise, this is a no-op.
*/
inline void forceValue(Value & v, const PosIdx pos);
template <typename Callable>
inline void forceValue(Value & v, Callable getPos);
/* Force a value, then recursively force list elements and
attributes. */
/**
* Force a value, then recursively force list elements and
* attributes.
*/
void forceValueDeep(Value & v);
/* Force `v', and then verify that it has the expected type. */
/**
* Force `v`, and then verify that it has the expected type.
*/
NixInt forceInt(Value & v, const PosIdx pos, std::string_view errorCtx);
NixFloat forceFloat(Value & v, const PosIdx pos, std::string_view errorCtx);
bool forceBool(Value & v, const PosIdx pos, std::string_view errorCtx);
@ -361,7 +421,10 @@ public:
inline void forceAttrs(Value & v, Callable getPos, std::string_view errorCtx);
inline void forceList(Value & v, const PosIdx pos, std::string_view errorCtx);
void forceFunction(Value & v, const PosIdx pos, std::string_view errorCtx); // either lambda or primop
/**
* @param v either lambda or primop
*/
void forceFunction(Value & v, const PosIdx pos, std::string_view errorCtx);
std::string_view forceString(Value & v, const PosIdx pos, std::string_view errorCtx);
std::string_view forceString(Value & v, PathSet & context, const PosIdx pos, std::string_view errorCtx);
std::string_view forceStringNoCtx(Value & v, const PosIdx pos, std::string_view errorCtx);
@ -372,17 +435,23 @@ public:
void addErrorTrace(Error & e, const PosIdx pos, const char * s, const std::string & s2, bool frame = false) const;
public:
/* Return true iff the value `v' denotes a derivation (i.e. a
set with attribute `type = "derivation"'). */
/**
* @return true iff the value `v` denotes a derivation (i.e. a
* set with attribute `type = "derivation"`).
*/
bool isDerivation(Value & v);
std::optional<std::string> tryAttrsToString(const PosIdx pos, Value & v,
PathSet & context, bool coerceMore = false, bool copyToStore = true);
/* String coercion. Converts strings, paths and derivations to a
string. If `coerceMore' is set, also converts nulls, integers,
booleans and lists to a string. If `copyToStore' is set,
referenced paths are copied to the Nix store as a side effect. */
/**
* String coercion.
*
* Converts strings, paths and derivations to a
* string. If `coerceMore` is set, also converts nulls, integers,
* booleans and lists to a string. If `copyToStore` is set,
* referenced paths are copied to the Nix store as a side effect.
*/
BackedStringView coerceToString(const PosIdx pos, Value & v, PathSet & context,
std::string_view errorCtx,
bool coerceMore = false, bool copyToStore = true,
@ -390,21 +459,31 @@ public:
StorePath copyPathToStore(PathSet & context, const SourcePath & path);
/* Path coercion. Converts strings, paths and derivations to a
path. The result is guaranteed to be a canonicalised, absolute
path. Nothing is copied to the store. */
/**
* Path coercion.
*
* Converts strings, paths and derivations to a
* path. The result is guaranteed to be a canonicalised, absolute
* path. Nothing is copied to the store.
*/
SourcePath coerceToPath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx);
/* Like coerceToPath, but the result must be a store path. */
/**
* Like coerceToPath, but the result must be a store path.
*/
StorePath coerceToStorePath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx);
public:
/* The base environment, containing the builtin functions and
values. */
/**
* The base environment, containing the builtin functions and
* values.
*/
Env & baseEnv;
/* The same, but used during parsing to resolve variables. */
/**
* The same, but used during parsing to resolve variables.
*/
std::shared_ptr<StaticEnv> staticBaseEnv; // !!! should be private
private:
@ -454,8 +533,10 @@ private:
public:
/* Do a deep equality test between two values. That is, list
elements and attributes are compared recursively. */
/**
* Do a deep equality test between two values. That is, list
* elements and attributes are compared recursively.
*/
bool eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx);
bool isFunctor(Value & fun);
@ -469,11 +550,15 @@ public:
callFunction(fun, 1, args, vRes, pos);
}
/* Automatically call a function for which each argument has a
default value or has a binding in the `args' map. */
/**
* Automatically call a function for which each argument has a
* default value or has a binding in the `args` map.
*/
void autoCallFunction(Bindings & args, Value & fun, Value & res);
/* Allocation primitives. */
/**
* Allocation primitives.
*/
inline Value * allocValue();
inline Env & allocEnv(size_t size);
@ -493,10 +578,13 @@ public:
void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx);
/* Print statistics. */
/**
* Print statistics.
*/
void printStats();
/* Realise the given context, and return a mapping from the placeholders
/**
* Realise the given context, and return a mapping from the placeholders
* used to construct the associated value to their final store path
*/
[[nodiscard]] StringMap realiseContext(const PathSet & context);
@ -556,11 +644,15 @@ struct DebugTraceStacker {
DebugTrace trace;
};
/* Return a string representing the type of the value `v'. */
/**
* @return A string representing the type of the value `v`.
*/
std::string_view showType(ValueType type);
std::string showType(const Value & v);
/* If `path' refers to a directory, then append "/default.nix". */
/**
* If `path` refers to a directory, then append "/default.nix".
*/
SourcePath resolveExprPath(const SourcePath & path);
struct InvalidPathError : EvalError

38
src/libexpr/flake/flake.cc
View file

@ -125,6 +125,9 @@ static FlakeInput parseFlakeInput(EvalState & state,
follows.insert(follows.begin(), lockRootPath.begin(), lockRootPath.end());
input.follows = follows;
} else {
// Allow selecting a subset of enum values
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wswitch-enum"
switch (attr.value->type()) {
case nString:
attrs.emplace(state.symbols[attr.name], attr.value->string.s);
@ -139,6 +142,7 @@ static FlakeInput parseFlakeInput(EvalState & state,
throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected",
state.symbols[attr.name], showType(*attr.value));
}
#pragma GCC diagnostic pop
}
} catch (Error & e) {
e.addTrace(
@ -334,10 +338,14 @@ LockedFlake lockFlake(
}
try {
if (!fetchSettings.allowDirty && lockFlags.referenceLockFilePath) {
throw Error("reference lock file was provided, but the `allow-dirty` setting is set to false");
}
// FIXME: symlink attack
auto oldLockFile = LockFile::read(
flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock");
lockFlags.referenceLockFilePath.value_or(
flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock"));
debug("old lock file: %s", oldLockFile);
@ -619,13 +627,20 @@ LockedFlake lockFlake(
debug("new lock file: %s", newLockFile);
auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock";
auto sourcePath = topRef.input.getSourcePath();
auto outputLockFilePath = sourcePath ? std::optional{*sourcePath + "/" + relPath} : std::nullopt;
if (lockFlags.outputLockFilePath) {
outputLockFilePath = lockFlags.outputLockFilePath;
}
/* Check whether we need to / can write the new lock file. */
if (!(newLockFile == oldLockFile)) {
if (newLockFile != oldLockFile || lockFlags.outputLockFilePath) {
auto diff = LockFile::diff(oldLockFile, newLockFile);
if (lockFlags.writeLockFile) {
if (auto sourcePath = topRef.input.getSourcePath()) {
if (outputLockFilePath) {
if (auto unlockedInput = newLockFile.isUnlocked()) {
if (fetchSettings.warnDirty)
warn("will not write lock file of flake '%s' because it has an unlocked input ('%s')", topRef, *unlockedInput);
@ -633,25 +648,24 @@ LockedFlake lockFlake(
if (!lockFlags.updateLockFile)
throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef);
auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock";
auto path = *sourcePath + "/" + relPath;
bool lockFileExists = pathExists(path);
bool lockFileExists = pathExists(*outputLockFilePath);
if (lockFileExists) {
auto s = chomp(diff);
if (s.empty())
warn("updating lock file '%s'", path);
warn("updating lock file '%s'", *outputLockFilePath);
else
warn("updating lock file '%s':\n%s", path, s);
warn("updating lock file '%s':\n%s", *outputLockFilePath, s);
} else
warn("creating lock file '%s'", path);
warn("creating lock file '%s'", *outputLockFilePath);
newLockFile.write(path);
newLockFile.write(*outputLockFilePath);
std::optional<std::string> commitMessage = std::nullopt;
if (lockFlags.commitLockFile) {
if (lockFlags.outputLockFilePath) {
throw Error("--commit-lock-file and --output-lock-file are currently incompatible");
}
std::string cm;
cm = fetchSettings.commitLockFileSummary.get();

113
src/libexpr/flake/flake.hh
View file

@ -18,7 +18,8 @@ struct FlakeInput;
typedef std::map<FlakeId, FlakeInput> FlakeInputs;
/* FlakeInput is the 'Flake'-level parsed form of the "input" entries
/**
* FlakeInput is the 'Flake'-level parsed form of the "input" entries
* in the flake file.
*
* A FlakeInput is normally constructed by the 'parseFlakeInput'
@ -42,7 +43,12 @@ typedef std::map<FlakeId, FlakeInput> FlakeInputs;
struct FlakeInput
{
std::optional<FlakeRef> ref;
bool isFlake = true; // true = process flake to get outputs, false = (fetched) static source path
/**
* true = process flake to get outputs
*
* false = (fetched) static source path
*/
bool isFlake = true;
std::optional<InputPath> follows;
FlakeInputs overrides;
};
@ -56,23 +62,42 @@ struct ConfigFile
void apply();
};
/* The contents of a flake.nix file. */
/**
* The contents of a flake.nix file.
*/
struct Flake
{
FlakeRef originalRef; // the original flake specification (by the user)
FlakeRef resolvedRef; // registry references and caching resolved to the specific underlying flake
FlakeRef lockedRef; // the specific local store result of invoking the fetcher
bool forceDirty = false; // pretend that 'lockedRef' is dirty
/**
* The original flake specification (by the user)
*/
FlakeRef originalRef;
/**
* registry references and caching resolved to the specific underlying flake
*/
FlakeRef resolvedRef;
/**
* the specific local store result of invoking the fetcher
*/
FlakeRef lockedRef;
/**
* pretend that 'lockedRef' is dirty
*/
bool forceDirty = false;
std::optional<std::string> description;
std::shared_ptr<const fetchers::Tree> sourceInfo;
FlakeInputs inputs;
ConfigFile config; // 'nixConfig' attribute
/**
* 'nixConfig' attribute
*/
ConfigFile config;
~Flake();
};
Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool allowLookup);
/* Fingerprint of a locked flake; used as a cache key. */
/**
* Fingerprint of a locked flake; used as a cache key.
*/
typedef Hash Fingerprint;
struct LockedFlake
@ -85,44 +110,72 @@ struct LockedFlake
struct LockFlags
{
/* Whether to ignore the existing lock file, creating a new one
from scratch. */
/**
* Whether to ignore the existing lock file, creating a new one
* from scratch.
*/
bool recreateLockFile = false;
/* Whether to update the lock file at all. If set to false, if any
change to the lock file is needed (e.g. when an input has been
added to flake.nix), you get a fatal error. */
/**
* Whether to update the lock file at all. If set to false, if any
* change to the lock file is needed (e.g. when an input has been
* added to flake.nix), you get a fatal error.
*/
bool updateLockFile = true;
/* Whether to write the lock file to disk. If set to true, if the
any changes to the lock file are needed and the flake is not
writable (i.e. is not a local Git working tree or similar), you
get a fatal error. If set to false, Nix will use the modified
lock file in memory only, without writing it to disk. */
/**
* Whether to write the lock file to disk. If set to true, if the
* any changes to the lock file are needed and the flake is not
* writable (i.e. is not a local Git working tree or similar), you
* get a fatal error. If set to false, Nix will use the modified
* lock file in memory only, without writing it to disk.
*/
bool writeLockFile = true;
/* Whether to use the registries to lookup indirect flake
references like 'nixpkgs'. */
/**
* Whether to use the registries to lookup indirect flake
* references like 'nixpkgs'.
*/
std::optional<bool> useRegistries = std::nullopt;
/* Whether to apply flake's nixConfig attribute to the configuration */
/**
* Whether to apply flake's nixConfig attribute to the configuration
*/
bool applyNixConfig = false;
/* Whether unlocked flake references (i.e. those without a Git
revision or similar) without a corresponding lock are
allowed. Unlocked flake references with a lock are always
allowed. */
/**
* Whether unlocked flake references (i.e. those without a Git
* revision or similar) without a corresponding lock are
* allowed. Unlocked flake references with a lock are always
* allowed.
*/
bool allowUnlocked = true;
/* Whether to commit changes to flake.lock. */
/**
* Whether to commit changes to flake.lock.
*/
bool commitLockFile = false;
/* Flake inputs to be overridden. */
/**
* The path to a lock file to read instead of the `flake.lock` file in the top-level flake
*/
std::optional<std::string> referenceLockFilePath;
/**
* The path to a lock file to write to instead of the `flake.lock` file in the top-level flake
*/
std::optional<Path> outputLockFilePath;
/**
* Flake inputs to be overridden.
*/
std::map<InputPath, FlakeRef> inputOverrides;
/* Flake inputs to be updated. This means that any existing lock
for those inputs will be ignored. */
/**
* Flake inputs to be updated. This means that any existing lock
* for those inputs will be ignored.
*/
std::set<InputPath> inputUpdates;
};

14
src/libexpr/flake/flakeref.hh
View file

@ -14,7 +14,8 @@ class Store;
typedef std::string FlakeId;
/* A flake reference specifies how to fetch a flake or raw source
/**
* A flake reference specifies how to fetch a flake or raw source
* (e.g. from a Git repository). It is created from a URL-like syntax
* (e.g. 'github:NixOS/patchelf'), an attrset representation (e.g. '{
* type="github"; owner = "NixOS"; repo = "patchelf"; }'), or a local
@ -33,14 +34,17 @@ typedef std::string FlakeId;
* be lazy), but the fetcher can be invoked at any time via the
* FlakeRef to ensure the store is populated with this input.
*/
struct FlakeRef
{
/* Fetcher-specific representation of the input, sufficient to
perform the fetch operation. */
/**
* Fetcher-specific representation of the input, sufficient to
* perform the fetch operation.
*/
fetchers::Input input;
/* sub-path within the fetched input that represents this input */
/**
* sub-path within the fetched input that represents this input
*/
Path subdir;
bool operator==(const FlakeRef & other) const;

5
src/libexpr/flake/lockfile.cc
View file

@ -234,6 +234,11 @@ bool LockFile::operator ==(const LockFile & other) const
return toJSON() == other.toJSON();
}
bool LockFile::operator !=(const LockFile & other) const
{
return !(*this == other);
}
InputPath parseInputPath(std::string_view s)
{
InputPath path;

23
src/libexpr/flake/lockfile.hh
View file

@ -16,9 +16,11 @@ typedef std::vector<FlakeId> InputPath;
struct LockedNode;
/* A node in the lock file. It has outgoing edges to other nodes (its
inputs). Only the root node has this type; all other nodes have
type LockedNode. */
/**
* A node in the lock file. It has outgoing edges to other nodes (its
* inputs). Only the root node has this type; all other nodes have
* type LockedNode.
*/
struct Node : std::enable_shared_from_this<Node>
{
typedef std::variant<ref<LockedNode>, InputPath> Edge;
@ -28,7 +30,9 @@ struct Node : std::enable_shared_from_this<Node>
virtual ~Node() { }
};
/* A non-root node in the lock file. */
/**
* A non-root node in the lock file.
*/
struct LockedNode : Node
{
FlakeRef lockedRef, originalRef;
@ -63,10 +67,15 @@ struct LockFile
void write(const Path & path) const;
/* Check whether this lock file has any unlocked inputs. */
/**
* Check whether this lock file has any unlocked inputs.
*/
std::optional<FlakeRef> isUnlocked() const;
bool operator ==(const LockFile & other) const;
// Needed for old gcc versions that don't synthesize it (like gcc 8.2.2
// that is still the default on aarch64-linux)
bool operator !=(const LockFile & other) const;
std::shared_ptr<Node> findInput(const InputPath & path);
@ -74,7 +83,9 @@ struct LockFile
static std::string diff(const LockFile & oldLocks, const LockFile & newLocks);
/* Check that every 'follows' input target exists. */
/**
* Check that every 'follows' input target exists.
*/
void check();
};

22
src/libexpr/get-drvs.hh
View file

@ -26,7 +26,10 @@ private:
mutable std::string outputName;
Outputs outputs;
bool failed = false; // set if we get an AssertionError
/**
* Set if we get an AssertionError
*/
bool failed = false;
Bindings * attrs = nullptr, * meta = nullptr;
@ -35,7 +38,10 @@ private:
bool checkMeta(Value & v);
public:
std::string attrPath; /* path towards the derivation */
/**
* path towards the derivation
*/
std::string attrPath;
DrvInfo(EvalState & state) : state(&state) { };
DrvInfo(EvalState & state, std::string attrPath, Bindings * attrs);
@ -47,8 +53,10 @@ public:
StorePath requireDrvPath() const;
StorePath queryOutPath() const;
std::string queryOutputName() const;
/** Return the unordered map of output names to (optional) output paths.
* The "outputs to install" are determined by `meta.outputsToInstall`. */
/**
* Return the unordered map of output names to (optional) output paths.
* The "outputs to install" are determined by `meta.outputsToInstall`.
*/
Outputs queryOutputs(bool withPaths = true, bool onlyOutputsToInstall = false);
StringSet queryMetaNames();
@ -80,8 +88,10 @@ typedef std::list<DrvInfo> DrvInfos;
#endif
/* If value `v' denotes a derivation, return a DrvInfo object
describing it. Otherwise return nothing. */
/**
* If value `v` denotes a derivation, return a DrvInfo object
* describing it. Otherwise return nothing.
*/
std::optional<DrvInfo> getDerivation(EvalState & state,
Value & v, bool ignoreAssertionFailures);

2
src/libexpr/local.mk
View file

@ -46,3 +46,5 @@ $(foreach i, $(wildcard src/libexpr/flake/*.hh), \
$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh
$(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh
src/libexpr/primops/fromTOML.o: ERROR_SWITCH_ENUM =

42
src/libexpr/nixexpr.cc
View file

@ -3,6 +3,7 @@
#include "eval.hh"
#include "symbol-table.hh"
#include "util.hh"
#include "print.hh"
#include <cstdlib>
@ -60,45 +61,12 @@ Pos::operator std::shared_ptr<AbstractPos>() const
return pos;
}
/* Displaying abstract syntax trees. */
static void showString(std::ostream & str, std::string_view s)
{
str << '"';
for (auto c : s)
if (c == '"' || c == '\\' || c == '$') str << "\\" << c;
else if (c == '\n') str << "\\n";
else if (c == '\r') str << "\\r";
else if (c == '\t') str << "\\t";
else str << c;
str << '"';
}
// FIXME: remove, because *symbols* are abstract and do not have a single
// textual representation; see printIdentifier()
std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol)
{
std::string_view s = symbol;
if (s.empty())
str << "\"\"";
else if (s == "if") // FIXME: handle other keywords
str << '"' << s << '"';
else {
char c = s[0];
if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) {
showString(str, s);
return str;
}
for (auto c : s)
if (!((c >= 'a' && c <= 'z') ||
(c >= 'A' && c <= 'Z') ||
(c >= '0' && c <= '9') ||
c == '_' || c == '\'' || c == '-')) {
showString(str, s);
return str;
}
str << s;
}
return str;
return printIdentifier(str, s);
}
void Expr::show(const SymbolTable & symbols, std::ostream & str) const
@ -118,7 +86,7 @@ void ExprFloat::show(const SymbolTable & symbols, std::ostream & str) const
void ExprString::show(const SymbolTable & symbols, std::ostream & str) const
{
showString(str, s);
printLiteralString(str, s);
}
void ExprPath::show(const SymbolTable & symbols, std::ostream & str) const

16
src/libexpr/nixexpr.hh
View file

@ -22,7 +22,9 @@ MakeError(UndefinedVarError, Error);
MakeError(MissingArgumentError, EvalError);
MakeError(RestrictedPathError, Error);
/* Position objects. */
/**
* Position objects.
*/
struct Pos
{
uint32_t line;
@ -133,7 +135,9 @@ class EvalState;
struct StaticEnv;
/* An attribute path is a sequence of attribute names. */
/**
* An attribute path is a sequence of attribute names.
*/
struct AttrName
{
Symbol symbol;
@ -213,11 +217,11 @@ struct ExprVar : Expr
or function argument) or from a "with". */
bool fromWith;
/* In the former case, the value is obtained by going `level'
/* In the former case, the value is obtained by going `level`
levels up from the current environment and getting the
`displ'th value in that environment. In the latter case, the
value is obtained by getting the attribute named `name' from
the set stored in the environment that is `level' levels up
`displ`th value in that environment. In the latter case, the
value is obtained by getting the attribute named `name` from
the set stored in the environment that is `level` levels up
from the current one.*/
Level level;
Displacement displ;

4
src/libexpr/primops.cc
View file

@ -578,6 +578,9 @@ struct CompareValues
return v1->integer < v2->fpoint;
if (v1->type() != v2->type())
state.error("cannot compare %s with %s", showType(*v1), showType(*v2)).debugThrow<EvalError>();
// Allow selecting a subset of enum values
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wswitch-enum"
switch (v1->type()) {
case nInt:
return v1->integer < v2->integer;
@ -600,6 +603,7 @@ struct CompareValues
}
default:
state.error("cannot compare %s with %s; values of that type are incomparable", showType(*v1), showType(*v2)).debugThrow<EvalError>();
#pragma GCC diagnostic pop
}
} catch (Error & e) {
if (!errorCtx.empty())

16
src/libexpr/primops.hh
View file

@ -23,9 +23,11 @@ struct RegisterPrimOp
typedef std::vector<Info> PrimOps;
static PrimOps * primOps;
/* You can register a constant by passing an arity of 0. fun
will get called during EvalState initialization, so there
may be primops not yet added and builtins is not yet sorted. */
/**
* You can register a constant by passing an arity of 0. fun
* will get called during EvalState initialization, so there
* may be primops not yet added and builtins is not yet sorted.
*/
RegisterPrimOp(
std::string name,
size_t arity,
@ -38,10 +40,14 @@ struct RegisterPrimOp
may wish to use them in limited contexts without globally enabling
them. */
/* Load a ValueInitializer from a DSO and return whatever it initializes */
/**
* Load a ValueInitializer from a DSO and return whatever it initializes
*/
void prim_importNative(EvalState & state, const PosIdx pos, Value * * args, Value & v);
/* Execute a program and parse its output */
/**
* Execute a program and parse its output
*/
void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v);
}

78
src/libexpr/print.cc Normal file
View file

@ -0,0 +1,78 @@
#include "print.hh"
namespace nix {
std::ostream &
printLiteralString(std::ostream & str, const std::string_view string)
{
str << "\"";
for (auto i = string.begin(); i != string.end(); ++i) {
if (*i == '\"' || *i == '\\') str << "\\" << *i;
else if (*i == '\n') str << "\\n";
else if (*i == '\r') str << "\\r";
else if (*i == '\t') str << "\\t";
else if (*i == '$' && *(i+1) == '{') str << "\\" << *i;
else str << *i;
}
str << "\"";
return str;
}
std::ostream &
printLiteralBool(std::ostream & str, bool boolean)
{
str << (boolean ? "true" : "false");
return str;
}
std::ostream &
printIdentifier(std::ostream & str, std::string_view s) {
if (s.empty())
str << "\"\"";
else if (s == "if") // FIXME: handle other keywords
str << '"' << s << '"';
else {
char c = s[0];
if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) {
printLiteralString(str, s);
return str;
}
for (auto c : s)
if (!((c >= 'a' && c <= 'z') ||
(c >= 'A' && c <= 'Z') ||
(c >= '0' && c <= '9') ||
c == '_' || c == '\'' || c == '-')) {
printLiteralString(str, s);
return str;
}
str << s;
}
return str;
}
// FIXME: keywords
static bool isVarName(std::string_view s)
{
if (s.size() == 0) return false;
char c = s[0];
if ((c >= '0' && c <= '9') || c == '-' || c == '\'') return false;
for (auto & i : s)
if (!((i >= 'a' && i <= 'z') ||
(i >= 'A' && i <= 'Z') ||
(i >= '0' && i <= '9') ||
i == '_' || i == '-' || i == '\''))
return false;
return true;
}
std::ostream &
printAttributeName(std::ostream & str, std::string_view name) {
if (isVarName(name))
str << name;
else
printLiteralString(str, name);
return str;
}
}

48
src/libexpr/print.hh Normal file
View file

@ -0,0 +1,48 @@
#pragma once
/**
* @file
* @brief Common printing functions for the Nix language
*
* While most types come with their own methods for printing, they share some
* functions that are placed here.
*/
#include <iostream>
namespace nix {
/**
* Print a string as a Nix string literal.
*
* Quotes and fairly minimal escaping are added.
*
* @param s The logical string
*/
std::ostream & printLiteralString(std::ostream & o, std::string_view s);
inline std::ostream & printLiteralString(std::ostream & o, const char * s) {
return printLiteralString(o, std::string_view(s));
}
inline std::ostream & printLiteralString(std::ostream & o, const std::string & s) {
return printLiteralString(o, std::string_view(s));
}
/** Print `true` or `false`. */
std::ostream & printLiteralBool(std::ostream & o, bool b);
/**
* Print a string as an attribute name in the Nix expression language syntax.
*
* Prints a quoted string if necessary.
*/
std::ostream & printAttributeName(std::ostream & o, std::string_view s);
/**
* Print a string as an identifier in the Nix expression language syntax.
*
* FIXME: "identifier" is ambiguous. Identifiers do not have a single
* textual representation. They can be used in variable references,
* let bindings, left-hand sides or attribute names in a select
* expression, or something else entirely, like JSON. Use one of the
* `print*` functions instead.
*/
std::ostream & printIdentifier(std::ostream & o, std::string_view s);
}

26
src/libexpr/symbol-table.hh
View file

@ -10,15 +10,11 @@
namespace nix {
/* Symbol table used by the parser and evaluator to represent and look
up identifiers and attributes efficiently. SymbolTable::create()
converts a string into a symbol. Symbols have the property that
they can be compared efficiently (using an equality test),
because the symbol table stores only one copy of each string. */
/* This class mainly exists to give us an operator<< for ostreams. We could also
return plain strings from SymbolTable, but then we'd have to wrap every
instance of a symbol that is fmt()ed, which is inconvenient and error-prone. */
/**
* This class mainly exists to give us an operator<< for ostreams. We could also
* return plain strings from SymbolTable, but then we'd have to wrap every
* instance of a symbol that is fmt()ed, which is inconvenient and error-prone.
*/
class SymbolStr
{
friend class SymbolTable;
@ -47,6 +43,11 @@ public:
friend std::ostream & operator <<(std::ostream & os, const SymbolStr & symbol);
};
/**
* Symbols have the property that they can be compared efficiently
* (using an equality test), because the symbol table stores only one
* copy of each string.
*/
class Symbol
{
friend class SymbolTable;
@ -66,6 +67,10 @@ public:
bool operator!=(const Symbol other) const { return id != other.id; }
};
/**
* Symbol table used by the parser and evaluator to represent and look
* up identifiers and attributes efficiently.
*/
class SymbolTable
{
private:
@ -74,6 +79,9 @@ private:
public:
/**
* converts a string into a symbol.
*/
Symbol create(std::string_view s)
{
// Most symbols are looked up more than once, so we trade off insertion performance

95
src/libexpr/value.hh
View file

@ -37,9 +37,11 @@ typedef enum {
tFloat
} InternalType;
// This type abstracts over all actual value types in the language,
// grouping together implementation details like tList*, different function
// types, and types in non-normal form (so thunks and co.)
/**
* This type abstracts over all actual value types in the language,
* grouping together implementation details like tList*, different function
* types, and types in non-normal form (so thunks and co.)
*/
typedef enum {
nThunk,
nInt,
@ -71,38 +73,51 @@ class XMLWriter;
typedef int64_t NixInt;
typedef double NixFloat;
/* External values must descend from ExternalValueBase, so that
/**
* External values must descend from ExternalValueBase, so that
* type-agnostic nix functions (e.g. showType) can be implemented
*/
class ExternalValueBase
{
friend std::ostream & operator << (std::ostream & str, const ExternalValueBase & v);
protected:
/* Print out the value */
/**
* Print out the value
*/
virtual std::ostream & print(std::ostream & str) const = 0;
public:
/* Return a simple string describing the type */
/**
* Return a simple string describing the type
*/
virtual std::string showType() const = 0;
/* Return a string to be used in builtins.typeOf */
/**
* Return a string to be used in builtins.typeOf
*/
virtual std::string typeOf() const = 0;
/* Coerce the value to a string. Defaults to uncoercable, i.e. throws an
/**
* Coerce the value to a string. Defaults to uncoercable, i.e. throws an
* error.
*/
virtual std::string coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const;
/* Compare to another value of the same type. Defaults to uncomparable,
/**
* Compare to another value of the same type. Defaults to uncomparable,
* i.e. always false.
*/
virtual bool operator ==(const ExternalValueBase & b) const;
/* Print the value as JSON. Defaults to unconvertable, i.e. throws an error */
/**
* Print the value as JSON. Defaults to unconvertable, i.e. throws an error
*/
virtual nlohmann::json printValueAsJSON(EvalState & state, bool strict,
PathSet & context, bool copyToStore = true) const;
/* Print the value as XML. Defaults to unevaluated */
/**
* Print the value as XML. Defaults to unevaluated
*/
virtual void printValueAsXML(EvalState & state, bool strict, bool location,
XMLWriter & doc, PathSet & context, PathSet & drvsSeen,
const PosIdx pos) const;
@ -147,26 +162,28 @@ public:
NixInt integer;
bool boolean;
/* Strings in the evaluator carry a so-called `context' which
is a list of strings representing store paths. This is to
allow users to write things like
/**
* Strings in the evaluator carry a so-called `context` which
* is a list of strings representing store paths. This is to
* allow users to write things like
"--with-freetype2-library=" + freetype + "/lib"
* "--with-freetype2-library=" + freetype + "/lib"
where `freetype' is a derivation (or a source to be copied
to the store). If we just concatenated the strings without
keeping track of the referenced store paths, then if the
string is used as a derivation attribute, the derivation
will not have the correct dependencies in its inputDrvs and
inputSrcs.
* where `freetype` is a derivation (or a source to be copied
* to the store). If we just concatenated the strings without
* keeping track of the referenced store paths, then if the
* string is used as a derivation attribute, the derivation
* will not have the correct dependencies in its inputDrvs and
* inputSrcs.
The semantics of the context is as follows: when a string
with context C is used as a derivation attribute, then the
derivations in C will be added to the inputDrvs of the
derivation, and the other store paths in C will be added to
the inputSrcs of the derivations.
* The semantics of the context is as follows: when a string
* with context C is used as a derivation attribute, then the
* derivations in C will be added to the inputDrvs of the
* derivation, and the other store paths in C will be added to
* the inputSrcs of the derivations.
For canonicity, the store paths should be in sorted order. */
* For canonicity, the store paths should be in sorted order.
*/
struct {
const char * s;
const char * * context; // must be in sorted order
@ -198,8 +215,10 @@ public:
NixFloat fpoint;
};
// Returns the normal type of a Value. This only returns nThunk if the
// Value hasn't been forceValue'd
/**
* Returns the normal type of a Value. This only returns nThunk if
* the Value hasn't been forceValue'd
*/
inline ValueType type() const
{
switch (internalType) {
@ -218,8 +237,10 @@ public:
abort();
}
/* After overwriting an app node, be sure to clear pointers in the
Value to ensure that the target isn't kept alive unnecessarily. */
/**
* After overwriting an app node, be sure to clear pointers in the
* Value to ensure that the target isn't kept alive unnecessarily.
*/
inline void clearValue()
{
app.left = app.right = 0;
@ -372,9 +393,11 @@ public:
PosIdx determinePos(const PosIdx pos) const;
/* Check whether forcing this value requires a trivial amount of
computation. In particular, function applications are
non-trivial. */
/**
* Check whether forcing this value requires a trivial amount of
* computation. In particular, function applications are
* non-trivial.
*/
bool isTrivial() const;
NixStringContext getContext(const Store &);
@ -432,7 +455,9 @@ typedef std::map<Symbol, ValueVector> ValueVectorMap;
#endif
/* A value allocated in traceable memory. */
/**
* A value allocated in traceable memory.
*/
typedef std::shared_ptr<Value *> RootValue;
RootValue allocRootValue(Value * v);

44
src/libexpr/value/context.hh
View file

@ -28,34 +28,37 @@ public:
class Store;
/* Plain opaque path to some store object.
Encoded as just the path: ‘<path>’.
*/
/**
* Plain opaque path to some store object.
*
* Encoded as just the path: ‘<path>’.
*/
struct NixStringContextElem_Opaque {
StorePath path;
GENERATE_CMP(NixStringContextElem_Opaque, me->path);
};
/* Path to a derivation and its entire build closure.
The path doesn't just refer to derivation itself and its closure, but
also all outputs of all derivations in that closure (including the
root derivation).
Encoded in the form ‘=<drvPath>’.
*/
/**
* Path to a derivation and its entire build closure.
*
* The path doesn't just refer to derivation itself and its closure, but
* also all outputs of all derivations in that closure (including the
* root derivation).
*
* Encoded in the form ‘=<drvPath>’.
*/
struct NixStringContextElem_DrvDeep {
StorePath drvPath;
GENERATE_CMP(NixStringContextElem_DrvDeep, me->drvPath);
};
/* Derivation output.
Encoded in the form ‘!<output>!<drvPath>’.
*/
/**
* Derivation output.
*
* Encoded in the form ‘!<output>!<drvPath>’.
*/
struct NixStringContextElem_Built {
StorePath drvPath;
std::string output;
@ -84,10 +87,11 @@ struct NixStringContextElem : _NixStringContextElem_Raw {
return static_cast<Raw &>(*this);
}
/* Decode a context string, one of:
- ‘<path>’
- ‘=<path>’
- ‘!<name>!<path>’
/**
* Decode a context string, one of:
* - ‘<path>’
* - ‘=<path>’
* - ‘!<name>!<path>’
*/
static NixStringContextElem parse(const Store & store, std::string_view s);
std::string to_string(const Store & store) const;

41
src/libfetchers/fetchers.hh
View file

@ -21,14 +21,14 @@ struct Tree
struct InputScheme;
/* The Input object is generated by a specific fetcher, based on the
/**
* The Input object is generated by a specific fetcher, based on the
* user-supplied input attribute in the flake.nix file, and contains
* the information that the specific fetcher needs to perform the
* actual fetch. The Input object is most commonly created via the
* "fromURL()" or "fromAttrs()" static functions which are provided
* the url or attrset specified in the flake file.
*/
struct Input
{
friend struct InputScheme;
@ -38,7 +38,9 @@ struct Input
bool locked = false;
bool direct = true;
/* path of the parent of this input, used for relative path resolution */
/**
* path of the parent of this input, used for relative path resolution
*/
std::optional<Path> parent;
public:
@ -56,27 +58,35 @@ public:
Attrs toAttrs() const;
/* Check whether this is a "direct" input, that is, not
one that goes through a registry. */
/**
* Check whether this is a "direct" input, that is, not
* one that goes through a registry.
*/
bool isDirect() const { return direct; }
/* Check whether this is a "locked" input, that is,
one that contains a commit hash or content hash. */
/**
* Check whether this is a "locked" input, that is,
* one that contains a commit hash or content hash.
*/
bool isLocked() const { return locked; }
/* Check whether the input carries all necessary info required
for cache insertion and substitution.
These fields are used to uniquely identify cached trees
within the "tarball TTL" window without necessarily
indicating that the input's origin is unchanged. */
/**
* Check whether the input carries all necessary info required
* for cache insertion and substitution.
* These fields are used to uniquely identify cached trees
* within the "tarball TTL" window without necessarily
* indicating that the input's origin is unchanged.
*/
bool hasAllInfo() const;
bool operator ==(const Input & other) const;
bool contains(const Input & other) const;
/* Fetch the input into the Nix store, returning the location in
the Nix store and the locked input. */
/**
* Fetch the input into the Nix store, returning the location in
* the Nix store and the locked input.
*/
std::pair<Tree, Input> fetch(ref<Store> store) const;
Input applyOverrides(
@ -105,7 +115,8 @@ public:
};
/* The InputScheme represents a type of fetcher. Each fetcher
/**
* The InputScheme represents a type of fetcher. Each fetcher
* registers with nix at startup time. When processing an input for a
* flake, each scheme is given an opportunity to "recognize" that
* input from the url or attributes in the flake file's specification

60
src/libmain/shared.hh
View file

@ -25,7 +25,9 @@ public:
int handleExceptions(const std::string & programName, std::function<void()> fun);
/* Don't forget to call initPlugins() after settings are initialized! */
/**
* Don't forget to call initPlugins() after settings are initialized!
*/
void initNix();
void parseCmdLine(int argc, char * * argv,
@ -36,7 +38,9 @@ void parseCmdLine(const std::string & programName, const Strings & args,
void printVersion(const std::string & programName);
/* Ugh. No better place to put this. */
/**
* Ugh. No better place to put this.
*/
void printGCWarning();
class Store;
@ -75,11 +79,15 @@ struct LegacyArgs : public MixCommonArgs
};
/* Show the manual page for the specified program. */
/**
* Show the manual page for the specified program.
*/
void showManPage(const std::string & name);
/* The constructor of this class starts a pager if stdout is a
terminal and $PAGER is set. Stdout is redirected to the pager. */
/**
* The constructor of this class starts a pager if stdout is a
* terminal and $PAGER is set. Stdout is redirected to the pager.
*/
class RunPager
{
public:
@ -110,28 +118,34 @@ struct PrintFreed
};
/* Install a SIGSEGV handler to detect stack overflows. */
/**
* Install a SIGSEGV handler to detect stack overflows.
*/
void detectStackOverflow();
/* Pluggable behavior to run in case of a stack overflow.
Default value: defaultStackOverflowHandler.
This is called by the handler installed by detectStackOverflow().
This gives Nix library consumers a limit opportunity to report the error
condition. The handler should exit the process.
See defaultStackOverflowHandler() for a reference implementation.
NOTE: Use with diligence, because this runs in the signal handler, with very
limited stack space and a potentially a corrupted heap, all while the failed
thread is blocked indefinitely. All functions called must be reentrant. */
/**
* Pluggable behavior to run in case of a stack overflow.
*
* Default value: defaultStackOverflowHandler.
*
* This is called by the handler installed by detectStackOverflow().
*
* This gives Nix library consumers a limit opportunity to report the error
* condition. The handler should exit the process.
* See defaultStackOverflowHandler() for a reference implementation.
*
* NOTE: Use with diligence, because this runs in the signal handler, with very
* limited stack space and a potentially a corrupted heap, all while the failed
* thread is blocked indefinitely. All functions called must be reentrant.
*/
extern std::function<void(siginfo_t * info, void * ctx)> stackOverflowHandler;
/* The default, robust implementation of stackOverflowHandler.
Prints an error message directly to stderr using a syscall instead of the
logger. Exits the process immediately after. */
/**
* The default, robust implementation of stackOverflowHandler.
*
* Prints an error message directly to stderr using a syscall instead of the
* logger. Exits the process immediately after.
*/
void defaultStackOverflowHandler(siginfo_t * info, void * ctx);
}

18
src/libstore/binary-cache-store.hh
View file

@ -46,6 +46,11 @@ struct BinaryCacheStoreConfig : virtual StoreConfig
)"};
};
/**
* @note subclasses must implement at least one of the two
* virtual getFile() methods.
*/
class BinaryCacheStore : public virtual BinaryCacheStoreConfig,
public virtual Store,
public virtual LogStore
@ -75,14 +80,15 @@ public:
std::string && data,
const std::string & mimeType);
/* Note: subclasses must implement at least one of the two
following getFile() methods. */
/* Dump the contents of the specified file to a sink. */
/**
* Dump the contents of the specified file to a sink.
*/
virtual void getFile(const std::string & path, Sink & sink);
/* Fetch the specified file and call the specified callback with
the result. A subclass may implement this asynchronously. */
/**
* Fetch the specified file and call the specified callback with
* the result. A subclass may implement this asynchronously.
*/
virtual void getFile(
const std::string & path,
Callback<std::optional<std::string>> callback) noexcept;

57
src/libstore/build-result.hh
View file

@ -12,9 +12,12 @@ namespace nix {
struct BuildResult
{
/* Note: don't remove status codes, and only add new status codes
at the end of the list, to prevent client/server
incompatibilities in the nix-store --serve protocol. */
/**
* @note This is directly used in the nix-store --serve protocol.
* That means we need to worry about compatability across versions.
* Therefore, don't remove status codes, and only add new status
* codes at the end of the list.
*/
enum Status {
Built = 0,
Substituted,
@ -22,8 +25,10 @@ struct BuildResult
PermanentFailure,
InputRejected,
OutputRejected,
TransientFailure, // possibly transient
CachedFailure, // no longer used
/// possibly transient
TransientFailure,
/// no longer used
CachedFailure,
TimedOut,
MiscFailure,
DependencyFailed,
@ -33,7 +38,12 @@ struct BuildResult
NoSubstituters,
} status = MiscFailure;
// FIXME: include entire ErrorInfo object.
/**
* Information about the error if the build failed.
*
* @todo This should be an entire ErrorInfo object, not just a
* string, for richer information.
*/
std::string errorMsg;
std::string toString() const {
@ -53,33 +63,46 @@ struct BuildResult
case LogLimitExceeded: return "LogLimitExceeded";
case NotDeterministic: return "NotDeterministic";
case ResolvesToAlreadyValid: return "ResolvesToAlreadyValid";
case NoSubstituters: return "NoSubstituters";
default: return "Unknown";
};
}();
return strStatus + ((errorMsg == "") ? "" : " : " + errorMsg);
}
/* How many times this build was performed. */
/**
* How many times this build was performed.
*/
unsigned int timesBuilt = 0;
/* If timesBuilt > 1, whether some builds did not produce the same
result. (Note that 'isNonDeterministic = false' does not mean
the build is deterministic, just that we don't have evidence of
non-determinism.) */
/**
* If timesBuilt > 1, whether some builds did not produce the same
* result. (Note that 'isNonDeterministic = false' does not mean
* the build is deterministic, just that we don't have evidence of
* non-determinism.)
*/
bool isNonDeterministic = false;
/* The derivation we built or the store path we substituted. */
/**
* The derivation we built or the store path we substituted.
*/
DerivedPath path;
/* For derivations, a mapping from the names of the wanted outputs
to actual paths. */
/**
* For derivations, a mapping from the names of the wanted outputs
* to actual paths.
*/
DrvOutputs builtOutputs;
/* The start/stop times of the build (or one of the rounds, if it
was repeated). */
/**
* The start/stop times of the build (or one of the rounds, if it
* was repeated).
*/
time_t startTime = 0, stopTime = 0;
/* User and system CPU time the build took. */
/**
* User and system CPU time the build took.
*/
std::optional<std::chrono::microseconds> cpuUser, cpuSystem;
bool success()

174
src/libstore/build/derivation-goal.hh
View file

@ -16,8 +16,10 @@ struct HookInstance;
typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
/* Unless we are repairing, we don't both to test validity and just assume it,
so the choices are `Absent` or `Valid`. */
/**
* Unless we are repairing, we don't both to test validity and just assume it,
* so the choices are `Absent` or `Valid`.
*/
enum struct PathStatus {
Corrupt,
Absent,
@ -27,11 +29,15 @@ enum struct PathStatus {
struct InitialOutputStatus {
StorePath path;
PathStatus status;
/* Valid in the store, and additionally non-corrupt if we are repairing */
/**
* Valid in the store, and additionally non-corrupt if we are repairing
*/
bool isValid() const {
return status == PathStatus::Valid;
}
/* Merely present, allowed to be corrupt */
/**
* Merely present, allowed to be corrupt
*/
bool isPresent() const {
return status == PathStatus::Corrupt
|| status == PathStatus::Valid;
@ -46,59 +52,87 @@ struct InitialOutput {
struct DerivationGoal : public Goal
{
/* Whether to use an on-disk .drv file. */
/**
* Whether to use an on-disk .drv file.
*/
bool useDerivation;
/* The path of the derivation. */
/** The path of the derivation. */
StorePath drvPath;
/* The goal for the corresponding resolved derivation */
/**
* The goal for the corresponding resolved derivation
*/
std::shared_ptr<DerivationGoal> resolvedDrvGoal;
/* The specific outputs that we need to build. Empty means all of
them. */
/**
* The specific outputs that we need to build. Empty means all of
* them.
*/
OutputsSpec wantedOutputs;
/* Mapping from input derivations + output names to actual store
paths. This is filled in by waiteeDone() as each dependency
finishes, before inputsRealised() is reached, */
/**
* Mapping from input derivations + output names to actual store
* paths. This is filled in by waiteeDone() as each dependency
* finishes, before inputsRealised() is reached.
*/
std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
/* Whether additional wanted outputs have been added. */
/**
* Whether additional wanted outputs have been added.
*/
bool needRestart = false;
/* Whether to retry substituting the outputs after building the
inputs. This is done in case of an incomplete closure. */
/**
* Whether to retry substituting the outputs after building the
* inputs. This is done in case of an incomplete closure.
*/
bool retrySubstitution = false;
/* Whether we've retried substitution, in which case we won't try
again. */
/**
* Whether we've retried substitution, in which case we won't try
* again.
*/
bool retriedSubstitution = false;
/* The derivation stored at drvPath. */
/**
* The derivation stored at drvPath.
*/
std::unique_ptr<Derivation> drv;
std::unique_ptr<ParsedDerivation> parsedDrv;
/* The remainder is state held during the build. */
/**
* The remainder is state held during the build.
*/
/* Locks on (fixed) output paths. */
/**
* Locks on (fixed) output paths.
*/
PathLocks outputLocks;
/* All input paths (that is, the union of FS closures of the
immediate input paths). */
/**
* All input paths (that is, the union of FS closures of the
* immediate input paths).
*/
StorePathSet inputPaths;
std::map<std::string, InitialOutput> initialOutputs;
/* File descriptor for the log file. */
/**
* File descriptor for the log file.
*/
AutoCloseFD fdLogFile;
std::shared_ptr<BufferedSink> logFileSink, logSink;
/* Number of bytes received from the builder's stdout/stderr. */
/**
* Number of bytes received from the builder's stdout/stderr.
*/
unsigned long logSize;
/* The most recent log lines. */
/**
* The most recent log lines.
*/
std::list<std::string> logTail;
std::string currentLogLine;
@ -106,10 +140,14 @@ struct DerivationGoal : public Goal
std::string currentHookLine;
/* The build hook. */
/**
* The build hook.
*/
std::unique_ptr<HookInstance> hook;
/* The sort of derivation we are building. */
/**
* The sort of derivation we are building.
*/
DerivationType derivationType;
typedef void (DerivationGoal::*GoalState)();
@ -121,12 +159,16 @@ struct DerivationGoal : public Goal
std::unique_ptr<Activity> act;
/* Activity that denotes waiting for a lock. */
/**
* Activity that denotes waiting for a lock.
*/
std::unique_ptr<Activity> actLock;
std::map<ActivityId, Activity> builderActivities;
/* The remote machine on which we're building. */
/**
* The remote machine on which we're building.
*/
std::string machineName;
DerivationGoal(const StorePath & drvPath,
@ -143,10 +185,14 @@ struct DerivationGoal : public Goal
void work() override;
/* Add wanted outputs to an already existing derivation goal. */
/**
* Add wanted outputs to an already existing derivation goal.
*/
void addWantedOutputs(const OutputsSpec & outputs);
/* The states. */
/**
* The states.
*/
void getDerivation();
void loadDerivation();
void haveDerivation();
@ -160,28 +206,42 @@ struct DerivationGoal : public Goal
void resolvedFinished();
/* Is the build hook willing to perform the build? */
/**
* Is the build hook willing to perform the build?
*/
HookReply tryBuildHook();
virtual int getChildStatus();
/* Check that the derivation outputs all exist and register them
as valid. */
/**
* Check that the derivation outputs all exist and register them
* as valid.
*/
virtual DrvOutputs registerOutputs();
/* Open a log file and a pipe to it. */
/**
* Open a log file and a pipe to it.
*/
Path openLogFile();
/* Sign the newly built realisation if the store allows it */
/**
* Sign the newly built realisation if the store allows it
*/
virtual void signRealisation(Realisation&) {}
/* Close the log file. */
/**
* Close the log file.
*/
void closeLogFile();
/* Close the read side of the logger pipe. */
/**
* Close the read side of the logger pipe.
*/
virtual void closeReadPipes();
/* Cleanup hooks for buildDone() */
/**
* Cleanup hooks for buildDone()
*/
virtual void cleanupHookFinally();
virtual void cleanupPreChildKill();
virtual void cleanupPostChildKill();
@ -191,30 +251,40 @@ struct DerivationGoal : public Goal
virtual bool isReadDesc(int fd);
/* Callback used by the worker to write to the log. */
/**
* Callback used by the worker to write to the log.
*/
void handleChildOutput(int fd, std::string_view data) override;
void handleEOF(int fd) override;
void flushLine();
/* Wrappers around the corresponding Store methods that first consult the
derivation. This is currently needed because when there is no drv file
there also is no DB entry. */
/**
* Wrappers around the corresponding Store methods that first consult the
* derivation. This is currently needed because when there is no drv file
* there also is no DB entry.
*/
std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap();
OutputPathMap queryDerivationOutputMap();
/* Update 'initialOutputs' to determine the current status of the
outputs of the derivation. Also returns a Boolean denoting
whether all outputs are valid and non-corrupt, and a
'DrvOutputs' structure containing the valid and wanted
outputs. */
/**
* Update 'initialOutputs' to determine the current status of the
* outputs of the derivation. Also returns a Boolean denoting
* whether all outputs are valid and non-corrupt, and a
* 'DrvOutputs' structure containing the valid and wanted
* outputs.
*/
std::pair<bool, DrvOutputs> checkPathValidity();
/* Aborts if any output is not valid or corrupt, and otherwise
returns a 'DrvOutputs' structure containing the wanted
outputs. */
/**
* Aborts if any output is not valid or corrupt, and otherwise
* returns a 'DrvOutputs' structure containing the wanted
* outputs.
*/
DrvOutputs assertPathValidity();
/* Forcibly kill the child process, if any. */
/**
* Forcibly kill the child process, if any.
*/
virtual void killChild();
void repairClosure();

34
src/libstore/build/drv-output-substitution-goal.hh
View file

@ -11,24 +11,34 @@ namespace nix {
class Worker;
// Substitution of a derivation output.
// This is done in three steps:
// 1. Fetch the output info from a substituter
// 2. Substitute the corresponding output path
// 3. Register the output info
/**
* Substitution of a derivation output.
* This is done in three steps:
* 1. Fetch the output info from a substituter
* 2. Substitute the corresponding output path
* 3. Register the output info
*/
class DrvOutputSubstitutionGoal : public Goal {
// The drv output we're trying to substitue
/**
* The drv output we're trying to substitue
*/
DrvOutput id;
// The realisation corresponding to the given output id.
// Will be filled once we can get it.
/**
* The realisation corresponding to the given output id.
* Will be filled once we can get it.
*/
std::shared_ptr<const Realisation> outputInfo;
/* The remaining substituters. */
/**
* The remaining substituters.
*/
std::list<ref<Store>> subs;
/* The current substituter. */
/**
* The current substituter.
*/
std::shared_ptr<Store> sub;
struct DownloadState
@ -39,7 +49,9 @@ class DrvOutputSubstitutionGoal : public Goal {
std::shared_ptr<DownloadState> downloadState;
/* Whether a substituter failed. */
/**
* Whether a substituter failed.
*/
bool substituterFailed = false;
public:

70
src/libstore/build/goal.hh
View file

@ -7,11 +7,15 @@
namespace nix {
/* Forward definition. */
/**
* Forward definition.
*/
struct Goal;
class Worker;
/* A pointer to a goal. */
/**
* A pointer to a goal.
*/
typedef std::shared_ptr<Goal> GoalPtr;
typedef std::weak_ptr<Goal> WeakGoalPtr;
@ -19,48 +23,72 @@ struct CompareGoalPtrs {
bool operator() (const GoalPtr & a, const GoalPtr & b) const;
};
/* Set of goals. */
/**
* Set of goals.
*/
typedef std::set<GoalPtr, CompareGoalPtrs> Goals;
typedef std::set<WeakGoalPtr, std::owner_less<WeakGoalPtr>> WeakGoals;
/* A map of paths to goals (and the other way around). */
/**
* A map of paths to goals (and the other way around).
*/
typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
struct Goal : public std::enable_shared_from_this<Goal>
{
typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
/* Backlink to the worker. */
/**
* Backlink to the worker.
*/
Worker & worker;
/* Goals that this goal is waiting for. */
/**
* Goals that this goal is waiting for.
*/
Goals waitees;
/* Goals waiting for this one to finish. Must use weak pointers
here to prevent cycles. */
/**
* Goals waiting for this one to finish. Must use weak pointers
* here to prevent cycles.
*/
WeakGoals waiters;
/* Number of goals we are/were waiting for that have failed. */
/**
* Number of goals we are/were waiting for that have failed.
*/
size_t nrFailed = 0;
/* Number of substitution goals we are/were waiting for that
failed because there are no substituters. */
/**
* Number of substitution goals we are/were waiting for that
* failed because there are no substituters.
*/
size_t nrNoSubstituters = 0;
/* Number of substitution goals we are/were waiting for that
failed because they had unsubstitutable references. */
/**
* Number of substitution goals we are/were waiting for that
* failed because they had unsubstitutable references.
*/
size_t nrIncompleteClosure = 0;
/* Name of this goal for debugging purposes. */
/**
* Name of this goal for debugging purposes.
*/
std::string name;
/* Whether the goal is finished. */
/**
* Whether the goal is finished.
*/
ExitCode exitCode = ecBusy;
/* Build result. */
/**
* Build result.
*/
BuildResult buildResult;
/* Exception containing an error message, if any. */
/**
* Exception containing an error message, if any.
*/
std::optional<Error> ex;
Goal(Worker & worker, DerivedPath path)
@ -96,9 +124,11 @@ struct Goal : public std::enable_shared_from_this<Goal>
return name;
}
/* Callback in case of a timeout. It should wake up its waiters,
get rid of any running child processes that are being monitored
by the worker (important!), etc. */
/**
* Callback in case of a timeout. It should wake up its waiters,
* get rid of any running child processes that are being monitored
* by the worker (important!), etc.
*/
virtual void timedOut(Error && ex) = 0;
virtual std::string key() = 0;

16
src/libstore/build/hook-instance.hh
View file

@ -8,16 +8,24 @@ namespace nix {
struct HookInstance
{
/* Pipes for talking to the build hook. */
/**
* Pipes for talking to the build hook.
*/
Pipe toHook;
/* Pipe for the hook's standard output/error. */
/**
* Pipe for the hook's standard output/error.
*/
Pipe fromHook;
/* Pipe for the builder's standard output/error. */
/**
* Pipe for the builder's standard output/error.
*/
Pipe builderOut;
/* The process ID of the hook. */
/**
* The process ID of the hook.
*/
Pid pid;
FdSink sink;

5
src/libstore/build/local-derivation-goal.cc
View file

@ -1415,6 +1415,9 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
virtual void addBuildLog(const StorePath & path, std::string_view log) override
{ unsupported("addBuildLog"); }
std::optional<TrustedFlag> isTrustedClient() override
{ return NotTrusted; }
};
@ -1467,7 +1470,7 @@ void LocalDerivationGoal::startDaemon()
FdSink to(remote.get());
try {
daemon::processConnection(store, from, to,
daemon::NotTrusted, daemon::Recursive);
NotTrusted, daemon::Recursive);
debug("terminated daemon connection");
} catch (SysError &) {
ignoreException();

221
src/libstore/build/local-derivation-goal.hh
View file

@ -10,49 +10,75 @@ struct LocalDerivationGoal : public DerivationGoal
{
LocalStore & getLocalStore();
/* User selected for running the builder. */
/**
* User selected for running the builder.
*/
std::unique_ptr<UserLock> buildUser;
/* The process ID of the builder. */
/**
* The process ID of the builder.
*/
Pid pid;
/* The cgroup of the builder, if any. */
/**
* The cgroup of the builder, if any.
*/
std::optional<Path> cgroup;
/* The temporary directory. */
/**
* The temporary directory.
*/
Path tmpDir;
/* The path of the temporary directory in the sandbox. */
/**
* The path of the temporary directory in the sandbox.
*/
Path tmpDirInSandbox;
/* Master side of the pseudoterminal used for the builder's
standard output/error. */
/**
* Master side of the pseudoterminal used for the builder's
* standard output/error.
*/
AutoCloseFD builderOut;
/* Pipe for synchronising updates to the builder namespaces. */
/**
* Pipe for synchronising updates to the builder namespaces.
*/
Pipe userNamespaceSync;
/* The mount namespace and user namespace of the builder, used to add additional
paths to the sandbox as a result of recursive Nix calls. */
/**
* The mount namespace and user namespace of the builder, used to add additional
* paths to the sandbox as a result of recursive Nix calls.
*/
AutoCloseFD sandboxMountNamespace;
AutoCloseFD sandboxUserNamespace;
/* On Linux, whether we're doing the build in its own user
namespace. */
/**
* On Linux, whether we're doing the build in its own user
* namespace.
*/
bool usingUserNamespace = true;
/* Whether we're currently doing a chroot build. */
/**
* Whether we're currently doing a chroot build.
*/
bool useChroot = false;
Path chrootRootDir;
/* RAII object to delete the chroot directory. */
/**
* RAII object to delete the chroot directory.
*/
std::shared_ptr<AutoDelete> autoDelChroot;
/* Whether to run the build in a private network namespace. */
/**
* Whether to run the build in a private network namespace.
*/
bool privateNetwork = false;
/* Stuff we need to pass to initChild(). */
/**
* Stuff we need to pass to initChild().
*/
struct ChrootPath {
Path source;
bool optional;
@ -71,30 +97,35 @@ struct LocalDerivationGoal : public DerivationGoal
SandboxProfile additionalSandboxProfile;
#endif
/* Hash rewriting. */
/**
* Hash rewriting.
*/
StringMap inputRewrites, outputRewrites;
typedef map<StorePath, StorePath> RedirectedOutputs;
RedirectedOutputs redirectedOutputs;
/* The outputs paths used during the build.
- Input-addressed derivations or fixed content-addressed outputs are
sometimes built when some of their outputs already exist, and can not
be hidden via sandboxing. We use temporary locations instead and
rewrite after the build. Otherwise the regular predetermined paths are
put here.
- Floating content-addressed derivations do not know their final build
output paths until the outputs are hashed, so random locations are
used, and then renamed. The randomness helps guard against hidden
self-references.
/**
* The outputs paths used during the build.
*
* - Input-addressed derivations or fixed content-addressed outputs are
* sometimes built when some of their outputs already exist, and can not
* be hidden via sandboxing. We use temporary locations instead and
* rewrite after the build. Otherwise the regular predetermined paths are
* put here.
*
* - Floating content-addressed derivations do not know their final build
* output paths until the outputs are hashed, so random locations are
* used, and then renamed. The randomness helps guard against hidden
* self-references.
*/
OutputPathMap scratchOutputs;
/* Path registration info from the previous round, if we're
building multiple times. Since this contains the hash, it
allows us to compare whether two rounds produced the same
result. */
/**
* Path registration info from the previous round, if we're
* building multiple times. Since this contains the hash, it
* allows us to compare whether two rounds produced the same
* result.
*/
std::map<Path, ValidPathInfo> prevInfos;
uid_t sandboxUid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); }
@ -102,25 +133,37 @@ struct LocalDerivationGoal : public DerivationGoal
const static Path homeDir;
/* The recursive Nix daemon socket. */
/**
* The recursive Nix daemon socket.
*/
AutoCloseFD daemonSocket;
/* The daemon main thread. */
/**
* The daemon main thread.
*/
std::thread daemonThread;
/* The daemon worker threads. */
/**
* The daemon worker threads.
*/
std::vector<std::thread> daemonWorkerThreads;
/* Paths that were added via recursive Nix calls. */
/**
* Paths that were added via recursive Nix calls.
*/
StorePathSet addedPaths;
/* Realisations that were added via recursive Nix calls. */
/**
* Realisations that were added via recursive Nix calls.
*/
std::set<DrvOutput> addedDrvOutputs;
/* Recursive Nix calls are only allowed to build or realize paths
in the original input closure or added via a recursive Nix call
(so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
/nix/store/<bla> is some arbitrary path in a binary cache). */
/**
* Recursive Nix calls are only allowed to build or realize paths
* in the original input closure or added via a recursive Nix call
* (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
* /nix/store/<bla> is some arbitrary path in a binary cache).
*/
bool isAllowed(const StorePath & path)
{
return inputPaths.count(path) || addedPaths.count(path);
@ -138,55 +181,81 @@ struct LocalDerivationGoal : public DerivationGoal
virtual ~LocalDerivationGoal() override;
/* Whether we need to perform hash rewriting if there are valid output paths. */
/**
* Whether we need to perform hash rewriting if there are valid output paths.
*/
bool needsHashRewrite();
/* The additional states. */
/**
* The additional states.
*/
void tryLocalBuild() override;
/* Start building a derivation. */
/**
* Start building a derivation.
*/
void startBuilder();
/* Fill in the environment for the builder. */
/**
* Fill in the environment for the builder.
*/
void initEnv();
/* Setup tmp dir location. */
/**
* Setup tmp dir location.
*/
void initTmpDir();
/* Write a JSON file containing the derivation attributes. */
/**
* Write a JSON file containing the derivation attributes.
*/
void writeStructuredAttrs();
void startDaemon();
void stopDaemon();
/* Add 'path' to the set of paths that may be referenced by the
outputs, and make it appear in the sandbox. */
/**
* Add 'path' to the set of paths that may be referenced by the
* outputs, and make it appear in the sandbox.
*/
void addDependency(const StorePath & path);
/* Make a file owned by the builder. */
/**
* Make a file owned by the builder.
*/
void chownToBuilder(const Path & path);
int getChildStatus() override;
/* Run the builder's process. */
/**
* Run the builder's process.
*/
void runChild();
/* Check that the derivation outputs all exist and register them
as valid. */
/**
* Check that the derivation outputs all exist and register them
* as valid.
*/
DrvOutputs registerOutputs() override;
void signRealisation(Realisation &) override;
/* Check that an output meets the requirements specified by the
'outputChecks' attribute (or the legacy
'{allowed,disallowed}{References,Requisites}' attributes). */
/**
* Check that an output meets the requirements specified by the
* 'outputChecks' attribute (or the legacy
* '{allowed,disallowed}{References,Requisites}' attributes).
*/
void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
/* Close the read side of the logger pipe. */
/**
* Close the read side of the logger pipe.
*/
void closeReadPipes() override;
/* Cleanup hooks for buildDone() */
/**
* Cleanup hooks for buildDone()
*/
void cleanupHookFinally() override;
void cleanupPreChildKill() override;
void cleanupPostChildKill() override;
@ -196,24 +265,36 @@ struct LocalDerivationGoal : public DerivationGoal
bool isReadDesc(int fd) override;
/* Delete the temporary directory, if we have one. */
/**
* Delete the temporary directory, if we have one.
*/
void deleteTmpDir(bool force);
/* Forcibly kill the child process, if any. */
/**
* Forcibly kill the child process, if any.
*/
void killChild() override;
/* Kill any processes running under the build user UID or in the
cgroup of the build. */
/**
* Kill any processes running under the build user UID or in the
* cgroup of the build.
*/
void killSandbox(bool getStats);
/* Create alternative path calculated from but distinct from the
input, so we can avoid overwriting outputs (or other store paths)
that already exist. */
/**
* Create alternative path calculated from but distinct from the
* input, so we can avoid overwriting outputs (or other store paths)
* that already exist.
*/
StorePath makeFallbackPath(const StorePath & path);
/* Make a path to another based on the output name along with the
derivation hash. */
/* FIXME add option to randomize, so we can audit whether our
rewrites caught everything */
/**
* Make a path to another based on the output name along with the
* derivation hash.
*
* @todo Add option to randomize, so we can audit whether our
* rewrites caught everything
*/
StorePath makeFallbackPath(std::string_view outputName);
};

62
src/libstore/build/substitution-goal.hh
View file

@ -11,38 +11,58 @@ class Worker;
struct PathSubstitutionGoal : public Goal
{
/* The store path that should be realised through a substitute. */
/**
* The store path that should be realised through a substitute.
*/
StorePath storePath;
/* The path the substituter refers to the path as. This will be
different when the stores have different names. */
/**
* The path the substituter refers to the path as. This will be
* different when the stores have different names.
*/
std::optional<StorePath> subPath;
/* The remaining substituters. */
/**
* The remaining substituters.
*/
std::list<ref<Store>> subs;
/* The current substituter. */
/**
* The current substituter.
*/
std::shared_ptr<Store> sub;
/* Whether a substituter failed. */
/**
* Whether a substituter failed.
*/
bool substituterFailed = false;
/* Path info returned by the substituter's query info operation. */
/**
* Path info returned by the substituter's query info operation.
*/
std::shared_ptr<const ValidPathInfo> info;
/* Pipe for the substituter's standard output. */
/**
* Pipe for the substituter's standard output.
*/
Pipe outPipe;
/* The substituter thread. */
/**
* The substituter thread.
*/
std::thread thr;
std::promise<void> promise;
/* Whether to try to repair a valid path. */
/**
* Whether to try to repair a valid path.
*/
RepairFlag repair;
/* Location where we're downloading the substitute. Differs from
storePath when doing a repair. */
/**
* Location where we're downloading the substitute. Differs from
* storePath when doing a repair.
*/
Path destPath;
std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions,
@ -51,7 +71,9 @@ struct PathSubstitutionGoal : public Goal
typedef void (PathSubstitutionGoal::*GoalState)();
GoalState state;
/* Content address for recomputing store path */
/**
* Content address for recomputing store path
*/
std::optional<ContentAddress> ca;
void done(
@ -65,16 +87,20 @@ public:
void timedOut(Error && ex) override { abort(); };
/**
* We prepend "a$" to the key name to ensure substitution goals
* happen before derivation goals.
*/
std::string key() override
{
/* "a$" ensures substitution goals happen before derivation
goals. */
return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath);
}
void work() override;
/* The states. */
/**
* The states.
*/
void init();
void tryNext();
void gotInfo();
@ -82,7 +108,9 @@ public:
void tryToRun();
void finished();
/* Callback used by the worker to write to the log. */
/**
* Callback used by the worker to write to the log.
*/
void handleChildOutput(int fd, std::string_view data) override;
void handleEOF(int fd) override;

186
src/libstore/build/worker.hh
View file

@ -17,24 +17,29 @@ struct DerivationGoal;
struct PathSubstitutionGoal;
class DrvOutputSubstitutionGoal;
/* Workaround for not being able to declare a something like
class PathSubstitutionGoal : public Goal;
even when Goal is a complete type.
This is still a static cast. The purpose of exporting it is to define it in
a place where `PathSubstitutionGoal` is concrete, and use it in a place where it
is opaque. */
/**
* Workaround for not being able to declare a something like
*
* ```c++
* class PathSubstitutionGoal : public Goal;
* ```
* even when Goal is a complete type.
*
* This is still a static cast. The purpose of exporting it is to define it in
* a place where `PathSubstitutionGoal` is concrete, and use it in a place where it
* is opaque.
*/
GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal);
GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal);
typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
/* A mapping used to remember for each child process to what goal it
belongs, and file descriptors for receiving log data and output
path creation commands. */
/**
* A mapping used to remember for each child process to what goal it
* belongs, and file descriptors for receiving log data and output
* path creation commands.
*/
struct Child
{
WeakGoalPtr goal;
@ -42,14 +47,19 @@ struct Child
std::set<int> fds;
bool respectTimeouts;
bool inBuildSlot;
steady_time_point lastOutput; /* time we last got output on stdout/stderr */
/**
* Time we last got output on stdout/stderr
*/
steady_time_point lastOutput;
steady_time_point timeStarted;
};
/* Forward definition. */
struct HookInstance;
/* The worker class. */
/**
* The worker class.
*/
class Worker
{
private:
@ -57,38 +67,58 @@ private:
/* Note: the worker should only have strong pointers to the
top-level goals. */
/* The top-level goals of the worker. */
/**
* The top-level goals of the worker.
*/
Goals topGoals;
/* Goals that are ready to do some work. */
/**
* Goals that are ready to do some work.
*/
WeakGoals awake;
/* Goals waiting for a build slot. */
/**
* Goals waiting for a build slot.
*/
WeakGoals wantingToBuild;
/* Child processes currently running. */
/**
* Child processes currently running.
*/
std::list<Child> children;
/* Number of build slots occupied. This includes local builds and
substitutions but not remote builds via the build hook. */
/**
* Number of build slots occupied. This includes local builds and
* substitutions but not remote builds via the build hook.
*/
unsigned int nrLocalBuilds;
/* Maps used to prevent multiple instantiations of a goal for the
same derivation / path. */
/**
* Maps used to prevent multiple instantiations of a goal for the
* same derivation / path.
*/
std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals;
std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals;
std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals;
/* Goals waiting for busy paths to be unlocked. */
/**
* Goals waiting for busy paths to be unlocked.
*/
WeakGoals waitingForAnyGoal;
/* Goals sleeping for a few seconds (polling a lock). */
/**
* Goals sleeping for a few seconds (polling a lock).
*/
WeakGoals waitingForAWhile;
/* Last time the goals in `waitingForAWhile' where woken up. */
/**
* Last time the goals in `waitingForAWhile` where woken up.
*/
steady_time_point lastWokenUp;
/* Cache for pathContentsGood(). */
/**
* Cache for pathContentsGood().
*/
std::map<StorePath, bool> pathContentsGoodCache;
public:
@ -97,17 +127,25 @@ public:
const Activity actDerivations;
const Activity actSubstitutions;
/* Set if at least one derivation had a BuildError (i.e. permanent
failure). */
/**
* Set if at least one derivation had a BuildError (i.e. permanent
* failure).
*/
bool permanentFailure;
/* Set if at least one derivation had a timeout. */
/**
* Set if at least one derivation had a timeout.
*/
bool timedOut;
/* Set if at least one derivation fails with a hash mismatch. */
/**
* Set if at least one derivation fails with a hash mismatch.
*/
bool hashMismatch;
/* Set if at least one derivation is not deterministic in check mode. */
/**
* Set if at least one derivation is not deterministic in check mode.
*/
bool checkMismatch;
Store & store;
@ -129,16 +167,22 @@ public:
uint64_t expectedNarSize = 0;
uint64_t doneNarSize = 0;
/* Whether to ask the build hook if it can build a derivation. If
it answers with "decline-permanently", we don't try again. */
/**
* Whether to ask the build hook if it can build a derivation. If
* it answers with "decline-permanently", we don't try again.
*/
bool tryBuildHook = true;
Worker(Store & store, Store & evalStore);
~Worker();
/* Make a goal (with caching). */
/**
* Make a goal (with caching).
*/
/* derivation goal */
/**
* derivation goal
*/
private:
std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
const StorePath & drvPath, const OutputsSpec & wantedOutputs,
@ -151,56 +195,80 @@ public:
const StorePath & drvPath, const BasicDerivation & drv,
const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal);
/* substitution goal */
/**
* substitution goal
*/
std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
std::shared_ptr<DrvOutputSubstitutionGoal> makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
/* Remove a dead goal. */
/**
* Remove a dead goal.
*/
void removeGoal(GoalPtr goal);
/* Wake up a goal (i.e., there is something for it to do). */
/**
* Wake up a goal (i.e., there is something for it to do).
*/
void wakeUp(GoalPtr goal);
/* Return the number of local build and substitution processes
currently running (but not remote builds via the build
hook). */
/**
* Return the number of local build and substitution processes
* currently running (but not remote builds via the build
* hook).
*/
unsigned int getNrLocalBuilds();
/* Registers a running child process. `inBuildSlot' means that
the process counts towards the jobs limit. */
/**
* Registers a running child process. `inBuildSlot` means that
* the process counts towards the jobs limit.
*/
void childStarted(GoalPtr goal, const std::set<int> & fds,
bool inBuildSlot, bool respectTimeouts);
/* Unregisters a running child process. `wakeSleepers' should be
false if there is no sense in waking up goals that are sleeping
because they can't run yet (e.g., there is no free build slot,
or the hook would still say `postpone'). */
/**
* Unregisters a running child process. `wakeSleepers` should be
* false if there is no sense in waking up goals that are sleeping
* because they can't run yet (e.g., there is no free build slot,
* or the hook would still say `postpone`).
*/
void childTerminated(Goal * goal, bool wakeSleepers = true);
/* Put `goal' to sleep until a build slot becomes available (which
might be right away). */
/**
* Put `goal` to sleep until a build slot becomes available (which
* might be right away).
*/
void waitForBuildSlot(GoalPtr goal);
/* Wait for any goal to finish. Pretty indiscriminate way to
wait for some resource that some other goal is holding. */
/**
* Wait for any goal to finish. Pretty indiscriminate way to
* wait for some resource that some other goal is holding.
*/
void waitForAnyGoal(GoalPtr goal);
/* Wait for a few seconds and then retry this goal. Used when
waiting for a lock held by another process. This kind of
polling is inefficient, but POSIX doesn't really provide a way
to wait for multiple locks in the main select() loop. */
/**
* Wait for a few seconds and then retry this goal. Used when
* waiting for a lock held by another process. This kind of
* polling is inefficient, but POSIX doesn't really provide a way
* to wait for multiple locks in the main select() loop.
*/
void waitForAWhile(GoalPtr goal);
/* Loop until the specified top-level goals have finished. */
/**
* Loop until the specified top-level goals have finished.
*/
void run(const Goals & topGoals);
/* Wait for input to become available. */
/**
* Wait for input to become available.
*/
void waitForInput();
unsigned int exitStatus();
/* Check whether the given valid path exists and has the right
contents. */
/**
* Check whether the given valid path exists and has the right
* contents.
*/
bool pathContentsGood(const StorePath & path);
void markContentsGood(const StorePath & path);

5
src/libstore/content-address.hh
View file

@ -3,6 +3,7 @@
#include <variant>
#include "hash.hh"
#include "comparator.hh"
namespace nix {
@ -30,6 +31,8 @@ struct TextHash {
* Hash of the contents of the text/file.
*/
Hash hash;
GENERATE_CMP(TextHash, me->hash);
};
/**
@ -46,6 +49,8 @@ struct FixedOutputHash {
Hash hash;
std::string printMethodAlgo() const;
GENERATE_CMP(FixedOutputHash, me->method, me->hash);
};
/**

16
src/libstore/crypto.hh
View file

@ -12,8 +12,10 @@ struct Key
std::string name;
std::string key;
/* Construct Key from a string in the format
‘<name>:<key-in-base64>’. */
/**
* Construct Key from a string in the format
* ‘<name>:<key-in-base64>’.
*/
Key(std::string_view s);
std::string to_string() const;
@ -29,7 +31,9 @@ struct SecretKey : Key
{
SecretKey(std::string_view s);
/* Return a detached signature of the given string. */
/**
* Return a detached signature of the given string.
*/
std::string signDetached(std::string_view s) const;
PublicKey toPublicKey() const;
@ -53,8 +57,10 @@ private:
typedef std::map<std::string, PublicKey> PublicKeys;
/* Return true iff ‘sig’ is a correct signature over ‘data’ using one
of the given public keys. */
/**
* @return true iff ‘sig’ is a correct signature over ‘data’ using one
* of the given public keys.
*/
bool verifyDetached(const std::string & data, const std::string & sig,
const PublicKeys & publicKeys);

9
src/libstore/daemon.cc
View file

@ -1032,6 +1032,15 @@ void processConnection(
if (GET_PROTOCOL_MINOR(clientVersion) >= 33)
to << nixVersion;
if (GET_PROTOCOL_MINOR(clientVersion) >= 35) {
// We and the underlying store both need to trust the client for
// it to be trusted.
auto temp = trusted
? store->isTrustedClient()
: std::optional { NotTrusted };
worker_proto::write(*store, to, temp);
}
/* Send startup error messages to the client. */
tunnelLogger->startWork();

1
src/libstore/daemon.hh
View file

@ -6,7 +6,6 @@
namespace nix::daemon {
enum TrustedFlag : bool { NotTrusted = false, Trusted = true };
enum RecursiveFlag : bool { NotRecursive = false, Recursive = true };
void processConnection(

178
src/libstore/derivations.cc
View file

@ -313,6 +313,15 @@ Derivation parseDerivation(const Store & store, std::string && s, std::string_vi
}
/**
* Print a derivation string literal to an `std::string`.
*
* This syntax does not generalize to the expression language, which needs to
* escape `$`.
*
* @param res Where to print to
* @param s Which logical string to print
*/
static void printString(std::string & res, std::string_view s)
{
boost::container::small_vector<char, 64 * 1024> buffer;
@ -889,6 +898,67 @@ std::optional<BasicDerivation> Derivation::tryResolve(
return resolved;
}
void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const
{
assert(drvPath.isDerivation());
std::string drvName(drvPath.name());
drvName = drvName.substr(0, drvName.size() - drvExtension.size());
if (drvName != name) {
throw Error("Derivation '%s' has name '%s' which does not match its path", store.printStorePath(drvPath), name);
}
auto envHasRightPath = [&](const StorePath & actual, const std::string & varName)
{
auto j = env.find(varName);
if (j == env.end() || store.parseStorePath(j->second) != actual)
throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'",
store.printStorePath(drvPath), varName, store.printStorePath(actual));
};
// Don't need the answer, but do this anyways to assert is proper
// combination. The code below is more general and naturally allows
// combinations that are currently prohibited.
type();
std::optional<DrvHash> hashesModulo;
for (auto & i : outputs) {
std::visit(overloaded {
[&](const DerivationOutput::InputAddressed & doia) {
if (!hashesModulo) {
// somewhat expensive so we do lazily
hashesModulo = hashDerivationModulo(store, *this, true);
}
auto currentOutputHash = get(hashesModulo->hashes, i.first);
if (!currentOutputHash)
throw Error("derivation '%s' has unexpected output '%s' (local-store / hashesModulo) named '%s'",
store.printStorePath(drvPath), store.printStorePath(doia.path), i.first);
StorePath recomputed = store.makeOutputPath(i.first, *currentOutputHash, drvName);
if (doia.path != recomputed)
throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
store.printStorePath(drvPath), store.printStorePath(doia.path), store.printStorePath(recomputed));
envHasRightPath(doia.path, i.first);
},
[&](const DerivationOutput::CAFixed & dof) {
StorePath path = store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName);
envHasRightPath(path, i.first);
},
[&](const DerivationOutput::CAFloating &) {
/* Nothing to check */
},
[&](const DerivationOutput::Deferred &) {
/* Nothing to check */
},
[&](const DerivationOutput::Impure &) {
/* Nothing to check */
},
}, i.second.raw());
}
}
const Hash impureOutputHash = hashString(htSHA256, "impure");
nlohmann::json DerivationOutput::toJSON(
@ -916,10 +986,79 @@ nlohmann::json DerivationOutput::toJSON(
return res;
}
DerivationOutput DerivationOutput::fromJSON(
const Store & store, std::string_view drvName, std::string_view outputName,
const nlohmann::json & _json)
{
std::set<std::string_view> keys;
auto json = (std::map<std::string, nlohmann::json>) _json;
for (const auto & [key, _] : json)
keys.insert(key);
auto methodAlgo = [&]() -> std::pair<FileIngestionMethod, HashType> {
std::string hashAlgo = json["hashAlgo"];
auto method = FileIngestionMethod::Flat;
if (hashAlgo.substr(0, 2) == "r:") {
method = FileIngestionMethod::Recursive;
hashAlgo = hashAlgo.substr(2);
}
auto hashType = parseHashType(hashAlgo);
return { method, hashType };
};
if (keys == (std::set<std::string_view> { "path" })) {
return DerivationOutput::InputAddressed {
.path = store.parseStorePath((std::string) json["path"]),
};
}
else if (keys == (std::set<std::string_view> { "path", "hashAlgo", "hash" })) {
auto [method, hashType] = methodAlgo();
auto dof = DerivationOutput::CAFixed {
.hash = {
.method = method,
.hash = Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType),
},
};
if (dof.path(store, drvName, outputName) != store.parseStorePath((std::string) json["path"]))
throw Error("Path doesn't match derivation output");
return dof;
}
else if (keys == (std::set<std::string_view> { "hashAlgo" })) {
auto [method, hashType] = methodAlgo();
return DerivationOutput::CAFloating {
.method = method,
.hashType = hashType,
};
}
else if (keys == (std::set<std::string_view> { })) {
return DerivationOutput::Deferred {};
}
else if (keys == (std::set<std::string_view> { "hashAlgo", "impure" })) {
auto [method, hashType] = methodAlgo();
return DerivationOutput::Impure {
.method = method,
.hashType = hashType,
};
}
else {
throw Error("invalid JSON for derivation output");
}
}
nlohmann::json Derivation::toJSON(const Store & store) const
{
nlohmann::json res = nlohmann::json::object();
res["name"] = name;
{
nlohmann::json & outputsObj = res["outputs"];
outputsObj = nlohmann::json::object();
@ -950,4 +1089,43 @@ nlohmann::json Derivation::toJSON(const Store & store) const
return res;
}
Derivation Derivation::fromJSON(
const Store & store,
const nlohmann::json & json)
{
Derivation res;
res.name = json["name"];
{
auto & outputsObj = json["outputs"];
for (auto & [outputName, output] : outputsObj.items()) {
res.outputs.insert_or_assign(
outputName,
DerivationOutput::fromJSON(store, res.name, outputName, output));
}
}
{
auto & inputsList = json["inputSrcs"];
for (auto & input : inputsList)
res.inputSrcs.insert(store.parseStorePath(static_cast<const std::string &>(input)));
}
{
auto & inputDrvsObj = json["inputDrvs"];
for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items())
res.inputDrvs[store.parseStorePath(inputDrvPath)] =
static_cast<const StringSet &>(inputOutputs);
}
res.platform = json["system"];
res.builder = json["builder"];
res.args = json["args"];
res.env = json["env"];
return res;
}
}

58
src/libstore/derivations.hh
View file

@ -7,6 +7,7 @@
#include "content-address.hh"
#include "repair-flag.hh"
#include "sync.hh"
#include "comparator.hh"
#include <map>
#include <variant>
@ -24,6 +25,8 @@ class Store;
struct DerivationOutputInputAddressed
{
StorePath path;
GENERATE_CMP(DerivationOutputInputAddressed, me->path);
};
/**
@ -44,6 +47,8 @@ struct DerivationOutputCAFixed
* @param outputName The name of this output.
*/
StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const;
GENERATE_CMP(DerivationOutputCAFixed, me->hash);
};
/**
@ -62,13 +67,17 @@ struct DerivationOutputCAFloating
* How the serialization will be hashed
*/
HashType hashType;
GENERATE_CMP(DerivationOutputCAFloating, me->method, me->hashType);
};
/**
* Input-addressed output which depends on a (CA) derivation whose hash
* isn't known yet.
*/
struct DerivationOutputDeferred {};
struct DerivationOutputDeferred {
GENERATE_CMP(DerivationOutputDeferred);
};
/**
* Impure output which is moved to a content-addressed location (like
@ -85,6 +94,8 @@ struct DerivationOutputImpure
* How the serialization will be hashed
*/
HashType hashType;
GENERATE_CMP(DerivationOutputImpure, me->method, me->hashType);
};
typedef std::variant<
@ -125,6 +136,11 @@ struct DerivationOutput : _DerivationOutputRaw
const Store & store,
std::string_view drvName,
std::string_view outputName) const;
static DerivationOutput fromJSON(
const Store & store,
std::string_view drvName,
std::string_view outputName,
const nlohmann::json & json);
};
typedef std::map<std::string, DerivationOutput> DerivationOutputs;
@ -242,8 +258,14 @@ struct DerivationType : _DerivationTypeRaw {
struct BasicDerivation
{
DerivationOutputs outputs; /* keyed on symbolic IDs */
StorePathSet inputSrcs; /* inputs that are sources */
/**
* keyed on symbolic IDs
*/
DerivationOutputs outputs;
/**
* inputs that are sources
*/
StorePathSet inputSrcs;
std::string platform;
Path builder;
Strings args;
@ -273,6 +295,15 @@ struct BasicDerivation
DerivationOutputsAndOptPaths outputsAndOptPaths(const Store & store) const;
static std::string_view nameFromPath(const StorePath & storePath);
GENERATE_CMP(BasicDerivation,
me->outputs,
me->inputSrcs,
me->platform,
me->builder,
me->args,
me->env,
me->name);
};
struct Derivation : BasicDerivation
@ -308,11 +339,26 @@ struct Derivation : BasicDerivation
Store & store,
const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const;
/* Check that the derivation is valid and does not present any
illegal states.
This is mainly a matter of checking the outputs, where our C++
representation supports all sorts of combinations we do not yet
allow. */
void checkInvariants(Store & store, const StorePath & drvPath) const;
Derivation() = default;
Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { }
Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { }
nlohmann::json toJSON(const Store & store) const;
static Derivation fromJSON(
const Store & store,
const nlohmann::json & json);
GENERATE_CMP(Derivation,
static_cast<const BasicDerivation &>(*me),
me->inputDrvs);
};
@ -389,12 +435,12 @@ void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept;
*
* A fixed-output derivation is a derivation whose outputs have a
* specified content hash and hash algorithm. (Currently they must have
* exactly one output (`out'), which is specified using the `outputHash'
* and `outputHashAlgo' attributes, but the algorithm doesn't assume
* exactly one output (`out`), which is specified using the `outputHash`
* and `outputHashAlgo` attributes, but the algorithm doesn't assume
* this.) We don't want changes to such derivations to propagate upwards
* through the dependency graph, changing output paths everywhere.
*
* For instance, if we change the url in a call to the `fetchurl'
* For instance, if we change the url in a call to the `fetchurl`
* function, we do not want to rebuild everything depending on it---after
* all, (the hash of) the file being downloaded is unchanged. So the
* *output paths* should not change. On the other hand, the *derivation

38
src/libstore/derived-path.cc
View file

@ -62,15 +62,31 @@ std::string DerivedPath::Opaque::to_string(const Store & store) const
std::string DerivedPath::Built::to_string(const Store & store) const
{
return store.printStorePath(drvPath)
+ "!"
+ '^'
+ outputs.to_string();
}
std::string DerivedPath::Built::to_string_legacy(const Store & store) const
{
return store.printStorePath(drvPath)
+ '!'
+ outputs.to_string();
}
std::string DerivedPath::to_string(const Store & store) const
{
return std::visit(
[&](const auto & req) { return req.to_string(store); },
this->raw());
return std::visit(overloaded {
[&](const DerivedPath::Built & req) { return req.to_string(store); },
[&](const DerivedPath::Opaque & req) { return req.to_string(store); },
}, this->raw());
}
std::string DerivedPath::to_string_legacy(const Store & store) const
{
return std::visit(overloaded {
[&](const DerivedPath::Built & req) { return req.to_string_legacy(store); },
[&](const DerivedPath::Opaque & req) { return req.to_string(store); },
}, this->raw());
}
@ -87,14 +103,24 @@ DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_vi
};
}
DerivedPath DerivedPath::parse(const Store & store, std::string_view s)
static inline DerivedPath parseWith(const Store & store, std::string_view s, std::string_view separator)
{
size_t n = s.find("!");
size_t n = s.find(separator);
return n == s.npos
? (DerivedPath) DerivedPath::Opaque::parse(store, s)
: (DerivedPath) DerivedPath::Built::parse(store, s.substr(0, n), s.substr(n + 1));
}
DerivedPath DerivedPath::parse(const Store & store, std::string_view s)
{
return parseWith(store, s, "^");
}
DerivedPath DerivedPath::parseLegacy(const Store & store, std::string_view s)
{
return parseWith(store, s, "!");
}
RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const
{
RealisedPath::Set res;

26
src/libstore/derived-path.hh
View file

@ -48,8 +48,18 @@ struct DerivedPathBuilt {
StorePath drvPath;
OutputsSpec outputs;
/**
* Uses `^` as the separator
*/
std::string to_string(const Store & store) const;
static DerivedPathBuilt parse(const Store & store, std::string_view, std::string_view);
/**
* Uses `!` as the separator
*/
std::string to_string_legacy(const Store & store) const;
/**
* The caller splits on the separator, so it works for both variants.
*/
static DerivedPathBuilt parse(const Store & store, std::string_view drvPath, std::string_view outputs);
nlohmann::json toJSON(ref<Store> store) const;
GENERATE_CMP(DerivedPathBuilt, me->drvPath, me->outputs);
@ -81,8 +91,22 @@ struct DerivedPath : _DerivedPathRaw {
return static_cast<const Raw &>(*this);
}
/**
* Uses `^` as the separator
*/
std::string to_string(const Store & store) const;
/**
* Uses `!` as the separator
*/
std::string to_string_legacy(const Store & store) const;
/**
* Uses `^` as the separator
*/
static DerivedPath parse(const Store & store, std::string_view);
/**
* Uses `!` as the separator
*/
static DerivedPath parseLegacy(const Store & store, std::string_view);
};
/**

11
src/libstore/dummy-store.cc
View file

@ -39,6 +39,14 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store
callback(nullptr);
}
/**
* The dummy store is incapable of *not* trusting! :)
*/
virtual std::optional<TrustedFlag> isTrustedClient() override
{
return Trusted;
}
static std::set<std::string> uriSchemes() {
return {"dummy"};
}
@ -63,6 +71,9 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store
void queryRealisationUncached(const DrvOutput &,
Callback<std::shared_ptr<const Realisation>> callback) noexcept override
{ callback(nullptr); }
virtual ref<FSAccessor> getFSAccessor() override
{ unsupported("getFSAccessor"); }
};
static RegisterStoreImplementation<DummyStore, DummyStoreConfig> regDummyStore;

5
src/libstore/filetransfer.cc
View file

@ -407,6 +407,10 @@ struct curlFileTransfer : public FileTransfer
err = Misc;
} else {
// Don't bother retrying on certain cURL errors either
// Allow selecting a subset of enum values
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wswitch-enum"
switch (code) {
case CURLE_FAILED_INIT:
case CURLE_URL_MALFORMAT:
@ -427,6 +431,7 @@ struct curlFileTransfer : public FileTransfer
default: // Shut up warnings
break;
}
#pragma GCC diagnostic pop
}
attempt++;

17
src/libstore/fs-accessor.hh
View file

@ -5,8 +5,10 @@
namespace nix {
/* An abstract class for accessing a filesystem-like structure, such
as a (possibly remote) Nix store or the contents of a NAR file. */
/**
* An abstract class for accessing a filesystem-like structure, such
* as a (possibly remote) Nix store or the contents of a NAR file.
*/
class FSAccessor
{
public:
@ -15,8 +17,17 @@ public:
struct Stat
{
Type type = tMissing;
uint64_t fileSize = 0; // regular files only
/**
* regular files only
*/
uint64_t fileSize = 0;
/**
* regular files only
*/
bool isExecutable = false; // regular files only
/**
* regular files only
*/
uint64_t narOffset = 0; // regular files only
};

83
src/libstore/gc-store.hh
View file

@ -12,18 +12,19 @@ typedef std::unordered_map<StorePath, std::unordered_set<std::string>> Roots;
struct GCOptions
{
/* Garbage collector operation:
- `gcReturnLive': return the set of paths reachable from
(i.e. in the closure of) the roots.
- `gcReturnDead': return the set of paths not reachable from
the roots.
- `gcDeleteDead': actually delete the latter set.
- `gcDeleteSpecific': delete the paths listed in
`pathsToDelete', insofar as they are not reachable.
/**
* Garbage collector operation:
*
* - `gcReturnLive`: return the set of paths reachable from
* (i.e. in the closure of) the roots.
*
* - `gcReturnDead`: return the set of paths not reachable from
* the roots.
*
* - `gcDeleteDead`: actually delete the latter set.
*
* - `gcDeleteSpecific`: delete the paths listed in
* `pathsToDelete`, insofar as they are not reachable.
*/
typedef enum {
gcReturnLive,
@ -34,28 +35,38 @@ struct GCOptions
GCAction action{gcDeleteDead};
/* If `ignoreLiveness' is set, then reachability from the roots is
ignored (dangerous!). However, the paths must still be
unreferenced *within* the store (i.e., there can be no other
store paths that depend on them). */
/**
* If `ignoreLiveness` is set, then reachability from the roots is
* ignored (dangerous!). However, the paths must still be
* unreferenced *within* the store (i.e., there can be no other
* store paths that depend on them).
*/
bool ignoreLiveness{false};
/* For `gcDeleteSpecific', the paths to delete. */
/**
* For `gcDeleteSpecific`, the paths to delete.
*/
StorePathSet pathsToDelete;
/* Stop after at least `maxFreed' bytes have been freed. */
/**
* Stop after at least `maxFreed` bytes have been freed.
*/
uint64_t maxFreed{std::numeric_limits<uint64_t>::max()};
};
struct GCResults
{
/* Depending on the action, the GC roots, or the paths that would
be or have been deleted. */
/**
* Depending on the action, the GC roots, or the paths that would
* be or have been deleted.
*/
PathSet paths;
/* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the
number of bytes that would be or was freed. */
/**
* For `gcReturnDead`, `gcDeleteDead` and `gcDeleteSpecific`, the
* number of bytes that would be or was freed.
*/
uint64_t bytesFreed = 0;
};
@ -64,21 +75,27 @@ struct GcStore : public virtual Store
{
inline static std::string operationName = "Garbage collection";
/* Add an indirect root, which is merely a symlink to `path' from
/nix/var/nix/gcroots/auto/<hash of `path'>. `path' is supposed
to be a symlink to a store path. The garbage collector will
automatically remove the indirect root when it finds that
`path' has disappeared. */
/**
* Add an indirect root, which is merely a symlink to `path` from
* `/nix/var/nix/gcroots/auto/<hash of path>`. `path` is supposed
* to be a symlink to a store path. The garbage collector will
* automatically remove the indirect root when it finds that
* `path` has disappeared.
*/
virtual void addIndirectRoot(const Path & path) = 0;
/* Find the roots of the garbage collector. Each root is a pair
(link, storepath) where `link' is the path of the symlink
outside of the Nix store that point to `storePath'. If
'censor' is true, privacy-sensitive information about roots
found in /proc is censored. */
/**
* Find the roots of the garbage collector. Each root is a pair
* `(link, storepath)` where `link` is the path of the symlink
* outside of the Nix store that point to `storePath`. If
* `censor` is true, privacy-sensitive information about roots
* found in `/proc` is censored.
*/
virtual Roots findRoots(bool censor) = 0;
/* Perform a garbage collection. */
/**
* Perform a garbage collection.
*/
virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0;
};

84
src/libstore/globals.hh
View file

@ -72,30 +72,46 @@ public:
Path nixPrefix;
/* The directory where we store sources and derived files. */
/**
* The directory where we store sources and derived files.
*/
Path nixStore;
Path nixDataDir; /* !!! fix */
/* The directory where we log various operations. */
/**
* The directory where we log various operations.
*/
Path nixLogDir;
/* The directory where state is stored. */
/**
* The directory where state is stored.
*/
Path nixStateDir;
/* The directory where system configuration files are stored. */
/**
* The directory where system configuration files are stored.
*/
Path nixConfDir;
/* A list of user configuration files to load. */
/**
* A list of user configuration files to load.
*/
std::vector<Path> nixUserConfFiles;
/* The directory where the main programs are stored. */
/**
* The directory where the main programs are stored.
*/
Path nixBinDir;
/* The directory where the man pages are stored. */
/**
* The directory where the man pages are stored.
*/
Path nixManDir;
/* File name of the socket the daemon listens to. */
/**
* File name of the socket the daemon listens to.
*/
Path nixDaemonSocketFile;
Setting<std::string> storeUri{this, getEnv("NIX_REMOTE").value_or("auto"), "store",
@ -121,7 +137,9 @@ public:
)",
{"build-fallback"}};
/* Whether to show build log output in real time. */
/**
* Whether to show build log output in real time.
*/
bool verboseBuild = true;
Setting<size_t> logLines{this, 10, "log-lines",
@ -157,8 +175,10 @@ public:
)",
{"build-cores"}, false};
/* Read-only mode. Don't copy stuff to the store, don't change
the database. */
/**
* Read-only mode. Don't copy stuff to the store, don't change
* the database.
*/
bool readOnlyMode = false;
Setting<std::string> thisSystem{
@ -308,16 +328,6 @@ public:
users in `build-users-group`.
UIDs are allocated starting at 872415232 (0x34000000) on Linux and 56930 on macOS.
> **Warning**
> This is an experimental feature.
To enable it, add the following to [`nix.conf`](#):
```
extra-experimental-features = auto-allocate-uids
auto-allocate-uids = true
```
)"};
Setting<uint32_t> startId{this,
@ -347,16 +357,6 @@ public:
Cgroups are required and enabled automatically for derivations
that require the `uid-range` system feature.
> **Warning**
> This is an experimental feature.
To enable it, add the following to [`nix.conf`](#):
```
extra-experimental-features = cgroups
use-cgroups = true
```
)"};
#endif
@ -458,7 +458,9 @@ public:
)",
{"env-keep-derivations"}};
/* Whether to lock the Nix client and worker to the same CPU. */
/**
* Whether to lock the Nix client and worker to the same CPU.
*/
bool lockCPU;
Setting<SandboxMode> sandboxMode{
@ -997,8 +999,10 @@ public:
// FIXME: don't use a global variable.
extern Settings settings;
/* This should be called after settings are initialized, but before
anything else */
/**
* This should be called after settings are initialized, but before
* anything else
*/
void initPlugins();
void loadConfFile();
@ -1008,12 +1012,16 @@ std::vector<Path> getUserConfigFiles();
extern const std::string nixVersion;
/* NB: This is not sufficient. You need to call initNix() */
/**
* NB: This is not sufficient. You need to call initNix()
*/
void initLibStore();
/* It's important to initialize before doing _anything_, which is why we
call upon the programmer to handle this correctly. However, we only add
this in a key locations, so as not to litter the code. */
/**
* It's important to initialize before doing _anything_, which is why we
* call upon the programmer to handle this correctly. However, we only add
* this in a key locations, so as not to litter the code.
*/
void assertLibStoreInitialized();
}

12
src/libstore/http-binary-cache-store.cc
View file

@ -194,6 +194,18 @@ protected:
}});
}
/**
* This isn't actually necessary read only. We support "upsert" now, so we
* have a notion of authentication via HTTP POST/PUT.
*
* For now, we conservatively say we don't know.
*
* \todo try to expose our HTTP authentication status.
*/
std::optional<TrustedFlag> isTrustedClient() override
{
return std::nullopt;
}
};
static RegisterStoreImplementation<HttpBinaryCacheStore, HttpBinaryCacheStoreConfig> regHttpBinaryCacheStore;

12
src/libstore/legacy-ssh-store.cc
View file

@ -342,6 +342,9 @@ public:
void ensurePath(const StorePath & path) override
{ unsupported("ensurePath"); }
virtual ref<FSAccessor> getFSAccessor() override
{ unsupported("getFSAccessor"); }
void computeFSClosure(const StorePathSet & paths,
StorePathSet & out, bool flipDirection = false,
bool includeOutputs = false, bool includeDerivers = false) override
@ -389,6 +392,15 @@ public:
return conn->remoteVersion;
}
/**
* The legacy ssh protocol doesn't support checking for trusted-user.
* Try using ssh-ng:// instead if you want to know.
*/
std::optional<TrustedFlag> isTrustedClient() override
{
return std::nullopt;
}
void queryRealisationUncached(const DrvOutput &,
Callback<std::shared_ptr<const Realisation>> callback) noexcept override
// TODO: Implement

4
src/libstore/local-binary-cache-store.cc
View file

@ -95,6 +95,10 @@ protected:
return paths;
}
std::optional<TrustedFlag> isTrustedClient() override
{
return Trusted;
}
};
void LocalBinaryCacheStore::init()

4
src/libstore/local-fs-store.hh
View file

@ -48,7 +48,9 @@ public:
void narFromPath(const StorePath & path, Sink & sink) override;
ref<FSAccessor> getFSAccessor() override;
/* Register a permanent GC root. */
/**
* Register a permanent GC root.
*/
Path addPermRoot(const StorePath & storePath, const Path & gcRoot);
virtual Path getRealStoreDir() { return realStoreDir; }

114
src/libstore/local-store.cc
View file

@ -710,62 +710,6 @@ void canonicalisePathMetaData(const Path & path,
canonicalisePathMetaData(path, uidRange, inodesSeen);
}
void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv)
{
assert(drvPath.isDerivation());
std::string drvName(drvPath.name());
drvName = drvName.substr(0, drvName.size() - drvExtension.size());
auto envHasRightPath = [&](const StorePath & actual, const std::string & varName)
{
auto j = drv.env.find(varName);
if (j == drv.env.end() || parseStorePath(j->second) != actual)
throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'",
printStorePath(drvPath), varName, printStorePath(actual));
};
// Don't need the answer, but do this anyways to assert is proper
// combination. The code below is more general and naturally allows
// combinations that are currently prohibited.
drv.type();
std::optional<DrvHash> hashesModulo;
for (auto & i : drv.outputs) {
std::visit(overloaded {
[&](const DerivationOutput::InputAddressed & doia) {
if (!hashesModulo) {
// somewhat expensive so we do lazily
hashesModulo = hashDerivationModulo(*this, drv, true);
}
auto currentOutputHash = get(hashesModulo->hashes, i.first);
if (!currentOutputHash)
throw Error("derivation '%s' has unexpected output '%s' (local-store / hashesModulo) named '%s'",
printStorePath(drvPath), printStorePath(doia.path), i.first);
StorePath recomputed = makeOutputPath(i.first, *currentOutputHash, drvName);
if (doia.path != recomputed)
throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed));
envHasRightPath(doia.path, i.first);
},
[&](const DerivationOutput::CAFixed & dof) {
StorePath path = makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName);
envHasRightPath(path, i.first);
},
[&](const DerivationOutput::CAFloating &) {
/* Nothing to check */
},
[&](const DerivationOutput::Deferred &) {
/* Nothing to check */
},
[&](const DerivationOutput::Impure &) {
/* Nothing to check */
},
}, i.second.raw());
}
}
void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs)
{
experimentalFeatureSettings.require(Xp::CaDerivations);
@ -876,7 +820,7 @@ uint64_t LocalStore::addValidPath(State & state,
derivations). Note that if this throws an error, then the
DB transaction is rolled back, so the path validity
registration above is undone. */
if (checkOutputs) checkDerivationOutputs(info.path, drv);
if (checkOutputs) drv.checkInvariants(*this, info.path);
for (auto & i : drv.outputsAndOptPaths(*this)) {
/* Floating CA derivations have indeterminate output paths until
@ -1134,54 +1078,6 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths)
}
// FIXME: move this, it's not specific to LocalStore.
void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos)
{
if (!settings.useSubstitutes) return;
for (auto & sub : getDefaultSubstituters()) {
for (auto & path : paths) {
if (infos.count(path.first))
// Choose first succeeding substituter.
continue;
auto subPath(path.first);
// Recompute store path so that we can use a different store root.
if (path.second) {
subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second);
if (sub->storeDir == storeDir)
assert(subPath == path.first);
if (subPath != path.first)
debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path.first), sub->printStorePath(subPath), sub->getUri());
} else if (sub->storeDir != storeDir) continue;
debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath));
try {
auto info = sub->queryPathInfo(subPath);
if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty()))
continue;
auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
std::shared_ptr<const ValidPathInfo>(info));
infos.insert_or_assign(path.first, SubstitutablePathInfo{
info->deriver,
info->references,
narInfo ? narInfo->fileSize : 0,
info->narSize});
} catch (InvalidPath &) {
} catch (SubstituterDisabled &) {
} catch (Error & e) {
if (settings.tryFallback)
logError(e.info());
else
throw;
}
}
}
}
void LocalStore::registerValidPath(const ValidPathInfo & info)
{
registerValidPaths({{info.path, info}});
@ -1223,8 +1119,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
for (auto & [_, i] : infos)
if (i.path.isDerivation()) {
// FIXME: inefficient; we already loaded the derivation in addValidPath().
checkDerivationOutputs(i.path,
readInvalidDerivation(i.path));
readInvalidDerivation(i.path).checkInvariants(*this, i.path);
}
/* Do a topological sort of the paths. This will throw an
@ -1733,6 +1628,11 @@ unsigned int LocalStore::getProtocol()
return PROTOCOL_VERSION;
}
std::optional<TrustedFlag> LocalStore::isTrustedClient()
{
return Trusted;
}
#if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL)

152
src/libstore/local-store.hh
View file

@ -19,10 +19,14 @@
namespace nix {
/* Nix store and database schema version. Version 1 (or 0) was Nix <=
0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10.
Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is
Nix 1.0. Version 7 is Nix 1.3. Version 10 is 2.0. */
/**
* Nix store and database schema version.
*
* Version 1 (or 0) was Nix <=
* 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10.
* Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is
* Nix 1.0. Version 7 is Nix 1.3. Version 10 is 2.0.
*/
const int nixSchemaVersion = 10;
@ -51,30 +55,40 @@ class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore,
{
private:
/* Lock file used for upgrading. */
/**
* Lock file used for upgrading.
*/
AutoCloseFD globalLock;
struct State
{
/* The SQLite database object. */
/**
* The SQLite database object.
*/
SQLite db;
struct Stmts;
std::unique_ptr<Stmts> stmts;
/* The last time we checked whether to do an auto-GC, or an
auto-GC finished. */
/**
* The last time we checked whether to do an auto-GC, or an
* auto-GC finished.
*/
std::chrono::time_point<std::chrono::steady_clock> lastGCCheck;
/* Whether auto-GC is running. If so, get gcFuture to wait for
the GC to finish. */
/**
* Whether auto-GC is running. If so, get gcFuture to wait for
* the GC to finish.
*/
bool gcRunning = false;
std::shared_future<void> gcFuture;
/* How much disk space was available after the previous
auto-GC. If the current available disk space is below
minFree but not much below availAfterGC, then there is no
point in starting a new GC. */
/**
* How much disk space was available after the previous
* auto-GC. If the current available disk space is below
* minFree but not much below availAfterGC, then there is no
* point in starting a new GC.
*/
uint64_t availAfterGC = std::numeric_limits<uint64_t>::max();
std::unique_ptr<PublicKeys> publicKeys;
@ -97,11 +111,15 @@ private:
public:
// Hack for build-remote.cc.
/**
* Hack for build-remote.cc.
*/
PathSet locksHeld;
/* Initialise the local store, upgrading the schema if
necessary. */
/**
* Initialise the local store, upgrading the schema if
* necessary.
*/
LocalStore(const Params & params);
LocalStore(std::string scheme, std::string path, const Params & params);
@ -110,7 +128,9 @@ public:
static std::set<std::string> uriSchemes()
{ return {}; }
/* Implementations of abstract store API methods. */
/**
* Implementations of abstract store API methods.
*/
std::string getUri() override;
@ -134,9 +154,6 @@ public:
StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
void querySubstitutablePathInfos(const StorePathCAMap & paths,
SubstitutablePathInfos & infos) override;
bool pathInfoIsUntrusted(const ValidPathInfo &) override;
bool realisationIsUntrusted(const Realisation & ) override;
@ -158,13 +175,19 @@ private:
void createTempRootsFile();
/* The file to which we write our temporary roots. */
/**
* The file to which we write our temporary roots.
*/
Sync<AutoCloseFD> _fdTempRoots;
/* The global GC lock. */
/**
* The global GC lock.
*/
Sync<AutoCloseFD> _fdGCLock;
/* Connection to the garbage collector. */
/**
* Connection to the garbage collector.
*/
Sync<AutoCloseFD> _fdRootsSocket;
public:
@ -183,42 +206,54 @@ public:
void collectGarbage(const GCOptions & options, GCResults & results) override;
/* Optimise the disk space usage of the Nix store by hard-linking
files with the same contents. */
/**
* Optimise the disk space usage of the Nix store by hard-linking
* files with the same contents.
*/
void optimiseStore(OptimiseStats & stats);
void optimiseStore() override;
/* Optimise a single store path. Optionally, test the encountered
symlinks for corruption. */
/**
* Optimise a single store path. Optionally, test the encountered
* symlinks for corruption.
*/
void optimisePath(const Path & path, RepairFlag repair);
bool verifyStore(bool checkContents, RepairFlag repair) override;
/* Register the validity of a path, i.e., that `path' exists, that
the paths referenced by it exists, and in the case of an output
path of a derivation, that it has been produced by a successful
execution of the derivation (or something equivalent). Also
register the hash of the file system contents of the path. The
hash must be a SHA-256 hash. */
/**
* Register the validity of a path, i.e., that `path` exists, that
* the paths referenced by it exists, and in the case of an output
* path of a derivation, that it has been produced by a successful
* execution of the derivation (or something equivalent). Also
* register the hash of the file system contents of the path. The
* hash must be a SHA-256 hash.
*/
void registerValidPath(const ValidPathInfo & info);
void registerValidPaths(const ValidPathInfos & infos);
unsigned int getProtocol() override;
std::optional<TrustedFlag> isTrustedClient() override;
void vacuumDB();
void repairPath(const StorePath & path) override;
void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
/* If free disk space in /nix/store if below minFree, delete
garbage until it exceeds maxFree. */
/**
* If free disk space in /nix/store if below minFree, delete
* garbage until it exceeds maxFree.
*/
void autoGC(bool sync = true);
/* Register the store path 'output' as the output named 'outputName' of
derivation 'deriver'. */
/**
* Register the store path 'output' as the output named 'outputName' of
* derivation 'deriver'.
*/
void registerDrvOutput(const Realisation & info) override;
void registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) override;
void cacheDrvOutputMapping(
@ -248,7 +283,9 @@ private:
void invalidatePath(State & state, const StorePath & path);
/* Delete a path from the Nix store. */
/**
* Delete a path from the Nix store.
*/
void invalidatePathChecked(const StorePath & path);
void verifyPath(const Path & path, const StringSet & store,
@ -271,8 +308,6 @@ private:
std::pair<Path, AutoCloseFD> createTempDirInStore();
void checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv);
typedef std::unordered_set<ino_t> InodeHash;
InodeHash loadInodeHash();
@ -283,8 +318,10 @@ private:
bool isValidPath_(State & state, const StorePath & path);
void queryReferrers(State & state, const StorePath & path, StorePathSet & referrers);
/* Add signatures to a ValidPathInfo or Realisation using the secret keys
specified by the ‘secret-key-files’ option. */
/**
* Add signatures to a ValidPathInfo or Realisation using the secret keys
* specified by the ‘secret-key-files’ option.
*/
void signPathInfo(ValidPathInfo & info);
void signRealisation(Realisation &);
@ -314,18 +351,23 @@ typedef std::pair<dev_t, ino_t> Inode;
typedef std::set<Inode> InodesSeen;
/* "Fix", or canonicalise, the meta-data of the files in a store path
after it has been built. In particular:
- the last modification date on each file is set to 1 (i.e.,
00:00:01 1/1/1970 UTC)
- the permissions are set of 444 or 555 (i.e., read-only with or
without execute permission; setuid bits etc. are cleared)
- the owner and group are set to the Nix user and group, if we're
running as root.
If uidRange is not empty, this function will throw an error if it
encounters files owned by a user outside of the closed interval
[uidRange->first, uidRange->second].
*/
/**
* "Fix", or canonicalise, the meta-data of the files in a store path
* after it has been built. In particular:
*
* - the last modification date on each file is set to 1 (i.e.,
* 00:00:01 1/1/1970 UTC)
*
* - the permissions are set of 444 or 555 (i.e., read-only with or
* without execute permission; setuid bits etc. are cleared)
*
* - the owner and group are set to the Nix user and group, if we're
* running as root.
*
* If uidRange is not empty, this function will throw an error if it
* encounters files owned by a user outside of the closed interval
* [uidRange->first, uidRange->second].
*/
void canonicalisePathMetaData(
const Path & path,
std::optional<std::pair<uid_t, uid_t>> uidRange,

14
src/libstore/lock.hh
View file

@ -13,14 +13,18 @@ struct UserLock
{
virtual ~UserLock() { }
/* Get the first and last UID. */
/**
* Get the first and last UID.
*/
std::pair<uid_t, uid_t> getUIDRange()
{
auto first = getUID();
return {first, first + getUIDCount() - 1};
}
/* Get the first UID. */
/**
* Get the first UID.
*/
virtual uid_t getUID() = 0;
virtual uid_t getUIDCount() = 0;
@ -30,8 +34,10 @@ struct UserLock
virtual std::vector<gid_t> getSupplementaryGIDs() = 0;
};
/* Acquire a user lock for a UID range of size `nrIds`. Note that this
may return nullptr if no user is available. */
/**
* Acquire a user lock for a UID range of size `nrIds`. Note that this
* may return nullptr if no user is available.
*/
std::unique_ptr<UserLock> acquireUserLock(uid_t nrIds, bool useUserNamespace);
bool useBuildUsers();

6
src/libstore/log-store.hh
View file

@ -10,8 +10,10 @@ struct LogStore : public virtual Store
{
inline static std::string operationName = "Build log storage and retrieval";
/* Return the build log of the specified store path, if available,
or null otherwise. */
/**
* Return the build log of the specified store path, if available,
* or null otherwise.
*/
std::optional<std::string> getBuildLog(const StorePath & path);
virtual std::optional<std::string> getBuildLogExact(const StorePath & path) = 0;

1
src/libstore/nar-accessor.cc
View file

@ -275,6 +275,7 @@ json listNar(ref<FSAccessor> accessor, const Path & path, bool recurse)
obj["type"] = "symlink";
obj["target"] = accessor->readLink(path);
break;
case FSAccessor::Type::tMissing:
default:
throw Error("path '%s' does not exist in NAR", path);
}

22
src/libstore/nar-accessor.hh
View file

@ -10,24 +10,30 @@ namespace nix {
struct Source;
/* Return an object that provides access to the contents of a NAR
file. */
/**
* Return an object that provides access to the contents of a NAR
* file.
*/
ref<FSAccessor> makeNarAccessor(std::string && nar);
ref<FSAccessor> makeNarAccessor(Source & source);
/* Create a NAR accessor from a NAR listing (in the format produced by
listNar()). The callback getNarBytes(offset, length) is used by the
readFile() method of the accessor to get the contents of files
inside the NAR. */
/**
* Create a NAR accessor from a NAR listing (in the format produced by
* listNar()). The callback getNarBytes(offset, length) is used by the
* readFile() method of the accessor to get the contents of files
* inside the NAR.
*/
typedef std::function<std::string(uint64_t, uint64_t)> GetNarBytes;
ref<FSAccessor> makeLazyNarAccessor(
const std::string & listing,
GetNarBytes getNarBytes);
/* Write a JSON representation of the contents of a NAR (except file
contents). */
/**
* Write a JSON representation of the contents of a NAR (except file
* contents).
*/
nlohmann::json listNar(ref<FSAccessor> accessor, const Path & path, bool recurse);
}

6
src/libstore/nar-info-disk-cache.hh
View file

@ -43,8 +43,10 @@ public:
const std::string & uri, const DrvOutput & id) = 0;
};
/* Return a singleton cache object that can be used concurrently by
multiple threads. */
/**
* Return a singleton cache object that can be used concurrently by
* multiple threads.
*/
ref<NarInfoDiskCache> getNarInfoDiskCache();
ref<NarInfoDiskCache> getTestNarInfoDiskCache(Path dbPath);

81
src/libstore/path-info.hh
View file

@ -19,8 +19,14 @@ struct SubstitutablePathInfo
{
std::optional<StorePath> deriver;
StorePathSet references;
uint64_t downloadSize; /* 0 = unknown or inapplicable */
uint64_t narSize; /* 0 = unknown */
/**
* 0 = unknown or inapplicable
*/
uint64_t downloadSize;
/**
* 0 = unknown
*/
uint64_t narSize;
};
typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos;
@ -30,34 +36,39 @@ struct ValidPathInfo
{
StorePath path;
std::optional<StorePath> deriver;
// TODO document this
/**
* \todo document this
*/
Hash narHash;
StorePathSet references;
time_t registrationTime = 0;
uint64_t narSize = 0; // 0 = unknown
uint64_t id; // internal use only
/* Whether the path is ultimately trusted, that is, it's a
derivation output that was built locally. */
/**
* Whether the path is ultimately trusted, that is, it's a
* derivation output that was built locally.
*/
bool ultimate = false;
StringSet sigs; // note: not necessarily verified
/* If non-empty, an assertion that the path is content-addressed,
i.e., that the store path is computed from a cryptographic hash
of the contents of the path, plus some other bits of data like
the "name" part of the path. Such a path doesn't need
signatures, since we don't have to trust anybody's claim that
the path is the output of a particular derivation. (In the
extensional store model, we have to trust that the *contents*
of an output path of a derivation were actually produced by
that derivation. In the intensional model, we have to trust
that a particular output path was produced by a derivation; the
path then implies the contents.)
Ideally, the content-addressability assertion would just be a Boolean,
and the store path would be computed from the name component, ‘narHash’
and ‘references’. However, we support many types of content addresses.
/**
* If non-empty, an assertion that the path is content-addressed,
* i.e., that the store path is computed from a cryptographic hash
* of the contents of the path, plus some other bits of data like
* the "name" part of the path. Such a path doesn't need
* signatures, since we don't have to trust anybody's claim that
* the path is the output of a particular derivation. (In the
* extensional store model, we have to trust that the *contents*
* of an output path of a derivation were actually produced by
* that derivation. In the intensional model, we have to trust
* that a particular output path was produced by a derivation; the
* path then implies the contents.)
*
* Ideally, the content-addressability assertion would just be a Boolean,
* and the store path would be computed from the name component, ‘narHash’
* and ‘references’. However, we support many types of content addresses.
*/
std::optional<ContentAddress> ca;
@ -69,27 +80,35 @@ struct ValidPathInfo
&& references == i.references;
}
/* Return a fingerprint of the store path to be used in binary
cache signatures. It contains the store path, the base-32
SHA-256 hash of the NAR serialisation of the path, the size of
the NAR, and the sorted references. The size field is strictly
speaking superfluous, but might prevent endless/excessive data
attacks. */
/**
* Return a fingerprint of the store path to be used in binary
* cache signatures. It contains the store path, the base-32
* SHA-256 hash of the NAR serialisation of the path, the size of
* the NAR, and the sorted references. The size field is strictly
* speaking superfluous, but might prevent endless/excessive data
* attacks.
*/
std::string fingerprint(const Store & store) const;
void sign(const Store & store, const SecretKey & secretKey);
/* Return true iff the path is verifiably content-addressed. */
/**
* @return true iff the path is verifiably content-addressed.
*/
bool isContentAddressed(const Store & store) const;
static const size_t maxSigs = std::numeric_limits<size_t>::max();
/* Return the number of signatures on this .narinfo that were
produced by one of the specified keys, or maxSigs if the path
is content-addressed. */
/**
* Return the number of signatures on this .narinfo that were
* produced by one of the specified keys, or maxSigs if the path
* is content-addressed.
*/
size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const;
/* Verify a single signature. */
/**
* Verify a single signature.
*/
bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const;
Strings shortRefs() const;

23
src/libstore/path-with-outputs.hh
View file

@ -6,13 +6,14 @@
namespace nix {
/* This is a deprecated old type just for use by the old CLI, and older
versions of the RPC protocols. In new code don't use it; you want
`DerivedPath` instead.
`DerivedPath` is better because it handles more cases, and does so more
explicitly without devious punning tricks.
*/
/**
* This is a deprecated old type just for use by the old CLI, and older
* versions of the RPC protocols. In new code don't use it; you want
* `DerivedPath` instead.
*
* `DerivedPath` is better because it handles more cases, and does so more
* explicitly without devious punning tricks.
*/
struct StorePathWithOutputs
{
StorePath path;
@ -31,9 +32,11 @@ std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s);
class Store;
/* Split a string specifying a derivation and a set of outputs
(/nix/store/hash-foo!out1,out2,...) into the derivation path
and the outputs. */
/**
* Split a string specifying a derivation and a set of outputs
* (/nix/store/hash-foo!out1,out2,...) into the derivation path
* and the outputs.
*/
StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs);
StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs);

12
src/libstore/pathlocks.hh
View file

@ -5,12 +5,16 @@
namespace nix {
/* Open (possibly create) a lock file and return the file descriptor.
-1 is returned if create is false and the lock could not be opened
because it doesn't exist. Any other error throws an exception. */
/**
* Open (possibly create) a lock file and return the file descriptor.
* -1 is returned if create is false and the lock could not be opened
* because it doesn't exist. Any other error throws an exception.
*/
AutoCloseFD openLockFile(const Path & path, bool create);
/* Delete an open lock file. */
/**
* Delete an open lock file.
*/
void deleteLockFile(const Path & path, int fd);
enum LockType { ltRead, ltWrite, ltNone };

42
src/libstore/profiles.hh
View file

@ -1,7 +1,7 @@
#pragma once
///@file
#include "types.hh"
#include "types.hh"
#include "pathlocks.hh"
#include <time.h>
@ -24,9 +24,11 @@ struct Generation
typedef std::list<Generation> Generations;
/* Returns the list of currently present generations for the specified
profile, sorted by generation number. Also returns the number of
the current generation. */
/**
* Returns the list of currently present generations for the specified
* profile, sorted by generation number. Also returns the number of
* the current generation.
*/
std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path profile);
class LocalFSStore;
@ -47,26 +49,32 @@ void deleteGenerationsOlderThan(const Path & profile, std::string_view timeSpec,
void switchLink(Path link, Path target);
/* Roll back a profile to the specified generation, or to the most
recent one older than the current. */
/**
* Roll back a profile to the specified generation, or to the most
* recent one older than the current.
*/
void switchGeneration(
const Path & profile,
std::optional<GenerationNumber> dstGen,
bool dryRun);
/* Ensure exclusive access to a profile. Any command that modifies
the profile first acquires this lock. */
/**
* Ensure exclusive access to a profile. Any command that modifies
* the profile first acquires this lock.
*/
void lockProfile(PathLocks & lock, const Path & profile);
/* Optimistic locking is used by long-running operations like `nix-env
-i'. Instead of acquiring the exclusive lock for the entire
duration of the operation, we just perform the operation
optimistically (without an exclusive lock), and check at the end
whether the profile changed while we were busy (i.e., the symlink
target changed). If so, the operation is restarted. Restarting is
generally cheap, since the build results are still in the Nix
store. Most of the time, only the user environment has to be
rebuilt. */
/**
* Optimistic locking is used by long-running operations like `nix-env
* -i'. Instead of acquiring the exclusive lock for the entire
* duration of the operation, we just perform the operation
* optimistically (without an exclusive lock), and check at the end
* whether the profile changed while we were busy (i.e., the symlink
* target changed). If so, the operation is restarted. Restarting is
* generally cheap, since the build results are still in the Nix
* store. Most of the time, only the user environment has to be
* rebuilt.
*/
std::string optimisticLockProfile(const Path & profile);
/**

50
src/libstore/remote-store.cc
View file

@ -42,6 +42,40 @@ void write(const Store & store, Sink & out, const StorePath & storePath)
}
std::optional<TrustedFlag> read(const Store & store, Source & from, Phantom<std::optional<TrustedFlag>> _)
{
auto temp = readNum<uint8_t>(from);
switch (temp) {
case 0:
return std::nullopt;
case 1:
return { Trusted };
case 2:
return { NotTrusted };
default:
throw Error("Invalid trusted status from remote");
}
}
void write(const Store & store, Sink & out, const std::optional<TrustedFlag> & optTrusted)
{
if (!optTrusted)
out << (uint8_t)0;
else {
switch (*optTrusted) {
case Trusted:
out << (uint8_t)1;
break;
case NotTrusted:
out << (uint8_t)2;
break;
default:
assert(false);
};
}
}
ContentAddress read(const Store & store, Source & from, Phantom<ContentAddress> _)
{
return parseContentAddress(readString(from));
@ -56,12 +90,12 @@ void write(const Store & store, Sink & out, const ContentAddress & ca)
DerivedPath read(const Store & store, Source & from, Phantom<DerivedPath> _)
{
auto s = readString(from);
return DerivedPath::parse(store, s);
return DerivedPath::parseLegacy(store, s);
}
void write(const Store & store, Sink & out, const DerivedPath & req)
{
out << req.to_string(store);
out << req.to_string_legacy(store);
}
@ -226,6 +260,13 @@ void RemoteStore::initConnection(Connection & conn)
conn.daemonNixVersion = readString(conn.from);
}
if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 35) {
conn.remoteTrustsUs = worker_proto::read(*this, conn.from, Phantom<std::optional<TrustedFlag>> {});
} else {
// We don't know the answer; protocol to old.
conn.remoteTrustsUs = std::nullopt;
}
auto ex = conn.processStderr();
if (ex) std::rethrow_exception(ex);
}
@ -1082,6 +1123,11 @@ unsigned int RemoteStore::getProtocol()
return conn->daemonVersion;
}
std::optional<TrustedFlag> RemoteStore::isTrustedClient()
{
auto conn(getConnection());
return conn->remoteTrustsUs;
}
void RemoteStore::flushBadConnections()
{

17
src/libstore/remote-store.hh
View file

@ -32,8 +32,10 @@ struct RemoteStoreConfig : virtual StoreConfig
"Maximum age of a connection before it is closed."};
};
/* FIXME: RemoteStore is a misnomer - should be something like
DaemonStore. */
/**
* \todo RemoteStore is a misnomer - should be something like
* DaemonStore.
*/
class RemoteStore : public virtual RemoteStoreConfig,
public virtual Store,
public virtual GcStore,
@ -69,7 +71,9 @@ public:
void querySubstitutablePathInfos(const StorePathCAMap & paths,
SubstitutablePathInfos & infos) override;
/* Add a content-addressable store path. `dump` will be drained. */
/**
* Add a content-addressable store path. `dump` will be drained.
*/
ref<const ValidPathInfo> addCAToStore(
Source & dump,
std::string_view name,
@ -77,7 +81,9 @@ public:
const StorePathSet & references,
RepairFlag repair);
/* Add a content-addressable store path. Does not support references. `dump` will be drained. */
/**
* Add a content-addressable store path. Does not support references. `dump` will be drained.
*/
StorePath addToStoreFromDump(Source & dump, std::string_view name,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override;
@ -144,6 +150,8 @@ public:
unsigned int getProtocol() override;
std::optional<TrustedFlag> isTrustedClient() override;
void flushBadConnections();
struct Connection
@ -151,6 +159,7 @@ public:
FdSink to;
FdSource from;
unsigned int daemonVersion;
std::optional<TrustedFlag> remoteTrustsUs;
std::optional<std::string> daemonNixVersion;
std::chrono::time_point<std::chrono::steady_clock> startTime;

10
src/libstore/s3-binary-cache-store.cc
View file

@ -509,6 +509,16 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
return paths;
}
/**
* For now, we conservatively say we don't know.
*
* \todo try to expose our S3 authentication status.
*/
std::optional<TrustedFlag> isTrustedClient() override
{
return std::nullopt;
}
static std::set<std::string> uriSchemes() { return {"s3"}; }
};

9
src/libstore/sqlite.cc
View file

@ -239,14 +239,11 @@ SQLiteTxn::~SQLiteTxn()
}
}
void handleSQLiteBusy(const SQLiteBusy & e)
void handleSQLiteBusy(const SQLiteBusy & e, time_t & nextWarning)
{
static std::atomic<time_t> lastWarned{0};
time_t now = time(0);
if (now > lastWarned + 10) {
lastWarned = now;
if (now > nextWarning) {
nextWarning = now + 10;
logWarning({
.msg = hintfmt(e.what())
});

48
src/libstore/sqlite.hh
View file

@ -11,7 +11,9 @@ struct sqlite3_stmt;
namespace nix {
/* RAII wrapper to close a SQLite database automatically. */
/**
* RAII wrapper to close a SQLite database automatically.
*/
struct SQLite
{
sqlite3 * db = 0;
@ -23,7 +25,9 @@ struct SQLite
~SQLite();
operator sqlite3 * () { return db; }
/* Disable synchronous mode, set truncate journal mode. */
/**
* Disable synchronous mode, set truncate journal mode.
*/
void isCache();
void exec(const std::string & stmt);
@ -31,7 +35,9 @@ struct SQLite
uint64_t getLastInsertedRowId();
};
/* RAII wrapper to create and destroy SQLite prepared statements. */
/**
* RAII wrapper to create and destroy SQLite prepared statements.
*/
struct SQLiteStmt
{
sqlite3 * db = 0;
@ -43,7 +49,9 @@ struct SQLiteStmt
~SQLiteStmt();
operator sqlite3_stmt * () { return stmt; }
/* Helper for binding / executing statements. */
/**
* Helper for binding / executing statements.
*/
class Use
{
friend struct SQLiteStmt;
@ -56,7 +64,9 @@ struct SQLiteStmt
~Use();
/* Bind the next parameter. */
/**
* Bind the next parameter.
*/
Use & operator () (std::string_view value, bool notNull = true);
Use & operator () (const unsigned char * data, size_t len, bool notNull = true);
Use & operator () (int64_t value, bool notNull = true);
@ -64,11 +74,15 @@ struct SQLiteStmt
int step();
/* Execute a statement that does not return rows. */
/**
* Execute a statement that does not return rows.
*/
void exec();
/* For statements that return 0 or more rows. Returns true iff
a row is available. */
/**
* For statements that return 0 or more rows. Returns true iff
* a row is available.
*/
bool next();
std::string getStr(int col);
@ -82,8 +96,10 @@ struct SQLiteStmt
}
};
/* RAII helper that ensures transactions are aborted unless explicitly
committed. */
/**
* RAII helper that ensures transactions are aborted unless explicitly
* committed.
*/
struct SQLiteTxn
{
bool active = false;
@ -123,18 +139,22 @@ protected:
MakeError(SQLiteBusy, SQLiteError);
void handleSQLiteBusy(const SQLiteBusy & e);
void handleSQLiteBusy(const SQLiteBusy & e, time_t & nextWarning);
/* Convenience function for retrying a SQLite transaction when the
database is busy. */
/**
* Convenience function for retrying a SQLite transaction when the
* database is busy.
*/
template<typename T, typename F>
T retrySQLite(F && fun)
{
time_t nextWarning = time(0) + 1;
while (true) {
try {
return fun();
} catch (SQLiteBusy & e) {
handleSQLiteBusy(e);
handleSQLiteBusy(e, nextWarning);
}
}
}

48
src/libstore/store-api.cc
View file

@ -507,6 +507,54 @@ StorePathSet Store::queryDerivationOutputs(const StorePath & path)
return outputPaths;
}
void Store::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos)
{
if (!settings.useSubstitutes) return;
for (auto & sub : getDefaultSubstituters()) {
for (auto & path : paths) {
if (infos.count(path.first))
// Choose first succeeding substituter.
continue;
auto subPath(path.first);
// Recompute store path so that we can use a different store root.
if (path.second) {
subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second);
if (sub->storeDir == storeDir)
assert(subPath == path.first);
if (subPath != path.first)
debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path.first), sub->printStorePath(subPath), sub->getUri());
} else if (sub->storeDir != storeDir) continue;
debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath));
try {
auto info = sub->queryPathInfo(subPath);
if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty()))
continue;
auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
std::shared_ptr<const ValidPathInfo>(info));
infos.insert_or_assign(path.first, SubstitutablePathInfo{
info->deriver,
info->references,
narInfo ? narInfo->fileSize : 0,
info->narSize});
} catch (InvalidPath &) {
} catch (SubstituterDisabled &) {
} catch (Error & e) {
if (settings.tryFallback)
logError(e.info());
else
throw;
}
}
}
}
bool Store::isValidPath(const StorePath & storePath)
{
{

39
src/libstore/store-api.hh
View file

@ -89,6 +89,7 @@ const uint32_t exportMagic = 0x4558494e;
enum BuildMode { bmNormal, bmRepair, bmCheck };
enum TrustedFlag : bool { NotTrusted = false, Trusted = true };
struct BuildResult;
@ -410,17 +411,17 @@ public:
{ unsupported("queryReferrers"); }
/**
* @return all currently valid derivations that have `path' as an
* @return all currently valid derivations that have `path` as an
* output.
*
* (Note that the result of `queryDeriver()' is the derivation that
* was actually used to produce `path', which may not exist
* (Note that the result of `queryDeriver()` is the derivation that
* was actually used to produce `path`, which may not exist
* anymore.)
*/
virtual StorePathSet queryValidDerivers(const StorePath & path) { return {}; };
/**
* Query the outputs of the derivation denoted by `path'.
* Query the outputs of the derivation denoted by `path`.
*/
virtual StorePathSet queryDerivationOutputs(const StorePath & path);
@ -456,7 +457,7 @@ public:
* resulting ‘infos’ map.
*/
virtual void querySubstitutablePathInfos(const StorePathCAMap & paths,
SubstitutablePathInfos & infos) { return; };
SubstitutablePathInfos & infos);
/**
* Import a path into the store.
@ -512,7 +513,7 @@ public:
/**
* Like addToStore(), but the contents of the path are contained
* in `dump', which is either a NAR serialisation (if recursive ==
* in `dump`, which is either a NAR serialisation (if recursive ==
* true) or simply the contents of a regular file (if recursive ==
* false).
* `dump` may be drained
@ -633,8 +634,8 @@ public:
/**
* @return a string representing information about the path that
* can be loaded into the database using `nix-store --load-db' or
* `nix-store --register-validity'.
* can be loaded into the database using `nix-store --load-db` or
* `nix-store --register-validity`.
*/
std::string makeValidityRegistration(const StorePathSet & paths,
bool showDerivers, bool showHash);
@ -677,8 +678,7 @@ public:
/**
* @return An object to access files in the Nix store.
*/
virtual ref<FSAccessor> getFSAccessor()
{ unsupported("getFSAccessor"); }
virtual ref<FSAccessor> getFSAccessor() = 0;
/**
* Repair the contents of the given path by redownloading it using
@ -714,12 +714,12 @@ public:
/**
* @param [out] out Place in here the set of all store paths in the
* file system closure of `storePath'; that is, all paths than can
* be directly or indirectly reached from it. `out' is not cleared.
* file system closure of `storePath`; that is, all paths than can
* be directly or indirectly reached from it. `out` is not cleared.
*
* @param flipDirection If true, the set of paths that can reach
* `storePath' is returned; that is, the closures under the
* `referrers' relation instead of the `references' relation is
* `storePath` is returned; that is, the closures under the
* `referrers` relation instead of the `references` relation is
* returned.
*/
virtual void computeFSClosure(const StorePathSet & paths,
@ -815,6 +815,17 @@ public:
return 0;
};
/**
* @return/ whether store trusts *us*.
*
* `std::nullopt` means we do not know.
*
* @note This is the opposite of the StoreConfig::isTrusted
* store setting. That is about whether *we* trust the store.
*/
virtual std::optional<TrustedFlag> isTrustedClient() = 0;
virtual Path toRealPath(const Path & storePath)
{
return storePath;

56
src/libstore/tests/derivation.cc
View file

@ -11,15 +11,29 @@ class DerivationTest : public LibStoreTest
{
};
#define TEST_JSON(TYPE, NAME, STR, VAL, ...) \
TEST_F(DerivationTest, TYPE ## _ ## NAME ## _to_json) { \
#define TEST_JSON(NAME, STR, VAL, DRV_NAME, OUTPUT_NAME) \
TEST_F(DerivationTest, DerivationOutput_ ## NAME ## _to_json) { \
using nlohmann::literals::operator "" _json; \
ASSERT_EQ( \
STR ## _json, \
(TYPE { VAL }).toJSON(*store __VA_OPT__(,) __VA_ARGS__)); \
(DerivationOutput { VAL }).toJSON( \
*store, \
DRV_NAME, \
OUTPUT_NAME)); \
} \
\
TEST_F(DerivationTest, DerivationOutput_ ## NAME ## _from_json) { \
using nlohmann::literals::operator "" _json; \
ASSERT_EQ( \
DerivationOutput { VAL }, \
DerivationOutput::fromJSON( \
*store, \
DRV_NAME, \
OUTPUT_NAME, \
STR ## _json)); \
}
TEST_JSON(DerivationOutput, inputAddressed,
TEST_JSON(inputAddressed,
R"({
"path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"
})",
@ -28,7 +42,7 @@ TEST_JSON(DerivationOutput, inputAddressed,
}),
"drv-name", "output-name")
TEST_JSON(DerivationOutput, caFixed,
TEST_JSON(caFixed,
R"({
"hashAlgo": "r:sha256",
"hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
@ -42,7 +56,7 @@ TEST_JSON(DerivationOutput, caFixed,
}),
"drv-name", "output-name")
TEST_JSON(DerivationOutput, caFloating,
TEST_JSON(caFloating,
R"({
"hashAlgo": "r:sha256"
})",
@ -52,12 +66,12 @@ TEST_JSON(DerivationOutput, caFloating,
}),
"drv-name", "output-name")
TEST_JSON(DerivationOutput, deferred,
TEST_JSON(deferred,
R"({ })",
DerivationOutput::Deferred { },
"drv-name", "output-name")
TEST_JSON(DerivationOutput, impure,
TEST_JSON(impure,
R"({
"hashAlgo": "r:sha256",
"impure": true
@ -68,8 +82,28 @@ TEST_JSON(DerivationOutput, impure,
}),
"drv-name", "output-name")
TEST_JSON(Derivation, impure,
#undef TEST_JSON
#define TEST_JSON(NAME, STR, VAL, DRV_NAME) \
TEST_F(DerivationTest, Derivation_ ## NAME ## _to_json) { \
using nlohmann::literals::operator "" _json; \
ASSERT_EQ( \
STR ## _json, \
(Derivation { VAL }).toJSON(*store)); \
} \
\
TEST_F(DerivationTest, Derivation_ ## NAME ## _from_json) { \
using nlohmann::literals::operator "" _json; \
ASSERT_EQ( \
Derivation { VAL }, \
Derivation::fromJSON( \
*store, \
STR ## _json)); \
}
TEST_JSON(simple,
R"({
"name": "my-derivation",
"inputSrcs": [
"/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"
],
@ -92,6 +126,7 @@ TEST_JSON(Derivation, impure,
})",
({
Derivation drv;
drv.name = "my-derivation";
drv.inputSrcs = {
store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"),
};
@ -117,7 +152,8 @@ TEST_JSON(Derivation, impure,
},
};
drv;
}))
}),
"drv-name")
#undef TEST_JSON

8
src/libstore/tests/derived-path.cc
View file

@ -51,6 +51,14 @@ TEST_F(DerivedPathTest, force_init)
{
}
RC_GTEST_FIXTURE_PROP(
DerivedPathTest,
prop_legacy_round_rip,
(const DerivedPath & o))
{
RC_ASSERT(o == DerivedPath::parseLegacy(*store, o.to_string_legacy(*store)));
}
RC_GTEST_FIXTURE_PROP(
DerivedPathTest,
prop_round_rip,

3
src/libstore/worker-protocol.hh
View file

@ -10,7 +10,7 @@ namespace nix {
#define WORKER_MAGIC_1 0x6e697863
#define WORKER_MAGIC_2 0x6478696f
#define PROTOCOL_VERSION (1 << 8 | 34)
#define PROTOCOL_VERSION (1 << 8 | 35)
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
@ -103,6 +103,7 @@ MAKE_WORKER_PROTO(, DerivedPath);
MAKE_WORKER_PROTO(, Realisation);
MAKE_WORKER_PROTO(, DrvOutput);
MAKE_WORKER_PROTO(, BuildResult);
MAKE_WORKER_PROTO(, std::optional<TrustedFlag>);
MAKE_WORKER_PROTO(template<typename T>, std::vector<T>);
MAKE_WORKER_PROTO(template<typename T>, std::set<T>);

7
src/libutil/ansicolor.hh
View file

@ -1,9 +1,12 @@
#pragma once
///@file
/**
* @file
*
* @brief Some ANSI escape sequences.
*/
namespace nix {
/* Some ANSI escape sequences. */
#define ANSI_NORMAL "\e[0m"
#define ANSI_BOLD "\e[1m"
#define ANSI_FAINT "\e[2m"

10
src/libutil/archive.hh
View file

@ -20,6 +20,7 @@ namespace nix {
*
* The format is as follows:
*
* ```
* IF path points to a REGULAR FILE:
* dump(path) = attrs(
* [ ("type", "regular")
@ -49,10 +50,11 @@ namespace nix {
*
* sort(strings) = lexicographic sort by 8-bit value (strcmp).
*
* entries(path) = the entries of a directory, without `.' and
* `..'.
* entries(path) = the entries of a directory, without `.` and
* `..`.
*
* `+' denotes string concatenation.
* `+` denotes string concatenation.
* ```
*/
void dumpPath(const Path & path, Sink & sink,
PathFilter & filter = defaultPathFilter);
@ -88,7 +90,7 @@ struct ParseSink
/**
* If the NAR archive contains a single file at top-level, then save
* the contents of the file to `s'. Otherwise barf.
* the contents of the file to `s`. Otherwise barf.
*/
struct RetrieveRegularNARSink : ParseSink
{

Some files were not shown because too many files have changed in this diff Show more