diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 60742cf6a..4488c7b7d 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -23,6 +23,7 @@ Maintainers: tick if completed or explain if not relevant - unit tests - `src/*/tests` - integration tests - `tests/nixos/*` - [ ] documentation in the manual + - [ ] documentation in the internal API docs - [ ] code and comments are self-explanatory - [ ] commit message explains why the change was made - [ ] new feature or incompatible change: updated release notes diff --git a/.github/labeler.yml b/.github/labeler.yml index dc502b6d5..fce0d3aeb 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -2,5 +2,22 @@ - doc/manual/* - src/nix/**/*.md +"store": + - src/libstore/store-api.* + - src/libstore/*-store.* + +"fetching": + - src/libfetchers/**/* + +"repl": + - src/libcmd/repl.* + - src/nix/repl.* + +"new-cli": + - src/nix/**/* + "tests": + # Unit tests + - src/*/tests/**/* + # Functional and integration tests - tests/**/* diff --git a/.gitignore b/.gitignore index e326966d6..8ceff4ef2 100644 --- a/.gitignore +++ b/.gitignore @@ -19,9 +19,12 @@ perl/Makefile.config /doc/manual/nix.json /doc/manual/conf-file.json /doc/manual/builtins.json +/doc/manual/xp-features.json /doc/manual/src/SUMMARY.md /doc/manual/src/command-ref/new-cli /doc/manual/src/command-ref/conf-file.md +/doc/manual/src/command-ref/experimental-features-shortlist.md +/doc/manual/src/contributing/experimental-feature-descriptions.md /doc/manual/src/language/builtins.md # /scripts/ diff --git a/.version b/.version index c910885a0..752490696 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.15.0 \ No newline at end of file +2.16.0 diff --git a/boehmgc-coroutine-sp-fallback.diff b/boehmgc-coroutine-sp-fallback.diff index 2826486fb..5066d8278 100644 --- a/boehmgc-coroutine-sp-fallback.diff +++ b/boehmgc-coroutine-sp-fallback.diff @@ -1,8 +1,8 @@ diff --git a/darwin_stop_world.c b/darwin_stop_world.c -index 3dbaa3fb..36a1d1f7 100644 +index 0468aaec..b348d869 100644 --- a/darwin_stop_world.c +++ b/darwin_stop_world.c -@@ -352,6 +352,7 @@ GC_INNER void GC_push_all_stacks(void) +@@ -356,6 +356,7 @@ GC_INNER void GC_push_all_stacks(void) int nthreads = 0; word total_size = 0; mach_msg_type_number_t listcount = (mach_msg_type_number_t)THREAD_TABLE_SZ; @@ -10,7 +10,7 @@ index 3dbaa3fb..36a1d1f7 100644 if (!EXPECT(GC_thr_initialized, TRUE)) GC_thr_init(); -@@ -407,6 +408,19 @@ GC_INNER void GC_push_all_stacks(void) +@@ -411,6 +412,19 @@ GC_INNER void GC_push_all_stacks(void) GC_push_all_stack_sections(lo, hi, p->traced_stack_sect); } if (altstack_lo) { @@ -30,6 +30,22 @@ index 3dbaa3fb..36a1d1f7 100644 total_size += altstack_hi - altstack_lo; GC_push_all_stack(altstack_lo, altstack_hi); } +diff --git a/include/gc.h b/include/gc.h +index edab6c22..f2c61282 100644 +--- a/include/gc.h ++++ b/include/gc.h +@@ -2172,6 +2172,11 @@ GC_API void GC_CALL GC_win32_free_heap(void); + (*GC_amiga_allocwrapper_do)(a,GC_malloc_atomic_ignore_off_page) + #endif /* _AMIGA && !GC_AMIGA_MAKINGLIB */ + ++#if !__APPLE__ ++/* Patch doesn't work on apple */ ++#define NIX_BOEHM_PATCH_VERSION 1 ++#endif ++ + #ifdef __cplusplus + } /* extern "C" */ + #endif diff --git a/pthread_stop_world.c b/pthread_stop_world.c index b5d71e62..aed7b0bf 100644 --- a/pthread_stop_world.c diff --git a/configure.ac b/configure.ac index f1f45f868..bb3f92e4d 100644 --- a/configure.ac +++ b/configure.ac @@ -184,7 +184,7 @@ fi # Look for OpenSSL, a required dependency. FIXME: this is only (maybe) # used by S3BinaryCacheStore. -PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) +PKG_CHECK_MODULES([OPENSSL], [libcrypto >= 1.1.1], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) # Look for libarchive. @@ -289,13 +289,24 @@ PKG_CHECK_MODULES([GTEST], [gtest_main]) # Look for rapidcheck. +AC_ARG_VAR([RAPIDCHECK_HEADERS], [include path of gtest headers shipped by RAPIDCHECK]) # No pkg-config yet, https://github.com/emil-e/rapidcheck/issues/302 AC_LANG_PUSH(C++) AC_SUBST(RAPIDCHECK_HEADERS) [CXXFLAGS="-I $RAPIDCHECK_HEADERS $CXXFLAGS"] +[LIBS="-lrapidcheck -lgtest $LIBS"] AC_CHECK_HEADERS([rapidcheck/gtest.h], [], [], [#include ]) -dnl No good for C++ libs with mangled symbols -dnl AC_CHECK_LIB([rapidcheck], []) +dnl AC_CHECK_LIB doesn't work for C++ libs with mangled symbols +AC_LINK_IFELSE([ + AC_LANG_PROGRAM([[ + #include + #include + ]], [[ + return RUN_ALL_TESTS(); + ]]) + ], + [], + [AC_MSG_ERROR([librapidcheck is not found.])]) AC_LANG_POP(C++) fi diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 86f2ca567..d04eecf55 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -10,7 +10,9 @@ let result = '' > **Warning** \ - > This program is **experimental** and its interface is subject to change. + > This program is + > [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) + > and its interface is subject to change. # Name @@ -29,19 +31,18 @@ let showSynopsis = command: args: let - showArgument = arg: "*${arg.label}*" + (if arg ? arity then "" else "..."); + showArgument = arg: "*${arg.label}*" + optionalString (! arg ? arity) "..."; arguments = concatStringsSep " " (map showArgument args); in '' `${command}` [*option*...] ${arguments} ''; - maybeSubcommands = if details ? commands && details.commands != {} - then '' + maybeSubcommands = optionalString (details ? commands && details.commands != {}) + '' where *subcommand* is one of the following: ${subcommands} - '' - else ""; + ''; subcommands = if length categories > 1 then listCategories @@ -63,12 +64,11 @@ let * [`${command} ${name}`](./${appendName filename name}.md) - ${subcmd.description} ''; - maybeDocumentation = - if details ? doc - then replaceStrings ["@stores@"] [storeDocs] details.doc - else ""; + maybeDocumentation = optionalString + (details ? doc) + (replaceStrings ["@stores@"] [storeDocs] details.doc); - maybeOptions = if details.flags == {} then "" else '' + maybeOptions = optionalString (details.flags != {}) '' # Options ${showOptions details.flags toplevel.flags} @@ -78,15 +78,19 @@ let let allOptions = options // commonOptions; showCategory = cat: '' - ${if cat != "" then "**${cat}:**" else ""} + ${optionalString (cat != "") "**${cat}:**"} ${listOptions (filterAttrs (n: v: v.category == cat) allOptions)} ''; listOptions = opts: concatStringsSep "\n" (attrValues (mapAttrs showOption opts)); showOption = name: option: let - shortName = if option ? shortName then "/ `-${option.shortName}`" else ""; - labels = if option ? labels then (concatStringsSep " " (map (s: "*${s}*") option.labels)) else ""; + shortName = optionalString + (option ? shortName) + ("/ `-${option.shortName}`"); + labels = optionalString + (option ? labels) + (concatStringsSep " " (map (s: "*${s}*") option.labels)); in trim '' - `--${name}` ${shortName} ${labels} diff --git a/doc/manual/generate-xp-features-shortlist.nix b/doc/manual/generate-xp-features-shortlist.nix new file mode 100644 index 000000000..30e211c96 --- /dev/null +++ b/doc/manual/generate-xp-features-shortlist.nix @@ -0,0 +1,9 @@ +with builtins; +with import ./utils.nix; + +let + showExperimentalFeature = name: doc: + '' + - [`${name}`](@docroot@/contributing/experimental-features.md#xp-feature-${name}) + ''; +in xps: indent " " (concatStrings (attrValues (mapAttrs showExperimentalFeature xps))) diff --git a/doc/manual/generate-xp-features.nix b/doc/manual/generate-xp-features.nix new file mode 100644 index 000000000..adb94355c --- /dev/null +++ b/doc/manual/generate-xp-features.nix @@ -0,0 +1,11 @@ +with builtins; +with import ./utils.nix; + +let + showExperimentalFeature = name: doc: + squash '' + ## [`${name}`]{#xp-feature-${name}} + + ${doc} + ''; +in xps: (concatStringsSep "\n" (attrValues (mapAttrs showExperimentalFeature xps))) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index df941d460..63e7e61e4 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -81,19 +81,20 @@ $(d)/%.8: $(d)/src/command-ref/%.md $(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md @printf "Title: %s\n\n" "$$(basename $@ .5)" > $^.tmp @cat $^ >> $^.tmp + @$(call process-includes,$^,$^.tmp) $(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@ @rm $^.tmp -$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli +$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md @cp $< $@ @$(call process-includes,$@,$@) -$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix +$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(bindir)/nix @rm -rf $@ $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix (builtins.readFile $<)' @mv $@.tmp $@ -$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix +$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp $(trace-gen) $(nix-eval) --expr '(import doc/manual/utils.nix).showSettings { useAnchors = true; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; @mv $@.tmp $@ @@ -106,6 +107,20 @@ $(d)/conf-file.json: $(bindir)/nix $(trace-gen) $(dummy-env) $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp @mv $@.tmp $@ +$(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(bindir)/nix + @rm -rf $@ $@.tmp + $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features.nix (builtins.fromJSON (builtins.readFile $<))' + @mv $@.tmp $@ + +$(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(bindir)/nix + @rm -rf $@ $@.tmp + $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features-shortlist.nix (builtins.fromJSON (builtins.readFile $<))' + @mv $@.tmp $@ + +$(d)/xp-features.json: $(bindir)/nix + $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-xp-features > $@.tmp + @mv $@.tmp $@ + $(d)/src/language/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix @cat doc/manual/src/language/builtins-prefix.md > $@.tmp $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; @@ -145,7 +160,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli done @touch $@ -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(trace-gen) \ tmp="$$(mktemp -d)"; \ cp -r doc/manual "$$tmp"; \ diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 4b654567f..f783d5908 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -95,9 +95,11 @@ - [Glossary](glossary.md) - [Contributing](contributing/contributing.md) - [Hacking](contributing/hacking.md) + - [Experimental Features](contributing/experimental-features.md) - [CLI guideline](contributing/cli-guideline.md) - [Release Notes](release-notes/release-notes.md) - [Release X.Y (202?-??-??)](release-notes/rl-next.md) + - [Release 2.15 (2023-04-11)](release-notes/rl-2.15.md) - [Release 2.14 (2023-02-28)](release-notes/rl-2.14.md) - [Release 2.13 (2023-01-17)](release-notes/rl-2.13.md) - [Release 2.12 (2022-12-06)](release-notes/rl-2.12.md) diff --git a/doc/manual/src/command-ref/experimental-commands.md b/doc/manual/src/command-ref/experimental-commands.md index cfa6f8b73..286ddc6d6 100644 --- a/doc/manual/src/command-ref/experimental-commands.md +++ b/doc/manual/src/command-ref/experimental-commands.md @@ -1,6 +1,6 @@ # Experimental Commands -This section lists experimental commands. +This section lists [experimental commands](@docroot@/contributing/experimental-features.md#xp-feature-nix-command). > **Warning** > diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index 9f36929f7..576e5ba0b 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -120,7 +120,8 @@ shell in which to build it: ```console $ nix-shell '' -A pan [nix-shell]$ eval ${unpackPhase:-unpackPhase} -[nix-shell]$ cd pan-* +[nix-shell]$ cd $sourceRoot +[nix-shell]$ eval ${patchPhase:-patchPhase} [nix-shell]$ eval ${configurePhase:-configurePhase} [nix-shell]$ eval ${buildPhase:-buildPhase} [nix-shell]$ ./pan/gui/pan diff --git a/doc/manual/src/contributing/experimental-features.md b/doc/manual/src/contributing/experimental-features.md new file mode 100644 index 000000000..ad5cffa91 --- /dev/null +++ b/doc/manual/src/contributing/experimental-features.md @@ -0,0 +1,95 @@ +This section describes the notion of *experimental features*, and how it fits into the big picture of the development of Nix. + +# What are experimental features? + +Experimental features are considered unstable, which means that they can be changed or removed at any time. +Users must explicitly enable them by toggling the associated [experimental feature flags](@docroot@/command-ref/conf-file.md#conf-experimental-features). +This allows accessing unstable functionality without unwittingly relying on it. + +Experimental feature flags were first introduced in [Nix 2.4](@docroot@/release-notes/rl-2.4.md). +Before that, Nix did have experimental features, but they were not guarded by flags and were merely documented as unstable. +This was a source of confusion and controversy. + +# When should a new feature be marked experimental? + +A change in the Nix codebase should be guarded by an experimental feature flag if it is considered likely to be reverted or adapted in a backwards-incompatible manner after gathering more experience with it in practice. + +Examples: + +- Changes to the Nix language, such as new built-ins, syntactic or semantic changes, etc. +- Changes to the command-line interface + +# Lifecycle of an experimental feature + +Experimental features have to be treated on a case-by-case basis. +However, the standard workflow for an experimental feature is as follows: + +- A new feature is implemented in a *pull request* + - It is guarded by an experimental feature flag that is disabled by default +- The pull request is merged, the *experimental* feature ends up in a release + - Using the feature requires explicitly enabling it, signifying awareness of the potential risks + - Being experimental, the feature can still be changed arbitrarily +- The feature can be *removed* + - The associated experimental feature flag is also removed +- The feature can be declared *stable* + - The associated experimental feature flag is removed + - There should be enough evidence of users having tried the feature, such as feedback, fixed bugs, demonstrations of how it is put to use + - Maintainers must feel confident that: + - The feature is designed and implemented sensibly, that it is fit for purpose + - Potential interactions are well-understood + - Stabilising the feature will not incur an outsized maintenance burden in the future + +The following diagram illustrates the process: + +``` + .------. + | idea | + '------' + | + discussion, design, implementation + | + | .-------. + | | | + v v | + .--------------. review + | pull request | | + '--------------' | + | ^ | | + | | '-------' + .---' '----. + | | + merge user feedback, + | (breaking) changes + | | + '---. .----' + | | + v | + +--------------+ + .---| experimental |----. + | +--------------+ | + | | +decision to stabilise decision against + | keeping the feature + | | + v v + +--------+ +---------+ + | stable | | removed | + +--------+ +---------+ +``` + +# Relation to the RFC process + +Experimental features and [RFCs](https://github.com/NixOS/rfcs/) both allow approaching substantial changes while minimizing the risk. +However they serve different purposes: + +- An experimental feature enables developers to iterate on and deliver a new idea without committing to it or requiring a costly long-running fork. + It is primarily an issue of *implementation*, targeting Nix developers and early testers. +- The goal of an RFC is to make explicit all the implications of a change: + Explain why it is wanted, which new use-cases it enables, which interface changes it requires, etc. + It is primarily an issue of *design* and *communication*, targeting the broader community. + +This means that experimental features and RFCs are orthogonal mechanisms, and can be used independently or together as needed. + +# Currently available experimental features + +{{#include ./experimental-feature-descriptions.md}} diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index b56d857d1..a9782be5c 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -15,7 +15,7 @@ Example: `/nix/store/g946hcz4c8mdvq2g8vxx42z51qb71rvp-git-2.38.1.drv` - See [`nix show-derivation`](./command-ref/new-cli/nix3-show-derivation.md) (experimental) for displaying the contents of store derivations. + See [`nix derivation show`](./command-ref/new-cli/nix3-derivation-show.md) (experimental) for displaying the contents of store derivations. [store derivation]: #gloss-store-derivation @@ -54,7 +54,7 @@ invoked, the Nix store can be referred to as a "_local_" or a "_remote_" one: - + A *local store* exists on the filesystem of + + A [local store]{#gloss-local-store} exists on the filesystem of the machine where Nix is invoked. You can use other local stores by passing the `--store` flag to the `nix` command. Local stores can be used for building derivations. @@ -65,17 +65,17 @@ served by the `nix-serve` Perl script. [store]: #gloss-store + [local store]: #gloss-local-store - [chroot store]{#gloss-chroot-store}\ - A local store whose canonical path is anything other than `/nix/store`. + A [local store] whose canonical path is anything other than `/nix/store`. - [binary cache]{#gloss-binary-cache}\ A *binary cache* is a Nix store which uses a different format: its metadata and signatures are kept in `.narinfo` files rather than in a - Nix database. This different format simplifies serving store objects - over the network, but cannot host builds. Examples of binary caches - include S3 buckets and the [NixOS binary - cache](https://cache.nixos.org). + [Nix database]. This different format simplifies serving store objects + over the network, but cannot host builds. Examples of binary caches + include S3 buckets and the [NixOS binary cache](https://cache.nixos.org). - [store path]{#gloss-store-path}\ The location of a [store object] in the file system, i.e., an @@ -108,7 +108,7 @@ [fixed-output derivations](#gloss-fixed-output-derivation). - [substitute]{#gloss-substitute}\ - A substitute is a command invocation stored in the Nix database that + A substitute is a command invocation stored in the [Nix database] that describes how to build a store object, bypassing the normal build mechanism (i.e., derivations). Typically, the substitute builds the store object by downloading a pre-built version of the store object @@ -127,6 +127,14 @@ builder can rely on external inputs such as the network or the system time) but the Nix model assumes it. + - Nix database{#gloss-nix-database}\ + An SQlite database to track [reference]s between [store object]s. + This is an implementation detail of the [local store]. + + Default location: `/nix/var/nix/db`. + + [Nix database]: #gloss-nix-database + - [Nix expression]{#gloss-nix-expression}\ A high-level description of software packages and compositions thereof. Deploying software using Nix entails writing Nix @@ -175,9 +183,9 @@ - [validity]{#gloss-validity}\ A store path is valid if all [store object]s in its [closure] can be read from the [store]. - For a local store, this means: + For a [local store], this means: - The store path leads to an existing [store object] in that [store]. - - The store path is listed in the Nix database as being valid. + - The store path is listed in the [Nix database] as being valid. - All paths in the store path's [closure] are valid. [validity]: #gloss-validity @@ -217,3 +225,9 @@ [string]: ./language/values.md#type-string [path]: ./language/values.md#type-path [attribute name]: ./language/values.md#attribute-set + + - [experimental feature]{#gloss-experimental-feature}\ + Not yet stabilized functionality guarded by named experimental feature flags. + These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting. + + See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/contributing/experimental-features.md). diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 3e8c48890..307971434 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -208,12 +208,26 @@ Derivations can declare some infrequently used optional attributes. about converting to and from base-32 notation.) - [`__contentAddressed`]{#adv-attr-__contentAddressed} - If this **experimental** attribute is set to true, then the derivation + > **Warning** + > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md). + > + > To use this attribute, you must enable the + > [`ca-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) experimental feature. + > For example, in [nix.conf](../command-ref/conf-file.md) you could add: + > + > ``` + > extra-experimental-features = ca-derivations + > ``` + + If this attribute is set to `true`, then the derivation outputs will be stored in a content-addressed location rather than the traditional input-addressed one. - This only has an effect if the `ca-derivations` experimental feature is enabled. - Setting this attribute also requires setting `outputHashMode` and `outputHashAlgo` like for *fixed-output derivations* (see above). + Setting this attribute also requires setting + [`outputHashMode`](#adv-attr-outputHashMode) + and + [`outputHashAlgo`](#adv-attr-outputHashAlgo) + like for *fixed-output derivations* (see above). - [`passAsFile`]{#adv-attr-passAsFile}\ A list of names of attributes that should be passed via files rather @@ -307,9 +321,11 @@ Derivations can declare some infrequently used optional attributes. - [`unsafeDiscardReferences`]{#adv-attr-unsafeDiscardReferences}\ > **Warning** - > This is an experimental feature. + > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md). > - > To enable it, add the following to [nix.conf](../command-ref/conf-file.md): + > To use this attribute, you must enable the + > [`discard-references`](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) experimental feature. + > For example, in [nix.conf](../command-ref/conf-file.md) you could add: > > ``` > extra-experimental-features = discard-references diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md index 651134c25..1d2688ede 100644 --- a/doc/manual/src/quick-start.md +++ b/doc/manual/src/quick-start.md @@ -19,7 +19,7 @@ to subsequent chapters. channel: ```console - $ nix-env -qaP + $ nix-env --query --available --attr-path nixpkgs.docbook_xml_dtd_43 docbook-xml-4.3 nixpkgs.docbook_xml_dtd_45 docbook-xml-4.5 nixpkgs.firefox firefox-33.0.2 @@ -31,7 +31,7 @@ to subsequent chapters. 1. Install some packages from the channel: ```console - $ nix-env -iA nixpkgs.hello + $ nix-env --install --attr nixpkgs.hello ``` This should download pre-built packages; it should not build them @@ -49,13 +49,13 @@ to subsequent chapters. 1. Uninstall a package: ```console - $ nix-env -e hello + $ nix-env --uninstall hello ``` 1. You can also test a package without installing it: ```console - $ nix-shell -p hello + $ nix-shell --packages hello ``` This builds or downloads GNU Hello and its dependencies, then drops @@ -76,7 +76,7 @@ to subsequent chapters. ```console $ nix-channel --update nixpkgs - $ nix-env -u '*' + $ nix-env --upgrade '*' ``` The latter command will upgrade each installed package for which @@ -95,5 +95,5 @@ to subsequent chapters. them: ```console - $ nix-collect-garbage -d + $ nix-collect-garbage --delete-old ``` diff --git a/doc/manual/src/release-notes/rl-2.15.md b/doc/manual/src/release-notes/rl-2.15.md new file mode 100644 index 000000000..133121999 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.15.md @@ -0,0 +1,58 @@ +# Release 2.15 (2023-04-11) + +* Commands which take installables on the command line can now read them from the standard input if + passed the `--stdin` flag. This is primarily useful when you have a large amount of paths which + exceed the OS argument limit. + +* The `nix-hash` command now supports Base64 and SRI. Use the flags `--base64` + or `--sri` to specify the format of output hash as Base64 or SRI, and `--to-base64` + or `--to-sri` to convert a hash to Base64 or SRI format, respectively. + + As the choice of hash formats is no longer binary, the `--base16` flag is also added + to explicitly specify the Base16 format, which is still the default. + +* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](../glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents. + + The new `^` syntax for store paths introduced in Nix 2.13 allows explicitly referencing output paths of a derivation. + Using this is better and more clear than relying on the now-removed `.drv` special handling. + + For example, + ```shell-session + $ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv + ``` + + now gives info about the derivation itself, while + + ```shell-session + $ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^* + ``` + provides information about each of its outputs. + +* The experimental command `nix describe-stores` has been removed. + +* Nix stores and their settings are now documented in [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md). + +* Documentation for operations of `nix-store` and `nix-env` are now available on separate pages of the manual. + They include all common options that can be specified and common environment variables that affect these commands. + + These pages can be viewed offline with `man` using + + * `man nix-store-` and `man nix-env-` + * `nix-store --help --` and `nix-env --help --`. + +* Nix when used as a client now checks whether the store (the server) trusts the client. + (The store always had to check whether it trusts the client, but now the client is informed of the store's decision.) + This is useful for scripting interactions with (non-legacy-ssh) remote Nix stores. + + `nix store ping` and `nix doctor` now display this information. + +* The new command `nix derivation add` allows adding derivations to the store without involving the Nix language. + It exists to round out our collection of basic utility/plumbing commands, and allow for a low barrier-to-entry way of experimenting with alternative front-ends to the Nix Store. + It uses the same JSON layout as `nix derivation show`, and is its inverse. + +* `nix show-derivation` has been renamed to `nix derivation show`. + This matches `nix derivation add`, and avoids bloating the top-level namespace. + The old name is still kept as an alias for compatibility, however. + +* The `nix derivation {add,show}` JSON format now includes the derivation name as a top-level field. + This is useful in general, but especially necessary for the `add` direction, as otherwise we would need to pass in the name out of band for certain cases. diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index ae159de8f..78ae99f4b 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,41 +1,2 @@ # Release X.Y (202?-??-??) -* Commands which take installables on the command line can now read them from the standard input if - passed the `--stdin` flag. This is primarily useful when you have a large amount of paths which - exceed the OS arg limit. - -* The `nix-hash` command now supports Base64 and SRI. Use the flags `--base64` - or `--sri` to specify the format of output hash as Base64 or SRI, and `--to-base64` - or `--to-sri` to convert a hash to Base64 or SRI format, respectively. - - As the choice of hash formats is no longer binary, the `--base16` flag is also added - to explicitly specify the Base16 format, which is still the default. - -* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](../glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents. - - The new `^` syntax for store paths introduced in Nix 2.13 allows explicitly referencing output paths of a derivation. - Using this is better and more clear than relying on the now-removed `.drv` special handling. - - For example, - ```shell-session - $ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv - ``` - - now gives info about the derivation itself, while - - ```shell-session - $ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^* - ``` - provides information about each of its outputs. - -* The experimental command `nix describe-stores` has been removed. - -* Nix stores and their settings are now documented in [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md). - -* Documentation for operations of `nix-store` and `nix-env` are now available on separate pages of the manual. - They include all common options that can be specified and common environment variables that affect these commands. - - These pages can be viewed offline with `man` using - - * `man nix-store-` and `man nix-env-` - * `nix-store --help --` and `nix-env --help --`. diff --git a/doc/manual/utils.nix b/doc/manual/utils.nix index 5eacce0dd..9043dd8cd 100644 --- a/doc/manual/utils.nix +++ b/doc/manual/utils.nix @@ -5,6 +5,9 @@ rec { concatStrings = concatStringsSep ""; + attrsToList = a: + map (name: { inherit name; value = a.${name}; }) (builtins.attrNames a); + replaceStringsRec = from: to: string: # recursively replace occurrences of `from` with `to` within `string` # example: @@ -39,7 +42,9 @@ rec { filterAttrs = pred: set: listToAttrs (concatMap (name: let v = set.${name}; in if pred name v then [(nameValuePair name v)] else []) (attrNames set)); - showSetting = { useAnchors }: name: { description, documentDefault, defaultValue, aliases, value }: + optionalString = cond: string: if cond then string else ""; + + showSetting = { useAnchors }: name: { description, documentDefault, defaultValue, aliases, value, experimentalFeature }: let result = squash '' - ${if useAnchors @@ -49,10 +54,28 @@ rec { ${indent " " body} ''; + experimentalFeatureNote = optionalString (experimentalFeature != null) '' + > **Warning** + > This setting is part of an + > [experimental feature](@docroot@/contributing/experimental-features.md). + + To change this setting, you need to make sure the corresponding experimental feature, + [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}), + is enabled. + For example, include the following in [`nix.conf`](#): + + ``` + extra-experimental-features = ${experimentalFeature} + ${name} = ... + ``` + ''; + # separate body to cleanly handle indentation body = '' ${description} + ${experimentalFeatureNote} + **Default:** ${showDefault documentDefault defaultValue} ${showAliases aliases} @@ -71,13 +94,13 @@ rec { else "*machine-specific*"; showAliases = aliases: - if aliases == [] then "" else + optionalString (aliases != []) "**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}"; - indent = prefix: s: - concatStringsSep "\n" (map (x: if x == "" then x else "${prefix}${x}") (splitLines s)); - in result; + indent = prefix: s: + concatStringsSep "\n" (map (x: if x == "" then x else "${prefix}${x}") (splitLines s)); + showSettings = args: settingsInfo: concatStrings (attrValues (mapAttrs (showSetting args) settingsInfo)); } diff --git a/flake.nix b/flake.nix index 2fa16f8e2..a4ee80b32 100644 --- a/flake.nix +++ b/flake.nix @@ -219,6 +219,7 @@ enableParallelBuilding = true; + configureFlags = testConfigureFlags; # otherwise configure fails dontBuild = true; doInstallCheck = true; @@ -472,8 +473,6 @@ }; in { - inherit nixpkgsFor; - # A Nixpkgs overlay that overrides the 'nix' and # 'nix.perl-bindings' packages. overlays.default = overlayFor (p: p.stdenv); @@ -583,6 +582,8 @@ tests.nix-copy-closure = runNixOSTestFor "x86_64-linux" ./tests/nixos/nix-copy-closure.nix; + tests.nix-copy = runNixOSTestFor "x86_64-linux" ./tests/nixos/nix-copy.nix; + tests.nssPreload = runNixOSTestFor "x86_64-linux" ./tests/nixos/nss-preload.nix; tests.githubFlakes = runNixOSTestFor "x86_64-linux" ./tests/nixos/github-flakes.nix; diff --git a/local.mk b/local.mk index 6a7074e8e..6951c179e 100644 --- a/local.mk +++ b/local.mk @@ -1,6 +1,8 @@ clean-files += Makefile.config -GLOBAL_CXXFLAGS += -Wno-deprecated-declarations +GLOBAL_CXXFLAGS += -Wno-deprecated-declarations -Werror=switch +# Allow switch-enum to be overridden for files that do not support it, usually because of dependency headers. +ERROR_SWITCH_ENUM = -Werror=switch-enum $(foreach i, config.h $(wildcard src/lib*/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) diff --git a/maintainers/README.md b/maintainers/README.md index 476a5f51e..618bfb4e4 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -56,7 +56,7 @@ Meeting notes are collected on a [collaborative scratchpad](https://pad.lassul.u The team uses a [GitHub project board](https://github.com/orgs/NixOS/projects/19/views/1) for tracking its work. -Issues on the board progress through the following states: +Items on the board progress through the following states: - No Status @@ -69,6 +69,7 @@ Issues on the board progress through the following states: 2. [security](https://github.com/NixOS/nix/labels/security) 3. [regression](https://github.com/NixOS/nix/labels/regression) 4. [bug](https://github.com/NixOS/nix/issues?q=is%3Aopen+label%3Abug+sort%3Areactions-%2B1-desc) + 5. [tests of existing functionality](https://github.com/NixOS/nix/issues?q=is%3Aopen+label%3Atests+-label%3Afeature+sort%3Areactions-%2B1-desc) - [oldest pull requests](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+sort%3Acreated-asc) - [most popular pull requests](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+sort%3Areactions-%2B1-desc) @@ -79,6 +80,9 @@ Issues on the board progress through the following states: If there is disagreement on the general idea behind an issue or pull request, it is moved to _To discuss_, otherwise to _In review_. + To ensure process quality and reliability, all non-trivial pull requests must be triaged before merging. + What constitutes a trivial pull request is up to maintainers' judgement. + - To discuss Pull requests and issues that are deemed important and controversial are discussed by the team during discussion meetings. @@ -91,7 +95,7 @@ Issues on the board progress through the following states: Contributors who took the time to implement concrete change proposals should not wait indefinitely. - - Prioritise fixing bugs over documentation, improvements or new features + - Prioritise fixing bugs and testing over documentation, improvements or new features The team values stability and accessibility higher than raw functionality. diff --git a/mk/patterns.mk b/mk/patterns.mk index 86a724806..c81150260 100644 --- a/mk/patterns.mk +++ b/mk/patterns.mk @@ -1,10 +1,10 @@ $(buildprefix)%.o: %.cc @mkdir -p "$(dir $@)" - $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep, $@) -MP $(buildprefix)%.o: %.cpp @mkdir -p "$(dir $@)" - $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep, $@) -MP $(buildprefix)%.o: %.c @mkdir -p "$(dir $@)" diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index bfe00d3e2..41ecbbeb4 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -27,8 +27,6 @@ static ref store() if (!_store) { try { initLibStore(); - loadConfFile(); - settings.lockCPU = false; _store = openStore(); } catch (Error & e) { croak("%s", e.what()); diff --git a/src/libcmd/command-installable-value.cc b/src/libcmd/command-installable-value.cc index d7581534b..7e0c15eb8 100644 --- a/src/libcmd/command-installable-value.cc +++ b/src/libcmd/command-installable-value.cc @@ -4,8 +4,8 @@ namespace nix { void InstallableValueCommand::run(ref store, ref installable) { - auto installableValue = InstallableValue::require(installable); - run(store, installableValue); + auto installableValue = InstallableValue::require(installable); + run(store, installableValue); } } diff --git a/src/libcmd/command-installable-value.hh b/src/libcmd/command-installable-value.hh index 8e31a0b92..7880d4119 100644 --- a/src/libcmd/command-installable-value.hh +++ b/src/libcmd/command-installable-value.hh @@ -1,10 +1,20 @@ +#pragma once +///@file + #include "installable-value.hh" #include "command.hh" namespace nix { +/** + * An InstallableCommand where the single positional argument must be an + * InstallableValue in particular. + */ struct InstallableValueCommand : InstallableCommand { + /** + * Entry point to this command + */ virtual void run(ref store, ref installable) = 0; void run(ref store, ref installable) override; diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index dbc155b79..96236b987 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "installable-value.hh" #include "args.hh" diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index 1ec800613..b69db11dd 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "args.hh" diff --git a/src/libcmd/editor-for.hh b/src/libcmd/editor-for.hh index 8fbd08792..c8c4e9d9b 100644 --- a/src/libcmd/editor-for.hh +++ b/src/libcmd/editor-for.hh @@ -1,11 +1,14 @@ #pragma once +///@file #include "types.hh" namespace nix { -/* Helper function to generate args that invoke $EDITOR on - filename:lineno. */ +/** + * Helper function to generate args that invoke $EDITOR on + * filename:lineno. + */ Strings editorFor(const Path & file, uint32_t line); } diff --git a/src/libcmd/installable-attr-path.hh b/src/libcmd/installable-attr-path.hh index c06132ec8..e9f0c33da 100644 --- a/src/libcmd/installable-attr-path.hh +++ b/src/libcmd/installable-attr-path.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include "globals.hh" #include "installable-value.hh" #include "outputs-spec.hh" diff --git a/src/libcmd/installable-derived-path.hh b/src/libcmd/installable-derived-path.hh index 042878b91..e0b4f18b3 100644 --- a/src/libcmd/installable-derived-path.hh +++ b/src/libcmd/installable-derived-path.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "installables.hh" diff --git a/src/libcmd/installable-flake.hh b/src/libcmd/installable-flake.hh index 313d2d7a3..afe64d977 100644 --- a/src/libcmd/installable-flake.hh +++ b/src/libcmd/installable-flake.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "installable-value.hh" diff --git a/src/libcmd/installable-value.cc b/src/libcmd/installable-value.cc index 30f80edb2..3a7ede4e2 100644 --- a/src/libcmd/installable-value.cc +++ b/src/libcmd/installable-value.cc @@ -22,7 +22,7 @@ InstallableValue::getCursor(EvalState & state) static UsageError nonValueInstallable(Installable & installable) { - return UsageError("installable '%s' does not correspond to a Nix language value", installable.what()); + return UsageError("installable '%s' does not correspond to a Nix language value", installable.what()); } InstallableValue & InstallableValue::require(Installable & installable) diff --git a/src/libcmd/installable-value.hh b/src/libcmd/installable-value.hh index 9e076cb10..bfb3bfeed 100644 --- a/src/libcmd/installable-value.hh +++ b/src/libcmd/installable-value.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "installables.hh" #include "flake/flake.hh" diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 67549b280..32ae46d9f 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -102,6 +102,28 @@ MixFlakeOptions::MixFlakeOptions() }} }); + addFlag({ + .longName = "reference-lock-file", + .description = "Read the given lock file instead of `flake.lock` within the top-level flake.", + .category = category, + .labels = {"flake-lock-path"}, + .handler = {[&](std::string lockFilePath) { + lockFlags.referenceLockFilePath = lockFilePath; + }}, + .completer = completePath + }); + + addFlag({ + .longName = "output-lock-file", + .description = "Write the given lock file instead of `flake.lock` within the top-level flake.", + .category = category, + .labels = {"flake-lock-path"}, + .handler = {[&](std::string lockFilePath) { + lockFlags.outputLockFilePath = lockFilePath; + }}, + .completer = completePath + }); + addFlag({ .longName = "inputs-from", .description = "Use the inputs of the specified flake as registry entries.", diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index b6efc0f17..42d6c7c7c 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "util.hh" #include "path.hh" diff --git a/src/libcmd/legacy.hh b/src/libcmd/legacy.hh index f503b0da3..357500a4d 100644 --- a/src/libcmd/legacy.hh +++ b/src/libcmd/legacy.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include diff --git a/src/libcmd/markdown.hh b/src/libcmd/markdown.hh index 78320fcf5..a04d32a4f 100644 --- a/src/libcmd/markdown.hh +++ b/src/libcmd/markdown.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include "types.hh" namespace nix { diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index e3afb1531..80c08bf1c 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -40,6 +40,7 @@ extern "C" { #include "markdown.hh" #include "local-fs-store.hh" #include "progress-bar.hh" +#include "print.hh" #if HAVE_BOEHMGC #define GC_INCLUDE_NEW @@ -252,7 +253,9 @@ void NixRepl::mainLoop() el_hist_size = 1000; #endif read_history(historyFile.c_str()); + auto oldRepl = curRepl; curRepl = this; + Finally restoreRepl([&] { curRepl = oldRepl; }); #ifndef READLINE rl_set_complete_func(completionCallback); rl_set_list_possib_func(listPossibleCallback); @@ -423,6 +426,7 @@ StringSet NixRepl::completePrefix(const std::string & prefix) } +// FIXME: DRY and match or use the parser static bool isVarName(std::string_view s) { if (s.size() == 0) return false; @@ -892,17 +896,6 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m } -std::ostream & printStringValue(std::ostream & str, const char * string) { - str << "\""; - for (const char * i = string; *i; i++) - if (*i == '\"' || *i == '\\') str << "\\" << *i; - else if (*i == '\n') str << "\\n"; - else if (*i == '\r') str << "\\r"; - else if (*i == '\t') str << "\\t"; - else str << *i; - str << "\""; - return str; -} // FIXME: lot of cut&paste from Nix's eval.cc. @@ -920,12 +913,14 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m break; case nBool: - str << ANSI_CYAN << (v.boolean ? "true" : "false") << ANSI_NORMAL; + str << ANSI_CYAN; + printLiteralBool(str, v.boolean); + str << ANSI_NORMAL; break; case nString: str << ANSI_WARNING; - printStringValue(str, v.string.s); + printLiteralString(str, v.string.s); str << ANSI_NORMAL; break; @@ -962,10 +957,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m sorted.emplace(state->symbols[i.name], i.value); for (auto & i : sorted) { - if (isVarName(i.first)) - str << i.first; - else - printStringValue(str, i.first.c_str()); + printAttributeName(str, i.first); str << " = "; if (seen.count(i.second)) str << "«repeated»"; @@ -1024,6 +1016,8 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m str << v.fpoint; break; + case nThunk: + case nExternal: default: str << ANSI_RED "«unknown»" ANSI_NORMAL; break; diff --git a/src/libcmd/repl.hh b/src/libcmd/repl.hh index dfccc93e7..731c8e6db 100644 --- a/src/libcmd/repl.hh +++ b/src/libcmd/repl.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "eval.hh" diff --git a/src/libexpr/attr-path.hh b/src/libexpr/attr-path.hh index 117e0051b..b2bfb5d04 100644 --- a/src/libexpr/attr-path.hh +++ b/src/libexpr/attr-path.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "eval.hh" @@ -16,7 +17,9 @@ std::pair findAlongAttrPath( Bindings & autoArgs, Value & vIn); -/* Heuristic to find the filename and lineno or a nix value. */ +/** + * Heuristic to find the filename and lineno or a nix value. + */ std::pair findPackageFilename(EvalState & state, Value & v, std::string what); std::vector parseAttrPath(EvalState & state, std::string_view s); diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh index dcc73b506..31215f880 100644 --- a/src/libexpr/attr-set.hh +++ b/src/libexpr/attr-set.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "nixexpr.hh" #include "symbol-table.hh" @@ -12,7 +13,9 @@ namespace nix { class EvalState; struct Value; -/* Map one attribute name to its value. */ +/** + * Map one attribute name to its value. + */ struct Attr { /* the placement of `name` and `pos` in this struct is important. @@ -36,10 +39,12 @@ static_assert(sizeof(Attr) == 2 * sizeof(uint32_t) + sizeof(Value *), "avoid introducing any padding into Attr if at all possible, and do not " "introduce new fields that need not be present for almost every instance."); -/* Bindings contains all the attributes of an attribute set. It is defined - by its size and its capacity, the capacity being the number of Attr - elements allocated after this structure, while the size corresponds to - the number of elements already inserted in this structure. */ +/** + * Bindings contains all the attributes of an attribute set. It is defined + * by its size and its capacity, the capacity being the number of Attr + * elements allocated after this structure, while the size corresponds to + * the number of elements already inserted in this structure. + */ class Bindings { public: @@ -94,7 +99,9 @@ public: size_t capacity() { return capacity_; } - /* Returns the attributes in lexicographically sorted order. */ + /** + * Returns the attributes in lexicographically sorted order. + */ std::vector lexicographicOrder(const SymbolTable & symbols) const { std::vector res; @@ -111,9 +118,11 @@ public: friend class EvalState; }; -/* A wrapper around Bindings that ensures that its always in sorted - order at the end. The only way to consume a BindingsBuilder is to - call finish(), which sorts the bindings. */ +/** + * A wrapper around Bindings that ensures that its always in sorted + * order at the end. The only way to consume a BindingsBuilder is to + * call finish(), which sorts the bindings. + */ class BindingsBuilder { Bindings * bindings; diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh index c93e55b93..46c4999c8 100644 --- a/src/libexpr/eval-cache.hh +++ b/src/libexpr/eval-cache.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "sync.hh" #include "hash.hh" @@ -109,8 +110,10 @@ public: ref getAttr(std::string_view name); - /* Get an attribute along a chain of attrsets. Note that this does - not auto-call functors or functions. */ + /** + * Get an attribute along a chain of attrsets. Note that this does + * not auto-call functors or functions. + */ OrSuggestions> findAlongAttrPath(const std::vector & attrPath, bool force = false); std::string getString(); @@ -129,7 +132,9 @@ public: Value & forceValue(); - /* Force creation of the .drv file in the Nix store. */ + /** + * Force creation of the .drv file in the Nix store. + */ StorePath forceDerivation(); }; diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index f0da688db..a988fa40c 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -1,10 +1,13 @@ #pragma once +///@file #include "eval.hh" namespace nix { -/* Note: Various places expect the allocated memory to be zeroed. */ +/** + * Note: Various places expect the allocated memory to be zeroed. + */ [[gnu::always_inline]] inline void * allocBytes(size_t n) { diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 584bbc879..6668add8c 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -9,6 +9,7 @@ #include "filetransfer.hh" #include "function-trace.hh" #include "profiles.hh" +#include "print.hh" #include #include @@ -104,18 +105,10 @@ void Value::print(const SymbolTable & symbols, std::ostream & str, str << integer; break; case tBool: - str << (boolean ? "true" : "false"); + printLiteralBool(str, boolean); break; case tString: - str << "\""; - for (const char * i = string.s; *i; i++) - if (*i == '\"' || *i == '\\') str << "\\" << *i; - else if (*i == '\n') str << "\\n"; - else if (*i == '\r') str << "\\r"; - else if (*i == '\t') str << "\\t"; - else if (*i == '$' && *(i+1) == '{') str << "\\" << *i; - else str << *i; - str << "\""; + printLiteralString(str, string.s); break; case tPath: str << path; // !!! escaping? @@ -173,7 +166,17 @@ void Value::print(const SymbolTable & symbols, std::ostream & str, case tFloat: str << fpoint; break; + case tBlackhole: + // Although we know for sure that it's going to be an infinite recursion + // when this value is accessed _in the current context_, it's likely + // that the user will misinterpret a simpler «infinite recursion» output + // as a definitive statement about the value, while in fact it may be + // a valid value after `builtins.trace` and perhaps some other steps + // have completed. + str << "«potential infinite recursion»"; + break; default: + printError("Nix evaluator internal error: Value::print(): invalid value type %1%", internalType); abort(); } } @@ -229,6 +232,9 @@ std::string_view showType(ValueType type) std::string showType(const Value & v) { + // Allow selecting a subset of enum values + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wswitch-enum" switch (v.internalType) { case tString: return v.string.context ? "a string with context" : "a string"; case tPrimOp: @@ -242,16 +248,21 @@ std::string showType(const Value & v) default: return std::string(showType(v.type())); } + #pragma GCC diagnostic pop } PosIdx Value::determinePos(const PosIdx pos) const { + // Allow selecting a subset of enum values + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wswitch-enum" switch (internalType) { case tAttrs: return attrs->pos; case tLambda: return lambda.fun->pos; case tApp: return app.left->determinePos(pos); default: return pos; } + #pragma GCC diagnostic pop } bool Value::isTrivial() const @@ -326,6 +337,22 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env) } } +#if HAVE_BOEHMGC +/* Disable GC while this object lives. Used by CoroutineContext. + * + * Boehm keeps a count of GC_disable() and GC_enable() calls, + * and only enables GC when the count matches. + */ +class BoehmDisableGC { +public: + BoehmDisableGC() { + GC_disable(); + }; + ~BoehmDisableGC() { + GC_enable(); + }; +}; +#endif static bool gcInitialised = false; @@ -350,6 +377,15 @@ void initGC() StackAllocator::defaultAllocator = &boehmGCStackAllocator; + +#if NIX_BOEHM_PATCH_VERSION != 1 + printTalkative("Unpatched BoehmGC, disabling GC inside coroutines"); + /* Used to disable GC when entering coroutines on macOS */ + create_coro_gc_hook = []() -> std::shared_ptr { + return std::make_shared(); + }; +#endif + /* Set the initial heap size to something fairly big (25% of physical RAM, up to a maximum of 384 MiB) so that in most cases we don't need to garbage collect at all. (Collection has a @@ -2327,6 +2363,7 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v case nFloat: return v1.fpoint == v2.fpoint; + case nThunk: // Must not be left by forceValue default: error("cannot compare %1% with %2%", showType(v1), showType(v2)).withTrace(pos, errorCtx).debugThrow(); } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index e4d5906bd..b3b985683 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "attr-set.hh" #include "types.hh" @@ -42,7 +43,10 @@ struct PrimOp struct Env { Env * up; - unsigned short prevWith:14; // nr of levels up to next `with' environment + /** + * Number of of levels up to next `with` environment + */ + unsigned short prevWith:14; enum { Plain = 0, HasWithExpr, HasWithAttrs } type:2; Value * values[0]; }; @@ -55,8 +59,10 @@ std::unique_ptr mapStaticEnvBindings(const SymbolTable & st, const Stati void copyContext(const Value & v, PathSet & context); -/* Cache for calls to addToStore(); maps source paths to the store - paths. */ +/** + * Cache for calls to addToStore(); maps source paths to the store + * paths. + */ typedef std::map SrcToStore; @@ -68,7 +74,9 @@ typedef std::pair SearchPathElem; typedef std::list SearchPath; -/* Initialise the Boehm GC, if applicable. */ +/** + * Initialise the Boehm GC, if applicable. + */ void initGC(); @@ -143,26 +151,36 @@ public: sOutputSpecified; Symbol sDerivationNix; - /* If set, force copying files to the Nix store even if they - already exist there. */ + /** + * If set, force copying files to the Nix store even if they + * already exist there. + */ RepairFlag repair; - /* The allowed filesystem paths in restricted or pure evaluation - mode. */ + /** + * The allowed filesystem paths in restricted or pure evaluation + * mode. + */ std::optional allowedPaths; Bindings emptyBindings; - /* Store used to materialise .drv files. */ + /** + * Store used to materialise .drv files. + */ const ref store; - /* Store used to build stuff. */ + /** + * Store used to build stuff. + */ const ref buildStore; RootValue vCallFlake = nullptr; RootValue vImportedDrvToDerivation = nullptr; - /* Debugger */ + /** + * Debugger + */ void (* debugRepl)(ref es, const ValMap & extraEnv); bool debugStop; bool debugQuit; @@ -218,7 +236,9 @@ public: private: SrcToStore srcToStore; - /* A cache from path names to parse trees. */ + /** + * A cache from path names to parse trees. + */ #if HAVE_BOEHMGC typedef std::map, traceable_allocator>> FileParseCache; #else @@ -226,7 +246,9 @@ private: #endif FileParseCache fileParseCache; - /* A cache from path names to values. */ + /** + * A cache from path names to values. + */ #if HAVE_BOEHMGC typedef std::map, traceable_allocator>> FileEvalCache; #else @@ -238,17 +260,25 @@ private: std::map> searchPathResolved; - /* Cache used by checkSourcePath(). */ + /** + * Cache used by checkSourcePath(). + */ std::unordered_map resolvedPaths; - /* Cache used by prim_match(). */ + /** + * Cache used by prim_match(). + */ std::shared_ptr regexCache; #if HAVE_BOEHMGC - /* Allocation cache for GC'd Value objects. */ + /** + * Allocation cache for GC'd Value objects. + */ std::shared_ptr valueAllocCache; - /* Allocation cache for size-1 Env objects. */ + /** + * Allocation cache for size-1 Env objects. + */ std::shared_ptr env1AllocCache; #endif @@ -264,47 +294,65 @@ public: SearchPath getSearchPath() { return searchPath; } - /* Allow access to a path. */ + /** + * Allow access to a path. + */ void allowPath(const Path & path); - /* Allow access to a store path. Note that this gets remapped to - the real store path if `store` is a chroot store. */ + /** + * Allow access to a store path. Note that this gets remapped to + * the real store path if `store` is a chroot store. + */ void allowPath(const StorePath & storePath); - /* Allow access to a store path and return it as a string. */ + /** + * Allow access to a store path and return it as a string. + */ void allowAndSetStorePathString(const StorePath & storePath, Value & v); - /* Check whether access to a path is allowed and throw an error if - not. Otherwise return the canonicalised path. */ + /** + * Check whether access to a path is allowed and throw an error if + * not. Otherwise return the canonicalised path. + */ Path checkSourcePath(const Path & path); void checkURI(const std::string & uri); - /* When using a diverted store and 'path' is in the Nix store, map - 'path' to the diverted location (e.g. /nix/store/foo is mapped - to /home/alice/my-nix/nix/store/foo). However, this is only - done if the context is not empty, since otherwise we're - probably trying to read from the actual /nix/store. This is - intended to distinguish between import-from-derivation and - sources stored in the actual /nix/store. */ + /** + * When using a diverted store and 'path' is in the Nix store, map + * 'path' to the diverted location (e.g. /nix/store/foo is mapped + * to /home/alice/my-nix/nix/store/foo). However, this is only + * done if the context is not empty, since otherwise we're + * probably trying to read from the actual /nix/store. This is + * intended to distinguish between import-from-derivation and + * sources stored in the actual /nix/store. + */ Path toRealPath(const Path & path, const PathSet & context); - /* Parse a Nix expression from the specified file. */ + /** + * Parse a Nix expression from the specified file. + */ Expr * parseExprFromFile(const Path & path); Expr * parseExprFromFile(const Path & path, std::shared_ptr & staticEnv); - /* Parse a Nix expression from the specified string. */ + /** + * Parse a Nix expression from the specified string. + */ Expr * parseExprFromString(std::string s, const Path & basePath, std::shared_ptr & staticEnv); Expr * parseExprFromString(std::string s, const Path & basePath); Expr * parseStdin(); - /* Evaluate an expression read from the given file to normal - form. Optionally enforce that the top-level expression is - trivial (i.e. doesn't require arbitrary computation). */ + /** + * Evaluate an expression read from the given file to normal + * form. Optionally enforce that the top-level expression is + * trivial (i.e. doesn't require arbitrary computation). + */ void evalFile(const Path & path, Value & v, bool mustBeTrivial = false); - /* Like `evalFile`, but with an already parsed expression. */ + /** + * Like `evalFile`, but with an already parsed expression. + */ void cacheFile( const Path & path, const Path & resolvedPath, @@ -314,37 +362,52 @@ public: void resetFileCache(); - /* Look up a file in the search path. */ + /** + * Look up a file in the search path. + */ Path findFile(const std::string_view path); Path findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos); - /* If the specified search path element is a URI, download it. */ + /** + * If the specified search path element is a URI, download it. + */ std::pair resolveSearchPathElem(const SearchPathElem & elem); - /* Evaluate an expression to normal form, storing the result in - value `v'. */ + /** + * Evaluate an expression to normal form + * + * @param [out] v The resulting is stored here. + */ void eval(Expr * e, Value & v); - /* Evaluation the expression, then verify that it has the expected - type. */ + /** + * Evaluation the expression, then verify that it has the expected + * type. + */ inline bool evalBool(Env & env, Expr * e); inline bool evalBool(Env & env, Expr * e, const PosIdx pos, std::string_view errorCtx); inline void evalAttrs(Env & env, Expr * e, Value & v, const PosIdx pos, std::string_view errorCtx); - /* If `v' is a thunk, enter it and overwrite `v' with the result - of the evaluation of the thunk. If `v' is a delayed function - application, call the function and overwrite `v' with the - result. Otherwise, this is a no-op. */ + /** + * If `v` is a thunk, enter it and overwrite `v` with the result + * of the evaluation of the thunk. If `v` is a delayed function + * application, call the function and overwrite `v` with the + * result. Otherwise, this is a no-op. + */ inline void forceValue(Value & v, const PosIdx pos); template inline void forceValue(Value & v, Callable getPos); - /* Force a value, then recursively force list elements and - attributes. */ + /** + * Force a value, then recursively force list elements and + * attributes. + */ void forceValueDeep(Value & v); - /* Force `v', and then verify that it has the expected type. */ + /** + * Force `v`, and then verify that it has the expected type. + */ NixInt forceInt(Value & v, const PosIdx pos, std::string_view errorCtx); NixFloat forceFloat(Value & v, const PosIdx pos, std::string_view errorCtx); bool forceBool(Value & v, const PosIdx pos, std::string_view errorCtx); @@ -355,7 +418,10 @@ public: inline void forceAttrs(Value & v, Callable getPos, std::string_view errorCtx); inline void forceList(Value & v, const PosIdx pos, std::string_view errorCtx); - void forceFunction(Value & v, const PosIdx pos, std::string_view errorCtx); // either lambda or primop + /** + * @param v either lambda or primop + */ + void forceFunction(Value & v, const PosIdx pos, std::string_view errorCtx); std::string_view forceString(Value & v, const PosIdx pos, std::string_view errorCtx); std::string_view forceString(Value & v, PathSet & context, const PosIdx pos, std::string_view errorCtx); std::string_view forceStringNoCtx(Value & v, const PosIdx pos, std::string_view errorCtx); @@ -366,17 +432,23 @@ public: void addErrorTrace(Error & e, const PosIdx pos, const char * s, const std::string & s2, bool frame = false) const; public: - /* Return true iff the value `v' denotes a derivation (i.e. a - set with attribute `type = "derivation"'). */ + /** + * @return true iff the value `v` denotes a derivation (i.e. a + * set with attribute `type = "derivation"`). + */ bool isDerivation(Value & v); std::optional tryAttrsToString(const PosIdx pos, Value & v, PathSet & context, bool coerceMore = false, bool copyToStore = true); - /* String coercion. Converts strings, paths and derivations to a - string. If `coerceMore' is set, also converts nulls, integers, - booleans and lists to a string. If `copyToStore' is set, - referenced paths are copied to the Nix store as a side effect. */ + /** + * String coercion. + * + * Converts strings, paths and derivations to a + * string. If `coerceMore` is set, also converts nulls, integers, + * booleans and lists to a string. If `copyToStore` is set, + * referenced paths are copied to the Nix store as a side effect. + */ BackedStringView coerceToString(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx, bool coerceMore = false, bool copyToStore = true, @@ -384,21 +456,31 @@ public: StorePath copyPathToStore(PathSet & context, const Path & path); - /* Path coercion. Converts strings, paths and derivations to a - path. The result is guaranteed to be a canonicalised, absolute - path. Nothing is copied to the store. */ + /** + * Path coercion. + * + * Converts strings, paths and derivations to a + * path. The result is guaranteed to be a canonicalised, absolute + * path. Nothing is copied to the store. + */ Path coerceToPath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx); - /* Like coerceToPath, but the result must be a store path. */ + /** + * Like coerceToPath, but the result must be a store path. + */ StorePath coerceToStorePath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx); public: - /* The base environment, containing the builtin functions and - values. */ + /** + * The base environment, containing the builtin functions and + * values. + */ Env & baseEnv; - /* The same, but used during parsing to resolve variables. */ + /** + * The same, but used during parsing to resolve variables. + */ std::shared_ptr staticBaseEnv; // !!! should be private private: @@ -448,8 +530,10 @@ private: public: - /* Do a deep equality test between two values. That is, list - elements and attributes are compared recursively. */ + /** + * Do a deep equality test between two values. That is, list + * elements and attributes are compared recursively. + */ bool eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx); bool isFunctor(Value & fun); @@ -463,11 +547,15 @@ public: callFunction(fun, 1, args, vRes, pos); } - /* Automatically call a function for which each argument has a - default value or has a binding in the `args' map. */ + /** + * Automatically call a function for which each argument has a + * default value or has a binding in the `args` map. + */ void autoCallFunction(Bindings & args, Value & fun, Value & res); - /* Allocation primitives. */ + /** + * Allocation primitives. + */ inline Value * allocValue(); inline Env & allocEnv(size_t size); @@ -487,10 +575,13 @@ public: void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx); - /* Print statistics. */ + /** + * Print statistics. + */ void printStats(); - /* Realise the given context, and return a mapping from the placeholders + /** + * Realise the given context, and return a mapping from the placeholders * used to construct the associated value to their final store path */ [[nodiscard]] StringMap realiseContext(const PathSet & context); @@ -550,11 +641,15 @@ struct DebugTraceStacker { DebugTrace trace; }; -/* Return a string representing the type of the value `v'. */ +/** + * @return A string representing the type of the value `v`. + */ std::string_view showType(ValueType type); std::string showType(const Value & v); -/* If `path' refers to a directory, then append "/default.nix". */ +/** + * If `path` refers to a directory, then append "/default.nix". + */ Path resolveExprPath(Path path); struct InvalidPathError : EvalError diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 81e94848a..ac396236f 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -125,6 +125,9 @@ static FlakeInput parseFlakeInput(EvalState & state, follows.insert(follows.begin(), lockRootPath.begin(), lockRootPath.end()); input.follows = follows; } else { + // Allow selecting a subset of enum values + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wswitch-enum" switch (attr.value->type()) { case nString: attrs.emplace(state.symbols[attr.name], attr.value->string.s); @@ -139,6 +142,7 @@ static FlakeInput parseFlakeInput(EvalState & state, throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected", state.symbols[attr.name], showType(*attr.value)); } + #pragma GCC diagnostic pop } } catch (Error & e) { e.addTrace( @@ -334,10 +338,14 @@ LockedFlake lockFlake( } try { + if (!fetchSettings.allowDirty && lockFlags.referenceLockFilePath) { + throw Error("reference lock file was provided, but the `allow-dirty` setting is set to false"); + } // FIXME: symlink attack auto oldLockFile = LockFile::read( - flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock"); + lockFlags.referenceLockFilePath.value_or( + flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock")); debug("old lock file: %s", oldLockFile); @@ -619,13 +627,20 @@ LockedFlake lockFlake( debug("new lock file: %s", newLockFile); + auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; + auto sourcePath = topRef.input.getSourcePath(); + auto outputLockFilePath = sourcePath ? std::optional{*sourcePath + "/" + relPath} : std::nullopt; + if (lockFlags.outputLockFilePath) { + outputLockFilePath = lockFlags.outputLockFilePath; + } + /* Check whether we need to / can write the new lock file. */ - if (!(newLockFile == oldLockFile)) { + if (newLockFile != oldLockFile || lockFlags.outputLockFilePath) { auto diff = LockFile::diff(oldLockFile, newLockFile); if (lockFlags.writeLockFile) { - if (auto sourcePath = topRef.input.getSourcePath()) { + if (outputLockFilePath) { if (auto unlockedInput = newLockFile.isUnlocked()) { if (fetchSettings.warnDirty) warn("will not write lock file of flake '%s' because it has an unlocked input ('%s')", topRef, *unlockedInput); @@ -633,25 +648,24 @@ LockedFlake lockFlake( if (!lockFlags.updateLockFile) throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef); - auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; - - auto path = *sourcePath + "/" + relPath; - - bool lockFileExists = pathExists(path); + bool lockFileExists = pathExists(*outputLockFilePath); if (lockFileExists) { auto s = chomp(diff); if (s.empty()) - warn("updating lock file '%s'", path); + warn("updating lock file '%s'", *outputLockFilePath); else - warn("updating lock file '%s':\n%s", path, s); + warn("updating lock file '%s':\n%s", *outputLockFilePath, s); } else - warn("creating lock file '%s'", path); + warn("creating lock file '%s'", *outputLockFilePath); - newLockFile.write(path); + newLockFile.write(*outputLockFilePath); std::optional commitMessage = std::nullopt; if (lockFlags.commitLockFile) { + if (lockFlags.outputLockFilePath) { + throw Error("--commit-lock-file and --output-lock-file are currently incompatible"); + } std::string cm; cm = fetchSettings.commitLockFileSummary.get(); diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index 10301d8aa..c1d1b71e5 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "flakeref.hh" @@ -17,7 +18,8 @@ struct FlakeInput; typedef std::map FlakeInputs; -/* FlakeInput is the 'Flake'-level parsed form of the "input" entries +/** + * FlakeInput is the 'Flake'-level parsed form of the "input" entries * in the flake file. * * A FlakeInput is normally constructed by the 'parseFlakeInput' @@ -41,7 +43,12 @@ typedef std::map FlakeInputs; struct FlakeInput { std::optional ref; - bool isFlake = true; // true = process flake to get outputs, false = (fetched) static source path + /** + * true = process flake to get outputs + * + * false = (fetched) static source path + */ + bool isFlake = true; std::optional follows; FlakeInputs overrides; }; @@ -55,23 +62,42 @@ struct ConfigFile void apply(); }; -/* The contents of a flake.nix file. */ +/** + * The contents of a flake.nix file. + */ struct Flake { - FlakeRef originalRef; // the original flake specification (by the user) - FlakeRef resolvedRef; // registry references and caching resolved to the specific underlying flake - FlakeRef lockedRef; // the specific local store result of invoking the fetcher - bool forceDirty = false; // pretend that 'lockedRef' is dirty + /** + * The original flake specification (by the user) + */ + FlakeRef originalRef; + /** + * registry references and caching resolved to the specific underlying flake + */ + FlakeRef resolvedRef; + /** + * the specific local store result of invoking the fetcher + */ + FlakeRef lockedRef; + /** + * pretend that 'lockedRef' is dirty + */ + bool forceDirty = false; std::optional description; std::shared_ptr sourceInfo; FlakeInputs inputs; - ConfigFile config; // 'nixConfig' attribute + /** + * 'nixConfig' attribute + */ + ConfigFile config; ~Flake(); }; Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool allowLookup); -/* Fingerprint of a locked flake; used as a cache key. */ +/** + * Fingerprint of a locked flake; used as a cache key. + */ typedef Hash Fingerprint; struct LockedFlake @@ -84,44 +110,72 @@ struct LockedFlake struct LockFlags { - /* Whether to ignore the existing lock file, creating a new one - from scratch. */ + /** + * Whether to ignore the existing lock file, creating a new one + * from scratch. + */ bool recreateLockFile = false; - /* Whether to update the lock file at all. If set to false, if any - change to the lock file is needed (e.g. when an input has been - added to flake.nix), you get a fatal error. */ + /** + * Whether to update the lock file at all. If set to false, if any + * change to the lock file is needed (e.g. when an input has been + * added to flake.nix), you get a fatal error. + */ bool updateLockFile = true; - /* Whether to write the lock file to disk. If set to true, if the - any changes to the lock file are needed and the flake is not - writable (i.e. is not a local Git working tree or similar), you - get a fatal error. If set to false, Nix will use the modified - lock file in memory only, without writing it to disk. */ + /** + * Whether to write the lock file to disk. If set to true, if the + * any changes to the lock file are needed and the flake is not + * writable (i.e. is not a local Git working tree or similar), you + * get a fatal error. If set to false, Nix will use the modified + * lock file in memory only, without writing it to disk. + */ bool writeLockFile = true; - /* Whether to use the registries to lookup indirect flake - references like 'nixpkgs'. */ + /** + * Whether to use the registries to lookup indirect flake + * references like 'nixpkgs'. + */ std::optional useRegistries = std::nullopt; - /* Whether to apply flake's nixConfig attribute to the configuration */ + /** + * Whether to apply flake's nixConfig attribute to the configuration + */ bool applyNixConfig = false; - /* Whether unlocked flake references (i.e. those without a Git - revision or similar) without a corresponding lock are - allowed. Unlocked flake references with a lock are always - allowed. */ + /** + * Whether unlocked flake references (i.e. those without a Git + * revision or similar) without a corresponding lock are + * allowed. Unlocked flake references with a lock are always + * allowed. + */ bool allowUnlocked = true; - /* Whether to commit changes to flake.lock. */ + /** + * Whether to commit changes to flake.lock. + */ bool commitLockFile = false; - /* Flake inputs to be overridden. */ + /** + * The path to a lock file to read instead of the `flake.lock` file in the top-level flake + */ + std::optional referenceLockFilePath; + + /** + * The path to a lock file to write to instead of the `flake.lock` file in the top-level flake + */ + std::optional outputLockFilePath; + + /** + * Flake inputs to be overridden. + */ std::map inputOverrides; - /* Flake inputs to be updated. This means that any existing lock - for those inputs will be ignored. */ + /** + * Flake inputs to be updated. This means that any existing lock + * for those inputs will be ignored. + */ std::set inputUpdates; }; diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index c4142fc20..a7c9208c0 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "hash.hh" @@ -13,7 +14,8 @@ class Store; typedef std::string FlakeId; -/* A flake reference specifies how to fetch a flake or raw source +/** + * A flake reference specifies how to fetch a flake or raw source * (e.g. from a Git repository). It is created from a URL-like syntax * (e.g. 'github:NixOS/patchelf'), an attrset representation (e.g. '{ * type="github"; owner = "NixOS"; repo = "patchelf"; }'), or a local @@ -32,14 +34,17 @@ typedef std::string FlakeId; * be lazy), but the fetcher can be invoked at any time via the * FlakeRef to ensure the store is populated with this input. */ - struct FlakeRef { - /* Fetcher-specific representation of the input, sufficient to - perform the fetch operation. */ + /** + * Fetcher-specific representation of the input, sufficient to + * perform the fetch operation. + */ fetchers::Input input; - /* sub-path within the fetched input that represents this input */ + /** + * sub-path within the fetched input that represents this input + */ Path subdir; bool operator==(const FlakeRef & other) const; diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index a74e68c9c..ba2fd46f0 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -234,6 +234,11 @@ bool LockFile::operator ==(const LockFile & other) const return toJSON() == other.toJSON(); } +bool LockFile::operator !=(const LockFile & other) const +{ + return !(*this == other); +} + InputPath parseInputPath(std::string_view s) { InputPath path; diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index 02e9bdfbc..ba4c0c848 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "flakeref.hh" @@ -15,9 +16,11 @@ typedef std::vector InputPath; struct LockedNode; -/* A node in the lock file. It has outgoing edges to other nodes (its - inputs). Only the root node has this type; all other nodes have - type LockedNode. */ +/** + * A node in the lock file. It has outgoing edges to other nodes (its + * inputs). Only the root node has this type; all other nodes have + * type LockedNode. + */ struct Node : std::enable_shared_from_this { typedef std::variant, InputPath> Edge; @@ -27,7 +30,9 @@ struct Node : std::enable_shared_from_this virtual ~Node() { } }; -/* A non-root node in the lock file. */ +/** + * A non-root node in the lock file. + */ struct LockedNode : Node { FlakeRef lockedRef, originalRef; @@ -62,10 +67,15 @@ struct LockFile void write(const Path & path) const; - /* Check whether this lock file has any unlocked inputs. */ + /** + * Check whether this lock file has any unlocked inputs. + */ std::optional isUnlocked() const; bool operator ==(const LockFile & other) const; + // Needed for old gcc versions that don't synthesize it (like gcc 8.2.2 + // that is still the default on aarch64-linux) + bool operator !=(const LockFile & other) const; std::shared_ptr findInput(const InputPath & path); @@ -73,7 +83,9 @@ struct LockFile static std::string diff(const LockFile & oldLocks, const LockFile & newLocks); - /* Check that every 'follows' input target exists. */ + /** + * Check that every 'follows' input target exists. + */ void check(); }; diff --git a/src/libexpr/function-trace.hh b/src/libexpr/function-trace.hh index e9a2526bd..91439b0aa 100644 --- a/src/libexpr/function-trace.hh +++ b/src/libexpr/function-trace.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "eval.hh" diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh index bbd2d3c47..584d64ac1 100644 --- a/src/libexpr/get-drvs.hh +++ b/src/libexpr/get-drvs.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "eval.hh" #include "path.hh" @@ -25,7 +26,10 @@ private: mutable std::string outputName; Outputs outputs; - bool failed = false; // set if we get an AssertionError + /** + * Set if we get an AssertionError + */ + bool failed = false; Bindings * attrs = nullptr, * meta = nullptr; @@ -34,7 +38,10 @@ private: bool checkMeta(Value & v); public: - std::string attrPath; /* path towards the derivation */ + /** + * path towards the derivation + */ + std::string attrPath; DrvInfo(EvalState & state) : state(&state) { }; DrvInfo(EvalState & state, std::string attrPath, Bindings * attrs); @@ -46,8 +53,10 @@ public: StorePath requireDrvPath() const; StorePath queryOutPath() const; std::string queryOutputName() const; - /** Return the unordered map of output names to (optional) output paths. - * The "outputs to install" are determined by `meta.outputsToInstall`. */ + /** + * Return the unordered map of output names to (optional) output paths. + * The "outputs to install" are determined by `meta.outputsToInstall`. + */ Outputs queryOutputs(bool withPaths = true, bool onlyOutputsToInstall = false); StringSet queryMetaNames(); @@ -79,8 +88,10 @@ typedef std::list DrvInfos; #endif -/* If value `v' denotes a derivation, return a DrvInfo object - describing it. Otherwise return nothing. */ +/** + * If value `v` denotes a derivation, return a DrvInfo object + * describing it. Otherwise return nothing. + */ std::optional getDerivation(EvalState & state, Value & v, bool ignoreAssertionFailures); diff --git a/src/libexpr/json-to-value.hh b/src/libexpr/json-to-value.hh index 84bec4eba..3b8ec000f 100644 --- a/src/libexpr/json-to-value.hh +++ b/src/libexpr/json-to-value.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "eval.hh" diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 2171e769b..d243b9cec 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -46,3 +46,5 @@ $(foreach i, $(wildcard src/libexpr/flake/*.hh), \ $(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh + +src/libexpr/primops/fromTOML.o: ERROR_SWITCH_ENUM = diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index eb6f062b4..1557cbbeb 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -3,6 +3,7 @@ #include "eval.hh" #include "symbol-table.hh" #include "util.hh" +#include "print.hh" #include @@ -60,45 +61,12 @@ Pos::operator std::shared_ptr() const return pos; } -/* Displaying abstract syntax trees. */ - -static void showString(std::ostream & str, std::string_view s) -{ - str << '"'; - for (auto c : s) - if (c == '"' || c == '\\' || c == '$') str << "\\" << c; - else if (c == '\n') str << "\\n"; - else if (c == '\r') str << "\\r"; - else if (c == '\t') str << "\\t"; - else str << c; - str << '"'; -} - +// FIXME: remove, because *symbols* are abstract and do not have a single +// textual representation; see printIdentifier() std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol) { std::string_view s = symbol; - - if (s.empty()) - str << "\"\""; - else if (s == "if") // FIXME: handle other keywords - str << '"' << s << '"'; - else { - char c = s[0]; - if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) { - showString(str, s); - return str; - } - for (auto c : s) - if (!((c >= 'a' && c <= 'z') || - (c >= 'A' && c <= 'Z') || - (c >= '0' && c <= '9') || - c == '_' || c == '\'' || c == '-')) { - showString(str, s); - return str; - } - str << s; - } - return str; + return printIdentifier(str, s); } void Expr::show(const SymbolTable & symbols, std::ostream & str) const @@ -118,7 +86,7 @@ void ExprFloat::show(const SymbolTable & symbols, std::ostream & str) const void ExprString::show(const SymbolTable & symbols, std::ostream & str) const { - showString(str, s); + printLiteralString(str, s); } void ExprPath::show(const SymbolTable & symbols, std::ostream & str) const diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 4a81eaa47..c2f817c9a 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -21,7 +22,9 @@ MakeError(UndefinedVarError, Error); MakeError(MissingArgumentError, EvalError); MakeError(RestrictedPathError, Error); -/* Position objects. */ +/** + * Position objects. + */ struct Pos { uint32_t line; @@ -132,7 +135,9 @@ class EvalState; struct StaticEnv; -/* An attribute path is a sequence of attribute names. */ +/** + * An attribute path is a sequence of attribute names. + */ struct AttrName { Symbol symbol; @@ -212,11 +217,11 @@ struct ExprVar : Expr or function argument) or from a "with". */ bool fromWith; - /* In the former case, the value is obtained by going `level' + /* In the former case, the value is obtained by going `level` levels up from the current environment and getting the - `displ'th value in that environment. In the latter case, the - value is obtained by getting the attribute named `name' from - the set stored in the environment that is `level' levels up + `displ`th value in that environment. In the latter case, the + value is obtained by getting the attribute named `name` from + the set stored in the environment that is `level` levels up from the current one.*/ Level level; Displacement displ; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index b6476e006..2476b7e73 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -577,6 +577,9 @@ struct CompareValues return v1->integer < v2->fpoint; if (v1->type() != v2->type()) state.error("cannot compare %s with %s", showType(*v1), showType(*v2)).debugThrow(); + // Allow selecting a subset of enum values + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wswitch-enum" switch (v1->type()) { case nInt: return v1->integer < v2->integer; @@ -599,6 +602,7 @@ struct CompareValues } default: state.error("cannot compare %s with %s; values of that type are incomparable", showType(*v1), showType(*v2)).debugThrow(); + #pragma GCC diagnostic pop } } catch (Error & e) { if (!errorCtx.empty()) diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index 1cfb4356b..4ae73fe1f 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "eval.hh" @@ -22,9 +23,11 @@ struct RegisterPrimOp typedef std::vector PrimOps; static PrimOps * primOps; - /* You can register a constant by passing an arity of 0. fun - will get called during EvalState initialization, so there - may be primops not yet added and builtins is not yet sorted. */ + /** + * You can register a constant by passing an arity of 0. fun + * will get called during EvalState initialization, so there + * may be primops not yet added and builtins is not yet sorted. + */ RegisterPrimOp( std::string name, size_t arity, @@ -37,10 +40,14 @@ struct RegisterPrimOp may wish to use them in limited contexts without globally enabling them. */ -/* Load a ValueInitializer from a DSO and return whatever it initializes */ +/** + * Load a ValueInitializer from a DSO and return whatever it initializes + */ void prim_importNative(EvalState & state, const PosIdx pos, Value * * args, Value & v); -/* Execute a program and parse its output */ +/** + * Execute a program and parse its output + */ void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v); } diff --git a/src/libexpr/print.cc b/src/libexpr/print.cc new file mode 100644 index 000000000..d08672cfc --- /dev/null +++ b/src/libexpr/print.cc @@ -0,0 +1,78 @@ +#include "print.hh" + +namespace nix { + +std::ostream & +printLiteralString(std::ostream & str, const std::string_view string) +{ + str << "\""; + for (auto i = string.begin(); i != string.end(); ++i) { + if (*i == '\"' || *i == '\\') str << "\\" << *i; + else if (*i == '\n') str << "\\n"; + else if (*i == '\r') str << "\\r"; + else if (*i == '\t') str << "\\t"; + else if (*i == '$' && *(i+1) == '{') str << "\\" << *i; + else str << *i; + } + str << "\""; + return str; +} + +std::ostream & +printLiteralBool(std::ostream & str, bool boolean) +{ + str << (boolean ? "true" : "false"); + return str; +} + +std::ostream & +printIdentifier(std::ostream & str, std::string_view s) { + if (s.empty()) + str << "\"\""; + else if (s == "if") // FIXME: handle other keywords + str << '"' << s << '"'; + else { + char c = s[0]; + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) { + printLiteralString(str, s); + return str; + } + for (auto c : s) + if (!((c >= 'a' && c <= 'z') || + (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || + c == '_' || c == '\'' || c == '-')) { + printLiteralString(str, s); + return str; + } + str << s; + } + return str; +} + +// FIXME: keywords +static bool isVarName(std::string_view s) +{ + if (s.size() == 0) return false; + char c = s[0]; + if ((c >= '0' && c <= '9') || c == '-' || c == '\'') return false; + for (auto & i : s) + if (!((i >= 'a' && i <= 'z') || + (i >= 'A' && i <= 'Z') || + (i >= '0' && i <= '9') || + i == '_' || i == '-' || i == '\'')) + return false; + return true; +} + +std::ostream & +printAttributeName(std::ostream & str, std::string_view name) { + if (isVarName(name)) + str << name; + else + printLiteralString(str, name); + return str; +} + + +} diff --git a/src/libexpr/print.hh b/src/libexpr/print.hh new file mode 100644 index 000000000..f9cfc3964 --- /dev/null +++ b/src/libexpr/print.hh @@ -0,0 +1,48 @@ +#pragma once +/** + * @file + * @brief Common printing functions for the Nix language + * + * While most types come with their own methods for printing, they share some + * functions that are placed here. + */ + +#include + +namespace nix { + /** + * Print a string as a Nix string literal. + * + * Quotes and fairly minimal escaping are added. + * + * @param s The logical string + */ + std::ostream & printLiteralString(std::ostream & o, std::string_view s); + inline std::ostream & printLiteralString(std::ostream & o, const char * s) { + return printLiteralString(o, std::string_view(s)); + } + inline std::ostream & printLiteralString(std::ostream & o, const std::string & s) { + return printLiteralString(o, std::string_view(s)); + } + + /** Print `true` or `false`. */ + std::ostream & printLiteralBool(std::ostream & o, bool b); + + /** + * Print a string as an attribute name in the Nix expression language syntax. + * + * Prints a quoted string if necessary. + */ + std::ostream & printAttributeName(std::ostream & o, std::string_view s); + + /** + * Print a string as an identifier in the Nix expression language syntax. + * + * FIXME: "identifier" is ambiguous. Identifiers do not have a single + * textual representation. They can be used in variable references, + * let bindings, left-hand sides or attribute names in a select + * expression, or something else entirely, like JSON. Use one of the + * `print*` functions instead. + */ + std::ostream & printIdentifier(std::ostream & o, std::string_view s); +} diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh index 288c15602..967a186dd 100644 --- a/src/libexpr/symbol-table.hh +++ b/src/libexpr/symbol-table.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -9,15 +10,11 @@ namespace nix { -/* Symbol table used by the parser and evaluator to represent and look - up identifiers and attributes efficiently. SymbolTable::create() - converts a string into a symbol. Symbols have the property that - they can be compared efficiently (using an equality test), - because the symbol table stores only one copy of each string. */ - -/* This class mainly exists to give us an operator<< for ostreams. We could also - return plain strings from SymbolTable, but then we'd have to wrap every - instance of a symbol that is fmt()ed, which is inconvenient and error-prone. */ +/** + * This class mainly exists to give us an operator<< for ostreams. We could also + * return plain strings from SymbolTable, but then we'd have to wrap every + * instance of a symbol that is fmt()ed, which is inconvenient and error-prone. + */ class SymbolStr { friend class SymbolTable; @@ -46,6 +43,11 @@ public: friend std::ostream & operator <<(std::ostream & os, const SymbolStr & symbol); }; +/** + * Symbols have the property that they can be compared efficiently + * (using an equality test), because the symbol table stores only one + * copy of each string. + */ class Symbol { friend class SymbolTable; @@ -65,6 +67,10 @@ public: bool operator!=(const Symbol other) const { return id != other.id; } }; +/** + * Symbol table used by the parser and evaluator to represent and look + * up identifiers and attributes efficiently. + */ class SymbolTable { private: @@ -73,6 +79,9 @@ private: public: + /** + * converts a string into a symbol. + */ Symbol create(std::string_view s) { // Most symbols are looked up more than once, so we trade off insertion performance diff --git a/src/libexpr/tests/libexpr.hh b/src/libexpr/tests/libexpr.hh index 8534d9567..69c932f05 100644 --- a/src/libexpr/tests/libexpr.hh +++ b/src/libexpr/tests/libexpr.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include #include diff --git a/src/libexpr/tests/value/context.hh b/src/libexpr/tests/value/context.hh index 54d21760e..c0bc97ba3 100644 --- a/src/libexpr/tests/value/context.hh +++ b/src/libexpr/tests/value/context.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include diff --git a/src/libexpr/value-to-json.hh b/src/libexpr/value-to-json.hh index 22f26b790..713356c7f 100644 --- a/src/libexpr/value-to-json.hh +++ b/src/libexpr/value-to-json.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "nixexpr.hh" #include "eval.hh" diff --git a/src/libexpr/value-to-xml.hh b/src/libexpr/value-to-xml.hh index 506f32b6b..ace7ead0f 100644 --- a/src/libexpr/value-to-xml.hh +++ b/src/libexpr/value-to-xml.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "nixexpr.hh" #include "eval.hh" diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 508dbe218..7739f99df 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include @@ -35,9 +36,11 @@ typedef enum { tFloat } InternalType; -// This type abstracts over all actual value types in the language, -// grouping together implementation details like tList*, different function -// types, and types in non-normal form (so thunks and co.) +/** + * This type abstracts over all actual value types in the language, + * grouping together implementation details like tList*, different function + * types, and types in non-normal form (so thunks and co.) + */ typedef enum { nThunk, nInt, @@ -69,38 +72,51 @@ class XMLWriter; typedef int64_t NixInt; typedef double NixFloat; -/* External values must descend from ExternalValueBase, so that +/** + * External values must descend from ExternalValueBase, so that * type-agnostic nix functions (e.g. showType) can be implemented */ class ExternalValueBase { friend std::ostream & operator << (std::ostream & str, const ExternalValueBase & v); protected: - /* Print out the value */ + /** + * Print out the value + */ virtual std::ostream & print(std::ostream & str) const = 0; public: - /* Return a simple string describing the type */ + /** + * Return a simple string describing the type + */ virtual std::string showType() const = 0; - /* Return a string to be used in builtins.typeOf */ + /** + * Return a string to be used in builtins.typeOf + */ virtual std::string typeOf() const = 0; - /* Coerce the value to a string. Defaults to uncoercable, i.e. throws an + /** + * Coerce the value to a string. Defaults to uncoercable, i.e. throws an * error. */ virtual std::string coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const; - /* Compare to another value of the same type. Defaults to uncomparable, + /** + * Compare to another value of the same type. Defaults to uncomparable, * i.e. always false. */ virtual bool operator ==(const ExternalValueBase & b) const; - /* Print the value as JSON. Defaults to unconvertable, i.e. throws an error */ + /** + * Print the value as JSON. Defaults to unconvertable, i.e. throws an error + */ virtual nlohmann::json printValueAsJSON(EvalState & state, bool strict, PathSet & context, bool copyToStore = true) const; - /* Print the value as XML. Defaults to unevaluated */ + /** + * Print the value as XML. Defaults to unevaluated + */ virtual void printValueAsXML(EvalState & state, bool strict, bool location, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, const PosIdx pos) const; @@ -145,26 +161,28 @@ public: NixInt integer; bool boolean; - /* Strings in the evaluator carry a so-called `context' which - is a list of strings representing store paths. This is to - allow users to write things like + /** + * Strings in the evaluator carry a so-called `context` which + * is a list of strings representing store paths. This is to + * allow users to write things like - "--with-freetype2-library=" + freetype + "/lib" + * "--with-freetype2-library=" + freetype + "/lib" - where `freetype' is a derivation (or a source to be copied - to the store). If we just concatenated the strings without - keeping track of the referenced store paths, then if the - string is used as a derivation attribute, the derivation - will not have the correct dependencies in its inputDrvs and - inputSrcs. + * where `freetype` is a derivation (or a source to be copied + * to the store). If we just concatenated the strings without + * keeping track of the referenced store paths, then if the + * string is used as a derivation attribute, the derivation + * will not have the correct dependencies in its inputDrvs and + * inputSrcs. - The semantics of the context is as follows: when a string - with context C is used as a derivation attribute, then the - derivations in C will be added to the inputDrvs of the - derivation, and the other store paths in C will be added to - the inputSrcs of the derivations. + * The semantics of the context is as follows: when a string + * with context C is used as a derivation attribute, then the + * derivations in C will be added to the inputDrvs of the + * derivation, and the other store paths in C will be added to + * the inputSrcs of the derivations. - For canonicity, the store paths should be in sorted order. */ + * For canonicity, the store paths should be in sorted order. + */ struct { const char * s; const char * * context; // must be in sorted order @@ -196,8 +214,10 @@ public: NixFloat fpoint; }; - // Returns the normal type of a Value. This only returns nThunk if the - // Value hasn't been forceValue'd + /** + * Returns the normal type of a Value. This only returns nThunk if + * the Value hasn't been forceValue'd + */ inline ValueType type() const { switch (internalType) { @@ -216,8 +236,10 @@ public: abort(); } - /* After overwriting an app node, be sure to clear pointers in the - Value to ensure that the target isn't kept alive unnecessarily. */ + /** + * After overwriting an app node, be sure to clear pointers in the + * Value to ensure that the target isn't kept alive unnecessarily. + */ inline void clearValue() { app.left = app.right = 0; @@ -365,9 +387,11 @@ public: PosIdx determinePos(const PosIdx pos) const; - /* Check whether forcing this value requires a trivial amount of - computation. In particular, function applications are - non-trivial. */ + /** + * Check whether forcing this value requires a trivial amount of + * computation. In particular, function applications are + * non-trivial. + */ bool isTrivial() const; NixStringContext getContext(const Store &); @@ -413,7 +437,9 @@ typedef std::map ValueVectorMap; #endif -/* A value allocated in traceable memory. */ +/** + * A value allocated in traceable memory. + */ typedef std::shared_ptr RootValue; RootValue allocRootValue(Value * v); diff --git a/src/libexpr/value/context.hh b/src/libexpr/value/context.hh index 721563cba..8719602d8 100644 --- a/src/libexpr/value/context.hh +++ b/src/libexpr/value/context.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "util.hh" #include "comparator.hh" @@ -27,34 +28,37 @@ public: class Store; -/* Plain opaque path to some store object. - - Encoded as just the path: ‘’. -*/ +/** + * Plain opaque path to some store object. + * + * Encoded as just the path: ‘’. + */ struct NixStringContextElem_Opaque { StorePath path; GENERATE_CMP(NixStringContextElem_Opaque, me->path); }; -/* Path to a derivation and its entire build closure. - - The path doesn't just refer to derivation itself and its closure, but - also all outputs of all derivations in that closure (including the - root derivation). - - Encoded in the form ‘=’. -*/ +/** + * Path to a derivation and its entire build closure. + * + * The path doesn't just refer to derivation itself and its closure, but + * also all outputs of all derivations in that closure (including the + * root derivation). + * + * Encoded in the form ‘=’. + */ struct NixStringContextElem_DrvDeep { StorePath drvPath; GENERATE_CMP(NixStringContextElem_DrvDeep, me->drvPath); }; -/* Derivation output. - - Encoded in the form ‘!!’. -*/ +/** + * Derivation output. + * + * Encoded in the form ‘!!’. + */ struct NixStringContextElem_Built { StorePath drvPath; std::string output; @@ -83,11 +87,12 @@ struct NixStringContextElem : _NixStringContextElem_Raw { return static_cast(*this); } - /* Decode a context string, one of: - - ‘’ - - ‘=’ - - ‘!!’ - */ + /** + * Decode a context string, one of: + * - ‘’ + * - ‘=’ + * - ‘!!’ + */ static NixStringContextElem parse(const Store & store, std::string_view s); std::string to_string(const Store & store) const; }; diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh index e41037633..1a14bb023 100644 --- a/src/libfetchers/attrs.hh +++ b/src/libfetchers/attrs.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh index 3763ee2a6..ae398d040 100644 --- a/src/libfetchers/cache.hh +++ b/src/libfetchers/cache.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "fetchers.hh" diff --git a/src/libfetchers/fetch-settings.hh b/src/libfetchers/fetch-settings.hh index 4bc2d0e1a..6108a179c 100644 --- a/src/libfetchers/fetch-settings.hh +++ b/src/libfetchers/fetch-settings.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "config.hh" diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 95c0f5974..498ad7e4d 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "hash.hh" @@ -20,14 +21,14 @@ struct Tree struct InputScheme; -/* The Input object is generated by a specific fetcher, based on the +/** + * The Input object is generated by a specific fetcher, based on the * user-supplied input attribute in the flake.nix file, and contains * the information that the specific fetcher needs to perform the * actual fetch. The Input object is most commonly created via the * "fromURL()" or "fromAttrs()" static functions which are provided * the url or attrset specified in the flake file. */ - struct Input { friend struct InputScheme; @@ -37,7 +38,9 @@ struct Input bool locked = false; bool direct = true; - /* path of the parent of this input, used for relative path resolution */ + /** + * path of the parent of this input, used for relative path resolution + */ std::optional parent; public: @@ -55,27 +58,35 @@ public: Attrs toAttrs() const; - /* Check whether this is a "direct" input, that is, not - one that goes through a registry. */ + /** + * Check whether this is a "direct" input, that is, not + * one that goes through a registry. + */ bool isDirect() const { return direct; } - /* Check whether this is a "locked" input, that is, - one that contains a commit hash or content hash. */ + /** + * Check whether this is a "locked" input, that is, + * one that contains a commit hash or content hash. + */ bool isLocked() const { return locked; } - /* Check whether the input carries all necessary info required - for cache insertion and substitution. - These fields are used to uniquely identify cached trees - within the "tarball TTL" window without necessarily - indicating that the input's origin is unchanged. */ + /** + * Check whether the input carries all necessary info required + * for cache insertion and substitution. + * These fields are used to uniquely identify cached trees + * within the "tarball TTL" window without necessarily + * indicating that the input's origin is unchanged. + */ bool hasAllInfo() const; bool operator ==(const Input & other) const; bool contains(const Input & other) const; - /* Fetch the input into the Nix store, returning the location in - the Nix store and the locked input. */ + /** + * Fetch the input into the Nix store, returning the location in + * the Nix store and the locked input. + */ std::pair fetch(ref store) const; Input applyOverrides( @@ -104,7 +115,8 @@ public: }; -/* The InputScheme represents a type of fetcher. Each fetcher +/** + * The InputScheme represents a type of fetcher. Each fetcher * registers with nix at startup time. When processing an input for a * flake, each scheme is given an opportunity to "recognize" that * input from the url or attributes in the flake file's specification diff --git a/src/libfetchers/registry.hh b/src/libfetchers/registry.hh index 260a2c460..f57ab1e6b 100644 --- a/src/libfetchers/registry.hh +++ b/src/libfetchers/registry.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "fetchers.hh" diff --git a/src/libmain/common-args.hh b/src/libmain/common-args.hh index f180d83ce..e7ed0d934 100644 --- a/src/libmain/common-args.hh +++ b/src/libmain/common-args.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "args.hh" diff --git a/src/libmain/loggers.hh b/src/libmain/loggers.hh index f3c759193..e5721420c 100644 --- a/src/libmain/loggers.hh +++ b/src/libmain/loggers.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 024259584..6600ec177 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -72,6 +72,7 @@ private: uint64_t corruptedPaths = 0, untrustedPaths = 0; bool active = true; + bool paused = false; bool haveUpdate = true; }; @@ -120,6 +121,18 @@ public: updateThread.join(); } + void pause() override { + state_.lock()->paused = true; + writeToStderr("\r\e[K"); + } + + void resume() override { + state_.lock()->paused = false; + writeToStderr("\r\e[K"); + state_.lock()->haveUpdate = true; + updateCV.notify_one(); + } + bool isVerbose() override { return printBuildLogs; @@ -339,7 +352,7 @@ public: auto nextWakeup = std::chrono::milliseconds::max(); state.haveUpdate = false; - if (!state.active) return nextWakeup; + if (state.paused || !state.active) return nextWakeup; std::string line; diff --git a/src/libmain/progress-bar.hh b/src/libmain/progress-bar.hh index 3a76f8448..c3c6e3833 100644 --- a/src/libmain/progress-bar.hh +++ b/src/libmain/progress-bar.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "logging.hh" diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 37664c065..56f47a4ac 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -10,7 +10,6 @@ #include #include #include -#include #include #include @@ -20,16 +19,9 @@ #ifdef __linux__ #include #endif -#ifdef __GLIBC__ -#include -#include -#include -#endif #include -#include - namespace nix { @@ -115,57 +107,6 @@ std::string getArg(const std::string & opt, return *i; } - -#if OPENSSL_VERSION_NUMBER < 0x10101000L -/* OpenSSL is not thread-safe by default - it will randomly crash - unless the user supplies a mutex locking function. So let's do - that. */ -static std::vector opensslLocks; - -static void opensslLockCallback(int mode, int type, const char * file, int line) -{ - if (mode & CRYPTO_LOCK) - opensslLocks[type].lock(); - else - opensslLocks[type].unlock(); -} -#endif - -static std::once_flag dns_resolve_flag; - -static void preloadNSS() { - /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of - one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already - been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to - load its lookup libraries in the parent before any child gets a chance to. */ - std::call_once(dns_resolve_flag, []() { -#ifdef __GLIBC__ - /* On linux, glibc will run every lookup through the nss layer. - * That means every lookup goes, by default, through nscd, which acts as a local - * cache. - * Because we run builds in a sandbox, we also remove access to nscd otherwise - * lookups would leak into the sandbox. - * - * But now we have a new problem, we need to make sure the nss_dns backend that - * does the dns lookups when nscd is not available is loaded or available. - * - * We can't make it available without leaking nix's environment, so instead we'll - * load the backend, and configure nss so it does not try to run dns lookups - * through nscd. - * - * This is technically only used for builtins:fetch* functions so we only care - * about dns. - * - * All other platforms are unaffected. - */ - if (!dlopen(LIBNSS_DNS_SO, RTLD_NOW)) - warn("unable to load nss_dns backend"); - // FIXME: get hosts entry from nsswitch.conf. - __nss_configure_lookup("hosts", "files dns"); -#endif - }); -} - static void sigHandler(int signo) { } @@ -177,16 +118,7 @@ void initNix() std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); #endif -#if OPENSSL_VERSION_NUMBER < 0x10101000L - /* Initialise OpenSSL locking. */ - opensslLocks = std::vector(CRYPTO_num_locks()); - CRYPTO_set_locking_callback(opensslLockCallback); -#endif - - if (sodium_init() == -1) - throw Error("could not initialise libsodium"); - - loadConfFile(); + initLibStore(); startSignalHandlerThread(); @@ -223,7 +155,10 @@ void initNix() if (sigaction(SIGTRAP, &act, 0)) throw SysError("handling SIGTRAP"); #endif - /* Register a SIGSEGV handler to detect stack overflows. */ + /* Register a SIGSEGV handler to detect stack overflows. + Why not initLibExpr()? initGC() is essentially that, but + detectStackOverflow is not an instance of the init function concept, as + it may have to be invoked more than once per process. */ detectStackOverflow(); /* There is no privacy in the Nix system ;-) At least not for @@ -236,16 +171,6 @@ void initNix() gettimeofday(&tv, 0); srandom(tv.tv_usec); - /* On macOS, don't use the per-session TMPDIR (as set e.g. by - sshd). This breaks build users because they don't have access - to the TMPDIR, in particular in ‘nix-store --serve’. */ -#if __APPLE__ - if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/")) - unsetenv("TMPDIR"); -#endif - - preloadNSS(); - initLibStore(); } diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 1715374a6..7a9e83c6c 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "util.hh" #include "args.hh" @@ -24,7 +25,9 @@ public: int handleExceptions(const std::string & programName, std::function fun); -/* Don't forget to call initPlugins() after settings are initialized! */ +/** + * Don't forget to call initPlugins() after settings are initialized! + */ void initNix(); void parseCmdLine(int argc, char * * argv, @@ -35,7 +38,9 @@ void parseCmdLine(const std::string & programName, const Strings & args, void printVersion(const std::string & programName); -/* Ugh. No better place to put this. */ +/** + * Ugh. No better place to put this. + */ void printGCWarning(); class Store; @@ -74,11 +79,15 @@ struct LegacyArgs : public MixCommonArgs }; -/* Show the manual page for the specified program. */ +/** + * Show the manual page for the specified program. + */ void showManPage(const std::string & name); -/* The constructor of this class starts a pager if stdout is a - terminal and $PAGER is set. Stdout is redirected to the pager. */ +/** + * The constructor of this class starts a pager if stdout is a + * terminal and $PAGER is set. Stdout is redirected to the pager. + */ class RunPager { public: @@ -109,28 +118,34 @@ struct PrintFreed }; -/* Install a SIGSEGV handler to detect stack overflows. */ +/** + * Install a SIGSEGV handler to detect stack overflows. + */ void detectStackOverflow(); -/* Pluggable behavior to run in case of a stack overflow. - - Default value: defaultStackOverflowHandler. - - This is called by the handler installed by detectStackOverflow(). - - This gives Nix library consumers a limit opportunity to report the error - condition. The handler should exit the process. - See defaultStackOverflowHandler() for a reference implementation. - - NOTE: Use with diligence, because this runs in the signal handler, with very - limited stack space and a potentially a corrupted heap, all while the failed - thread is blocked indefinitely. All functions called must be reentrant. */ +/** + * Pluggable behavior to run in case of a stack overflow. + * + * Default value: defaultStackOverflowHandler. + * + * This is called by the handler installed by detectStackOverflow(). + * + * This gives Nix library consumers a limit opportunity to report the error + * condition. The handler should exit the process. + * See defaultStackOverflowHandler() for a reference implementation. + * + * NOTE: Use with diligence, because this runs in the signal handler, with very + * limited stack space and a potentially a corrupted heap, all while the failed + * thread is blocked indefinitely. All functions called must be reentrant. + */ extern std::function stackOverflowHandler; -/* The default, robust implementation of stackOverflowHandler. - - Prints an error message directly to stderr using a syscall instead of the - logger. Exits the process immediately after. */ +/** + * The default, robust implementation of stackOverflowHandler. + * + * Prints an error message directly to stderr using a syscall instead of the + * logger. Exits the process immediately after. + */ void defaultStackOverflowHandler(siginfo_t * info, void * ctx); } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 628e9b9db..fcd763a9d 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -315,6 +315,7 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n }, .references = { .others = references, + // caller is not capable of creating a self-reference, because this is content-addressed without modulus .self = false, }, }, @@ -433,6 +434,7 @@ StorePath BinaryCacheStore::addToStore( }, .references = { .others = references, + // caller is not capable of creating a self-reference, because this is content-addressed without modulus .self = false, }, }, diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index c1d08926d..49f271d24 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "crypto.hh" #include "store-api.hh" @@ -45,6 +46,11 @@ struct BinaryCacheStoreConfig : virtual StoreConfig )"}; }; + +/** + * @note subclasses must implement at least one of the two + * virtual getFile() methods. + */ class BinaryCacheStore : public virtual BinaryCacheStoreConfig, public virtual Store, public virtual LogStore @@ -74,14 +80,15 @@ public: std::string && data, const std::string & mimeType); - /* Note: subclasses must implement at least one of the two - following getFile() methods. */ - - /* Dump the contents of the specified file to a sink. */ + /** + * Dump the contents of the specified file to a sink. + */ virtual void getFile(const std::string & path, Sink & sink); - /* Fetch the specified file and call the specified callback with - the result. A subclass may implement this asynchronously. */ + /** + * Fetch the specified file and call the specified callback with + * the result. A subclass may implement this asynchronously. + */ virtual void getFile( const std::string & path, Callback> callback) noexcept; diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh index a5749cf33..27d1a1b6c 100644 --- a/src/libstore/build-result.hh +++ b/src/libstore/build-result.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "realisation.hh" #include "derived-path.hh" @@ -11,9 +12,12 @@ namespace nix { struct BuildResult { - /* Note: don't remove status codes, and only add new status codes - at the end of the list, to prevent client/server - incompatibilities in the nix-store --serve protocol. */ + /** + * @note This is directly used in the nix-store --serve protocol. + * That means we need to worry about compatability across versions. + * Therefore, don't remove status codes, and only add new status + * codes at the end of the list. + */ enum Status { Built = 0, Substituted, @@ -21,8 +25,10 @@ struct BuildResult PermanentFailure, InputRejected, OutputRejected, - TransientFailure, // possibly transient - CachedFailure, // no longer used + /// possibly transient + TransientFailure, + /// no longer used + CachedFailure, TimedOut, MiscFailure, DependencyFailed, @@ -32,7 +38,12 @@ struct BuildResult NoSubstituters, } status = MiscFailure; - // FIXME: include entire ErrorInfo object. + /** + * Information about the error if the build failed. + * + * @todo This should be an entire ErrorInfo object, not just a + * string, for richer information. + */ std::string errorMsg; std::string toString() const { @@ -52,33 +63,46 @@ struct BuildResult case LogLimitExceeded: return "LogLimitExceeded"; case NotDeterministic: return "NotDeterministic"; case ResolvesToAlreadyValid: return "ResolvesToAlreadyValid"; + case NoSubstituters: return "NoSubstituters"; default: return "Unknown"; }; }(); return strStatus + ((errorMsg == "") ? "" : " : " + errorMsg); } - /* How many times this build was performed. */ + /** + * How many times this build was performed. + */ unsigned int timesBuilt = 0; - /* If timesBuilt > 1, whether some builds did not produce the same - result. (Note that 'isNonDeterministic = false' does not mean - the build is deterministic, just that we don't have evidence of - non-determinism.) */ + /** + * If timesBuilt > 1, whether some builds did not produce the same + * result. (Note that 'isNonDeterministic = false' does not mean + * the build is deterministic, just that we don't have evidence of + * non-determinism.) + */ bool isNonDeterministic = false; - /* The derivation we built or the store path we substituted. */ + /** + * The derivation we built or the store path we substituted. + */ DerivedPath path; - /* For derivations, a mapping from the names of the wanted outputs - to actual paths. */ + /** + * For derivations, a mapping from the names of the wanted outputs + * to actual paths. + */ DrvOutputs builtOutputs; - /* The start/stop times of the build (or one of the rounds, if it - was repeated). */ + /** + * The start/stop times of the build (or one of the rounds, if it + * was repeated). + */ time_t startTime = 0, stopTime = 0; - /* User and system CPU time the build took. */ + /** + * User and system CPU time the build took. + */ std::optional cpuUser, cpuSystem; bool success() diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 596034c0f..26faf8c8e 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -911,7 +911,11 @@ void DerivationGoal::buildDone() msg += line; msg += "\n"; } - msg += fmt("For full logs, run '" ANSI_BOLD "nix log %s" ANSI_NORMAL "'.", + auto nixLogCommand = experimentalFeatureSettings.isEnabled(Xp::NixCommand) + ? "nix log" + : "nix-store -l"; + msg += fmt("For full logs, run '" ANSI_BOLD "%s %s" ANSI_NORMAL "'.", + nixLogCommand, worker.store.printStorePath(drvPath)); } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 707e38b4b..3a6f0c2d9 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "parsed-derivations.hh" #include "lock.hh" @@ -15,8 +16,10 @@ struct HookInstance; typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; -/* Unless we are repairing, we don't both to test validity and just assume it, - so the choices are `Absent` or `Valid`. */ +/** + * Unless we are repairing, we don't both to test validity and just assume it, + * so the choices are `Absent` or `Valid`. + */ enum struct PathStatus { Corrupt, Absent, @@ -26,11 +29,15 @@ enum struct PathStatus { struct InitialOutputStatus { StorePath path; PathStatus status; - /* Valid in the store, and additionally non-corrupt if we are repairing */ + /** + * Valid in the store, and additionally non-corrupt if we are repairing + */ bool isValid() const { return status == PathStatus::Valid; } - /* Merely present, allowed to be corrupt */ + /** + * Merely present, allowed to be corrupt + */ bool isPresent() const { return status == PathStatus::Corrupt || status == PathStatus::Valid; @@ -45,59 +52,87 @@ struct InitialOutput { struct DerivationGoal : public Goal { - /* Whether to use an on-disk .drv file. */ + /** + * Whether to use an on-disk .drv file. + */ bool useDerivation; - /* The path of the derivation. */ + /** The path of the derivation. */ StorePath drvPath; - /* The goal for the corresponding resolved derivation */ + /** + * The goal for the corresponding resolved derivation + */ std::shared_ptr resolvedDrvGoal; - /* The specific outputs that we need to build. Empty means all of - them. */ + /** + * The specific outputs that we need to build. Empty means all of + * them. + */ OutputsSpec wantedOutputs; - /* Mapping from input derivations + output names to actual store - paths. This is filled in by waiteeDone() as each dependency - finishes, before inputsRealised() is reached, */ + /** + * Mapping from input derivations + output names to actual store + * paths. This is filled in by waiteeDone() as each dependency + * finishes, before inputsRealised() is reached. + */ std::map, StorePath> inputDrvOutputs; - /* Whether additional wanted outputs have been added. */ + /** + * Whether additional wanted outputs have been added. + */ bool needRestart = false; - /* Whether to retry substituting the outputs after building the - inputs. This is done in case of an incomplete closure. */ + /** + * Whether to retry substituting the outputs after building the + * inputs. This is done in case of an incomplete closure. + */ bool retrySubstitution = false; - /* Whether we've retried substitution, in which case we won't try - again. */ + /** + * Whether we've retried substitution, in which case we won't try + * again. + */ bool retriedSubstitution = false; - /* The derivation stored at drvPath. */ + /** + * The derivation stored at drvPath. + */ std::unique_ptr drv; std::unique_ptr parsedDrv; - /* The remainder is state held during the build. */ + /** + * The remainder is state held during the build. + */ - /* Locks on (fixed) output paths. */ + /** + * Locks on (fixed) output paths. + */ PathLocks outputLocks; - /* All input paths (that is, the union of FS closures of the - immediate input paths). */ + /** + * All input paths (that is, the union of FS closures of the + * immediate input paths). + */ StorePathSet inputPaths; std::map initialOutputs; - /* File descriptor for the log file. */ + /** + * File descriptor for the log file. + */ AutoCloseFD fdLogFile; std::shared_ptr logFileSink, logSink; - /* Number of bytes received from the builder's stdout/stderr. */ + /** + * Number of bytes received from the builder's stdout/stderr. + */ unsigned long logSize; - /* The most recent log lines. */ + /** + * The most recent log lines. + */ std::list logTail; std::string currentLogLine; @@ -105,10 +140,14 @@ struct DerivationGoal : public Goal std::string currentHookLine; - /* The build hook. */ + /** + * The build hook. + */ std::unique_ptr hook; - /* The sort of derivation we are building. */ + /** + * The sort of derivation we are building. + */ DerivationType derivationType; typedef void (DerivationGoal::*GoalState)(); @@ -120,12 +159,16 @@ struct DerivationGoal : public Goal std::unique_ptr act; - /* Activity that denotes waiting for a lock. */ + /** + * Activity that denotes waiting for a lock. + */ std::unique_ptr actLock; std::map builderActivities; - /* The remote machine on which we're building. */ + /** + * The remote machine on which we're building. + */ std::string machineName; DerivationGoal(const StorePath & drvPath, @@ -142,10 +185,14 @@ struct DerivationGoal : public Goal void work() override; - /* Add wanted outputs to an already existing derivation goal. */ + /** + * Add wanted outputs to an already existing derivation goal. + */ void addWantedOutputs(const OutputsSpec & outputs); - /* The states. */ + /** + * The states. + */ void getDerivation(); void loadDerivation(); void haveDerivation(); @@ -159,28 +206,42 @@ struct DerivationGoal : public Goal void resolvedFinished(); - /* Is the build hook willing to perform the build? */ + /** + * Is the build hook willing to perform the build? + */ HookReply tryBuildHook(); virtual int getChildStatus(); - /* Check that the derivation outputs all exist and register them - as valid. */ + /** + * Check that the derivation outputs all exist and register them + * as valid. + */ virtual DrvOutputs registerOutputs(); - /* Open a log file and a pipe to it. */ + /** + * Open a log file and a pipe to it. + */ Path openLogFile(); - /* Sign the newly built realisation if the store allows it */ + /** + * Sign the newly built realisation if the store allows it + */ virtual void signRealisation(Realisation&) {} - /* Close the log file. */ + /** + * Close the log file. + */ void closeLogFile(); - /* Close the read side of the logger pipe. */ + /** + * Close the read side of the logger pipe. + */ virtual void closeReadPipes(); - /* Cleanup hooks for buildDone() */ + /** + * Cleanup hooks for buildDone() + */ virtual void cleanupHookFinally(); virtual void cleanupPreChildKill(); virtual void cleanupPostChildKill(); @@ -190,30 +251,40 @@ struct DerivationGoal : public Goal virtual bool isReadDesc(int fd); - /* Callback used by the worker to write to the log. */ + /** + * Callback used by the worker to write to the log. + */ void handleChildOutput(int fd, std::string_view data) override; void handleEOF(int fd) override; void flushLine(); - /* Wrappers around the corresponding Store methods that first consult the - derivation. This is currently needed because when there is no drv file - there also is no DB entry. */ + /** + * Wrappers around the corresponding Store methods that first consult the + * derivation. This is currently needed because when there is no drv file + * there also is no DB entry. + */ std::map> queryPartialDerivationOutputMap(); OutputPathMap queryDerivationOutputMap(); - /* Update 'initialOutputs' to determine the current status of the - outputs of the derivation. Also returns a Boolean denoting - whether all outputs are valid and non-corrupt, and a - 'DrvOutputs' structure containing the valid and wanted - outputs. */ + /** + * Update 'initialOutputs' to determine the current status of the + * outputs of the derivation. Also returns a Boolean denoting + * whether all outputs are valid and non-corrupt, and a + * 'DrvOutputs' structure containing the valid and wanted + * outputs. + */ std::pair checkPathValidity(); - /* Aborts if any output is not valid or corrupt, and otherwise - returns a 'DrvOutputs' structure containing the wanted - outputs. */ + /** + * Aborts if any output is not valid or corrupt, and otherwise + * returns a 'DrvOutputs' structure containing the wanted + * outputs. + */ DrvOutputs assertPathValidity(); - /* Forcibly kill the child process, if any. */ + /** + * Forcibly kill the child process, if any. + */ virtual void killChild(); void repairClosure(); diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh index e4b044790..697ddb283 100644 --- a/src/libstore/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" #include "goal.hh" @@ -10,24 +11,34 @@ namespace nix { class Worker; -// Substitution of a derivation output. -// This is done in three steps: -// 1. Fetch the output info from a substituter -// 2. Substitute the corresponding output path -// 3. Register the output info +/** + * Substitution of a derivation output. + * This is done in three steps: + * 1. Fetch the output info from a substituter + * 2. Substitute the corresponding output path + * 3. Register the output info + */ class DrvOutputSubstitutionGoal : public Goal { - // The drv output we're trying to substitue + /** + * The drv output we're trying to substitue + */ DrvOutput id; - // The realisation corresponding to the given output id. - // Will be filled once we can get it. + /** + * The realisation corresponding to the given output id. + * Will be filled once we can get it. + */ std::shared_ptr outputInfo; - /* The remaining substituters. */ + /** + * The remaining substituters. + */ std::list> subs; - /* The current substituter. */ + /** + * The current substituter. + */ std::shared_ptr sub; struct DownloadState @@ -38,7 +49,9 @@ class DrvOutputSubstitutionGoal : public Goal { std::shared_ptr downloadState; - /* Whether a substituter failed. */ + /** + * Whether a substituter failed. + */ bool substituterFailed = false; public: diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 776eb86bc..f4bf6f38b 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "store-api.hh" @@ -6,11 +7,15 @@ namespace nix { -/* Forward definition. */ +/** + * Forward definition. + */ struct Goal; class Worker; -/* A pointer to a goal. */ +/** + * A pointer to a goal. + */ typedef std::shared_ptr GoalPtr; typedef std::weak_ptr WeakGoalPtr; @@ -18,48 +23,72 @@ struct CompareGoalPtrs { bool operator() (const GoalPtr & a, const GoalPtr & b) const; }; -/* Set of goals. */ +/** + * Set of goals. + */ typedef std::set Goals; typedef std::set> WeakGoals; -/* A map of paths to goals (and the other way around). */ +/** + * A map of paths to goals (and the other way around). + */ typedef std::map WeakGoalMap; struct Goal : public std::enable_shared_from_this { typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; - /* Backlink to the worker. */ + /** + * Backlink to the worker. + */ Worker & worker; - /* Goals that this goal is waiting for. */ + /** + * Goals that this goal is waiting for. + */ Goals waitees; - /* Goals waiting for this one to finish. Must use weak pointers - here to prevent cycles. */ + /** + * Goals waiting for this one to finish. Must use weak pointers + * here to prevent cycles. + */ WeakGoals waiters; - /* Number of goals we are/were waiting for that have failed. */ + /** + * Number of goals we are/were waiting for that have failed. + */ size_t nrFailed = 0; - /* Number of substitution goals we are/were waiting for that - failed because there are no substituters. */ + /** + * Number of substitution goals we are/were waiting for that + * failed because there are no substituters. + */ size_t nrNoSubstituters = 0; - /* Number of substitution goals we are/were waiting for that - failed because they had unsubstitutable references. */ + /** + * Number of substitution goals we are/were waiting for that + * failed because they had unsubstitutable references. + */ size_t nrIncompleteClosure = 0; - /* Name of this goal for debugging purposes. */ + /** + * Name of this goal for debugging purposes. + */ std::string name; - /* Whether the goal is finished. */ + /** + * Whether the goal is finished. + */ ExitCode exitCode = ecBusy; - /* Build result. */ + /** + * Build result. + */ BuildResult buildResult; - /* Exception containing an error message, if any. */ + /** + * Exception containing an error message, if any. + */ std::optional ex; Goal(Worker & worker, DerivedPath path) @@ -95,9 +124,11 @@ struct Goal : public std::enable_shared_from_this return name; } - /* Callback in case of a timeout. It should wake up its waiters, - get rid of any running child processes that are being monitored - by the worker (important!), etc. */ + /** + * Callback in case of a timeout. It should wake up its waiters, + * get rid of any running child processes that are being monitored + * by the worker (important!), etc. + */ virtual void timedOut(Error && ex) = 0; virtual std::string key() = 0; diff --git a/src/libstore/build/hook-instance.hh b/src/libstore/build/hook-instance.hh index 9e8cff128..d84f62877 100644 --- a/src/libstore/build/hook-instance.hh +++ b/src/libstore/build/hook-instance.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "logging.hh" #include "serialise.hh" @@ -7,16 +8,24 @@ namespace nix { struct HookInstance { - /* Pipes for talking to the build hook. */ + /** + * Pipes for talking to the build hook. + */ Pipe toHook; - /* Pipe for the hook's standard output/error. */ + /** + * Pipe for the hook's standard output/error. + */ Pipe fromHook; - /* Pipe for the builder's standard output/error. */ + /** + * Pipe for the builder's standard output/error. + */ Pipe builderOut; - /* The process ID of the hook. */ + /** + * The process ID of the hook. + */ Pid pid; FdSink sink; diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 7cb80977d..bbec4aea0 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1415,6 +1415,9 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo virtual void addBuildLog(const StorePath & path, std::string_view log) override { unsupported("addBuildLog"); } + + std::optional isTrustedClient() override + { return NotTrusted; } }; @@ -1467,7 +1470,7 @@ void LocalDerivationGoal::startDaemon() FdSink to(remote.get()); try { daemon::processConnection(store, from, to, - daemon::NotTrusted, daemon::Recursive); + NotTrusted, daemon::Recursive); debug("terminated daemon connection"); } catch (SysError &) { ignoreException(); diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index c9ecc8828..42d32a31a 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "derivation-goal.hh" #include "local-store.hh" @@ -9,49 +10,75 @@ struct LocalDerivationGoal : public DerivationGoal { LocalStore & getLocalStore(); - /* User selected for running the builder. */ + /** + * User selected for running the builder. + */ std::unique_ptr buildUser; - /* The process ID of the builder. */ + /** + * The process ID of the builder. + */ Pid pid; - /* The cgroup of the builder, if any. */ + /** + * The cgroup of the builder, if any. + */ std::optional cgroup; - /* The temporary directory. */ + /** + * The temporary directory. + */ Path tmpDir; - /* The path of the temporary directory in the sandbox. */ + /** + * The path of the temporary directory in the sandbox. + */ Path tmpDirInSandbox; - /* Master side of the pseudoterminal used for the builder's - standard output/error. */ + /** + * Master side of the pseudoterminal used for the builder's + * standard output/error. + */ AutoCloseFD builderOut; - /* Pipe for synchronising updates to the builder namespaces. */ + /** + * Pipe for synchronising updates to the builder namespaces. + */ Pipe userNamespaceSync; - /* The mount namespace and user namespace of the builder, used to add additional - paths to the sandbox as a result of recursive Nix calls. */ + /** + * The mount namespace and user namespace of the builder, used to add additional + * paths to the sandbox as a result of recursive Nix calls. + */ AutoCloseFD sandboxMountNamespace; AutoCloseFD sandboxUserNamespace; - /* On Linux, whether we're doing the build in its own user - namespace. */ + /** + * On Linux, whether we're doing the build in its own user + * namespace. + */ bool usingUserNamespace = true; - /* Whether we're currently doing a chroot build. */ + /** + * Whether we're currently doing a chroot build. + */ bool useChroot = false; Path chrootRootDir; - /* RAII object to delete the chroot directory. */ + /** + * RAII object to delete the chroot directory. + */ std::shared_ptr autoDelChroot; - /* Whether to run the build in a private network namespace. */ + /** + * Whether to run the build in a private network namespace. + */ bool privateNetwork = false; - /* Stuff we need to pass to initChild(). */ + /** + * Stuff we need to pass to initChild(). + */ struct ChrootPath { Path source; bool optional; @@ -70,30 +97,35 @@ struct LocalDerivationGoal : public DerivationGoal SandboxProfile additionalSandboxProfile; #endif - /* Hash rewriting. */ + /** + * Hash rewriting. + */ StringMap inputRewrites, outputRewrites; typedef map RedirectedOutputs; RedirectedOutputs redirectedOutputs; - /* The outputs paths used during the build. - - - Input-addressed derivations or fixed content-addressed outputs are - sometimes built when some of their outputs already exist, and can not - be hidden via sandboxing. We use temporary locations instead and - rewrite after the build. Otherwise the regular predetermined paths are - put here. - - - Floating content-addressed derivations do not know their final build - output paths until the outputs are hashed, so random locations are - used, and then renamed. The randomness helps guard against hidden - self-references. + /** + * The outputs paths used during the build. + * + * - Input-addressed derivations or fixed content-addressed outputs are + * sometimes built when some of their outputs already exist, and can not + * be hidden via sandboxing. We use temporary locations instead and + * rewrite after the build. Otherwise the regular predetermined paths are + * put here. + * + * - Floating content-addressed derivations do not know their final build + * output paths until the outputs are hashed, so random locations are + * used, and then renamed. The randomness helps guard against hidden + * self-references. */ OutputPathMap scratchOutputs; - /* Path registration info from the previous round, if we're - building multiple times. Since this contains the hash, it - allows us to compare whether two rounds produced the same - result. */ + /** + * Path registration info from the previous round, if we're + * building multiple times. Since this contains the hash, it + * allows us to compare whether two rounds produced the same + * result. + */ std::map prevInfos; uid_t sandboxUid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); } @@ -101,25 +133,37 @@ struct LocalDerivationGoal : public DerivationGoal const static Path homeDir; - /* The recursive Nix daemon socket. */ + /** + * The recursive Nix daemon socket. + */ AutoCloseFD daemonSocket; - /* The daemon main thread. */ + /** + * The daemon main thread. + */ std::thread daemonThread; - /* The daemon worker threads. */ + /** + * The daemon worker threads. + */ std::vector daemonWorkerThreads; - /* Paths that were added via recursive Nix calls. */ + /** + * Paths that were added via recursive Nix calls. + */ StorePathSet addedPaths; - /* Realisations that were added via recursive Nix calls. */ + /** + * Realisations that were added via recursive Nix calls. + */ std::set addedDrvOutputs; - /* Recursive Nix calls are only allowed to build or realize paths - in the original input closure or added via a recursive Nix call - (so e.g. you can't do 'nix-store -r /nix/store/' where - /nix/store/ is some arbitrary path in a binary cache). */ + /** + * Recursive Nix calls are only allowed to build or realize paths + * in the original input closure or added via a recursive Nix call + * (so e.g. you can't do 'nix-store -r /nix/store/' where + * /nix/store/ is some arbitrary path in a binary cache). + */ bool isAllowed(const StorePath & path) { return inputPaths.count(path) || addedPaths.count(path); @@ -137,55 +181,81 @@ struct LocalDerivationGoal : public DerivationGoal virtual ~LocalDerivationGoal() override; - /* Whether we need to perform hash rewriting if there are valid output paths. */ + /** + * Whether we need to perform hash rewriting if there are valid output paths. + */ bool needsHashRewrite(); - /* The additional states. */ + /** + * The additional states. + */ void tryLocalBuild() override; - /* Start building a derivation. */ + /** + * Start building a derivation. + */ void startBuilder(); - /* Fill in the environment for the builder. */ + /** + * Fill in the environment for the builder. + */ void initEnv(); - /* Setup tmp dir location. */ + /** + * Setup tmp dir location. + */ void initTmpDir(); - /* Write a JSON file containing the derivation attributes. */ + /** + * Write a JSON file containing the derivation attributes. + */ void writeStructuredAttrs(); void startDaemon(); void stopDaemon(); - /* Add 'path' to the set of paths that may be referenced by the - outputs, and make it appear in the sandbox. */ + /** + * Add 'path' to the set of paths that may be referenced by the + * outputs, and make it appear in the sandbox. + */ void addDependency(const StorePath & path); - /* Make a file owned by the builder. */ + /** + * Make a file owned by the builder. + */ void chownToBuilder(const Path & path); int getChildStatus() override; - /* Run the builder's process. */ + /** + * Run the builder's process. + */ void runChild(); - /* Check that the derivation outputs all exist and register them - as valid. */ + /** + * Check that the derivation outputs all exist and register them + * as valid. + */ DrvOutputs registerOutputs() override; void signRealisation(Realisation &) override; - /* Check that an output meets the requirements specified by the - 'outputChecks' attribute (or the legacy - '{allowed,disallowed}{References,Requisites}' attributes). */ + /** + * Check that an output meets the requirements specified by the + * 'outputChecks' attribute (or the legacy + * '{allowed,disallowed}{References,Requisites}' attributes). + */ void checkOutputs(const std::map & outputs); - /* Close the read side of the logger pipe. */ + /** + * Close the read side of the logger pipe. + */ void closeReadPipes() override; - /* Cleanup hooks for buildDone() */ + /** + * Cleanup hooks for buildDone() + */ void cleanupHookFinally() override; void cleanupPreChildKill() override; void cleanupPostChildKill() override; @@ -195,24 +265,36 @@ struct LocalDerivationGoal : public DerivationGoal bool isReadDesc(int fd) override; - /* Delete the temporary directory, if we have one. */ + /** + * Delete the temporary directory, if we have one. + */ void deleteTmpDir(bool force); - /* Forcibly kill the child process, if any. */ + /** + * Forcibly kill the child process, if any. + */ void killChild() override; - /* Kill any processes running under the build user UID or in the - cgroup of the build. */ + /** + * Kill any processes running under the build user UID or in the + * cgroup of the build. + */ void killSandbox(bool getStats); - /* Create alternative path calculated from but distinct from the - input, so we can avoid overwriting outputs (or other store paths) - that already exist. */ + /** + * Create alternative path calculated from but distinct from the + * input, so we can avoid overwriting outputs (or other store paths) + * that already exist. + */ StorePath makeFallbackPath(const StorePath & path); - /* Make a path to another based on the output name along with the - derivation hash. */ - /* FIXME add option to randomize, so we can audit whether our - rewrites caught everything */ + + /** + * Make a path to another based on the output name along with the + * derivation hash. + * + * @todo Add option to randomize, so we can audit whether our + * rewrites caught everything + */ StorePath makeFallbackPath(std::string_view outputName); }; diff --git a/src/libstore/build/personality.hh b/src/libstore/build/personality.hh index 30e4f4062..91b730fab 100644 --- a/src/libstore/build/personality.hh +++ b/src/libstore/build/personality.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index a73f8e666..c2b7fc95a 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "lock.hh" #include "store-api.hh" @@ -10,38 +11,58 @@ class Worker; struct PathSubstitutionGoal : public Goal { - /* The store path that should be realised through a substitute. */ + /** + * The store path that should be realised through a substitute. + */ StorePath storePath; - /* The path the substituter refers to the path as. This will be - different when the stores have different names. */ + /** + * The path the substituter refers to the path as. This will be + * different when the stores have different names. + */ std::optional subPath; - /* The remaining substituters. */ + /** + * The remaining substituters. + */ std::list> subs; - /* The current substituter. */ + /** + * The current substituter. + */ std::shared_ptr sub; - /* Whether a substituter failed. */ + /** + * Whether a substituter failed. + */ bool substituterFailed = false; - /* Path info returned by the substituter's query info operation. */ + /** + * Path info returned by the substituter's query info operation. + */ std::shared_ptr info; - /* Pipe for the substituter's standard output. */ + /** + * Pipe for the substituter's standard output. + */ Pipe outPipe; - /* The substituter thread. */ + /** + * The substituter thread. + */ std::thread thr; std::promise promise; - /* Whether to try to repair a valid path. */ + /** + * Whether to try to repair a valid path. + */ RepairFlag repair; - /* Location where we're downloading the substitute. Differs from - storePath when doing a repair. */ + /** + * Location where we're downloading the substitute. Differs from + * storePath when doing a repair. + */ Path destPath; std::unique_ptr> maintainExpectedSubstitutions, @@ -50,7 +71,9 @@ struct PathSubstitutionGoal : public Goal typedef void (PathSubstitutionGoal::*GoalState)(); GoalState state; - /* Content address for recomputing store path */ + /** + * Content address for recomputing store path + */ std::optional ca; void done( @@ -64,16 +87,20 @@ public: void timedOut(Error && ex) override { abort(); }; + /** + * We prepend "a$" to the key name to ensure substitution goals + * happen before derivation goals. + */ std::string key() override { - /* "a$" ensures substitution goals happen before derivation - goals. */ return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath); } void work() override; - /* The states. */ + /** + * The states. + */ void init(); void tryNext(); void gotInfo(); @@ -81,7 +108,9 @@ public: void tryToRun(); void finished(); - /* Callback used by the worker to write to the log. */ + /** + * Callback used by the worker to write to the log. + */ void handleChildOutput(int fd, std::string_view data) override; void handleEOF(int fd) override; diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 6d68d3cf1..48a1a27fa 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "lock.hh" @@ -16,24 +17,29 @@ struct DerivationGoal; struct PathSubstitutionGoal; class DrvOutputSubstitutionGoal; -/* Workaround for not being able to declare a something like - - class PathSubstitutionGoal : public Goal; - - even when Goal is a complete type. - - This is still a static cast. The purpose of exporting it is to define it in - a place where `PathSubstitutionGoal` is concrete, and use it in a place where it - is opaque. */ +/** + * Workaround for not being able to declare a something like + * + * ```c++ + * class PathSubstitutionGoal : public Goal; + * ``` + * even when Goal is a complete type. + * + * This is still a static cast. The purpose of exporting it is to define it in + * a place where `PathSubstitutionGoal` is concrete, and use it in a place where it + * is opaque. + */ GoalPtr upcast_goal(std::shared_ptr subGoal); GoalPtr upcast_goal(std::shared_ptr subGoal); typedef std::chrono::time_point steady_time_point; -/* A mapping used to remember for each child process to what goal it - belongs, and file descriptors for receiving log data and output - path creation commands. */ +/** + * A mapping used to remember for each child process to what goal it + * belongs, and file descriptors for receiving log data and output + * path creation commands. + */ struct Child { WeakGoalPtr goal; @@ -41,14 +47,19 @@ struct Child std::set fds; bool respectTimeouts; bool inBuildSlot; - steady_time_point lastOutput; /* time we last got output on stdout/stderr */ + /** + * Time we last got output on stdout/stderr + */ + steady_time_point lastOutput; steady_time_point timeStarted; }; /* Forward definition. */ struct HookInstance; -/* The worker class. */ +/** + * The worker class. + */ class Worker { private: @@ -56,38 +67,58 @@ private: /* Note: the worker should only have strong pointers to the top-level goals. */ - /* The top-level goals of the worker. */ + /** + * The top-level goals of the worker. + */ Goals topGoals; - /* Goals that are ready to do some work. */ + /** + * Goals that are ready to do some work. + */ WeakGoals awake; - /* Goals waiting for a build slot. */ + /** + * Goals waiting for a build slot. + */ WeakGoals wantingToBuild; - /* Child processes currently running. */ + /** + * Child processes currently running. + */ std::list children; - /* Number of build slots occupied. This includes local builds and - substitutions but not remote builds via the build hook. */ + /** + * Number of build slots occupied. This includes local builds and + * substitutions but not remote builds via the build hook. + */ unsigned int nrLocalBuilds; - /* Maps used to prevent multiple instantiations of a goal for the - same derivation / path. */ + /** + * Maps used to prevent multiple instantiations of a goal for the + * same derivation / path. + */ std::map> derivationGoals; std::map> substitutionGoals; std::map> drvOutputSubstitutionGoals; - /* Goals waiting for busy paths to be unlocked. */ + /** + * Goals waiting for busy paths to be unlocked. + */ WeakGoals waitingForAnyGoal; - /* Goals sleeping for a few seconds (polling a lock). */ + /** + * Goals sleeping for a few seconds (polling a lock). + */ WeakGoals waitingForAWhile; - /* Last time the goals in `waitingForAWhile' where woken up. */ + /** + * Last time the goals in `waitingForAWhile` where woken up. + */ steady_time_point lastWokenUp; - /* Cache for pathContentsGood(). */ + /** + * Cache for pathContentsGood(). + */ std::map pathContentsGoodCache; public: @@ -96,17 +127,25 @@ public: const Activity actDerivations; const Activity actSubstitutions; - /* Set if at least one derivation had a BuildError (i.e. permanent - failure). */ + /** + * Set if at least one derivation had a BuildError (i.e. permanent + * failure). + */ bool permanentFailure; - /* Set if at least one derivation had a timeout. */ + /** + * Set if at least one derivation had a timeout. + */ bool timedOut; - /* Set if at least one derivation fails with a hash mismatch. */ + /** + * Set if at least one derivation fails with a hash mismatch. + */ bool hashMismatch; - /* Set if at least one derivation is not deterministic in check mode. */ + /** + * Set if at least one derivation is not deterministic in check mode. + */ bool checkMismatch; Store & store; @@ -128,16 +167,22 @@ public: uint64_t expectedNarSize = 0; uint64_t doneNarSize = 0; - /* Whether to ask the build hook if it can build a derivation. If - it answers with "decline-permanently", we don't try again. */ + /** + * Whether to ask the build hook if it can build a derivation. If + * it answers with "decline-permanently", we don't try again. + */ bool tryBuildHook = true; Worker(Store & store, Store & evalStore); ~Worker(); - /* Make a goal (with caching). */ + /** + * Make a goal (with caching). + */ - /* derivation goal */ + /** + * derivation goal + */ private: std::shared_ptr makeDerivationGoalCommon( const StorePath & drvPath, const OutputsSpec & wantedOutputs, @@ -150,56 +195,80 @@ public: const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal); - /* substitution goal */ + /** + * substitution goal + */ std::shared_ptr makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); std::shared_ptr makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); - /* Remove a dead goal. */ + /** + * Remove a dead goal. + */ void removeGoal(GoalPtr goal); - /* Wake up a goal (i.e., there is something for it to do). */ + /** + * Wake up a goal (i.e., there is something for it to do). + */ void wakeUp(GoalPtr goal); - /* Return the number of local build and substitution processes - currently running (but not remote builds via the build - hook). */ + /** + * Return the number of local build and substitution processes + * currently running (but not remote builds via the build + * hook). + */ unsigned int getNrLocalBuilds(); - /* Registers a running child process. `inBuildSlot' means that - the process counts towards the jobs limit. */ + /** + * Registers a running child process. `inBuildSlot` means that + * the process counts towards the jobs limit. + */ void childStarted(GoalPtr goal, const std::set & fds, bool inBuildSlot, bool respectTimeouts); - /* Unregisters a running child process. `wakeSleepers' should be - false if there is no sense in waking up goals that are sleeping - because they can't run yet (e.g., there is no free build slot, - or the hook would still say `postpone'). */ + /** + * Unregisters a running child process. `wakeSleepers` should be + * false if there is no sense in waking up goals that are sleeping + * because they can't run yet (e.g., there is no free build slot, + * or the hook would still say `postpone`). + */ void childTerminated(Goal * goal, bool wakeSleepers = true); - /* Put `goal' to sleep until a build slot becomes available (which - might be right away). */ + /** + * Put `goal` to sleep until a build slot becomes available (which + * might be right away). + */ void waitForBuildSlot(GoalPtr goal); - /* Wait for any goal to finish. Pretty indiscriminate way to - wait for some resource that some other goal is holding. */ + /** + * Wait for any goal to finish. Pretty indiscriminate way to + * wait for some resource that some other goal is holding. + */ void waitForAnyGoal(GoalPtr goal); - /* Wait for a few seconds and then retry this goal. Used when - waiting for a lock held by another process. This kind of - polling is inefficient, but POSIX doesn't really provide a way - to wait for multiple locks in the main select() loop. */ + /** + * Wait for a few seconds and then retry this goal. Used when + * waiting for a lock held by another process. This kind of + * polling is inefficient, but POSIX doesn't really provide a way + * to wait for multiple locks in the main select() loop. + */ void waitForAWhile(GoalPtr goal); - /* Loop until the specified top-level goals have finished. */ + /** + * Loop until the specified top-level goals have finished. + */ void run(const Goals & topGoals); - /* Wait for input to become available. */ + /** + * Wait for input to become available. + */ void waitForInput(); unsigned int exitStatus(); - /* Check whether the given valid path exists and has the right - contents. */ + /** + * Check whether the given valid path exists and has the right + * contents. + */ bool pathContentsGood(const StorePath & path); void markContentsGood(const StorePath & path); diff --git a/src/libstore/builtins.hh b/src/libstore/builtins.hh index 66597e456..d201fb3ac 100644 --- a/src/libstore/builtins.hh +++ b/src/libstore/builtins.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "derivations.hh" diff --git a/src/libstore/builtins/buildenv.hh b/src/libstore/builtins/buildenv.hh index a018de3af..0923c2adb 100644 --- a/src/libstore/builtins/buildenv.hh +++ b/src/libstore/builtins/buildenv.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "derivations.hh" #include "store-api.hh" diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 7704d2f00..962b63e83 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include "hash.hh" diff --git a/src/libstore/crypto.hh b/src/libstore/crypto.hh index 03f85c103..35216d470 100644 --- a/src/libstore/crypto.hh +++ b/src/libstore/crypto.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" @@ -11,8 +12,10 @@ struct Key std::string name; std::string key; - /* Construct Key from a string in the format - ‘:’. */ + /** + * Construct Key from a string in the format + * ‘:’. + */ Key(std::string_view s); std::string to_string() const; @@ -28,7 +31,9 @@ struct SecretKey : Key { SecretKey(std::string_view s); - /* Return a detached signature of the given string. */ + /** + * Return a detached signature of the given string. + */ std::string signDetached(std::string_view s) const; PublicKey toPublicKey() const; @@ -52,8 +57,10 @@ private: typedef std::map PublicKeys; -/* Return true iff ‘sig’ is a correct signature over ‘data’ using one - of the given public keys. */ +/** + * @return true iff ‘sig’ is a correct signature over ‘data’ using one + * of the given public keys. + */ bool verifyDetached(const std::string & data, const std::string & sig, const PublicKeys & publicKeys); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 0e2169035..139a05587 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -1036,6 +1036,15 @@ void processConnection( if (GET_PROTOCOL_MINOR(clientVersion) >= 33) to << nixVersion; + if (GET_PROTOCOL_MINOR(clientVersion) >= 35) { + // We and the underlying store both need to trust the client for + // it to be trusted. + auto temp = trusted + ? store->isTrustedClient() + : std::optional { NotTrusted }; + worker_proto::write(*store, to, temp); + } + /* Send startup error messages to the client. */ tunnelLogger->startWork(); diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh index 8c765615c..1964c0d99 100644 --- a/src/libstore/daemon.hh +++ b/src/libstore/daemon.hh @@ -1,11 +1,11 @@ #pragma once +///@file #include "serialise.hh" #include "store-api.hh" namespace nix::daemon { -enum TrustedFlag : bool { NotTrusted = false, Trusted = true }; enum RecursiveFlag : bool { NotRecursive = false, Recursive = true }; void processConnection( diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index aaae864f4..56f30f2e4 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -312,6 +312,15 @@ Derivation parseDerivation(const Store & store, std::string && s, std::string_vi } +/** + * Print a derivation string literal to an `std::string`. + * + * This syntax does not generalize to the expression language, which needs to + * escape `$`. + * + * @param res Where to print to + * @param s Which logical string to print + */ static void printString(std::string & res, std::string_view s) { boost::container::small_vector buffer; @@ -888,6 +897,67 @@ std::optional Derivation::tryResolve( return resolved; } + +void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const +{ + assert(drvPath.isDerivation()); + std::string drvName(drvPath.name()); + drvName = drvName.substr(0, drvName.size() - drvExtension.size()); + + if (drvName != name) { + throw Error("Derivation '%s' has name '%s' which does not match its path", store.printStorePath(drvPath), name); + } + + auto envHasRightPath = [&](const StorePath & actual, const std::string & varName) + { + auto j = env.find(varName); + if (j == env.end() || store.parseStorePath(j->second) != actual) + throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'", + store.printStorePath(drvPath), varName, store.printStorePath(actual)); + }; + + + // Don't need the answer, but do this anyways to assert is proper + // combination. The code below is more general and naturally allows + // combinations that are currently prohibited. + type(); + + std::optional hashesModulo; + for (auto & i : outputs) { + std::visit(overloaded { + [&](const DerivationOutput::InputAddressed & doia) { + if (!hashesModulo) { + // somewhat expensive so we do lazily + hashesModulo = hashDerivationModulo(store, *this, true); + } + auto currentOutputHash = get(hashesModulo->hashes, i.first); + if (!currentOutputHash) + throw Error("derivation '%s' has unexpected output '%s' (local-store / hashesModulo) named '%s'", + store.printStorePath(drvPath), store.printStorePath(doia.path), i.first); + StorePath recomputed = store.makeOutputPath(i.first, *currentOutputHash, drvName); + if (doia.path != recomputed) + throw Error("derivation '%s' has incorrect output '%s', should be '%s'", + store.printStorePath(drvPath), store.printStorePath(doia.path), store.printStorePath(recomputed)); + envHasRightPath(doia.path, i.first); + }, + [&](const DerivationOutput::CAFixed & dof) { + auto path = dof.path(store, drvName, i.first); + envHasRightPath(path, i.first); + }, + [&](const DerivationOutput::CAFloating &) { + /* Nothing to check */ + }, + [&](const DerivationOutput::Deferred &) { + /* Nothing to check */ + }, + [&](const DerivationOutput::Impure &) { + /* Nothing to check */ + }, + }, i.second.raw()); + } +} + + const Hash impureOutputHash = hashString(htSHA256, "impure"); nlohmann::json DerivationOutput::toJSON( @@ -916,10 +986,79 @@ nlohmann::json DerivationOutput::toJSON( return res; } + +DerivationOutput DerivationOutput::fromJSON( + const Store & store, std::string_view drvName, std::string_view outputName, + const nlohmann::json & _json) +{ + std::set keys; + auto json = (std::map) _json; + + for (const auto & [key, _] : json) + keys.insert(key); + + auto methodAlgo = [&]() -> std::pair { + std::string hashAlgo = json["hashAlgo"]; + auto method = FileIngestionMethod::Flat; + if (hashAlgo.substr(0, 2) == "r:") { + method = FileIngestionMethod::Recursive; + hashAlgo = hashAlgo.substr(2); + } + auto hashType = parseHashType(hashAlgo); + return { method, hashType }; + }; + + if (keys == (std::set { "path" })) { + return DerivationOutput::InputAddressed { + .path = store.parseStorePath((std::string) json["path"]), + }; + } + + else if (keys == (std::set { "path", "hashAlgo", "hash" })) { + auto [method, hashType] = methodAlgo(); + auto dof = DerivationOutput::CAFixed { + .ca = ContentAddressWithReferences::fromParts( + method, + Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType), + {}), + }; + if (dof.path(store, drvName, outputName) != store.parseStorePath((std::string) json["path"])) + throw Error("Path doesn't match derivation output"); + return dof; + } + + else if (keys == (std::set { "hashAlgo" })) { + auto [method, hashType] = methodAlgo(); + return DerivationOutput::CAFloating { + .method = method, + .hashType = hashType, + }; + } + + else if (keys == (std::set { })) { + return DerivationOutput::Deferred {}; + } + + else if (keys == (std::set { "hashAlgo", "impure" })) { + auto [method, hashType] = methodAlgo(); + return DerivationOutput::Impure { + .method = method, + .hashType = hashType, + }; + } + + else { + throw Error("invalid JSON for derivation output"); + } +} + + nlohmann::json Derivation::toJSON(const Store & store) const { nlohmann::json res = nlohmann::json::object(); + res["name"] = name; + { nlohmann::json & outputsObj = res["outputs"]; outputsObj = nlohmann::json::object(); @@ -950,4 +1089,43 @@ nlohmann::json Derivation::toJSON(const Store & store) const return res; } + +Derivation Derivation::fromJSON( + const Store & store, + const nlohmann::json & json) +{ + Derivation res; + + res.name = json["name"]; + + { + auto & outputsObj = json["outputs"]; + for (auto & [outputName, output] : outputsObj.items()) { + res.outputs.insert_or_assign( + outputName, + DerivationOutput::fromJSON(store, res.name, outputName, output)); + } + } + + { + auto & inputsList = json["inputSrcs"]; + for (auto & input : inputsList) + res.inputSrcs.insert(store.parseStorePath(static_cast(input))); + } + + { + auto & inputDrvsObj = json["inputDrvs"]; + for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items()) + res.inputDrvs[store.parseStorePath(inputDrvPath)] = + static_cast(inputOutputs); + } + + res.platform = json["system"]; + res.builder = json["builder"]; + res.args = json["args"]; + res.env = json["env"]; + + return res; +} + } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index dff6b472c..65901ec6d 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "path.hh" #include "types.hh" @@ -6,6 +7,7 @@ #include "content-address.hh" #include "repair-flag.hh" #include "sync.hh" +#include "comparator.hh" #include #include @@ -23,6 +25,8 @@ class Store; struct DerivationOutputInputAddressed { StorePath path; + + GENERATE_CMP(DerivationOutputInputAddressed, me->path); }; /** @@ -43,6 +47,8 @@ struct DerivationOutputCAFixed * @param outputName The name of this output. */ StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const; + + GENERATE_CMP(DerivationOutputCAFixed, me->ca); }; /** @@ -61,13 +67,17 @@ struct DerivationOutputCAFloating * How the serialization will be hashed */ HashType hashType; + + GENERATE_CMP(DerivationOutputCAFloating, me->method, me->hashType); }; /** * Input-addressed output which depends on a (CA) derivation whose hash * isn't known yet. */ -struct DerivationOutputDeferred {}; +struct DerivationOutputDeferred { + GENERATE_CMP(DerivationOutputDeferred); +}; /** * Impure output which is moved to a content-addressed location (like @@ -84,6 +94,8 @@ struct DerivationOutputImpure * How the serialization will be hashed */ HashType hashType; + + GENERATE_CMP(DerivationOutputImpure, me->method, me->hashType); }; typedef std::variant< @@ -124,6 +136,11 @@ struct DerivationOutput : _DerivationOutputRaw const Store & store, std::string_view drvName, std::string_view outputName) const; + static DerivationOutput fromJSON( + const Store & store, + std::string_view drvName, + std::string_view outputName, + const nlohmann::json & json); }; typedef std::map DerivationOutputs; @@ -241,8 +258,14 @@ struct DerivationType : _DerivationTypeRaw { struct BasicDerivation { - DerivationOutputs outputs; /* keyed on symbolic IDs */ - StorePathSet inputSrcs; /* inputs that are sources */ + /** + * keyed on symbolic IDs + */ + DerivationOutputs outputs; + /** + * inputs that are sources + */ + StorePathSet inputSrcs; std::string platform; Path builder; Strings args; @@ -272,6 +295,15 @@ struct BasicDerivation DerivationOutputsAndOptPaths outputsAndOptPaths(const Store & store) const; static std::string_view nameFromPath(const StorePath & storePath); + + GENERATE_CMP(BasicDerivation, + me->outputs, + me->inputSrcs, + me->platform, + me->builder, + me->args, + me->env, + me->name); }; struct Derivation : BasicDerivation @@ -307,11 +339,26 @@ struct Derivation : BasicDerivation Store & store, const std::map, StorePath> & inputDrvOutputs) const; + /* Check that the derivation is valid and does not present any + illegal states. + + This is mainly a matter of checking the outputs, where our C++ + representation supports all sorts of combinations we do not yet + allow. */ + void checkInvariants(Store & store, const StorePath & drvPath) const; + Derivation() = default; Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { } Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { } nlohmann::json toJSON(const Store & store) const; + static Derivation fromJSON( + const Store & store, + const nlohmann::json & json); + + GENERATE_CMP(Derivation, + static_cast(*me), + me->inputDrvs); }; @@ -388,12 +435,12 @@ void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept; * * A fixed-output derivation is a derivation whose outputs have a * specified content hash and hash algorithm. (Currently they must have - * exactly one output (`out'), which is specified using the `outputHash' - * and `outputHashAlgo' attributes, but the algorithm doesn't assume + * exactly one output (`out`), which is specified using the `outputHash` + * and `outputHashAlgo` attributes, but the algorithm doesn't assume * this.) We don't want changes to such derivations to propagate upwards * through the dependency graph, changing output paths everywhere. * - * For instance, if we change the url in a call to the `fetchurl' + * For instance, if we change the url in a call to the `fetchurl` * function, we do not want to rebuild everything depending on it---after * all, (the hash of) the file being downloaded is unchanged. So the * *output paths* should not change. On the other hand, the *derivation diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index e5f0f1b33..9a2ffda39 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -62,15 +62,31 @@ std::string DerivedPath::Opaque::to_string(const Store & store) const std::string DerivedPath::Built::to_string(const Store & store) const { return store.printStorePath(drvPath) - + "!" + + '^' + + outputs.to_string(); +} + +std::string DerivedPath::Built::to_string_legacy(const Store & store) const +{ + return store.printStorePath(drvPath) + + '!' + outputs.to_string(); } std::string DerivedPath::to_string(const Store & store) const { - return std::visit( - [&](const auto & req) { return req.to_string(store); }, - this->raw()); + return std::visit(overloaded { + [&](const DerivedPath::Built & req) { return req.to_string(store); }, + [&](const DerivedPath::Opaque & req) { return req.to_string(store); }, + }, this->raw()); +} + +std::string DerivedPath::to_string_legacy(const Store & store) const +{ + return std::visit(overloaded { + [&](const DerivedPath::Built & req) { return req.to_string_legacy(store); }, + [&](const DerivedPath::Opaque & req) { return req.to_string(store); }, + }, this->raw()); } @@ -87,14 +103,24 @@ DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_vi }; } -DerivedPath DerivedPath::parse(const Store & store, std::string_view s) +static inline DerivedPath parseWith(const Store & store, std::string_view s, std::string_view separator) { - size_t n = s.find("!"); + size_t n = s.find(separator); return n == s.npos ? (DerivedPath) DerivedPath::Opaque::parse(store, s) : (DerivedPath) DerivedPath::Built::parse(store, s.substr(0, n), s.substr(n + 1)); } +DerivedPath DerivedPath::parse(const Store & store, std::string_view s) +{ + return parseWith(store, s, "^"); +} + +DerivedPath DerivedPath::parseLegacy(const Store & store, std::string_view s) +{ + return parseWith(store, s, "!"); +} + RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const { RealisedPath::Set res; diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 72dbcc128..5f7acbebc 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "util.hh" #include "path.hh" @@ -47,8 +48,18 @@ struct DerivedPathBuilt { StorePath drvPath; OutputsSpec outputs; + /** + * Uses `^` as the separator + */ std::string to_string(const Store & store) const; - static DerivedPathBuilt parse(const Store & store, std::string_view, std::string_view); + /** + * Uses `!` as the separator + */ + std::string to_string_legacy(const Store & store) const; + /** + * The caller splits on the separator, so it works for both variants. + */ + static DerivedPathBuilt parse(const Store & store, std::string_view drvPath, std::string_view outputs); nlohmann::json toJSON(ref store) const; GENERATE_CMP(DerivedPathBuilt, me->drvPath, me->outputs); @@ -80,8 +91,22 @@ struct DerivedPath : _DerivedPathRaw { return static_cast(*this); } + /** + * Uses `^` as the separator + */ std::string to_string(const Store & store) const; + /** + * Uses `!` as the separator + */ + std::string to_string_legacy(const Store & store) const; + /** + * Uses `^` as the separator + */ static DerivedPath parse(const Store & store, std::string_view); + /** + * Uses `!` as the separator + */ + static DerivedPath parseLegacy(const Store & store, std::string_view); }; /** diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 16e5fafd7..74d6ed3b5 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -39,6 +39,14 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store callback(nullptr); } + /** + * The dummy store is incapable of *not* trusting! :) + */ + virtual std::optional isTrustedClient() override + { + return Trusted; + } + static std::set uriSchemes() { return {"dummy"}; } @@ -63,6 +71,9 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store void queryRealisationUncached(const DrvOutput &, Callback> callback) noexcept override { callback(nullptr); } + + virtual ref getFSAccessor() override + { unsupported("getFSAccessor"); } }; static RegisterStoreImplementation regDummyStore; diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 1ba399a29..2346accbe 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -407,6 +407,10 @@ struct curlFileTransfer : public FileTransfer err = Misc; } else { // Don't bother retrying on certain cURL errors either + + // Allow selecting a subset of enum values + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wswitch-enum" switch (code) { case CURLE_FAILED_INIT: case CURLE_URL_MALFORMAT: @@ -427,6 +431,7 @@ struct curlFileTransfer : public FileTransfer default: // Shut up warnings break; } + #pragma GCC diagnostic pop } attempt++; diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 07d58f53a..378c6ff78 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "hash.hh" @@ -87,39 +88,56 @@ struct FileTransfer { virtual ~FileTransfer() { } - /* Enqueue a data transfer request, returning a future to the result of - the download. The future may throw a FileTransferError - exception. */ + /** + * Enqueue a data transfer request, returning a future to the result of + * the download. The future may throw a FileTransferError + * exception. + */ virtual void enqueueFileTransfer(const FileTransferRequest & request, Callback callback) = 0; std::future enqueueFileTransfer(const FileTransferRequest & request); - /* Synchronously download a file. */ + /** + * Synchronously download a file. + */ FileTransferResult download(const FileTransferRequest & request); - /* Synchronously upload a file. */ + /** + * Synchronously upload a file. + */ FileTransferResult upload(const FileTransferRequest & request); - /* Download a file, writing its data to a sink. The sink will be - invoked on the thread of the caller. */ + /** + * Download a file, writing its data to a sink. The sink will be + * invoked on the thread of the caller. + */ void download(FileTransferRequest && request, Sink & sink); enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; -/* Return a shared FileTransfer object. Using this object is preferred - because it enables connection reuse and HTTP/2 multiplexing. */ +/** + * @return a shared FileTransfer object. + * + * Using this object is preferred because it enables connection reuse + * and HTTP/2 multiplexing. + */ ref getFileTransfer(); -/* Return a new FileTransfer object. */ +/** + * @return a new FileTransfer object + * + * Prefer getFileTransfer() to this; see its docs for why. + */ ref makeFileTransfer(); class FileTransferError : public Error { public: FileTransfer::Error error; - std::optional response; // intentionally optional + /// intentionally optional + std::optional response; template FileTransferError(FileTransfer::Error error, std::optional response, const Args & ... args); diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh index c825e84f2..1df19e647 100644 --- a/src/libstore/fs-accessor.hh +++ b/src/libstore/fs-accessor.hh @@ -1,11 +1,14 @@ #pragma once +///@file #include "types.hh" namespace nix { -/* An abstract class for accessing a filesystem-like structure, such - as a (possibly remote) Nix store or the contents of a NAR file. */ +/** + * An abstract class for accessing a filesystem-like structure, such + * as a (possibly remote) Nix store or the contents of a NAR file. + */ class FSAccessor { public: @@ -14,8 +17,17 @@ public: struct Stat { Type type = tMissing; - uint64_t fileSize = 0; // regular files only + /** + * regular files only + */ + uint64_t fileSize = 0; + /** + * regular files only + */ bool isExecutable = false; // regular files only + /** + * regular files only + */ uint64_t narOffset = 0; // regular files only }; diff --git a/src/libstore/gc-store.hh b/src/libstore/gc-store.hh index b3cbbad74..2c26c65c4 100644 --- a/src/libstore/gc-store.hh +++ b/src/libstore/gc-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" @@ -11,19 +12,20 @@ typedef std::unordered_map> Roots; struct GCOptions { - /* Garbage collector operation: - - - `gcReturnLive': return the set of paths reachable from - (i.e. in the closure of) the roots. - - - `gcReturnDead': return the set of paths not reachable from - the roots. - - - `gcDeleteDead': actually delete the latter set. - - - `gcDeleteSpecific': delete the paths listed in - `pathsToDelete', insofar as they are not reachable. - */ + /** + * Garbage collector operation: + * + * - `gcReturnLive`: return the set of paths reachable from + * (i.e. in the closure of) the roots. + * + * - `gcReturnDead`: return the set of paths not reachable from + * the roots. + * + * - `gcDeleteDead`: actually delete the latter set. + * + * - `gcDeleteSpecific`: delete the paths listed in + * `pathsToDelete`, insofar as they are not reachable. + */ typedef enum { gcReturnLive, gcReturnDead, @@ -33,28 +35,38 @@ struct GCOptions GCAction action{gcDeleteDead}; - /* If `ignoreLiveness' is set, then reachability from the roots is - ignored (dangerous!). However, the paths must still be - unreferenced *within* the store (i.e., there can be no other - store paths that depend on them). */ + /** + * If `ignoreLiveness` is set, then reachability from the roots is + * ignored (dangerous!). However, the paths must still be + * unreferenced *within* the store (i.e., there can be no other + * store paths that depend on them). + */ bool ignoreLiveness{false}; - /* For `gcDeleteSpecific', the paths to delete. */ + /** + * For `gcDeleteSpecific`, the paths to delete. + */ StorePathSet pathsToDelete; - /* Stop after at least `maxFreed' bytes have been freed. */ + /** + * Stop after at least `maxFreed` bytes have been freed. + */ uint64_t maxFreed{std::numeric_limits::max()}; }; struct GCResults { - /* Depending on the action, the GC roots, or the paths that would - be or have been deleted. */ + /** + * Depending on the action, the GC roots, or the paths that would + * be or have been deleted. + */ PathSet paths; - /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the - number of bytes that would be or was freed. */ + /** + * For `gcReturnDead`, `gcDeleteDead` and `gcDeleteSpecific`, the + * number of bytes that would be or was freed. + */ uint64_t bytesFreed = 0; }; @@ -63,21 +75,27 @@ struct GcStore : public virtual Store { inline static std::string operationName = "Garbage collection"; - /* Add an indirect root, which is merely a symlink to `path' from - /nix/var/nix/gcroots/auto/. `path' is supposed - to be a symlink to a store path. The garbage collector will - automatically remove the indirect root when it finds that - `path' has disappeared. */ + /** + * Add an indirect root, which is merely a symlink to `path` from + * `/nix/var/nix/gcroots/auto/`. `path` is supposed + * to be a symlink to a store path. The garbage collector will + * automatically remove the indirect root when it finds that + * `path` has disappeared. + */ virtual void addIndirectRoot(const Path & path) = 0; - /* Find the roots of the garbage collector. Each root is a pair - (link, storepath) where `link' is the path of the symlink - outside of the Nix store that point to `storePath'. If - 'censor' is true, privacy-sensitive information about roots - found in /proc is censored. */ + /** + * Find the roots of the garbage collector. Each root is a pair + * `(link, storepath)` where `link` is the path of the symlink + * outside of the Nix store that point to `storePath`. If + * `censor` is true, privacy-sensitive information about roots + * found in `/proc` is censored. + */ virtual Roots findRoots(bool censor) = 0; - /* Perform a garbage collection. */ + /** + * Perform a garbage collection. + */ virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0; }; diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 823b4af74..1b38e32fb 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -7,12 +7,20 @@ #include #include +#include #include #include #include #include +#include + +#ifdef __GLIBC__ +#include +#include +#include +#endif namespace nix { @@ -41,7 +49,6 @@ Settings::Settings() , nixDaemonSocketFile(canonPath(getEnvNonEmpty("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH))) { buildUsersGroup = getuid() == 0 ? "nixbld" : ""; - lockCPU = getEnv("NIX_AFFINITY_HACK") == "1"; allowSymlinkedStore = getEnv("NIX_IGNORE_SYMLINK_STORE") == "1"; auto sslOverride = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or("")); @@ -281,6 +288,42 @@ void initPlugins() settings.pluginFiles.pluginsLoaded = true; } +static void preloadNSS() +{ + /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of + one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already + been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to + load its lookup libraries in the parent before any child gets a chance to. */ + static std::once_flag dns_resolve_flag; + + std::call_once(dns_resolve_flag, []() { +#ifdef __GLIBC__ + /* On linux, glibc will run every lookup through the nss layer. + * That means every lookup goes, by default, through nscd, which acts as a local + * cache. + * Because we run builds in a sandbox, we also remove access to nscd otherwise + * lookups would leak into the sandbox. + * + * But now we have a new problem, we need to make sure the nss_dns backend that + * does the dns lookups when nscd is not available is loaded or available. + * + * We can't make it available without leaking nix's environment, so instead we'll + * load the backend, and configure nss so it does not try to run dns lookups + * through nscd. + * + * This is technically only used for builtins:fetch* functions so we only care + * about dns. + * + * All other platforms are unaffected. + */ + if (!dlopen(LIBNSS_DNS_SO, RTLD_NOW)) + warn("unable to load nss_dns backend"); + // FIXME: get hosts entry from nsswitch.conf. + __nss_configure_lookup("hosts", "files dns"); +#endif + }); +} + static bool initLibStoreDone = false; void assertLibStoreInitialized() { @@ -291,6 +334,24 @@ void assertLibStoreInitialized() { } void initLibStore() { + + initLibUtil(); + + if (sodium_init() == -1) + throw Error("could not initialise libsodium"); + + loadConfFile(); + + preloadNSS(); + + /* On macOS, don't use the per-session TMPDIR (as set e.g. by + sshd). This breaks build users because they don't have access + to the TMPDIR, in particular in ‘nix-store --serve’. */ +#if __APPLE__ + if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/")) + unsetenv("TMPDIR"); +#endif + initLibStoreDone = true; } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 299584f99..d6c5d437a 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "config.hh" @@ -71,30 +72,46 @@ public: Path nixPrefix; - /* The directory where we store sources and derived files. */ + /** + * The directory where we store sources and derived files. + */ Path nixStore; Path nixDataDir; /* !!! fix */ - /* The directory where we log various operations. */ + /** + * The directory where we log various operations. + */ Path nixLogDir; - /* The directory where state is stored. */ + /** + * The directory where state is stored. + */ Path nixStateDir; - /* The directory where system configuration files are stored. */ + /** + * The directory where system configuration files are stored. + */ Path nixConfDir; - /* A list of user configuration files to load. */ + /** + * A list of user configuration files to load. + */ std::vector nixUserConfFiles; - /* The directory where the main programs are stored. */ + /** + * The directory where the main programs are stored. + */ Path nixBinDir; - /* The directory where the man pages are stored. */ + /** + * The directory where the man pages are stored. + */ Path nixManDir; - /* File name of the socket the daemon listens to. */ + /** + * File name of the socket the daemon listens to. + */ Path nixDaemonSocketFile; Setting storeUri{this, getEnv("NIX_REMOTE").value_or("auto"), "store", @@ -120,7 +137,9 @@ public: )", {"build-fallback"}}; - /* Whether to show build log output in real time. */ + /** + * Whether to show build log output in real time. + */ bool verboseBuild = true; Setting logLines{this, 10, "log-lines", @@ -156,8 +175,10 @@ public: )", {"build-cores"}, false}; - /* Read-only mode. Don't copy stuff to the store, don't change - the database. */ + /** + * Read-only mode. Don't copy stuff to the store, don't change + * the database. + */ bool readOnlyMode = false; Setting thisSystem{ @@ -307,16 +328,6 @@ public: users in `build-users-group`. UIDs are allocated starting at 872415232 (0x34000000) on Linux and 56930 on macOS. - - > **Warning** - > This is an experimental feature. - - To enable it, add the following to [`nix.conf`](#): - - ``` - extra-experimental-features = auto-allocate-uids - auto-allocate-uids = true - ``` )"}; Setting startId{this, @@ -346,16 +357,6 @@ public: Cgroups are required and enabled automatically for derivations that require the `uid-range` system feature. - - > **Warning** - > This is an experimental feature. - - To enable it, add the following to [`nix.conf`](#): - - ``` - extra-experimental-features = cgroups - use-cgroups = true - ``` )"}; #endif @@ -457,9 +458,6 @@ public: )", {"env-keep-derivations"}}; - /* Whether to lock the Nix client and worker to the same CPU. */ - bool lockCPU; - Setting sandboxMode{ this, #if __linux__ @@ -996,8 +994,10 @@ public: // FIXME: don't use a global variable. extern Settings settings; -/* This should be called after settings are initialized, but before - anything else */ +/** + * This should be called after settings are initialized, but before + * anything else + */ void initPlugins(); void loadConfFile(); @@ -1007,12 +1007,16 @@ std::vector getUserConfigFiles(); extern const std::string nixVersion; -/* NB: This is not sufficient. You need to call initNix() */ +/** + * NB: This is not sufficient. You need to call initNix() + */ void initLibStore(); -/* It's important to initialize before doing _anything_, which is why we - call upon the programmer to handle this correctly. However, we only add - this in a key locations, so as not to litter the code. */ +/** + * It's important to initialize before doing _anything_, which is why we + * call upon the programmer to handle this correctly. However, we only add + * this in a key locations, so as not to litter the code. + */ void assertLibStoreInitialized(); } diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 238fd1d98..85c5eed4c 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -194,6 +194,18 @@ protected: }}); } + /** + * This isn't actually necessary read only. We support "upsert" now, so we + * have a notion of authentication via HTTP POST/PUT. + * + * For now, we conservatively say we don't know. + * + * \todo try to expose our HTTP authentication status. + */ + std::optional isTrustedClient() override + { + return std::nullopt; + } }; static RegisterStoreImplementation regHttpBinaryCacheStore; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index a1c38d180..c3cb3032a 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -342,6 +342,9 @@ public: void ensurePath(const StorePath & path) override { unsupported("ensurePath"); } + virtual ref getFSAccessor() override + { unsupported("getFSAccessor"); } + void computeFSClosure(const StorePathSet & paths, StorePathSet & out, bool flipDirection = false, bool includeOutputs = false, bool includeDerivers = false) override @@ -389,6 +392,15 @@ public: return conn->remoteVersion; } + /** + * The legacy ssh protocol doesn't support checking for trusted-user. + * Try using ssh-ng:// instead if you want to know. + */ + std::optional isTrustedClient() override + { + return std::nullopt; + } + void queryRealisationUncached(const DrvOutput &, Callback> callback) noexcept override // TODO: Implement diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index e5ee6fc15..5481dd762 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -95,6 +95,10 @@ protected: return paths; } + std::optional isTrustedClient() override + { + return Trusted; + } }; void LocalBinaryCacheStore::init() diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 796e72045..a03bb88f5 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" #include "gc-store.hh" @@ -47,7 +48,9 @@ public: void narFromPath(const StorePath & path, Sink & sink) override; ref getFSAccessor() override; - /* Register a permanent GC root. */ + /** + * Register a permanent GC root. + */ Path addPermRoot(const StorePath & storePath, const Path & gcRoot); virtual Path getRealStoreDir() { return realStoreDir; } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 63039e6ad..7fb312c37 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -711,61 +711,6 @@ void canonicalisePathMetaData(const Path & path, } -void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv) -{ - assert(drvPath.isDerivation()); - std::string drvName(drvPath.name()); - drvName = drvName.substr(0, drvName.size() - drvExtension.size()); - - auto envHasRightPath = [&](const StorePath & actual, const std::string & varName) - { - auto j = drv.env.find(varName); - if (j == drv.env.end() || parseStorePath(j->second) != actual) - throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'", - printStorePath(drvPath), varName, printStorePath(actual)); - }; - - - // Don't need the answer, but do this anyways to assert is proper - // combination. The code below is more general and naturally allows - // combinations that are currently prohibited. - drv.type(); - - std::optional hashesModulo; - for (auto & i : drv.outputs) { - std::visit(overloaded { - [&](const DerivationOutput::InputAddressed & doia) { - if (!hashesModulo) { - // somewhat expensive so we do lazily - hashesModulo = hashDerivationModulo(*this, drv, true); - } - auto currentOutputHash = get(hashesModulo->hashes, i.first); - if (!currentOutputHash) - throw Error("derivation '%s' has unexpected output '%s' (local-store / hashesModulo) named '%s'", - printStorePath(drvPath), printStorePath(doia.path), i.first); - StorePath recomputed = makeOutputPath(i.first, *currentOutputHash, drvName); - if (doia.path != recomputed) - throw Error("derivation '%s' has incorrect output '%s', should be '%s'", - printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed)); - envHasRightPath(doia.path, i.first); - }, - [&](const DerivationOutput::CAFixed & dof) { - auto path = dof.path(*this, drvName, i.first); - envHasRightPath(path, i.first); - }, - [&](const DerivationOutput::CAFloating &) { - /* Nothing to check */ - }, - [&](const DerivationOutput::Deferred &) { - /* Nothing to check */ - }, - [&](const DerivationOutput::Impure &) { - /* Nothing to check */ - }, - }, i.second.raw()); - } -} - void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) { experimentalFeatureSettings.require(Xp::CaDerivations); @@ -876,7 +821,7 @@ uint64_t LocalStore::addValidPath(State & state, derivations). Note that if this throws an error, then the DB transaction is rolled back, so the path validity registration above is undone. */ - if (checkOutputs) checkDerivationOutputs(info.path, drv); + if (checkOutputs) drv.checkInvariants(*this, info.path); for (auto & i : drv.outputsAndOptPaths(*this)) { /* Floating CA derivations have indeterminate output paths until @@ -1134,57 +1079,6 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) } -// FIXME: move this, it's not specific to LocalStore. -void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) -{ - if (!settings.useSubstitutes) return; - for (auto & sub : getDefaultSubstituters()) { - for (auto & path : paths) { - if (infos.count(path.first)) - // Choose first succeeding substituter. - continue; - - auto subPath(path.first); - - // Recompute store path so that we can use a different store root. - if (path.second) { - subPath = makeFixedOutputPathFromCA( - path.first.name(), - ContentAddressWithReferences::withoutRefs(*path.second)); - if (sub->storeDir == storeDir) - assert(subPath == path.first); - if (subPath != path.first) - debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path.first), sub->printStorePath(subPath), sub->getUri()); - } else if (sub->storeDir != storeDir) continue; - - debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath)); - try { - auto info = sub->queryPathInfo(subPath); - - if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty())) - continue; - - auto narInfo = std::dynamic_pointer_cast( - std::shared_ptr(info)); - infos.insert_or_assign(path.first, SubstitutablePathInfo{ - .deriver = info->deriver, - .references = info->references, - .downloadSize = narInfo ? narInfo->fileSize : 0, - .narSize = info->narSize, - }); - } catch (InvalidPath &) { - } catch (SubstituterDisabled &) { - } catch (Error & e) { - if (settings.tryFallback) - logError(e.info()); - else - throw; - } - } - } -} - - void LocalStore::registerValidPath(const ValidPathInfo & info) { registerValidPaths({{info.path, info}}); @@ -1226,8 +1120,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) for (auto & [_, i] : infos) if (i.path.isDerivation()) { // FIXME: inefficient; we already loaded the derivation in addValidPath(). - checkDerivationOutputs(i.path, - readInvalidDerivation(i.path)); + readInvalidDerivation(i.path).checkInvariants(*this, i.path); } /* Do a topological sort of the paths. This will throw an @@ -1435,6 +1328,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name }, .references = { .others = references, + // caller is not capable of creating a self-reference, because this is content-addressed without modulus .self = false, }, }; @@ -1753,6 +1647,11 @@ unsigned int LocalStore::getProtocol() return PROTOCOL_VERSION; } +std::optional LocalStore::isTrustedClient() +{ + return Trusted; +} + #if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 639772b36..55add18dd 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "sqlite.hh" @@ -18,10 +19,14 @@ namespace nix { -/* Nix store and database schema version. Version 1 (or 0) was Nix <= - 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10. - Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is - Nix 1.0. Version 7 is Nix 1.3. Version 10 is 2.0. */ +/** + * Nix store and database schema version. + * + * Version 1 (or 0) was Nix <= + * 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10. + * Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is + * Nix 1.0. Version 7 is Nix 1.3. Version 10 is 2.0. + */ const int nixSchemaVersion = 10; @@ -50,30 +55,40 @@ class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore, { private: - /* Lock file used for upgrading. */ + /** + * Lock file used for upgrading. + */ AutoCloseFD globalLock; struct State { - /* The SQLite database object. */ + /** + * The SQLite database object. + */ SQLite db; struct Stmts; std::unique_ptr stmts; - /* The last time we checked whether to do an auto-GC, or an - auto-GC finished. */ + /** + * The last time we checked whether to do an auto-GC, or an + * auto-GC finished. + */ std::chrono::time_point lastGCCheck; - /* Whether auto-GC is running. If so, get gcFuture to wait for - the GC to finish. */ + /** + * Whether auto-GC is running. If so, get gcFuture to wait for + * the GC to finish. + */ bool gcRunning = false; std::shared_future gcFuture; - /* How much disk space was available after the previous - auto-GC. If the current available disk space is below - minFree but not much below availAfterGC, then there is no - point in starting a new GC. */ + /** + * How much disk space was available after the previous + * auto-GC. If the current available disk space is below + * minFree but not much below availAfterGC, then there is no + * point in starting a new GC. + */ uint64_t availAfterGC = std::numeric_limits::max(); std::unique_ptr publicKeys; @@ -96,11 +111,15 @@ private: public: - // Hack for build-remote.cc. + /** + * Hack for build-remote.cc. + */ PathSet locksHeld; - /* Initialise the local store, upgrading the schema if - necessary. */ + /** + * Initialise the local store, upgrading the schema if + * necessary. + */ LocalStore(const Params & params); LocalStore(std::string scheme, std::string path, const Params & params); @@ -109,7 +128,9 @@ public: static std::set uriSchemes() { return {}; } - /* Implementations of abstract store API methods. */ + /** + * Implementations of abstract store API methods. + */ std::string getUri() override; @@ -133,9 +154,6 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; - void querySubstitutablePathInfos(const StorePathCAMap & paths, - SubstitutablePathInfos & infos) override; - bool pathInfoIsUntrusted(const ValidPathInfo &) override; bool realisationIsUntrusted(const Realisation & ) override; @@ -157,13 +175,19 @@ private: void createTempRootsFile(); - /* The file to which we write our temporary roots. */ + /** + * The file to which we write our temporary roots. + */ Sync _fdTempRoots; - /* The global GC lock. */ + /** + * The global GC lock. + */ Sync _fdGCLock; - /* Connection to the garbage collector. */ + /** + * Connection to the garbage collector. + */ Sync _fdRootsSocket; public: @@ -182,42 +206,54 @@ public: void collectGarbage(const GCOptions & options, GCResults & results) override; - /* Optimise the disk space usage of the Nix store by hard-linking - files with the same contents. */ + /** + * Optimise the disk space usage of the Nix store by hard-linking + * files with the same contents. + */ void optimiseStore(OptimiseStats & stats); void optimiseStore() override; - /* Optimise a single store path. Optionally, test the encountered - symlinks for corruption. */ + /** + * Optimise a single store path. Optionally, test the encountered + * symlinks for corruption. + */ void optimisePath(const Path & path, RepairFlag repair); bool verifyStore(bool checkContents, RepairFlag repair) override; - /* Register the validity of a path, i.e., that `path' exists, that - the paths referenced by it exists, and in the case of an output - path of a derivation, that it has been produced by a successful - execution of the derivation (or something equivalent). Also - register the hash of the file system contents of the path. The - hash must be a SHA-256 hash. */ + /** + * Register the validity of a path, i.e., that `path` exists, that + * the paths referenced by it exists, and in the case of an output + * path of a derivation, that it has been produced by a successful + * execution of the derivation (or something equivalent). Also + * register the hash of the file system contents of the path. The + * hash must be a SHA-256 hash. + */ void registerValidPath(const ValidPathInfo & info); void registerValidPaths(const ValidPathInfos & infos); unsigned int getProtocol() override; + std::optional isTrustedClient() override; + void vacuumDB(); void repairPath(const StorePath & path) override; void addSignatures(const StorePath & storePath, const StringSet & sigs) override; - /* If free disk space in /nix/store if below minFree, delete - garbage until it exceeds maxFree. */ + /** + * If free disk space in /nix/store if below minFree, delete + * garbage until it exceeds maxFree. + */ void autoGC(bool sync = true); - /* Register the store path 'output' as the output named 'outputName' of - derivation 'deriver'. */ + /** + * Register the store path 'output' as the output named 'outputName' of + * derivation 'deriver'. + */ void registerDrvOutput(const Realisation & info) override; void registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) override; void cacheDrvOutputMapping( @@ -247,7 +283,9 @@ private: void invalidatePath(State & state, const StorePath & path); - /* Delete a path from the Nix store. */ + /** + * Delete a path from the Nix store. + */ void invalidatePathChecked(const StorePath & path); void verifyPath(const Path & path, const StringSet & store, @@ -270,8 +308,6 @@ private: std::pair createTempDirInStore(); - void checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv); - typedef std::unordered_set InodeHash; InodeHash loadInodeHash(); @@ -282,8 +318,10 @@ private: bool isValidPath_(State & state, const StorePath & path); void queryReferrers(State & state, const StorePath & path, StorePathSet & referrers); - /* Add signatures to a ValidPathInfo or Realisation using the secret keys - specified by the ‘secret-key-files’ option. */ + /** + * Add signatures to a ValidPathInfo or Realisation using the secret keys + * specified by the ‘secret-key-files’ option. + */ void signPathInfo(ValidPathInfo & info); void signRealisation(Realisation &); @@ -313,18 +351,23 @@ typedef std::pair Inode; typedef std::set InodesSeen; -/* "Fix", or canonicalise, the meta-data of the files in a store path - after it has been built. In particular: - - the last modification date on each file is set to 1 (i.e., - 00:00:01 1/1/1970 UTC) - - the permissions are set of 444 or 555 (i.e., read-only with or - without execute permission; setuid bits etc. are cleared) - - the owner and group are set to the Nix user and group, if we're - running as root. - If uidRange is not empty, this function will throw an error if it - encounters files owned by a user outside of the closed interval - [uidRange->first, uidRange->second]. -*/ +/** + * "Fix", or canonicalise, the meta-data of the files in a store path + * after it has been built. In particular: + * + * - the last modification date on each file is set to 1 (i.e., + * 00:00:01 1/1/1970 UTC) + * + * - the permissions are set of 444 or 555 (i.e., read-only with or + * without execute permission; setuid bits etc. are cleared) + * + * - the owner and group are set to the Nix user and group, if we're + * running as root. + * + * If uidRange is not empty, this function will throw an error if it + * encounters files owned by a user outside of the closed interval + * [uidRange->first, uidRange->second]. + */ void canonicalisePathMetaData( const Path & path, std::optional> uidRange, diff --git a/src/libstore/lock.hh b/src/libstore/lock.hh index 7f1934510..1c268e1fb 100644 --- a/src/libstore/lock.hh +++ b/src/libstore/lock.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" @@ -12,14 +13,18 @@ struct UserLock { virtual ~UserLock() { } - /* Get the first and last UID. */ + /** + * Get the first and last UID. + */ std::pair getUIDRange() { auto first = getUID(); return {first, first + getUIDCount() - 1}; } - /* Get the first UID. */ + /** + * Get the first UID. + */ virtual uid_t getUID() = 0; virtual uid_t getUIDCount() = 0; @@ -29,8 +34,10 @@ struct UserLock virtual std::vector getSupplementaryGIDs() = 0; }; -/* Acquire a user lock for a UID range of size `nrIds`. Note that this - may return nullptr if no user is available. */ +/** + * Acquire a user lock for a UID range of size `nrIds`. Note that this + * may return nullptr if no user is available. + */ std::unique_ptr acquireUserLock(uid_t nrIds, bool useUserNamespace); bool useBuildUsers(); diff --git a/src/libstore/log-store.hh b/src/libstore/log-store.hh index e4d95bab6..a84f7dbeb 100644 --- a/src/libstore/log-store.hh +++ b/src/libstore/log-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" @@ -9,8 +10,10 @@ struct LogStore : public virtual Store { inline static std::string operationName = "Build log storage and retrieval"; - /* Return the build log of the specified store path, if available, - or null otherwise. */ + /** + * Return the build log of the specified store path, if available, + * or null otherwise. + */ std::optional getBuildLog(const StorePath & path); virtual std::optional getBuildLogExact(const StorePath & path) = 0; diff --git a/src/libstore/machines.hh b/src/libstore/machines.hh index 834626de9..1adeaf1f0 100644 --- a/src/libstore/machines.hh +++ b/src/libstore/machines.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" diff --git a/src/libstore/make-content-addressed.hh b/src/libstore/make-content-addressed.hh index c4a66ed41..2ce6ec7bc 100644 --- a/src/libstore/make-content-addressed.hh +++ b/src/libstore/make-content-addressed.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" diff --git a/src/libstore/names.hh b/src/libstore/names.hh index 3977fc6cc..d82b99bb4 100644 --- a/src/libstore/names.hh +++ b/src/libstore/names.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 9a0003588..f0dfcb19b 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -275,6 +275,7 @@ json listNar(ref accessor, const Path & path, bool recurse) obj["type"] = "symlink"; obj["target"] = accessor->readLink(path); break; + case FSAccessor::Type::tMissing: default: throw Error("path '%s' does not exist in NAR", path); } diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh index 7d998ae0b..5e19bd3c7 100644 --- a/src/libstore/nar-accessor.hh +++ b/src/libstore/nar-accessor.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include @@ -9,24 +10,30 @@ namespace nix { struct Source; -/* Return an object that provides access to the contents of a NAR - file. */ +/** + * Return an object that provides access to the contents of a NAR + * file. + */ ref makeNarAccessor(std::string && nar); ref makeNarAccessor(Source & source); -/* Create a NAR accessor from a NAR listing (in the format produced by - listNar()). The callback getNarBytes(offset, length) is used by the - readFile() method of the accessor to get the contents of files - inside the NAR. */ +/** + * Create a NAR accessor from a NAR listing (in the format produced by + * listNar()). The callback getNarBytes(offset, length) is used by the + * readFile() method of the accessor to get the contents of files + * inside the NAR. + */ typedef std::function GetNarBytes; ref makeLazyNarAccessor( const std::string & listing, GetNarBytes getNarBytes); -/* Write a JSON representation of the contents of a NAR (except file - contents). */ +/** + * Write a JSON representation of the contents of a NAR (except file + * contents). + */ nlohmann::json listNar(ref accessor, const Path & path, bool recurse); } diff --git a/src/libstore/nar-info-disk-cache.hh b/src/libstore/nar-info-disk-cache.hh index 4877f56d8..bbd1d05d5 100644 --- a/src/libstore/nar-info-disk-cache.hh +++ b/src/libstore/nar-info-disk-cache.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "ref.hh" #include "nar-info.hh" @@ -42,8 +43,10 @@ public: const std::string & uri, const DrvOutput & id) = 0; }; -/* Return a singleton cache object that can be used concurrently by - multiple threads. */ +/** + * Return a singleton cache object that can be used concurrently by + * multiple threads. + */ ref getNarInfoDiskCache(); ref getTestNarInfoDiskCache(Path dbPath); diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index a4dccb397..5dbdafac3 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "hash.hh" diff --git a/src/libstore/outputs-spec.hh b/src/libstore/outputs-spec.hh index 0b7c98ac9..5a726fe90 100644 --- a/src/libstore/outputs-spec.hh +++ b/src/libstore/outputs-spec.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index bfb3857c0..71085a604 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "derivations.hh" #include "store-api.hh" diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 97eb6638b..221523622 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "crypto.hh" #include "path.hh" @@ -18,8 +19,14 @@ struct SubstitutablePathInfo { std::optional deriver; StorePathSet references; - uint64_t downloadSize; /* 0 = unknown or inapplicable */ - uint64_t narSize; /* 0 = unknown */ + /** + * 0 = unknown or inapplicable + */ + uint64_t downloadSize; + /** + * 0 = unknown + */ + uint64_t narSize; }; typedef std::map SubstitutablePathInfos; @@ -29,35 +36,40 @@ struct ValidPathInfo { StorePath path; std::optional deriver; - // TODO document this + /** + * \todo document this + */ Hash narHash; StorePathSet references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown uint64_t id; // internal use only - /* Whether the path is ultimately trusted, that is, it's a - derivation output that was built locally. */ + /** + * Whether the path is ultimately trusted, that is, it's a + * derivation output that was built locally. + */ bool ultimate = false; StringSet sigs; // note: not necessarily verified - /* If non-empty, an assertion that the path is content-addressed, - i.e., that the store path is computed from a cryptographic hash - of the contents of the path, plus some other bits of data like - the "name" part of the path. Such a path doesn't need - signatures, since we don't have to trust anybody's claim that - the path is the output of a particular derivation. (In the - extensional store model, we have to trust that the *contents* - of an output path of a derivation were actually produced by - that derivation. In the intensional model, we have to trust - that a particular output path was produced by a derivation; the - path then implies the contents.) - - Ideally, the content-addressability assertion would just be a Boolean, - and the store path would be computed from the name component, ‘narHash’ - and ‘references’. However, we support many types of content addresses. - */ + /** + * If non-empty, an assertion that the path is content-addressed, + * i.e., that the store path is computed from a cryptographic hash + * of the contents of the path, plus some other bits of data like + * the "name" part of the path. Such a path doesn't need + * signatures, since we don't have to trust anybody's claim that + * the path is the output of a particular derivation. (In the + * extensional store model, we have to trust that the *contents* + * of an output path of a derivation were actually produced by + * that derivation. In the intensional model, we have to trust + * that a particular output path was produced by a derivation; the + * path then implies the contents.) + * + * Ideally, the content-addressability assertion would just be a Boolean, + * and the store path would be computed from the name component, ‘narHash’ + * and ‘references’. However, we support many types of content addresses. + */ std::optional ca; bool operator == (const ValidPathInfo & i) const @@ -68,29 +80,42 @@ struct ValidPathInfo && references == i.references; } - /* Return a fingerprint of the store path to be used in binary - cache signatures. It contains the store path, the base-32 - SHA-256 hash of the NAR serialisation of the path, the size of - the NAR, and the sorted references. The size field is strictly - speaking superfluous, but might prevent endless/excessive data - attacks. */ + /** + * Return a fingerprint of the store path to be used in binary + * cache signatures. It contains the store path, the base-32 + * SHA-256 hash of the NAR serialisation of the path, the size of + * the NAR, and the sorted references. The size field is strictly + * speaking superfluous, but might prevent endless/excessive data + * attacks. + */ std::string fingerprint(const Store & store) const; void sign(const Store & store, const SecretKey & secretKey); + /** + * @return The `ContentAddressWithReferences` that determines the + * store path for a content-addressed store object, `std::nullopt` + * for an input-addressed store object. + */ std::optional contentAddressWithReferences() const; - /* Return true iff the path is verifiably content-addressed. */ + /** + * @return true iff the path is verifiably content-addressed. + */ bool isContentAddressed(const Store & store) const; static const size_t maxSigs = std::numeric_limits::max(); - /* Return the number of signatures on this .narinfo that were - produced by one of the specified keys, or maxSigs if the path - is content-addressed. */ + /** + * Return the number of signatures on this .narinfo that were + * produced by one of the specified keys, or maxSigs if the path + * is content-addressed. + */ size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const; - /* Verify a single signature. */ + /** + * Verify a single signature. + */ bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const; Strings shortRefs() const; diff --git a/src/libstore/path-regex.hh b/src/libstore/path-regex.hh index 6893c3876..4f8dc4c1f 100644 --- a/src/libstore/path-regex.hh +++ b/src/libstore/path-regex.hh @@ -1,4 +1,5 @@ #pragma once +///@file namespace nix { diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index 5d25656a5..d75850868 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -1,17 +1,19 @@ #pragma once +///@file #include "path.hh" #include "derived-path.hh" namespace nix { -/* This is a deprecated old type just for use by the old CLI, and older - versions of the RPC protocols. In new code don't use it; you want - `DerivedPath` instead. - - `DerivedPath` is better because it handles more cases, and does so more - explicitly without devious punning tricks. -*/ +/** + * This is a deprecated old type just for use by the old CLI, and older + * versions of the RPC protocols. In new code don't use it; you want + * `DerivedPath` instead. + * + * `DerivedPath` is better because it handles more cases, and does so more + * explicitly without devious punning tricks. + */ struct StorePathWithOutputs { StorePath path; @@ -30,9 +32,11 @@ std::pair parsePathWithOutputs(std::string_view s); class Store; -/* Split a string specifying a derivation and a set of outputs - (/nix/store/hash-foo!out1,out2,...) into the derivation path - and the outputs. */ +/** + * Split a string specifying a derivation and a set of outputs + * (/nix/store/hash-foo!out1,out2,...) into the derivation path + * and the outputs. + */ StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs); StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs); diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 2730541c6..4ca6747b3 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index 5e3a734b4..4921df352 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -1,15 +1,20 @@ #pragma once +///@file #include "util.hh" namespace nix { -/* Open (possibly create) a lock file and return the file descriptor. - -1 is returned if create is false and the lock could not be opened - because it doesn't exist. Any other error throws an exception. */ +/** + * Open (possibly create) a lock file and return the file descriptor. + * -1 is returned if create is false and the lock could not be opened + * because it doesn't exist. Any other error throws an exception. + */ AutoCloseFD openLockFile(const Path & path, bool create); -/* Delete an open lock file. */ +/** + * Delete an open lock file. + */ void deleteLockFile(const Path & path, int fd); enum LockType { ltRead, ltWrite, ltNone }; diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh index 3cadd5c2a..4e1f42e83 100644 --- a/src/libstore/profiles.hh +++ b/src/libstore/profiles.hh @@ -1,6 +1,7 @@ #pragma once +///@file -#include "types.hh" + #include "types.hh" #include "pathlocks.hh" #include @@ -23,9 +24,11 @@ struct Generation typedef std::list Generations; -/* Returns the list of currently present generations for the specified - profile, sorted by generation number. Also returns the number of - the current generation. */ +/** + * Returns the list of currently present generations for the specified + * profile, sorted by generation number. Also returns the number of + * the current generation. + */ std::pair> findGenerations(Path profile); class LocalFSStore; @@ -46,26 +49,32 @@ void deleteGenerationsOlderThan(const Path & profile, std::string_view timeSpec, void switchLink(Path link, Path target); -/* Roll back a profile to the specified generation, or to the most - recent one older than the current. */ +/** + * Roll back a profile to the specified generation, or to the most + * recent one older than the current. + */ void switchGeneration( const Path & profile, std::optional dstGen, bool dryRun); -/* Ensure exclusive access to a profile. Any command that modifies - the profile first acquires this lock. */ +/** + * Ensure exclusive access to a profile. Any command that modifies + * the profile first acquires this lock. + */ void lockProfile(PathLocks & lock, const Path & profile); -/* Optimistic locking is used by long-running operations like `nix-env - -i'. Instead of acquiring the exclusive lock for the entire - duration of the operation, we just perform the operation - optimistically (without an exclusive lock), and check at the end - whether the profile changed while we were busy (i.e., the symlink - target changed). If so, the operation is restarted. Restarting is - generally cheap, since the build results are still in the Nix - store. Most of the time, only the user environment has to be - rebuilt. */ +/** + * Optimistic locking is used by long-running operations like `nix-env + * -i'. Instead of acquiring the exclusive lock for the entire + * duration of the operation, we just perform the operation + * optimistically (without an exclusive lock), and check at the end + * whether the profile changed while we were busy (i.e., the symlink + * target changed). If so, the operation is restarted. Restarting is + * generally cheap, since the build results are still in the Nix + * store. Most of the time, only the user environment has to be + * rebuilt. + */ std::string optimisticLockProfile(const Path & profile); /** diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 48d0283de..a18cf2aa8 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include diff --git a/src/libstore/references.hh b/src/libstore/references.hh index 6f381f96c..52d71b333 100644 --- a/src/libstore/references.hh +++ b/src/libstore/references.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "hash.hh" #include "path.hh" diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh index 99f5544ef..e2673b6f6 100644 --- a/src/libstore/remote-fs-accessor.hh +++ b/src/libstore/remote-fs-accessor.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "fs-accessor.hh" #include "ref.hh" diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 1c6b8530d..2abd3aa51 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -42,6 +42,40 @@ void write(const Store & store, Sink & out, const StorePath & storePath) } +std::optional read(const Store & store, Source & from, Phantom> _) +{ + auto temp = readNum(from); + switch (temp) { + case 0: + return std::nullopt; + case 1: + return { Trusted }; + case 2: + return { NotTrusted }; + default: + throw Error("Invalid trusted status from remote"); + } +} + +void write(const Store & store, Sink & out, const std::optional & optTrusted) +{ + if (!optTrusted) + out << (uint8_t)0; + else { + switch (*optTrusted) { + case Trusted: + out << (uint8_t)1; + break; + case NotTrusted: + out << (uint8_t)2; + break; + default: + assert(false); + }; + } +} + + ContentAddress read(const Store & store, Source & from, Phantom _) { return ContentAddress::parse(readString(from)); @@ -56,12 +90,12 @@ void write(const Store & store, Sink & out, const ContentAddress & ca) DerivedPath read(const Store & store, Source & from, Phantom _) { auto s = readString(from); - return DerivedPath::parse(store, s); + return DerivedPath::parseLegacy(store, s); } void write(const Store & store, Sink & out, const DerivedPath & req) { - out << req.to_string(store); + out << req.to_string_legacy(store); } @@ -226,6 +260,13 @@ void RemoteStore::initConnection(Connection & conn) conn.daemonNixVersion = readString(conn.from); } + if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 35) { + conn.remoteTrustsUs = worker_proto::read(*this, conn.from, Phantom> {}); + } else { + // We don't know the answer; protocol to old. + conn.remoteTrustsUs = std::nullopt; + } + auto ex = conn.processStderr(); if (ex) std::rethrow_exception(ex); } @@ -1086,6 +1127,11 @@ unsigned int RemoteStore::getProtocol() return conn->daemonVersion; } +std::optional RemoteStore::isTrustedClient() +{ + auto conn(getConnection()); + return conn->remoteTrustsUs; +} void RemoteStore::flushBadConnections() { diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 3ff22ed66..3babf8e21 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -31,8 +32,10 @@ struct RemoteStoreConfig : virtual StoreConfig "Maximum age of a connection before it is closed."}; }; -/* FIXME: RemoteStore is a misnomer - should be something like - DaemonStore. */ +/** + * \todo RemoteStore is a misnomer - should be something like + * DaemonStore. + */ class RemoteStore : public virtual RemoteStoreConfig, public virtual Store, public virtual GcStore, @@ -68,7 +71,9 @@ public: void querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) override; - /* Add a content-addressable store path. `dump` will be drained. */ + /** + * Add a content-addressable store path. `dump` will be drained. + */ ref addCAToStore( Source & dump, std::string_view name, @@ -77,7 +82,9 @@ public: const StorePathSet & references, RepairFlag repair); - /* Add a content-addressable store path. Does not support references. `dump` will be drained. */ + /** + * Add a content-addressable store path. Does not support references. `dump` will be drained. + */ StorePath addToStoreFromDump(Source & dump, std::string_view name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override; @@ -144,6 +151,8 @@ public: unsigned int getProtocol() override; + std::optional isTrustedClient() override; + void flushBadConnections(); struct Connection @@ -151,6 +160,7 @@ public: FdSink to; FdSource from; unsigned int daemonVersion; + std::optional remoteTrustsUs; std::optional daemonNixVersion; std::chrono::time_point startTime; diff --git a/src/libstore/repair-flag.hh b/src/libstore/repair-flag.hh index a13cda312..f412d6a20 100644 --- a/src/libstore/repair-flag.hh +++ b/src/libstore/repair-flag.hh @@ -1,4 +1,5 @@ #pragma once +///@file namespace nix { diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index ac82147ee..d2fc6abaf 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -509,6 +509,16 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual return paths; } + /** + * For now, we conservatively say we don't know. + * + * \todo try to expose our S3 authentication status. + */ + std::optional isTrustedClient() override + { + return std::nullopt; + } + static std::set uriSchemes() { return {"s3"}; } }; diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index bce828b11..c62ea5147 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "binary-cache-store.hh" diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh index cdb3e5908..f0aeb3bed 100644 --- a/src/libstore/s3.hh +++ b/src/libstore/s3.hh @@ -1,4 +1,5 @@ #pragma once +///@file #if ENABLE_S3 diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index 3f76baa82..553fd3a09 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -1,4 +1,5 @@ #pragma once +///@file namespace nix { diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 871f2f3be..df334c23c 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -239,14 +239,11 @@ SQLiteTxn::~SQLiteTxn() } } -void handleSQLiteBusy(const SQLiteBusy & e) +void handleSQLiteBusy(const SQLiteBusy & e, time_t & nextWarning) { - static std::atomic lastWarned{0}; - time_t now = time(0); - - if (now > lastWarned + 10) { - lastWarned = now; + if (now > nextWarning) { + nextWarning = now + 10; logWarning({ .msg = hintfmt(e.what()) }); diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index 1853731a2..6e14852cb 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -10,7 +11,9 @@ struct sqlite3_stmt; namespace nix { -/* RAII wrapper to close a SQLite database automatically. */ +/** + * RAII wrapper to close a SQLite database automatically. + */ struct SQLite { sqlite3 * db = 0; @@ -22,7 +25,9 @@ struct SQLite ~SQLite(); operator sqlite3 * () { return db; } - /* Disable synchronous mode, set truncate journal mode. */ + /** + * Disable synchronous mode, set truncate journal mode. + */ void isCache(); void exec(const std::string & stmt); @@ -30,7 +35,9 @@ struct SQLite uint64_t getLastInsertedRowId(); }; -/* RAII wrapper to create and destroy SQLite prepared statements. */ +/** + * RAII wrapper to create and destroy SQLite prepared statements. + */ struct SQLiteStmt { sqlite3 * db = 0; @@ -42,7 +49,9 @@ struct SQLiteStmt ~SQLiteStmt(); operator sqlite3_stmt * () { return stmt; } - /* Helper for binding / executing statements. */ + /** + * Helper for binding / executing statements. + */ class Use { friend struct SQLiteStmt; @@ -55,7 +64,9 @@ struct SQLiteStmt ~Use(); - /* Bind the next parameter. */ + /** + * Bind the next parameter. + */ Use & operator () (std::string_view value, bool notNull = true); Use & operator () (const unsigned char * data, size_t len, bool notNull = true); Use & operator () (int64_t value, bool notNull = true); @@ -63,11 +74,15 @@ struct SQLiteStmt int step(); - /* Execute a statement that does not return rows. */ + /** + * Execute a statement that does not return rows. + */ void exec(); - /* For statements that return 0 or more rows. Returns true iff - a row is available. */ + /** + * For statements that return 0 or more rows. Returns true iff + * a row is available. + */ bool next(); std::string getStr(int col); @@ -81,8 +96,10 @@ struct SQLiteStmt } }; -/* RAII helper that ensures transactions are aborted unless explicitly - committed. */ +/** + * RAII helper that ensures transactions are aborted unless explicitly + * committed. + */ struct SQLiteTxn { bool active = false; @@ -122,18 +139,22 @@ protected: MakeError(SQLiteBusy, SQLiteError); -void handleSQLiteBusy(const SQLiteBusy & e); +void handleSQLiteBusy(const SQLiteBusy & e, time_t & nextWarning); -/* Convenience function for retrying a SQLite transaction when the - database is busy. */ +/** + * Convenience function for retrying a SQLite transaction when the + * database is busy. + */ template T retrySQLite(F && fun) { + time_t nextWarning = time(0) + 1; + while (true) { try { return fun(); } catch (SQLiteBusy & e) { - handleSQLiteBusy(e); + handleSQLiteBusy(e, nextWarning); } } } diff --git a/src/libstore/ssh-store-config.hh b/src/libstore/ssh-store-config.hh index c4232df34..c27a5d00f 100644 --- a/src/libstore/ssh-store-config.hh +++ b/src/libstore/ssh-store-config.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include "store-api.hh" namespace nix { diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 69bfe3418..6f6deda51 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -1,4 +1,5 @@ #include "ssh.hh" +#include "finally.hh" namespace nix { @@ -35,6 +36,9 @@ void SSHMaster::addCommonSSHOpts(Strings & args) } if (compress) args.push_back("-C"); + + args.push_back("-oPermitLocalCommand=yes"); + args.push_back("-oLocalCommand=echo started"); } std::unique_ptr SSHMaster::startCommand(const std::string & command) @@ -49,6 +53,11 @@ std::unique_ptr SSHMaster::startCommand(const std::string ProcessOptions options; options.dieWithParent = false; + if (!fakeSSH && !useMaster) { + logger->pause(); + } + Finally cleanup = [&]() { logger->resume(); }; + conn->sshPid = startProcess([&]() { restoreProcessContext(); @@ -86,6 +95,18 @@ std::unique_ptr SSHMaster::startCommand(const std::string in.readSide = -1; out.writeSide = -1; + // Wait for the SSH connection to be established, + // So that we don't overwrite the password prompt with our progress bar. + if (!fakeSSH && !useMaster) { + std::string reply; + try { + reply = readLine(out.readSide.get()); + } catch (EndOfFile & e) { } + + if (reply != "started") + throw Error("failed to start SSH connection to '%s'", host); + } + conn->out = std::move(out.readSide); conn->in = std::move(in.writeSide); @@ -109,6 +130,9 @@ Path SSHMaster::startMaster() ProcessOptions options; options.dieWithParent = false; + logger->pause(); + Finally cleanup = [&]() { logger->resume(); }; + state->sshMaster = startProcess([&]() { restoreProcessContext(); @@ -117,11 +141,7 @@ Path SSHMaster::startMaster() if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1) throw SysError("duping over stdout"); - Strings args = - { "ssh", host.c_str(), "-M", "-N", "-S", state->socketPath - , "-o", "LocalCommand=echo started" - , "-o", "PermitLocalCommand=yes" - }; + Strings args = { "ssh", host.c_str(), "-M", "-N", "-S", state->socketPath }; if (verbosity >= lvlChatty) args.push_back("-v"); addCommonSSHOpts(args); diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh index dabbcedda..c86a8a986 100644 --- a/src/libstore/ssh.hh +++ b/src/libstore/ssh.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "util.hh" #include "sync.hh" diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 78b0d907e..5bee1af9f 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -527,6 +527,57 @@ StorePathSet Store::queryDerivationOutputs(const StorePath & path) return outputPaths; } + +void Store::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) +{ + if (!settings.useSubstitutes) return; + for (auto & sub : getDefaultSubstituters()) { + for (auto & path : paths) { + if (infos.count(path.first)) + // Choose first succeeding substituter. + continue; + + auto subPath(path.first); + + // Recompute store path so that we can use a different store root. + if (path.second) { + subPath = makeFixedOutputPathFromCA( + path.first.name(), + ContentAddressWithReferences::withoutRefs(*path.second)); + if (sub->storeDir == storeDir) + assert(subPath == path.first); + if (subPath != path.first) + debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path.first), sub->printStorePath(subPath), sub->getUri()); + } else if (sub->storeDir != storeDir) continue; + + debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath)); + try { + auto info = sub->queryPathInfo(subPath); + + if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty())) + continue; + + auto narInfo = std::dynamic_pointer_cast( + std::shared_ptr(info)); + infos.insert_or_assign(path.first, SubstitutablePathInfo{ + .deriver = info->deriver, + .references = info->references, + .downloadSize = narInfo ? narInfo->fileSize : 0, + .narSize = info->narSize, + }); + } catch (InvalidPath &) { + } catch (SubstituterDisabled &) { + } catch (Error & e) { + if (settings.tryFallback) + logError(e.info()); + else + throw; + } + } + } +} + + bool Store::isValidPath(const StorePath & storePath) { { @@ -1125,7 +1176,8 @@ std::map copyPaths( return storePathForDst; }; - uint64_t total = 0; + // total is accessed by each copy, which are each handled in separate threads + std::atomic total = 0; for (auto & missingPath : sortedMissing) { auto info = srcStore.queryPathInfo(missingPath); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 5edcc0f36..74f50a00d 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "nar-info.hh" #include "realisation.hh" @@ -88,6 +89,7 @@ const uint32_t exportMagic = 0x4558494e; enum BuildMode { bmNormal, bmRepair, bmCheck }; +enum TrustedFlag : bool { NotTrusted = false, Trusted = true }; struct BuildResult; @@ -403,17 +405,17 @@ public: { unsupported("queryReferrers"); } /** - * @return all currently valid derivations that have `path' as an + * @return all currently valid derivations that have `path` as an * output. * - * (Note that the result of `queryDeriver()' is the derivation that - * was actually used to produce `path', which may not exist + * (Note that the result of `queryDeriver()` is the derivation that + * was actually used to produce `path`, which may not exist * anymore.) */ virtual StorePathSet queryValidDerivers(const StorePath & path) { return {}; }; /** - * Query the outputs of the derivation denoted by `path'. + * Query the outputs of the derivation denoted by `path`. */ virtual StorePathSet queryDerivationOutputs(const StorePath & path); @@ -449,7 +451,7 @@ public: * resulting ‘infos’ map. */ virtual void querySubstitutablePathInfos(const StorePathCAMap & paths, - SubstitutablePathInfos & infos) { return; }; + SubstitutablePathInfos & infos); /** * Import a path into the store. @@ -505,7 +507,7 @@ public: /** * Like addToStore(), but the contents of the path are contained - * in `dump', which is either a NAR serialisation (if recursive == + * in `dump`, which is either a NAR serialisation (if recursive == * true) or simply the contents of a regular file (if recursive == * false). * `dump` may be drained @@ -626,8 +628,8 @@ public: /** * @return a string representing information about the path that - * can be loaded into the database using `nix-store --load-db' or - * `nix-store --register-validity'. + * can be loaded into the database using `nix-store --load-db` or + * `nix-store --register-validity`. */ std::string makeValidityRegistration(const StorePathSet & paths, bool showDerivers, bool showHash); @@ -670,8 +672,7 @@ public: /** * @return An object to access files in the Nix store. */ - virtual ref getFSAccessor() - { unsupported("getFSAccessor"); } + virtual ref getFSAccessor() = 0; /** * Repair the contents of the given path by redownloading it using @@ -707,12 +708,12 @@ public: /** * @param [out] out Place in here the set of all store paths in the - * file system closure of `storePath'; that is, all paths than can - * be directly or indirectly reached from it. `out' is not cleared. + * file system closure of `storePath`; that is, all paths than can + * be directly or indirectly reached from it. `out` is not cleared. * * @param flipDirection If true, the set of paths that can reach - * `storePath' is returned; that is, the closures under the - * `referrers' relation instead of the `references' relation is + * `storePath` is returned; that is, the closures under the + * `referrers` relation instead of the `references` relation is * returned. */ virtual void computeFSClosure(const StorePathSet & paths, @@ -808,6 +809,17 @@ public: return 0; }; + /** + * @return/ whether store trusts *us*. + * + * `std::nullopt` means we do not know. + * + * @note This is the opposite of the StoreConfig::isTrusted + * store setting. That is about whether *we* trust the store. + */ + virtual std::optional isTrustedClient() = 0; + + virtual Path toRealPath(const Path & storePath) { return storePath; diff --git a/src/libstore/store-cast.hh b/src/libstore/store-cast.hh index ff62fc359..2473e72c5 100644 --- a/src/libstore/store-cast.hh +++ b/src/libstore/store-cast.hh @@ -1,9 +1,17 @@ #pragma once +///@file #include "store-api.hh" namespace nix { +/** + * Helper to try downcasting a Store with a nice method if it fails. + * + * This is basically an alternative to the user-facing part of + * Store::unsupported that allows us to still have a nice message but + * better interface design. + */ template T & require(Store & store) { diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc index 4dd14dcce..1cab68e06 100644 --- a/src/libstore/tests/derivation.cc +++ b/src/libstore/tests/derivation.cc @@ -11,15 +11,29 @@ class DerivationTest : public LibStoreTest { }; -#define TEST_JSON(TYPE, NAME, STR, VAL, ...) \ - TEST_F(DerivationTest, TYPE ## _ ## NAME ## _to_json) { \ - using nlohmann::literals::operator "" _json; \ - ASSERT_EQ( \ - STR ## _json, \ - (TYPE { VAL }).toJSON(*store __VA_OPT__(,) __VA_ARGS__)); \ +#define TEST_JSON(NAME, STR, VAL, DRV_NAME, OUTPUT_NAME) \ + TEST_F(DerivationTest, DerivationOutput_ ## NAME ## _to_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + STR ## _json, \ + (DerivationOutput { VAL }).toJSON( \ + *store, \ + DRV_NAME, \ + OUTPUT_NAME)); \ + } \ + \ + TEST_F(DerivationTest, DerivationOutput_ ## NAME ## _from_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + DerivationOutput { VAL }, \ + DerivationOutput::fromJSON( \ + *store, \ + DRV_NAME, \ + OUTPUT_NAME, \ + STR ## _json)); \ } -TEST_JSON(DerivationOutput, inputAddressed, +TEST_JSON(inputAddressed, R"({ "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" })", @@ -28,7 +42,7 @@ TEST_JSON(DerivationOutput, inputAddressed, }), "drv-name", "output-name") -TEST_JSON(DerivationOutput, caFixed, +TEST_JSON(caFixed, R"({ "hashAlgo": "r:sha256", "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", @@ -45,7 +59,7 @@ TEST_JSON(DerivationOutput, caFixed, }), "drv-name", "output-name") -TEST_JSON(DerivationOutput, caFixedText, +TEST_JSON(caFixedText, R"({ "hashAlgo": "text:sha256", "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", @@ -61,7 +75,7 @@ TEST_JSON(DerivationOutput, caFixedText, }), "drv-name", "output-name") -TEST_JSON(DerivationOutput, caFloating, +TEST_JSON(caFloating, R"({ "hashAlgo": "r:sha256" })", @@ -71,12 +85,12 @@ TEST_JSON(DerivationOutput, caFloating, }), "drv-name", "output-name") -TEST_JSON(DerivationOutput, deferred, +TEST_JSON(deferred, R"({ })", DerivationOutput::Deferred { }, "drv-name", "output-name") -TEST_JSON(DerivationOutput, impure, +TEST_JSON(impure, R"({ "hashAlgo": "r:sha256", "impure": true @@ -87,8 +101,28 @@ TEST_JSON(DerivationOutput, impure, }), "drv-name", "output-name") -TEST_JSON(Derivation, impure, +#undef TEST_JSON + +#define TEST_JSON(NAME, STR, VAL, DRV_NAME) \ + TEST_F(DerivationTest, Derivation_ ## NAME ## _to_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + STR ## _json, \ + (Derivation { VAL }).toJSON(*store)); \ + } \ + \ + TEST_F(DerivationTest, Derivation_ ## NAME ## _from_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + Derivation { VAL }, \ + Derivation::fromJSON( \ + *store, \ + STR ## _json)); \ + } + +TEST_JSON(simple, R"({ + "name": "my-derivation", "inputSrcs": [ "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" ], @@ -111,6 +145,7 @@ TEST_JSON(Derivation, impure, })", ({ Derivation drv; + drv.name = "my-derivation"; drv.inputSrcs = { store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), }; @@ -136,7 +171,8 @@ TEST_JSON(Derivation, impure, }, }; drv; - })) + }), + "drv-name") #undef TEST_JSON diff --git a/src/libstore/tests/derived-path.cc b/src/libstore/tests/derived-path.cc index d1ac2c5e7..e6d32dbd0 100644 --- a/src/libstore/tests/derived-path.cc +++ b/src/libstore/tests/derived-path.cc @@ -51,6 +51,14 @@ TEST_F(DerivedPathTest, force_init) { } +RC_GTEST_FIXTURE_PROP( + DerivedPathTest, + prop_legacy_round_rip, + (const DerivedPath & o)) +{ + RC_ASSERT(o == DerivedPath::parseLegacy(*store, o.to_string_legacy(*store))); +} + RC_GTEST_FIXTURE_PROP( DerivedPathTest, prop_round_rip, diff --git a/src/libstore/tests/derived-path.hh b/src/libstore/tests/derived-path.hh index 3bc812440..506f3ccb1 100644 --- a/src/libstore/tests/derived-path.hh +++ b/src/libstore/tests/derived-path.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include diff --git a/src/libstore/tests/libstore.hh b/src/libstore/tests/libstore.hh index 05397659b..ef93457b5 100644 --- a/src/libstore/tests/libstore.hh +++ b/src/libstore/tests/libstore.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include #include diff --git a/src/libstore/tests/outputs-spec.hh b/src/libstore/tests/outputs-spec.hh index 2d455c817..ded331b33 100644 --- a/src/libstore/tests/outputs-spec.hh +++ b/src/libstore/tests/outputs-spec.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include diff --git a/src/libstore/tests/path.hh b/src/libstore/tests/path.hh index d7f1a8988..21cb62310 100644 --- a/src/libstore/tests/path.hh +++ b/src/libstore/tests/path.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index caa452919..bd1dcb67c 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "remote-store.hh" #include "local-fs-store.hh" diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 87088a3ac..c7a6f8688 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" #include "serialise.hh" @@ -9,11 +10,15 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION (1 << 8 | 34) +#define PROTOCOL_VERSION (1 << 8 | 35) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) +/** + * Enumeration of all the request types for the "worker protocol", used + * by unix:// and ssh-ng:// stores. + */ typedef enum { wopIsValidPath = 1, wopHasSubstitutes = 3, @@ -74,7 +79,12 @@ typedef enum { class Store; struct Source; -/* To guide overloading */ +/** + * Used to guide overloading + * + * See https://en.cppreference.com/w/cpp/language/adl for the broader + * concept of what is going on here. + */ template struct Phantom {}; @@ -93,6 +103,7 @@ MAKE_WORKER_PROTO(, DerivedPath); MAKE_WORKER_PROTO(, Realisation); MAKE_WORKER_PROTO(, DrvOutput); MAKE_WORKER_PROTO(, BuildResult); +MAKE_WORKER_PROTO(, std::optional); MAKE_WORKER_PROTO(template, std::vector); MAKE_WORKER_PROTO(template, std::set); @@ -103,18 +114,19 @@ MAKE_WORKER_PROTO(X_, Y_); #undef X_ #undef Y_ -/* These use the empty string for the null case, relying on the fact - that the underlying types never serialize to the empty string. - - We do this instead of a generic std::optional instance because - ordinal tags (0 or 1, here) are a bit of a compatability hazard. For - the same reason, we don't have a std::variant instances (ordinal - tags 0...n). - - We could the generic instances and then these as specializations for - compatability, but that's proven a bit finnicky, and also makes the - worker protocol harder to implement in other languages where such - specializations may not be allowed. +/** + * These use the empty string for the null case, relying on the fact + * that the underlying types never serialize to the empty string. + * + * We do this instead of a generic std::optional instance because + * ordinal tags (0 or 1, here) are a bit of a compatability hazard. For + * the same reason, we don't have a std::variant instances (ordinal + * tags 0...n). + * + * We could the generic instances and then these as specializations for + * compatability, but that's proven a bit finnicky, and also makes the + * worker protocol harder to implement in other languages where such + * specializations may not be allowed. */ MAKE_WORKER_PROTO(, std::optional); MAKE_WORKER_PROTO(, std::optional); diff --git a/src/libutil/abstract-setting-to-json.hh b/src/libutil/abstract-setting-to-json.hh index 2d82b54e7..7b6c3fcb5 100644 --- a/src/libutil/abstract-setting-to-json.hh +++ b/src/libutil/abstract-setting-to-json.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include "config.hh" diff --git a/src/libutil/ansicolor.hh b/src/libutil/ansicolor.hh index 38305e71c..86becafa6 100644 --- a/src/libutil/ansicolor.hh +++ b/src/libutil/ansicolor.hh @@ -1,8 +1,12 @@ #pragma once +/** + * @file + * + * @brief Some ANSI escape sequences. + */ namespace nix { -/* Some ANSI escape sequences. */ #define ANSI_NORMAL "\e[0m" #define ANSI_BOLD "\e[1m" #define ANSI_FAINT "\e[2m" diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index e42dea540..2cf164a41 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "serialise.hh" @@ -7,54 +8,73 @@ namespace nix { -/* dumpPath creates a Nix archive of the specified path. The format - is as follows: - - IF path points to a REGULAR FILE: - dump(path) = attrs( - [ ("type", "regular") - , ("contents", contents(path)) - ]) - - IF path points to a DIRECTORY: - dump(path) = attrs( - [ ("type", "directory") - , ("entries", concat(map(f, sort(entries(path))))) - ]) - where f(fn) = attrs( - [ ("name", fn) - , ("file", dump(path + "/" + fn)) - ]) - - where: - - attrs(as) = concat(map(attr, as)) + encN(0) - attrs((a, b)) = encS(a) + encS(b) - - encS(s) = encN(len(s)) + s + (padding until next 64-bit boundary) - - encN(n) = 64-bit little-endian encoding of n. - - contents(path) = the contents of a regular file. - - sort(strings) = lexicographic sort by 8-bit value (strcmp). - - entries(path) = the entries of a directory, without `.' and - `..'. - - `+' denotes string concatenation. */ - - +/** + * dumpPath creates a Nix archive of the specified path. + * + * @param path the file system data to dump. Dumping is recursive so if + * this is a directory we dump it and all its children. + * + * @param [out] sink The serialised archive is fed into this sink. + * + * @param filter Can be used to skip certain files. + * + * The format is as follows: + * + * ``` + * IF path points to a REGULAR FILE: + * dump(path) = attrs( + * [ ("type", "regular") + * , ("contents", contents(path)) + * ]) + * + * IF path points to a DIRECTORY: + * dump(path) = attrs( + * [ ("type", "directory") + * , ("entries", concat(map(f, sort(entries(path))))) + * ]) + * where f(fn) = attrs( + * [ ("name", fn) + * , ("file", dump(path + "/" + fn)) + * ]) + * + * where: + * + * attrs(as) = concat(map(attr, as)) + encN(0) + * attrs((a, b)) = encS(a) + encS(b) + * + * encS(s) = encN(len(s)) + s + (padding until next 64-bit boundary) + * + * encN(n) = 64-bit little-endian encoding of n. + * + * contents(path) = the contents of a regular file. + * + * sort(strings) = lexicographic sort by 8-bit value (strcmp). + * + * entries(path) = the entries of a directory, without `.` and + * `..`. + * + * `+` denotes string concatenation. + * ``` + */ void dumpPath(const Path & path, Sink & sink, PathFilter & filter = defaultPathFilter); -/* Same as `void dumpPath()`, but returns the last modified date of the path */ +/** + * Same as dumpPath(), but returns the last modified date of the path. + */ time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter = defaultPathFilter); +/** + * Dump an archive with a single file with these contents. + * + * @param s Contents of the file. + */ void dumpString(std::string_view s, Sink & sink); -/* FIXME: fix this API, it sucks. */ +/** + * \todo Fix this API, it sucks. + */ struct ParseSink { virtual void createDirectory(const Path & path) { }; @@ -68,8 +88,10 @@ struct ParseSink virtual void createSymlink(const Path & path, const std::string & target) { }; }; -/* If the NAR archive contains a single file at top-level, then save - the contents of the file to `s'. Otherwise barf. */ +/** + * If the NAR archive contains a single file at top-level, then save + * the contents of the file to `s`. Otherwise barf. + */ struct RetrieveRegularNARSink : ParseSink { bool regular = true; @@ -97,7 +119,9 @@ void parseDump(ParseSink & sink, Source & source); void restorePath(const Path & path, Source & source); -/* Read a NAR from 'source' and write it to 'sink'. */ +/** + * Read a NAR from 'source' and write it to 'sink'. + */ void copyNAR(Source & source, Sink & sink); void copyPath(const Path & from, const Path & to); diff --git a/src/libutil/args.cc b/src/libutil/args.cc index fc009592c..081dbeb28 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -236,8 +236,6 @@ nlohmann::json Args::toJSON() auto flags = nlohmann::json::object(); for (auto & [name, flag] : longFlags) { - /* Skip experimental flags when listing flags. */ - if (!experimentalFeatureSettings.isEnabled(flag->experimentalFeature)) continue; auto j = nlohmann::json::object(); if (flag->aliases.count(name)) continue; if (flag->shortName) @@ -249,6 +247,11 @@ nlohmann::json Args::toJSON() j["arity"] = flag->handler.arity; if (!flag->labels.empty()) j["labels"] = flag->labels; + // TODO With C++23 use `std::optional::tranform` + if (auto & xp = flag->experimentalFeature) + j["experimental-feature"] = showExperimentalFeature(*xp); + else + j["experimental-feature"] = nullptr; flags[name] = std::move(j); } @@ -345,6 +348,11 @@ Strings argvToStrings(int argc, char * * argv) return args; } +std::optional Command::experimentalFeature () +{ + return { Xp::NixCommand }; +} + MultiCommand::MultiCommand(const Commands & commands_) : commands(commands_) { @@ -408,6 +416,11 @@ nlohmann::json MultiCommand::toJSON() cat["id"] = command->category(); cat["description"] = trim(categories[command->category()]); j["category"] = std::move(cat); + // TODO With C++23 use `std::optional::tranform` + if (auto xp = command->experimentalFeature()) + cat["experimental-feature"] = showExperimentalFeature(*xp); + else + cat["experimental-feature"] = nullptr; cmds[name] = std::move(j); } diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 2969806dd..d90129796 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -18,16 +19,22 @@ class Args { public: - /* Parse the command line, throwing a UsageError if something goes - wrong. */ + /** + * Parse the command line, throwing a UsageError if something goes + * wrong. + */ void parseCmdline(const Strings & cmdline); - /* Return a short one-line description of the command. */ + /** + * Return a short one-line description of the command. + */ virtual std::string description() { return ""; } virtual bool forceImpureByDefault() { return false; } - /* Return documentation about this command, in Markdown format. */ + /** + * Return documentation about this command, in Markdown format. + */ virtual std::string doc() { return ""; } protected: @@ -146,13 +153,17 @@ protected: std::set hiddenCategories; - /* Called after all command line flags before the first non-flag - argument (if any) have been processed. */ + /** + * Called after all command line flags before the first non-flag + * argument (if any) have been processed. + */ virtual void initialFlagsProcessed() {} - /* Called after the command line has been processed if we need to generate - completions. Useful for commands that need to know the whole command line - in order to know what completions to generate. */ + /** + * Called after the command line has been processed if we need to generate + * completions. Useful for commands that need to know the whole command line + * in order to know what completions to generate. + */ virtual void completionHook() { } public: @@ -166,7 +177,9 @@ public: expectedArgs.emplace_back(std::move(arg)); } - /* Expect a string argument. */ + /** + * Expect a string argument. + */ void expectArg(const std::string & label, std::string * dest, bool optional = false) { expectArgs({ @@ -176,7 +189,9 @@ public: }); } - /* Expect 0 or more arguments. */ + /** + * Expect 0 or more arguments. + */ void expectArgs(const std::string & label, std::vector * dest) { expectArgs({ @@ -202,27 +217,36 @@ private: std::set flagExperimentalFeatures; }; -/* A command is an argument parser that can be executed by calling its - run() method. */ +/** + * A command is an argument parser that can be executed by calling its + * run() method. + */ struct Command : virtual public Args { friend class MultiCommand; virtual ~Command() { } + /** + * Entry point to the command + */ virtual void run() = 0; typedef int Category; static constexpr Category catDefault = 0; + virtual std::optional experimentalFeature (); + virtual Category category() { return catDefault; } }; typedef std::map()>> Commands; -/* An argument parser that supports multiple subcommands, - i.e. ‘ ’. */ +/** + * An argument parser that supports multiple subcommands, + * i.e. ‘ ’. + */ class MultiCommand : virtual public Args { public: @@ -230,7 +254,9 @@ public: std::map categories; - // Selected command, if any. + /** + * Selected command, if any. + */ std::optional>> command; MultiCommand(const Commands & commands); diff --git a/src/libutil/callback.hh b/src/libutil/callback.hh index ef31794be..3710d1239 100644 --- a/src/libutil/callback.hh +++ b/src/libutil/callback.hh @@ -1,13 +1,16 @@ #pragma once +///@file #include #include namespace nix { -/* A callback is a wrapper around a lambda that accepts a valid of - type T or an exception. (We abuse std::future to pass the value or - exception.) */ +/** + * A callback is a wrapper around a lambda that accepts a valid of + * type T or an exception. (We abuse std::future to pass the value or + * exception.) + */ template class Callback { diff --git a/src/libutil/canon-path.cc b/src/libutil/canon-path.cc index b132b4262..ddf6db6d1 100644 --- a/src/libutil/canon-path.cc +++ b/src/libutil/canon-path.cc @@ -100,4 +100,30 @@ std::ostream & operator << (std::ostream & stream, const CanonPath & path) return stream; } +std::string CanonPath::makeRelative(const CanonPath & path) const +{ + auto p1 = begin(); + auto p2 = path.begin(); + + for (; p1 != end() && p2 != path.end() && *p1 == *p2; ++p1, ++p2) ; + + if (p1 == end() && p2 == path.end()) + return "."; + else if (p1 == end()) + return std::string(p2.remaining); + else { + std::string res; + while (p1 != end()) { + ++p1; + if (!res.empty()) res += '/'; + res += ".."; + } + if (p2 != path.end()) { + if (!res.empty()) res += '/'; + res += p2.remaining; + } + return res; + } +} + } diff --git a/src/libutil/canon-path.hh b/src/libutil/canon-path.hh index 9d5984584..614883c06 100644 --- a/src/libutil/canon-path.hh +++ b/src/libutil/canon-path.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -8,28 +9,31 @@ namespace nix { -/* A canonical representation of a path. It ensures the following: - - - It always starts with a slash. - - - It never ends with a slash, except if the path is "/". - - - A slash is never followed by a slash (i.e. no empty components). - - - There are no components equal to '.' or '..'. - - Note that the path does not need to correspond to an actually - existing path, and there is no guarantee that symlinks are - resolved. -*/ +/** + * A canonical representation of a path. It ensures the following: + * + * - It always starts with a slash. + * + * - It never ends with a slash, except if the path is "/". + * + * - A slash is never followed by a slash (i.e. no empty components). + * + * - There are no components equal to '.' or '..'. + * + * Note that the path does not need to correspond to an actually + * existing path, and there is no guarantee that symlinks are + * resolved. + */ class CanonPath { std::string path; public: - /* Construct a canon path from a non-canonical path. Any '.', '..' - or empty components are removed. */ + /** + * Construct a canon path from a non-canonical path. Any '.', '..' + * or empty components are removed. + */ CanonPath(std::string_view raw); explicit CanonPath(const char * raw) @@ -44,9 +48,11 @@ public: static CanonPath root; - /* If `raw` starts with a slash, return - `CanonPath(raw)`. Otherwise return a `CanonPath` representing - `root + "/" + raw`. */ + /** + * If `raw` starts with a slash, return + * `CanonPath(raw)`. Otherwise return a `CanonPath` representing + * `root + "/" + raw`. + */ CanonPath(std::string_view raw, const CanonPath & root); bool isRoot() const @@ -58,8 +64,10 @@ public: const std::string & abs() const { return path; } - /* Like abs(), but return an empty string if this path is - '/'. Thus the returned string never ends in a slash. */ + /** + * Like abs(), but return an empty string if this path is + * '/'. Thus the returned string never ends in a slash. + */ const std::string & absOrEmpty() const { const static std::string epsilon; @@ -85,6 +93,9 @@ public: bool operator != (const Iterator & x) const { return remaining.data() != x.remaining.data(); } + bool operator == (const Iterator & x) const + { return !(*this != x); } + const std::string_view operator * () const { return remaining.substr(0, slash); } @@ -104,7 +115,9 @@ public: std::optional parent() const; - /* Remove the last component. Panics if this path is the root. */ + /** + * Remove the last component. Panics if this path is the root. + */ void pop(); std::optional dirOf() const @@ -125,10 +138,12 @@ public: bool operator != (const CanonPath & x) const { return path != x.path; } - /* Compare paths lexicographically except that path separators - are sorted before any other character. That is, in the sorted order - a directory is always followed directly by its children. For - instance, 'foo' < 'foo/bar' < 'foo!'. */ + /** + * Compare paths lexicographically except that path separators + * are sorted before any other character. That is, in the sorted order + * a directory is always followed directly by its children. For + * instance, 'foo' < 'foo/bar' < 'foo!'. + */ bool operator < (const CanonPath & x) const { auto i = path.begin(); @@ -144,28 +159,44 @@ public: return i == path.end() && j != x.path.end(); } - /* Return true if `this` is equal to `parent` or a child of - `parent`. */ + /** + * Return true if `this` is equal to `parent` or a child of + * `parent`. + */ bool isWithin(const CanonPath & parent) const; CanonPath removePrefix(const CanonPath & prefix) const; - /* Append another path to this one. */ + /** + * Append another path to this one. + */ void extend(const CanonPath & x); - /* Concatenate two paths. */ + /** + * Concatenate two paths. + */ CanonPath operator + (const CanonPath & x) const; - /* Add a path component to this one. It must not contain any slashes. */ + /** + * Add a path component to this one. It must not contain any slashes. + */ void push(std::string_view c); CanonPath operator + (std::string_view c) const; - /* Check whether access to this path is allowed, which is the case - if 1) `this` is within any of the `allowed` paths; or 2) any of - the `allowed` paths are within `this`. (The latter condition - ensures access to the parents of allowed paths.) */ + /** + * Check whether access to this path is allowed, which is the case + * if 1) `this` is within any of the `allowed` paths; or 2) any of + * the `allowed` paths are within `this`. (The latter condition + * ensures access to the parents of allowed paths.) + */ bool isAllowed(const std::set & allowed) const; + + /** + * Return a representation `x` of `path` relative to `this`, i.e. + * `CanonPath(this.makeRelative(x), this) == path`. + */ + std::string makeRelative(const CanonPath & path) const; }; std::ostream & operator << (std::ostream & stream, const CanonPath & path); diff --git a/src/libutil/cgroup.hh b/src/libutil/cgroup.hh index d08c8ad29..574ae8e5b 100644 --- a/src/libutil/cgroup.hh +++ b/src/libutil/cgroup.hh @@ -1,4 +1,5 @@ #pragma once +///@file #if __linux__ @@ -18,10 +19,12 @@ struct CgroupStats std::optional cpuUser, cpuSystem; }; -/* Destroy the cgroup denoted by 'path'. The postcondition is that - 'path' does not exist, and thus any processes in the cgroup have - been killed. Also return statistics from the cgroup just before - destruction. */ +/** + * Destroy the cgroup denoted by 'path'. The postcondition is that + * 'path' does not exist, and thus any processes in the cgroup have + * been killed. Also return statistics from the cgroup just before + * destruction. + */ CgroupStats destroyCgroup(const Path & cgroup); } diff --git a/src/libutil/chunked-vector.hh b/src/libutil/chunked-vector.hh index 0a4f0b400..d914e2542 100644 --- a/src/libutil/chunked-vector.hh +++ b/src/libutil/chunked-vector.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -7,20 +8,24 @@ namespace nix { -/* Provides an indexable container like vector<> with memory overhead - guarantees like list<> by allocating storage in chunks of ChunkSize - elements instead of using a contiguous memory allocation like vector<> - does. Not using a single vector that is resized reduces memory overhead - on large data sets by on average (growth factor)/2, mostly - eliminates copies within the vector during resizing, and provides stable - references to its elements. */ +/** + * Provides an indexable container like vector<> with memory overhead + * guarantees like list<> by allocating storage in chunks of ChunkSize + * elements instead of using a contiguous memory allocation like vector<> + * does. Not using a single vector that is resized reduces memory overhead + * on large data sets by on average (growth factor)/2, mostly + * eliminates copies within the vector during resizing, and provides stable + * references to its elements. + */ template class ChunkedVector { private: uint32_t size_ = 0; std::vector> chunks; - /* keep this out of the ::add hot path */ + /** + * Keep this out of the ::add hot path + */ [[gnu::noinline]] auto & addChunk() { diff --git a/src/libutil/closure.hh b/src/libutil/closure.hh index 779b9b2d5..16e3b93e4 100644 --- a/src/libutil/closure.hh +++ b/src/libutil/closure.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include #include #include "sync.hh" diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh index eecd5b819..9f661c5c3 100644 --- a/src/libutil/comparator.hh +++ b/src/libutil/comparator.hh @@ -1,6 +1,8 @@ #pragma once +///@file -/* Awfull hacky generation of the comparison operators by doing a lexicographic +/** + * Awful hacky generation of the comparison operators by doing a lexicographic * comparison between the choosen fields. * * ``` @@ -15,12 +17,12 @@ * } * ``` */ -#define GENERATE_ONE_CMP(COMPARATOR, MY_TYPE, FIELDS...) \ +#define GENERATE_ONE_CMP(COMPARATOR, MY_TYPE, ...) \ bool operator COMPARATOR(const MY_TYPE& other) const { \ - const MY_TYPE* me = this; \ - auto fields1 = std::make_tuple( FIELDS ); \ - me = &other; \ - auto fields2 = std::make_tuple( FIELDS ); \ + __VA_OPT__(const MY_TYPE* me = this;) \ + auto fields1 = std::make_tuple( __VA_ARGS__ ); \ + __VA_OPT__(me = &other;) \ + auto fields2 = std::make_tuple( __VA_ARGS__ ); \ return fields1 COMPARATOR fields2; \ } #define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args) diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index 89180e7a7..ba0847cde 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -23,7 +23,7 @@ struct ChunkedCompressionSink : CompressionSink { uint8_t outbuf[32 * 1024]; - void write(std::string_view data) override + void writeUnbuffered(std::string_view data) override { const size_t CHUNK_SIZE = sizeof(outbuf) << 2; while (!data.empty()) { @@ -103,7 +103,7 @@ struct ArchiveCompressionSink : CompressionSink throw Error(reason, archive_error_string(this->archive)); } - void write(std::string_view data) override + void writeUnbuffered(std::string_view data) override { ssize_t result = archive_write_data(archive, data.data(), data.length()); if (result <= 0) check(result); @@ -136,7 +136,7 @@ struct NoneSink : CompressionSink warn("requested compression level '%d' not supported by compression method 'none'", level); } void finish() override { flush(); } - void write(std::string_view data) override { nextSink(data); } + void writeUnbuffered(std::string_view data) override { nextSink(data); } }; struct BrotliDecompressionSink : ChunkedCompressionSink diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh index c470b82a5..4e53a7b3c 100644 --- a/src/libutil/compression.hh +++ b/src/libutil/compression.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "ref.hh" #include "types.hh" @@ -11,7 +12,7 @@ namespace nix { struct CompressionSink : BufferedSink, FinishSink { using BufferedSink::operator (); - using BufferedSink::write; + using BufferedSink::writeUnbuffered; using FinishSink::finish; }; diff --git a/src/libutil/compute-levels.hh b/src/libutil/compute-levels.hh index 8ded295f9..093e7a915 100644 --- a/src/libutil/compute-levels.hh +++ b/src/libutil/compute-levels.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include "types.hh" namespace nix { diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 8d63536d6..a42f3a849 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -70,17 +70,10 @@ void AbstractConfig::reapplyUnknownSettings() set(s.first, s.second); } -// Whether we should process the option. Excludes aliases, which are handled elsewhere, and disabled features. -static bool applicable(const Config::SettingData & sd) -{ - return !sd.isAlias - && experimentalFeatureSettings.isEnabled(sd.setting->experimentalFeature); -} - void Config::getSettings(std::map & res, bool overriddenOnly) { for (auto & opt : _settings) - if (applicable(opt.second) && (!overriddenOnly || opt.second.setting->overridden)) + if (!opt.second.isAlias && (!overriddenOnly || opt.second.setting->overridden)) res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description}); } @@ -154,7 +147,7 @@ nlohmann::json Config::toJSON() { auto res = nlohmann::json::object(); for (auto & s : _settings) - if (applicable(s.second)) + if (!s.second.isAlias) res.emplace(s.first, s.second.setting->toJSON()); return res; } @@ -163,7 +156,7 @@ std::string Config::toKeyValue() { auto res = std::string(); for (auto & s : _settings) - if (applicable(s.second)) + if (s.second.isAlias) res += fmt("%s = %s\n", s.first, s.second.setting->to_string()); return res; } @@ -171,9 +164,6 @@ std::string Config::toKeyValue() void Config::convertToArgs(Args & args, const std::string & category) { for (auto & s : _settings) { - /* We do include args for settings gated on disabled - experimental-features. The args themselves however will also be - gated on any experimental feature the underlying setting is. */ if (!s.second.isAlias) s.second.setting->convertToArg(args, category); } @@ -201,6 +191,10 @@ std::map AbstractSetting::toJSONObject() std::map obj; obj.emplace("description", description); obj.emplace("aliases", aliases); + if (experimentalFeature) + obj.emplace("experimentalFeature", *experimentalFeature); + else + obj.emplace("experimentalFeature", nullptr); return obj; } diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 748d6043b..162626791 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -124,21 +125,21 @@ public: void reapplyUnknownSettings(); }; -/* A class to simplify providing configuration settings. The typical - use is to inherit Config and add Setting members: - - class MyClass : private Config - { - Setting foo{this, 123, "foo", "the number of foos to use"}; - Setting bar{this, "blabla", "bar", "the name of the bar"}; - - MyClass() : Config(readConfigFile("/etc/my-app.conf")) - { - std::cout << foo << "\n"; // will print 123 unless overridden - } - }; -*/ - +/** + * A class to simplify providing configuration settings. The typical + * use is to inherit Config and add Setting members: + * + * class MyClass : private Config + * { + * Setting foo{this, 123, "foo", "the number of foos to use"}; + * Setting bar{this, "blabla", "bar", "the name of the bar"}; + * + * MyClass() : Config(readConfigFile("/etc/my-app.conf")) + * { + * std::cout << foo << "\n"; // will print 123 unless overridden + * } + * }; + */ class Config : public AbstractConfig { friend class AbstractSetting; @@ -228,7 +229,9 @@ protected: bool isOverridden() const { return overridden; } }; -/* A setting of type T. */ +/** + * A setting of type T. + */ template class BaseSetting : public AbstractSetting { @@ -311,8 +314,10 @@ public: void operator =(const T & v) { this->assign(v); } }; -/* A special setting for Paths. These are automatically canonicalised - (e.g. "/foo//bar/" becomes "/foo/bar"). */ +/** + * A special setting for Paths. These are automatically canonicalised + * (e.g. "/foo//bar/" becomes "/foo/bar"). + */ class PathSetting : public BaseSetting { bool allowEmpty; @@ -366,8 +371,23 @@ extern GlobalConfig globalConfig; struct ExperimentalFeatureSettings : Config { - Setting> experimentalFeatures{this, {}, "experimental-features", - "Experimental Nix features to enable."}; + Setting> experimentalFeatures{ + this, {}, "experimental-features", + R"( + Experimental features that are enabled. + + Example: + + ``` + experimental-features = nix-command flakes + ``` + + The following experimental features are available: + + {{#include experimental-features-shortlist.md}} + + Experimental features are [further documented in the manual](@docroot@/contributing/experimental-features.md). + )"}; /** * Check whether the given experimental feature is enabled. diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 0ebeaba61..6a0923081 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -1,4 +1,19 @@ #pragma once +/** + * @file + * + * @brief This file defines two main structs/classes used in nix error handling. + * + * ErrorInfo provides a standard payload of error information, with conversion to string + * happening in the logger rather than at the call site. + * + * BaseError is the ancestor of nix specific exceptions (and Interrupted), and contains + * an ErrorInfo. + * + * ErrorInfo structs are sent to the logger as part of an exception, or directly with the + * logError or logWarning macros. + * See libutil/tests/logging.cc for usage examples. + */ #include "suggestions.hh" #include "ref.hh" @@ -26,22 +41,6 @@ namespace nix { -/* - - This file defines two main structs/classes used in nix error handling. - - ErrorInfo provides a standard payload of error information, with conversion to string - happening in the logger rather than at the call site. - - BaseError is the ancestor of nix specific exceptions (and Interrupted), and contains - an ErrorInfo. - - ErrorInfo structs are sent to the logger as part of an exception, or directly with the - logError or logWarning macros. - - See libutil/tests/logging.cc for usage examples. - - */ typedef enum { lvlError = 0, @@ -54,20 +53,26 @@ typedef enum { lvlVomit } Verbosity; -// the lines of code surrounding an error. +/** + * The lines of code surrounding an error. + */ struct LinesOfCode { std::optional prevLineOfCode; std::optional errLineOfCode; std::optional nextLineOfCode; }; -/* An abstract type that represents a location in a source file. */ +/** + * An abstract type that represents a location in a source file. + */ struct AbstractPos { uint32_t line = 0; uint32_t column = 0; - /* Return the contents of the source file. */ + /** + * Return the contents of the source file. + */ virtual std::optional getSource() const { return std::nullopt; }; @@ -104,8 +109,10 @@ struct ErrorInfo { std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool showTrace); -/* BaseError should generally not be caught, as it has Interrupted as - a subclass. Catch Error instead. */ +/** + * BaseError should generally not be caught, as it has Interrupted as + * a subclass. Catch Error instead. + */ class BaseError : public std::exception { protected: diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 58d762ebb..5b4418714 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -5,29 +5,209 @@ namespace nix { -std::map stringifiedXpFeatures = { - { Xp::CaDerivations, "ca-derivations" }, - { Xp::ImpureDerivations, "impure-derivations" }, - { Xp::Flakes, "flakes" }, - { Xp::NixCommand, "nix-command" }, - { Xp::RecursiveNix, "recursive-nix" }, - { Xp::NoUrlLiterals, "no-url-literals" }, - { Xp::FetchClosure, "fetch-closure" }, - { Xp::ReplFlake, "repl-flake" }, - { Xp::AutoAllocateUids, "auto-allocate-uids" }, - { Xp::Cgroups, "cgroups" }, - { Xp::DiscardReferences, "discard-references" }, +struct ExperimentalFeatureDetails +{ + ExperimentalFeature tag; + std::string_view name; + std::string_view description; }; +constexpr std::array xpFeatureDetails = {{ + { + .tag = Xp::CaDerivations, + .name = "ca-derivations", + .description = R"( + Allow derivations to be content-addressed in order to prevent + rebuilds when changes to the derivation do not result in changes to + the derivation's output. See + [__contentAddressed](@docroot@/language/advanced-attributes.md#adv-attr-__contentAddressed) + for details. + )", + }, + { + .tag = Xp::ImpureDerivations, + .name = "impure-derivations", + .description = R"( + Allow derivations to produce non-fixed outputs by setting the + `__impure` derivation attribute to `true`. An impure derivation can + have differing outputs each time it is built. + + Example: + + ``` + derivation { + name = "impure"; + builder = /bin/sh; + __impure = true; # mark this derivation as impure + args = [ "-c" "read -n 10 random < /dev/random; echo $random > $out" ]; + system = builtins.currentSystem; + } + ``` + + Each time this derivation is built, it can produce a different + output (as the builder outputs random bytes to `$out`). Impure + derivations also have access to the network, and only fixed-output + or other impure derivations can rely on impure derivations. Finally, + an impure derivation cannot also be + [content-addressed](#xp-feature-ca-derivations). + )", + }, + { + .tag = Xp::Flakes, + .name = "flakes", + .description = R"( + Enable flakes. See the manual entry for [`nix + flake`](@docroot@/command-ref/new-cli/nix3-flake.md) for details. + )", + }, + { + .tag = Xp::NixCommand, + .name = "nix-command", + .description = R"( + Enable the new `nix` subcommands. See the manual on + [`nix`](@docroot@/command-ref/new-cli/nix.md) for details. + )", + }, + { + .tag = Xp::RecursiveNix, + .name = "recursive-nix", + .description = R"( + Allow derivation builders to call Nix, and thus build derivations + recursively. + + Example: + + ``` + with import {}; + + runCommand "foo" + { + buildInputs = [ nix jq ]; + NIX_PATH = "nixpkgs=${}"; + } + '' + hello=$(nix-build -E '(import {}).hello.overrideDerivation (args: { name = "recursive-hello"; })') + + mkdir -p $out/bin + ln -s $hello/bin/hello $out/bin/hello + '' + ``` + + An important restriction on recursive builders is disallowing + arbitrary substitutions. For example, running + + ``` + nix-store -r /nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10 + ``` + + in the above `runCommand` script would be disallowed, as this could + lead to derivations with hidden dependencies or breaking + reproducibility by relying on the current state of the Nix store. An + exception would be if + `/nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10` were + already in the build inputs or built by a previous recursive Nix + call. + )", + }, + { + .tag = Xp::NoUrlLiterals, + .name = "no-url-literals", + .description = R"( + Disallow unquoted URLs as part of the Nix language syntax. The Nix + language allows for URL literals, like so: + + ``` + $ nix repl + Welcome to Nix 2.15.0. Type :? for help. + + nix-repl> http://foo + "http://foo" + ``` + + But enabling this experimental feature will cause the Nix parser to + throw an error when encountering a URL literal: + + ``` + $ nix repl --extra-experimental-features 'no-url-literals' + Welcome to Nix 2.15.0. Type :? for help. + + nix-repl> http://foo + error: URL literals are disabled + + at «string»:1:1: + + 1| http://foo + | ^ + + ``` + + While this is currently an experimental feature, unquoted URLs are + being deprecated and their usage is discouraged. + + The reason is that, as opposed to path literals, URLs have no + special properties that distinguish them from regular strings, URLs + containing parameters have to be quoted anyway, and unquoted URLs + may confuse external tooling. + )", + }, + { + .tag = Xp::FetchClosure, + .name = "fetch-closure", + .description = R"( + Enable the use of the [`fetchClosure`](@docroot@/language/builtins.md#builtins-fetchClosure) built-in function in the Nix language. + )", + }, + { + .tag = Xp::ReplFlake, + .name = "repl-flake", + .description = R"( + Allow passing [installables](@docroot@/command-ref/new-cli/nix.md#installables) to `nix repl`, making its interface consistent with the other experimental commands. + )", + }, + { + .tag = Xp::AutoAllocateUids, + .name = "auto-allocate-uids", + .description = R"( + Allows Nix to automatically pick UIDs for builds, rather than creating + `nixbld*` user accounts. See the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting for details. + )", + }, + { + .tag = Xp::Cgroups, + .name = "cgroups", + .description = R"( + Allows Nix to execute builds inside cgroups. See + the [`use-cgroups`](#conf-use-cgroups) setting for details. + )", + }, + { + .tag = Xp::DiscardReferences, + .name = "discard-references", + .description = R"( + Allow the use of the [`unsafeDiscardReferences`](@docroot@/language/advanced-attributes.html#adv-attr-unsafeDiscardReferences) attribute in derivations + that use [structured attributes](@docroot@/language/advanced-attributes.html#adv-attr-structuredAttrs). This disables scanning of outputs for + runtime dependencies. + )", + }, +}}; + +static_assert( + []() constexpr { + for (auto [index, feature] : enumerate(xpFeatureDetails)) + if (index != (size_t)feature.tag) + return false; + return true; + }(), + "array order does not match enum tag order"); + const std::optional parseExperimentalFeature(const std::string_view & name) { using ReverseXpMap = std::map; - static auto reverseXpMap = []() - { + static std::unique_ptr reverseXpMap = []() { auto reverseXpMap = std::make_unique(); - for (auto & [feature, name] : stringifiedXpFeatures) - (*reverseXpMap)[name] = feature; + for (auto & xpFeature : xpFeatureDetails) + (*reverseXpMap)[xpFeature.name] = xpFeature.tag; return reverseXpMap; }(); @@ -37,20 +217,27 @@ const std::optional parseExperimentalFeature(const std::str return std::nullopt; } -std::string_view showExperimentalFeature(const ExperimentalFeature feature) +std::string_view showExperimentalFeature(const ExperimentalFeature tag) { - const auto ret = get(stringifiedXpFeatures, feature); - assert(ret); - return *ret; + assert((size_t)tag < xpFeatureDetails.size()); + return xpFeatureDetails[(size_t)tag].name; +} + +nlohmann::json documentExperimentalFeatures() +{ + StringMap res; + for (auto & xpFeature : xpFeatureDetails) + res[std::string { xpFeature.name }] = + trim(stripIndentation(xpFeature.description)); + return (nlohmann::json) res; } std::set parseFeatures(const std::set & rawFeatures) { std::set res; - for (auto & rawFeature : rawFeatures) { + for (auto & rawFeature : rawFeatures) if (auto feature = parseExperimentalFeature(rawFeature)) res.insert(*feature); - } return res; } diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index ac372e03e..8ef66263a 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "comparator.hh" #include "error.hh" @@ -10,9 +11,10 @@ namespace nix { /** * The list of available experimental features. * - * If you update this, don’t forget to also change the map defining their - * string representation in the corresponding `.cc` file. - **/ + * If you update this, don’t forget to also change the map defining + * their string representation and documentation in the corresponding + * `.cc` file as well. + */ enum struct ExperimentalFeature { CaDerivations, @@ -33,26 +35,52 @@ enum struct ExperimentalFeature */ using Xp = ExperimentalFeature; +/** + * Parse an experimental feature (enum value) from its name. Experimental + * feature flag names are hyphenated and do not contain spaces. + */ const std::optional parseExperimentalFeature( const std::string_view & name); + +/** + * Show the name of an experimental feature. This is the opposite of + * parseExperimentalFeature(). + */ std::string_view showExperimentalFeature(const ExperimentalFeature); +/** + * Compute the documentation of all experimental features. + * + * See `doc/manual` for how this information is used. + */ +nlohmann::json documentExperimentalFeatures(); + +/** + * Shorthand for `str << showExperimentalFeature(feature)`. + */ std::ostream & operator<<( std::ostream & str, const ExperimentalFeature & feature); /** - * Parse a set of strings to the corresponding set of experimental features, - * ignoring (but warning for) any unkwown feature. + * Parse a set of strings to the corresponding set of experimental + * features, ignoring (but warning for) any unknown feature. */ std::set parseFeatures(const std::set &); +/** + * An experimental feature was required for some (experimental) + * operation, but was not enabled. + */ class MissingExperimentalFeature : public Error { public: + /** + * The experimental feature that was required but not enabled. + */ ExperimentalFeature missingFeature; - MissingExperimentalFeature(ExperimentalFeature); + MissingExperimentalFeature(ExperimentalFeature missingFeature); }; /** diff --git a/src/libutil/finally.hh b/src/libutil/finally.hh index dee2e8d2f..db654301f 100644 --- a/src/libutil/finally.hh +++ b/src/libutil/finally.hh @@ -1,6 +1,9 @@ #pragma once +///@file -/* A trivial class to run a function at the end of a scope. */ +/** + * A trivial class to run a function at the end of a scope. + */ template class Finally { diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index e11426b88..727255b45 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -8,20 +9,25 @@ namespace nix { -/* Inherit some names from other namespaces for convenience. */ +/** + * Inherit some names from other namespaces for convenience. + */ using boost::format; -/* A variadic template that does nothing. Useful to call a function - for all variadic arguments but ignoring the result. */ +/** + * A variadic template that does nothing. Useful to call a function + * for all variadic arguments but ignoring the result. + */ struct nop { template nop(T...) {} }; -/* A helper for formatting strings. ‘fmt(format, a_0, ..., a_n)’ is - equivalent to ‘boost::format(format) % a_0 % ... % - ... a_n’. However, ‘fmt(s)’ is equivalent to ‘s’ (so no %-expansion - takes place). */ - +/** + * A helper for formatting strings. ‘fmt(format, a_0, ..., a_n)’ is + * equivalent to ‘boost::format(format) % a_0 % ... % + * ... a_n’. However, ‘fmt(s)’ is equivalent to ‘s’ (so no %-expansion + * takes place). + */ template inline void formatHelper(F & f) { diff --git a/src/libutil/git.hh b/src/libutil/git.hh index cb13ef0e5..bf2b9a286 100644 --- a/src/libutil/git.hh +++ b/src/libutil/git.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -8,21 +9,23 @@ namespace nix { namespace git { -// A line from the output of `git ls-remote --symref`. -// -// These can be of two kinds: -// -// - Symbolic references of the form -// -// ref: {target} {reference} -// -// where {target} is itself a reference and {reference} is optional -// -// - Object references of the form -// -// {target} {reference} -// -// where {target} is a commit id and {reference} is mandatory +/** + * A line from the output of `git ls-remote --symref`. + * + * These can be of two kinds: + * + * - Symbolic references of the form + * + * ref: {target} {reference} + * + * where {target} is itself a reference and {reference} is optional + * + * - Object references of the form + * + * {target} {reference} + * + * where {target} is a commit id and {reference} is mandatory + */ struct LsRemoteRefLine { enum struct Kind { Symbolic, diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 5735e4715..2c36d9d94 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -1,6 +1,7 @@ #include #include +#include #include #include @@ -16,7 +17,6 @@ namespace nix { - static size_t regularHashSize(HashType type) { switch (type) { case htMD5: return md5HashSize; @@ -343,7 +343,7 @@ HashSink::~HashSink() delete ctx; } -void HashSink::write(std::string_view data) +void HashSink::writeUnbuffered(std::string_view data) { bytes += data.size(); update(ht, *ctx, data); diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 38d09646e..ae3ee40f4 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "serialise.hh" @@ -33,62 +34,86 @@ struct Hash HashType type; - /* Create a zero-filled hash object. */ + /** + * Create a zero-filled hash object. + */ Hash(HashType type); - /* Parse the hash from a string representation in the format - "[:]" or "-" (a - Subresource Integrity hash expression). If the 'type' argument - is not present, then the hash type must be specified in the - string. */ + /** + * Parse the hash from a string representation in the format + * "[:]" or "-" (a + * Subresource Integrity hash expression). If the 'type' argument + * is not present, then the hash type must be specified in the + * string. + */ static Hash parseAny(std::string_view s, std::optional type); - /* Parse a hash from a string representation like the above, except the - type prefix is mandatory is there is no separate arguement. */ + /** + * Parse a hash from a string representation like the above, except the + * type prefix is mandatory is there is no separate arguement. + */ static Hash parseAnyPrefixed(std::string_view s); - /* Parse a plain hash that musst not have any prefix indicating the type. - The type is passed in to disambiguate. */ + /** + * Parse a plain hash that musst not have any prefix indicating the type. + * The type is passed in to disambiguate. + */ static Hash parseNonSRIUnprefixed(std::string_view s, HashType type); static Hash parseSRI(std::string_view original); private: - /* The type must be provided, the string view must not include - prefix. `isSRI` helps disambigate the various base-* encodings. */ + /** + * The type must be provided, the string view must not include + * prefix. `isSRI` helps disambigate the various base-* encodings. + */ Hash(std::string_view s, HashType type, bool isSRI); public: - /* Check whether two hash are equal. */ + /** + * Check whether two hash are equal. + */ bool operator == (const Hash & h2) const; - /* Check whether two hash are not equal. */ + /** + * Check whether two hash are not equal. + */ bool operator != (const Hash & h2) const; - /* For sorting. */ + /** + * For sorting. + */ bool operator < (const Hash & h) const; - /* Returns the length of a base-16 representation of this hash. */ + /** + * Returns the length of a base-16 representation of this hash. + */ size_t base16Len() const { return hashSize * 2; } - /* Returns the length of a base-32 representation of this hash. */ + /** + * Returns the length of a base-32 representation of this hash. + */ size_t base32Len() const { return (hashSize * 8 - 1) / 5 + 1; } - /* Returns the length of a base-64 representation of this hash. */ + /** + * Returns the length of a base-64 representation of this hash. + */ size_t base64Len() const { return ((4 * hashSize / 3) + 3) & ~3; } - /* Return a string representation of the hash, in base-16, base-32 - or base-64. By default, this is prefixed by the hash type - (e.g. "sha256:"). */ + /** + * Return a string representation of the hash, in base-16, base-32 + * or base-64. By default, this is prefixed by the hash type + * (e.g. "sha256:"). + */ std::string to_string(Base base, bool includeType) const; std::string gitRev() const @@ -104,35 +129,53 @@ public: static Hash dummy; }; -/* Helper that defaults empty hashes to the 0 hash. */ +/** + * Helper that defaults empty hashes to the 0 hash. + */ Hash newHashAllowEmpty(std::string_view hashStr, std::optional ht); -/* Print a hash in base-16 if it's MD5, or base-32 otherwise. */ +/** + * Print a hash in base-16 if it's MD5, or base-32 otherwise. + */ std::string printHash16or32(const Hash & hash); -/* Compute the hash of the given string. */ +/** + * Compute the hash of the given string. + */ Hash hashString(HashType ht, std::string_view s); -/* Compute the hash of the given file. */ +/** + * Compute the hash of the given file. + */ Hash hashFile(HashType ht, const Path & path); -/* Compute the hash of the given path. The hash is defined as - (essentially) hashString(ht, dumpPath(path)). */ +/** + * Compute the hash of the given path. The hash is defined as + * (essentially) hashString(ht, dumpPath(path)). + */ typedef std::pair HashResult; HashResult hashPath(HashType ht, const Path & path, PathFilter & filter = defaultPathFilter); -/* Compress a hash to the specified number of bytes by cyclically - XORing bytes together. */ +/** + * Compress a hash to the specified number of bytes by cyclically + * XORing bytes together. + */ Hash compressHash(const Hash & hash, unsigned int newSize); -/* Parse a string representing a hash type. */ +/** + * Parse a string representing a hash type. + */ HashType parseHashType(std::string_view s); -/* Will return nothing on parse error */ +/** + * Will return nothing on parse error + */ std::optional parseHashTypeOpt(std::string_view s); -/* And the reverse. */ +/** + * And the reverse. + */ std::string_view printHashType(HashType ht); @@ -154,7 +197,7 @@ public: HashSink(HashType ht); HashSink(const HashSink & h); ~HashSink(); - void write(std::string_view data) override; + void writeUnbuffered(std::string_view data) override; HashResult finish() override; HashResult currentHash(); }; diff --git a/src/libutil/hilite.hh b/src/libutil/hilite.hh index f8bdbfc55..2d5cf7c6f 100644 --- a/src/libutil/hilite.hh +++ b/src/libutil/hilite.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -6,11 +7,13 @@ namespace nix { -/* Highlight all the given matches in the given string `s` by wrapping - them between `prefix` and `postfix`. - - If some matches overlap, then their union will be wrapped rather - than the individual matches. */ +/** + * Highlight all the given matches in the given string `s` by wrapping + * them between `prefix` and `postfix`. + * + * If some matches overlap, then their union will be wrapped rather + * than the individual matches. + */ std::string hiliteMatches( std::string_view s, std::vector matches, diff --git a/src/libutil/json-impls.hh b/src/libutil/json-impls.hh index bd75748ad..b26163a04 100644 --- a/src/libutil/json-impls.hh +++ b/src/libutil/json-impls.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "nlohmann/json_fwd.hpp" diff --git a/src/libutil/json-utils.hh b/src/libutil/json-utils.hh index b8a031227..eb00e954f 100644 --- a/src/libutil/json-utils.hh +++ b/src/libutil/json-utils.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 7cac75ce1..5a2dd99af 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -65,9 +65,10 @@ public: switch (lvl) { case lvlError: c = '3'; break; case lvlWarn: c = '4'; break; - case lvlInfo: c = '5'; break; + case lvlNotice: case lvlInfo: c = '5'; break; case lvlTalkative: case lvlChatty: c = '6'; break; - default: c = '7'; + case lvlDebug: case lvlVomit: c = '7'; + default: c = '7'; break; // should not happen, and missing enum case is reported by -Werror=switch-enum } prefix = std::string("<") + c + ">"; } diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 59a707eef..5aa6bee95 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "error.hh" @@ -72,6 +73,9 @@ public: virtual void stop() { }; + virtual void pause() { }; + virtual void resume() { }; + // Whether the logger prints the whole build log virtual bool isVerbose() { return false; } @@ -179,12 +183,17 @@ bool handleJSONLogMessage(const std::string & msg, const Activity & act, std::map & activities, bool trusted); -extern Verbosity verbosity; /* suppress msgs > this */ +/** + * suppress msgs > this + */ +extern Verbosity verbosity; -/* Print a message with the standard ErrorInfo format. - In general, use these 'log' macros for reporting problems that may require user - intervention or that need more explanation. Use the 'print' macros for more - lightweight status messages. */ +/** + * Print a message with the standard ErrorInfo format. + * In general, use these 'log' macros for reporting problems that may require user + * intervention or that need more explanation. Use the 'print' macros for more + * lightweight status messages. + */ #define logErrorInfo(level, errorInfo...) \ do { \ if ((level) <= nix::verbosity) { \ @@ -195,9 +204,11 @@ extern Verbosity verbosity; /* suppress msgs > this */ #define logError(errorInfo...) logErrorInfo(lvlError, errorInfo) #define logWarning(errorInfo...) logErrorInfo(lvlWarn, errorInfo) -/* Print a string message if the current log level is at least the specified - level. Note that this has to be implemented as a macro to ensure that the - arguments are evaluated lazily. */ +/** + * Print a string message if the current log level is at least the specified + * level. Note that this has to be implemented as a macro to ensure that the + * arguments are evaluated lazily. + */ #define printMsgUsing(loggerParam, level, args...) \ do { \ auto __lvl = level; \ @@ -214,7 +225,9 @@ extern Verbosity verbosity; /* suppress msgs > this */ #define debug(args...) printMsg(lvlDebug, args) #define vomit(args...) printMsg(lvlVomit, args) -/* if verbosity >= lvlWarn, print a message with a yellow 'warning:' prefix. */ +/** + * if verbosity >= lvlWarn, print a message with a yellow 'warning:' prefix. + */ template inline void warn(const std::string & fs, const Args & ... args) { diff --git a/src/libutil/lru-cache.hh b/src/libutil/lru-cache.hh index 6ef4a3e06..0e19517ed 100644 --- a/src/libutil/lru-cache.hh +++ b/src/libutil/lru-cache.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -7,7 +8,9 @@ namespace nix { -/* A simple least-recently used cache. Not thread-safe. */ +/** + * A simple least-recently used cache. Not thread-safe. + */ template class LRUCache { @@ -31,7 +34,9 @@ public: LRUCache(size_t capacity) : capacity(capacity) { } - /* Insert or upsert an item in the cache. */ + /** + * Insert or upsert an item in the cache. + */ void upsert(const Key & key, const Value & value) { if (capacity == 0) return; @@ -39,7 +44,9 @@ public: erase(key); if (data.size() >= capacity) { - /* Retire the oldest item. */ + /** + * Retire the oldest item. + */ auto oldest = lru.begin(); data.erase(*oldest); lru.erase(oldest); @@ -63,14 +70,18 @@ public: return true; } - /* Look up an item in the cache. If it exists, it becomes the most - recently used item. */ + /** + * Look up an item in the cache. If it exists, it becomes the most + * recently used item. + * */ std::optional get(const Key & key) { auto i = data.find(key); if (i == data.end()) return {}; - /* Move this item to the back of the LRU list. */ + /** + * Move this item to the back of the LRU list. + */ lru.erase(i->second.first.it); auto j = lru.insert(lru.end(), i); i->second.first.it = j; diff --git a/src/libutil/monitor-fd.hh b/src/libutil/monitor-fd.hh index 9518cf8aa..86d0115fc 100644 --- a/src/libutil/monitor-fd.hh +++ b/src/libutil/monitor-fd.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include diff --git a/src/libutil/namespaces.hh b/src/libutil/namespaces.hh index e82379b9c..0b7eeb66c 100644 --- a/src/libutil/namespaces.hh +++ b/src/libutil/namespaces.hh @@ -1,4 +1,5 @@ #pragma once +///@file namespace nix { diff --git a/src/libutil/pool.hh b/src/libutil/pool.hh index d49067bb9..6247b6125 100644 --- a/src/libutil/pool.hh +++ b/src/libutil/pool.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -11,33 +12,37 @@ namespace nix { -/* This template class implements a simple pool manager of resources - of some type R, such as database connections. It is used as - follows: - - class Connection { ... }; - - Pool pool; - - { - auto conn(pool.get()); - conn->exec("select ..."); - } - - Here, the Connection object referenced by ‘conn’ is automatically - returned to the pool when ‘conn’ goes out of scope. -*/ - +/** + * This template class implements a simple pool manager of resources + * of some type R, such as database connections. It is used as + * follows: + * + * class Connection { ... }; + * + * Pool pool; + * + * { + * auto conn(pool.get()); + * conn->exec("select ..."); + * } + * + * Here, the Connection object referenced by ‘conn’ is automatically + * returned to the pool when ‘conn’ goes out of scope. + */ template class Pool { public: - /* A function that produces new instances of R on demand. */ + /** + * A function that produces new instances of R on demand. + */ typedef std::function()> Factory; - /* A function that checks whether an instance of R is still - usable. Unusable instances are removed from the pool. */ + /** + * A function that checks whether an instance of R is still + * usable. Unusable instances are removed from the pool. + */ typedef std::function &)> Validator; private: diff --git a/src/libutil/ref.hh b/src/libutil/ref.hh index 7d38b059c..af5f8304c 100644 --- a/src/libutil/ref.hh +++ b/src/libutil/ref.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -6,8 +7,10 @@ namespace nix { -/* A simple non-nullable reference-counted pointer. Actually a wrapper - around std::shared_ptr that prevents null constructions. */ +/** + * A simple non-nullable reference-counted pointer. Actually a wrapper + * around std::shared_ptr that prevents null constructions. + */ template class ref { diff --git a/src/libutil/regex-combinators.hh b/src/libutil/regex-combinators.hh index 0b997b25a..87d6aa678 100644 --- a/src/libutil/regex-combinators.hh +++ b/src/libutil/regex-combinators.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 7476e6f6c..3d5121a19 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -20,7 +20,7 @@ void BufferedSink::operator () (std::string_view data) buffer size. */ if (bufPos + data.size() >= bufSize) { flush(); - write(data); + writeUnbuffered(data); break; } /* Otherwise, copy the bytes to the buffer. Flush the buffer @@ -38,7 +38,7 @@ void BufferedSink::flush() if (bufPos == 0) return; size_t n = bufPos; bufPos = 0; // don't trigger the assert() in ~BufferedSink() - write({buffer.get(), n}); + writeUnbuffered({buffer.get(), n}); } @@ -48,7 +48,7 @@ FdSink::~FdSink() } -void FdSink::write(std::string_view data) +void FdSink::writeUnbuffered(std::string_view data) { written += data.size(); try { @@ -186,6 +186,22 @@ static DefaultStackAllocator defaultAllocatorSingleton; StackAllocator *StackAllocator::defaultAllocator = &defaultAllocatorSingleton; +std::shared_ptr (*create_coro_gc_hook)() = []() -> std::shared_ptr { + return {}; +}; + +/* This class is used for entry and exit hooks on coroutines */ +class CoroutineContext { + /* Disable GC when entering the coroutine without the boehm patch, + * since it doesn't find the main thread stack in this case. + * std::shared_ptr performs type-erasure, so it will call the right + * deleter. */ + const std::shared_ptr coro_gc_hook = create_coro_gc_hook(); +public: + CoroutineContext() {}; + ~CoroutineContext() {}; +}; + std::unique_ptr sourceToSink(std::function fun) { struct SourceToSink : FinishSink @@ -206,7 +222,8 @@ std::unique_ptr sourceToSink(std::function fun) if (in.empty()) return; cur = in; - if (!coro) + if (!coro) { + CoroutineContext ctx; coro = coro_t::push_type(VirtualStackAllocator{}, [&](coro_t::pull_type & yield) { LambdaSource source([&](char *out, size_t out_len) { if (cur.empty()) { @@ -223,17 +240,24 @@ std::unique_ptr sourceToSink(std::function fun) }); fun(source); }); + } if (!*coro) { abort(); } - if (!cur.empty()) (*coro)(false); + if (!cur.empty()) { + CoroutineContext ctx; + (*coro)(false); + } } void finish() override { if (!coro) return; if (!*coro) abort(); - (*coro)(true); + { + CoroutineContext ctx; + (*coro)(true); + } if (*coro) abort(); } }; @@ -264,18 +288,23 @@ std::unique_ptr sinkToSource( size_t read(char * data, size_t len) override { - if (!coro) + if (!coro) { + CoroutineContext ctx; coro = coro_t::pull_type(VirtualStackAllocator{}, [&](coro_t::push_type & yield) { LambdaSink sink([&](std::string_view data) { if (!data.empty()) yield(std::string(data)); }); fun(sink); }); + } if (!*coro) { eof(); abort(); } if (pos == cur.size()) { - if (!cur.empty()) (*coro)(); + if (!cur.empty()) { + CoroutineContext ctx; + (*coro)(); + } cur = coro->get(); pos = 0; } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 7da5b07fd..333c254ea 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include @@ -10,7 +11,9 @@ namespace boost::context { struct stack_context; } namespace nix { -/* Abstract destination of binary data. */ +/** + * Abstract destination of binary data. + */ struct Sink { virtual ~Sink() { } @@ -18,7 +21,9 @@ struct Sink virtual bool good() { return true; } }; -/* Just throws away data. */ +/** + * Just throws away data. + */ struct NullSink : Sink { void operator () (std::string_view data) override @@ -32,8 +37,10 @@ struct FinishSink : virtual Sink }; -/* A buffered abstract sink. Warning: a BufferedSink should not be - used from multiple threads concurrently. */ +/** + * A buffered abstract sink. Warning: a BufferedSink should not be + * used from multiple threads concurrently. + */ struct BufferedSink : virtual Sink { size_t bufSize, bufPos; @@ -46,23 +53,31 @@ struct BufferedSink : virtual Sink void flush(); - virtual void write(std::string_view data) = 0; +protected: + + virtual void writeUnbuffered(std::string_view data) = 0; }; -/* Abstract source of binary data. */ +/** + * Abstract source of binary data. + */ struct Source { virtual ~Source() { } - /* Store exactly ‘len’ bytes in the buffer pointed to by ‘data’. - It blocks until all the requested data is available, or throws - an error if it is not going to be available. */ + /** + * Store exactly ‘len’ bytes in the buffer pointed to by ‘data’. + * It blocks until all the requested data is available, or throws + * an error if it is not going to be available. + */ void operator () (char * data, size_t len); - /* Store up to ‘len’ in the buffer pointed to by ‘data’, and - return the number of bytes stored. It blocks until at least - one byte is available. */ + /** + * Store up to ‘len’ in the buffer pointed to by ‘data’, and + * return the number of bytes stored. It blocks until at least + * one byte is available. + */ virtual size_t read(char * data, size_t len) = 0; virtual bool good() { return true; } @@ -73,8 +88,10 @@ struct Source }; -/* A buffered abstract source. Warning: a BufferedSource should not be - used from multiple threads concurrently. */ +/** + * A buffered abstract source. Warning: a BufferedSource should not be + * used from multiple threads concurrently. + */ struct BufferedSource : Source { size_t bufSize, bufPosIn, bufPosOut; @@ -88,12 +105,16 @@ struct BufferedSource : Source bool hasData(); protected: - /* Underlying read call, to be overridden. */ + /** + * Underlying read call, to be overridden. + */ virtual size_t readUnbuffered(char * data, size_t len) = 0; }; -/* A sink that writes data to a file descriptor. */ +/** + * A sink that writes data to a file descriptor. + */ struct FdSink : BufferedSink { int fd; @@ -114,7 +135,7 @@ struct FdSink : BufferedSink ~FdSink(); - void write(std::string_view data) override; + void writeUnbuffered(std::string_view data) override; bool good() override; @@ -123,7 +144,9 @@ private: }; -/* A source that reads data from a file descriptor. */ +/** + * A source that reads data from a file descriptor. + */ struct FdSource : BufferedSource { int fd; @@ -149,7 +172,9 @@ private: }; -/* A sink that writes data to a string. */ +/** + * A sink that writes data to a string. + */ struct StringSink : Sink { std::string s; @@ -163,7 +188,9 @@ struct StringSink : Sink }; -/* A source that reads data from a string. */ +/** + * A source that reads data from a string. + */ struct StringSource : Source { std::string_view s; @@ -173,7 +200,9 @@ struct StringSource : Source }; -/* A sink that writes all incoming data to two other sinks. */ +/** + * A sink that writes all incoming data to two other sinks. + */ struct TeeSink : Sink { Sink & sink1, & sink2; @@ -186,7 +215,9 @@ struct TeeSink : Sink }; -/* Adapter class of a Source that saves all data read to a sink. */ +/** + * Adapter class of a Source that saves all data read to a sink. + */ struct TeeSource : Source { Source & orig; @@ -201,7 +232,9 @@ struct TeeSource : Source } }; -/* A reader that consumes the original Source until 'size'. */ +/** + * A reader that consumes the original Source until 'size'. + */ struct SizedSource : Source { Source & orig; @@ -219,7 +252,9 @@ struct SizedSource : Source return n; } - /* Consume the original source until no remain data is left to consume. */ + /** + * Consume the original source until no remain data is left to consume. + */ size_t drainAll() { std::vector buf(8192); @@ -232,7 +267,9 @@ struct SizedSource : Source } }; -/* A sink that that just counts the number of bytes given to it */ +/** + * A sink that that just counts the number of bytes given to it + */ struct LengthSink : Sink { uint64_t length = 0; @@ -243,7 +280,9 @@ struct LengthSink : Sink } }; -/* Convert a function into a sink. */ +/** + * Convert a function into a sink. + */ struct LambdaSink : Sink { typedef std::function lambda_t; @@ -259,7 +298,9 @@ struct LambdaSink : Sink }; -/* Convert a function into a source. */ +/** + * Convert a function into a source. + */ struct LambdaSource : Source { typedef std::function lambda_t; @@ -274,8 +315,10 @@ struct LambdaSource : Source } }; -/* Chain two sources together so after the first is exhausted, the second is - used */ +/** + * Chain two sources together so after the first is exhausted, the second is + * used + */ struct ChainSource : Source { Source & source1, & source2; @@ -289,8 +332,10 @@ struct ChainSource : Source std::unique_ptr sourceToSink(std::function fun); -/* Convert a function that feeds data into a Sink into a Source. The - Source executes the function as a coroutine. */ +/** + * Convert a function that feeds data into a Sink into a Source. The + * Source executes the function as a coroutine. + */ std::unique_ptr sinkToSource( std::function fun, std::function eof = []() { @@ -376,7 +421,9 @@ Source & operator >> (Source & in, bool & b) Error readError(Source & source); -/* An adapter that converts a std::basic_istream into a source. */ +/** + * An adapter that converts a std::basic_istream into a source. + */ struct StreamToSourceAdapter : Source { std::shared_ptr> istream; @@ -399,13 +446,14 @@ struct StreamToSourceAdapter : Source }; -/* A source that reads a distinct format of concatenated chunks back into its - logical form, in order to guarantee a known state to the original stream, - even in the event of errors. - - Use with FramedSink, which also allows the logical stream to be terminated - in the event of an exception. -*/ +/** + * A source that reads a distinct format of concatenated chunks back into its + * logical form, in order to guarantee a known state to the original stream, + * even in the event of errors. + * + * Use with FramedSink, which also allows the logical stream to be terminated + * in the event of an exception. + */ struct FramedSource : Source { Source & from; @@ -450,11 +498,12 @@ struct FramedSource : Source } }; -/* Write as chunks in the format expected by FramedSource. - - The exception_ptr reference can be used to terminate the stream when you - detect that an error has occurred on the remote end. -*/ +/** + * Write as chunks in the format expected by FramedSource. + * + * The exception_ptr reference can be used to terminate the stream when you + * detect that an error has occurred on the remote end. + */ struct FramedSink : nix::BufferedSink { BufferedSink & to; @@ -473,7 +522,7 @@ struct FramedSink : nix::BufferedSink } } - void write(std::string_view data) override + void writeUnbuffered(std::string_view data) override { /* Don't send more data if the remote has encountered an error. */ @@ -487,18 +536,27 @@ struct FramedSink : nix::BufferedSink }; }; -/* Stack allocation strategy for sinkToSource. - Mutable to avoid a boehm gc dependency in libutil. - - boost::context doesn't provide a virtual class, so we define our own. +/** + * Stack allocation strategy for sinkToSource. + * Mutable to avoid a boehm gc dependency in libutil. + * + * boost::context doesn't provide a virtual class, so we define our own. */ struct StackAllocator { virtual boost::context::stack_context allocate() = 0; virtual void deallocate(boost::context::stack_context sctx) = 0; - /* The stack allocator to use in sinkToSource and potentially elsewhere. - It is reassigned by the initGC() method in libexpr. */ + /** + * The stack allocator to use in sinkToSource and potentially elsewhere. + * It is reassigned by the initGC() method in libexpr. + */ static StackAllocator *defaultAllocator; }; +/* Disabling GC when entering a coroutine (without the boehm patch). + mutable to avoid boehm gc dependency in libutil. + */ +extern std::shared_ptr (*create_coro_gc_hook)(); + + } diff --git a/src/libutil/split.hh b/src/libutil/split.hh index 87a23b13e..3b9b2b83b 100644 --- a/src/libutil/split.hh +++ b/src/libutil/split.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -7,10 +8,12 @@ namespace nix { -// If `separator` is found, we return the portion of the string before the -// separator, and modify the string argument to contain only the part after the -// separator. Otherwise, we return `std::nullopt`, and we leave the argument -// string alone. +/** + * If `separator` is found, we return the portion of the string before the + * separator, and modify the string argument to contain only the part after the + * separator. Otherwise, we return `std::nullopt`, and we leave the argument + * string alone. + */ static inline std::optional splitPrefixTo(std::string_view & string, char separator) { auto sepInstance = string.find(separator); diff --git a/src/libutil/suggestions.hh b/src/libutil/suggestions.hh index d54dd8e31..9abf5ee5f 100644 --- a/src/libutil/suggestions.hh +++ b/src/libutil/suggestions.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "comparator.hh" #include "types.hh" @@ -13,7 +14,8 @@ int levenshteinDistance(std::string_view first, std::string_view second); */ class Suggestion { public: - int distance; // The smaller the better + /// The smaller the better + int distance; std::string suggestion; std::string to_string() const; @@ -43,7 +45,9 @@ public: std::ostream & operator<<(std::ostream & str, const Suggestion &); std::ostream & operator<<(std::ostream & str, const Suggestions &); -// Either a value of type `T`, or some suggestions +/** + * Either a value of type `T`, or some suggestions + */ template class OrSuggestions { public: diff --git a/src/libutil/sync.hh b/src/libutil/sync.hh index e1d591d77..47e4512b1 100644 --- a/src/libutil/sync.hh +++ b/src/libutil/sync.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -7,22 +8,22 @@ namespace nix { -/* This template class ensures synchronized access to a value of type - T. It is used as follows: - - struct Data { int x; ... }; - - Sync data; - - { - auto data_(data.lock()); - data_->x = 123; - } - - Here, "data" is automatically unlocked when "data_" goes out of - scope. -*/ - +/** + * This template class ensures synchronized access to a value of type + * T. It is used as follows: + * + * struct Data { int x; ... }; + * + * Sync data; + * + * { + * auto data_(data.lock()); + * data_->x = 123; + * } + * + * Here, "data" is automatically unlocked when "data_" goes out of + * scope. + */ template class Sync { diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh index 4d9141fd4..24afb710a 100644 --- a/src/libutil/tarfile.hh +++ b/src/libutil/tarfile.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include "serialise.hh" #include @@ -14,7 +17,7 @@ struct TarArchive { TarArchive(const Path & path); - // disable copy constructor + /// disable copy constructor TarArchive(const TarArchive &) = delete; void close(); diff --git a/src/libutil/tests/canon-path.cc b/src/libutil/tests/canon-path.cc index c1c5adadf..fc94ccc3d 100644 --- a/src/libutil/tests/canon-path.cc +++ b/src/libutil/tests/canon-path.cc @@ -107,15 +107,13 @@ namespace nix { } TEST(CanonPath, within) { - { - ASSERT_TRUE(CanonPath("foo").isWithin(CanonPath("foo"))); - ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("bar"))); - ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("fo"))); - ASSERT_TRUE(CanonPath("foo/bar").isWithin(CanonPath("foo"))); - ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("foo/bar"))); - ASSERT_TRUE(CanonPath("/foo/bar/default.nix").isWithin(CanonPath("/"))); - ASSERT_TRUE(CanonPath("/").isWithin(CanonPath("/"))); - } + ASSERT_TRUE(CanonPath("foo").isWithin(CanonPath("foo"))); + ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("bar"))); + ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("fo"))); + ASSERT_TRUE(CanonPath("foo/bar").isWithin(CanonPath("foo"))); + ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("foo/bar"))); + ASSERT_TRUE(CanonPath("/foo/bar/default.nix").isWithin(CanonPath("/"))); + ASSERT_TRUE(CanonPath("/").isWithin(CanonPath("/"))); } TEST(CanonPath, sort) { @@ -127,29 +125,38 @@ namespace nix { } TEST(CanonPath, allowed) { - { - std::set allowed { - CanonPath("foo/bar"), - CanonPath("foo!"), - CanonPath("xyzzy"), - CanonPath("a/b/c"), - }; + std::set allowed { + CanonPath("foo/bar"), + CanonPath("foo!"), + CanonPath("xyzzy"), + CanonPath("a/b/c"), + }; - ASSERT_TRUE (CanonPath("foo/bar").isAllowed(allowed)); - ASSERT_TRUE (CanonPath("foo/bar/bla").isAllowed(allowed)); - ASSERT_TRUE (CanonPath("foo").isAllowed(allowed)); - ASSERT_FALSE(CanonPath("bar").isAllowed(allowed)); - ASSERT_FALSE(CanonPath("bar/a").isAllowed(allowed)); - ASSERT_TRUE (CanonPath("a").isAllowed(allowed)); - ASSERT_TRUE (CanonPath("a/b").isAllowed(allowed)); - ASSERT_TRUE (CanonPath("a/b/c").isAllowed(allowed)); - ASSERT_TRUE (CanonPath("a/b/c/d").isAllowed(allowed)); - ASSERT_TRUE (CanonPath("a/b/c/d/e").isAllowed(allowed)); - ASSERT_FALSE(CanonPath("a/b/a").isAllowed(allowed)); - ASSERT_FALSE(CanonPath("a/b/d").isAllowed(allowed)); - ASSERT_FALSE(CanonPath("aaa").isAllowed(allowed)); - ASSERT_FALSE(CanonPath("zzz").isAllowed(allowed)); - ASSERT_TRUE (CanonPath("/").isAllowed(allowed)); - } + ASSERT_TRUE (CanonPath("foo/bar").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("foo/bar/bla").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("foo").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("bar").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("bar/a").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("a").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("a/b").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("a/b/c").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("a/b/c/d").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("a/b/c/d/e").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("a/b/a").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("a/b/d").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("aaa").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("zzz").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("/").isAllowed(allowed)); + } + + TEST(CanonPath, makeRelative) { + CanonPath d("/foo/bar"); + ASSERT_EQ(d.makeRelative(CanonPath("/foo/bar")), "."); + ASSERT_EQ(d.makeRelative(CanonPath("/foo")), ".."); + ASSERT_EQ(d.makeRelative(CanonPath("/")), "../.."); + ASSERT_EQ(d.makeRelative(CanonPath("/foo/bar/xyzzy")), "xyzzy"); + ASSERT_EQ(d.makeRelative(CanonPath("/foo/bar/xyzzy/bla")), "xyzzy/bla"); + ASSERT_EQ(d.makeRelative(CanonPath("/foo/xyzzy/bla")), "../xyzzy/bla"); + ASSERT_EQ(d.makeRelative(CanonPath("/xyzzy/bla")), "../../xyzzy/bla"); } } diff --git a/src/libutil/tests/config.cc b/src/libutil/tests/config.cc index 8be6730dd..f250e934e 100644 --- a/src/libutil/tests/config.cc +++ b/src/libutil/tests/config.cc @@ -156,12 +156,54 @@ namespace nix { } TEST(Config, toJSONOnNonEmptyConfig) { + using nlohmann::literals::operator "" _json; Config config; - std::map settings; - Setting setting{&config, "", "name-of-the-setting", "description"}; + Setting setting{ + &config, + "", + "name-of-the-setting", + "description", + }; setting.assign("value"); - ASSERT_EQ(config.toJSON().dump(), R"#({"name-of-the-setting":{"aliases":[],"defaultValue":"","description":"description\n","documentDefault":true,"value":"value"}})#"); + ASSERT_EQ(config.toJSON(), + R"#({ + "name-of-the-setting": { + "aliases": [], + "defaultValue": "", + "description": "description\n", + "documentDefault": true, + "value": "value", + "experimentalFeature": null + } + })#"_json); + } + + TEST(Config, toJSONOnNonEmptyConfigWithExperimentalSetting) { + using nlohmann::literals::operator "" _json; + Config config; + Setting setting{ + &config, + "", + "name-of-the-setting", + "description", + {}, + true, + Xp::Flakes, + }; + setting.assign("value"); + + ASSERT_EQ(config.toJSON(), + R"#({ + "name-of-the-setting": { + "aliases": [], + "defaultValue": "", + "description": "description\n", + "documentDefault": true, + "value": "value", + "experimentalFeature": "flakes" + } + })#"_json); } TEST(Config, setSettingAlias) { diff --git a/src/libutil/tests/hash.hh b/src/libutil/tests/hash.hh index 9e9650e6e..1f9fa59ae 100644 --- a/src/libutil/tests/hash.hh +++ b/src/libutil/tests/hash.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include diff --git a/src/libutil/thread-pool.hh b/src/libutil/thread-pool.hh index b22e0d162..0e09fae97 100644 --- a/src/libutil/thread-pool.hh +++ b/src/libutil/thread-pool.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "sync.hh" #include "util.hh" @@ -13,8 +14,10 @@ namespace nix { MakeError(ThreadPoolShutDown, Error); -/* A simple thread pool that executes a queue of work items - (lambdas). */ +/** + * A simple thread pool that executes a queue of work items + * (lambdas). + */ class ThreadPool { public: @@ -23,19 +26,30 @@ public: ~ThreadPool(); - // FIXME: use std::packaged_task? + /** + * An individual work item. + * + * \todo use std::packaged_task? + */ typedef std::function work_t; - /* Enqueue a function to be executed by the thread pool. */ + /** + * Enqueue a function to be executed by the thread pool. + */ void enqueue(const work_t & t); - /* Execute work items until the queue is empty. Note that work - items are allowed to add new items to the queue; this is - handled correctly. Queue processing stops prematurely if any - work item throws an exception. This exception is propagated to - the calling thread. If multiple work items throw an exception - concurrently, only one item is propagated; the others are - printed on stderr and otherwise ignored. */ + /** + * Execute work items until the queue is empty. + * + * \note Note that work items are allowed to add new items to the + * queue; this is handled correctly. + * + * Queue processing stops prematurely if any work item throws an + * exception. This exception is propagated to the calling thread. If + * multiple work items throw an exception concurrently, only one + * item is propagated; the others are printed on stderr and + * otherwise ignored. + */ void process(); private: @@ -62,9 +76,11 @@ private: void shutdown(); }; -/* Process in parallel a set of items of type T that have a partial - ordering between them. Thus, any item is only processed after all - its dependencies have been processed. */ +/** + * Process in parallel a set of items of type T that have a partial + * ordering between them. Thus, any item is only processed after all + * its dependencies have been processed. + */ template void processGraph( ThreadPool & pool, diff --git a/src/libutil/topo-sort.hh b/src/libutil/topo-sort.hh index 7418be5e0..a52811fbf 100644 --- a/src/libutil/topo-sort.hh +++ b/src/libutil/topo-sort.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "error.hh" diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 6bcbd7e1d..c86f52175 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "ref.hh" @@ -17,7 +18,9 @@ typedef std::set StringSet; typedef std::map StringMap; typedef std::map StringPairs; -/* Paths are just strings. */ +/** + * Paths are just strings. + */ typedef std::string Path; typedef std::string_view PathView; typedef std::list Paths; @@ -25,15 +28,19 @@ typedef std::set PathSet; typedef std::vector> Headers; -/* Helper class to run code at startup. */ +/** + * Helper class to run code at startup. + */ template struct OnStartup { OnStartup(T && t) { t(); } }; -/* Wrap bools to prevent string literals (i.e. 'char *') from being - cast to a bool in Attr. */ +/** + * Wrap bools to prevent string literals (i.e. 'char *') from being + * cast to a bool in Attr. + */ template struct Explicit { T t; @@ -45,21 +52,25 @@ struct Explicit { }; -/* This wants to be a little bit like rust's Cow type. - Some parts of the evaluator benefit greatly from being able to reuse - existing allocations for strings, but have to be able to also use - newly allocated storage for values. - - We do not define implicit conversions, even with ref qualifiers, - since those can easily become ambiguous to the reader and can degrade - into copying behaviour we want to avoid. */ +/** + * This wants to be a little bit like rust's Cow type. + * Some parts of the evaluator benefit greatly from being able to reuse + * existing allocations for strings, but have to be able to also use + * newly allocated storage for values. + * + * We do not define implicit conversions, even with ref qualifiers, + * since those can easily become ambiguous to the reader and can degrade + * into copying behaviour we want to avoid. + */ class BackedStringView { private: std::variant data; - /* Needed to introduce a temporary since operator-> must return - a pointer. Without this we'd need to store the view object - even when we already own a string. */ + /** + * Needed to introduce a temporary since operator-> must return + * a pointer. Without this we'd need to store the view object + * even when we already own a string. + */ class Ptr { private: std::string_view view; @@ -77,8 +88,10 @@ public: BackedStringView(const BackedStringView &) = delete; BackedStringView & operator=(const BackedStringView &) = delete; - /* We only want move operations defined since the sole purpose of - this type is to avoid copies. */ + /** + * We only want move operations defined since the sole purpose of + * this type is to avoid copies. + */ BackedStringView(BackedStringView && other) = default; BackedStringView & operator=(BackedStringView && other) = default; diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh index afc1df98a..98162b0f7 100644 --- a/src/libutil/url-parts.hh +++ b/src/libutil/url-parts.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -22,21 +23,22 @@ const static std::string segmentRegex = "(?:" + pcharRegex + "*)"; const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)"; const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)"; -// A Git ref (i.e. branch or tag name). -const static std::string refRegexS = "[a-zA-Z0-9@][a-zA-Z0-9_.\\/@-]*"; // FIXME: check +/// A Git ref (i.e. branch or tag name). +/// \todo check that this is correct. +const static std::string refRegexS = "[a-zA-Z0-9@][a-zA-Z0-9_.\\/@-]*"; extern std::regex refRegex; -// Instead of defining what a good Git Ref is, we define what a bad Git Ref is -// This is because of the definition of a ref in refs.c in https://github.com/git/git -// See tests/fetchGitRefs.sh for the full definition +/// Instead of defining what a good Git Ref is, we define what a bad Git Ref is +/// This is because of the definition of a ref in refs.c in https://github.com/git/git +/// See tests/fetchGitRefs.sh for the full definition const static std::string badGitRefRegexS = "//|^[./]|/\\.|\\.\\.|[[:cntrl:][:space:]:?^~\[]|\\\\|\\*|\\.lock$|\\.lock/|@\\{|[/.]$|^@$|^$"; extern std::regex badGitRefRegex; -// A Git revision (a SHA-1 commit hash). +/// A Git revision (a SHA-1 commit hash). const static std::string revRegexS = "[0-9a-fA-F]{40}"; extern std::regex revRegex; -// A ref or revision, or a ref followed by a revision. +/// A ref or revision, or a ref followed by a revision. const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))"; const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*"; diff --git a/src/libutil/url.hh b/src/libutil/url.hh index ddd673d65..d2413ec0e 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "error.hh" @@ -7,7 +8,8 @@ namespace nix { struct ParsedURL { std::string url; - std::string base; // URL without query/fragment + /// URL without query/fragment + std::string base; std::string scheme; std::optional authority; std::string path; @@ -28,7 +30,7 @@ std::map decodeQuery(const std::string & query); ParsedURL parseURL(const std::string & url); -/* +/** * Although that’s not really standardized anywhere, an number of tools * use a scheme of the form 'x+y' in urls, where y is the “transport layer” * scheme, and x is the “application layer” scheme. diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 843a10eab..21d1c8dcd 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -47,6 +47,9 @@ extern char * * environ __attribute__((weak)); namespace nix { +void initLibUtil() { +} + std::optional getEnv(const std::string & key) { char * value = getenv(key.c_str()); @@ -1744,13 +1747,39 @@ void triggerInterrupt() } static sigset_t savedSignalMask; +static bool savedSignalMaskIsSet = false; + +void setChildSignalMask(sigset_t * sigs) +{ + assert(sigs); // C style function, but think of sigs as a reference + +#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE + sigemptyset(&savedSignalMask); + // There's no "assign" or "copy" function, so we rely on (math) idempotence + // of the or operator: a or a = a. + sigorset(&savedSignalMask, sigs, sigs); +#else + // Without sigorset, our best bet is to assume that sigset_t is a type that + // can be assigned directly, such as is the case for a sigset_t defined as + // an integer type. + savedSignalMask = *sigs; +#endif + + savedSignalMaskIsSet = true; +} + +void saveSignalMask() { + if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask)) + throw SysError("querying signal mask"); + + savedSignalMaskIsSet = true; +} void startSignalHandlerThread() { updateWindowSize(); - if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask)) - throw SysError("querying signal mask"); + saveSignalMask(); sigset_t set; sigemptyset(&set); @@ -1767,6 +1796,20 @@ void startSignalHandlerThread() static void restoreSignals() { + // If startSignalHandlerThread wasn't called, that means we're not running + // in a proper libmain process, but a process that presumably manages its + // own signal handlers. Such a process should call either + // - initNix(), to be a proper libmain process + // - startSignalHandlerThread(), to resemble libmain regarding signal + // handling only + // - saveSignalMask(), for processes that define their own signal handling + // thread + // TODO: Warn about this? Have a default signal mask? The latter depends on + // whether we should generally inherit signal masks from the caller. + // I don't know what the larger unix ecosystem expects from us here. + if (!savedSignalMaskIsSet) + return; + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) throw SysError("restoring signals"); } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 5a3976d02..040fed68f 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "error.hh" @@ -31,78 +32,114 @@ namespace nix { struct Sink; struct Source; +void initLibUtil(); -/* The system for which Nix is compiled. */ +/** + * The system for which Nix is compiled. + */ extern const std::string nativeSystem; -/* Return an environment variable. */ +/** + * @return an environment variable. + */ std::optional getEnv(const std::string & key); -/* Return a non empty environment variable. Returns nullopt if the env -variable is set to "" */ +/** + * @return a non empty environment variable. Returns nullopt if the env + * variable is set to "" + */ std::optional getEnvNonEmpty(const std::string & key); -/* Get the entire environment. */ +/** + * Get the entire environment. + */ std::map getEnv(); -/* Clear the environment. */ +/** + * Clear the environment. + */ void clearEnv(); -/* Return an absolutized path, resolving paths relative to the - specified directory, or the current directory otherwise. The path - is also canonicalised. */ +/** + * @return An absolutized path, resolving paths relative to the + * specified directory, or the current directory otherwise. The path + * is also canonicalised. + */ Path absPath(Path path, std::optional dir = {}, bool resolveSymlinks = false); -/* Canonicalise a path by removing all `.' or `..' components and - double or trailing slashes. Optionally resolves all symlink - components such that each component of the resulting path is *not* - a symbolic link. */ +/** + * Canonicalise a path by removing all `.` or `..` components and + * double or trailing slashes. Optionally resolves all symlink + * components such that each component of the resulting path is *not* + * a symbolic link. + */ Path canonPath(PathView path, bool resolveSymlinks = false); -/* Return the directory part of the given canonical path, i.e., - everything before the final `/'. If the path is the root or an - immediate child thereof (e.g., `/foo'), this means `/' - is returned.*/ +/** + * @return The directory part of the given canonical path, i.e., + * everything before the final `/`. If the path is the root or an + * immediate child thereof (e.g., `/foo`), this means `/` + * is returned. + */ Path dirOf(const PathView path); -/* Return the base name of the given canonical path, i.e., everything - following the final `/' (trailing slashes are removed). */ +/** + * @return the base name of the given canonical path, i.e., everything + * following the final `/` (trailing slashes are removed). + */ std::string_view baseNameOf(std::string_view path); -/* Perform tilde expansion on a path. */ +/** + * Perform tilde expansion on a path. + */ std::string expandTilde(std::string_view path); -/* Check whether 'path' is a descendant of 'dir'. Both paths must be - canonicalized. */ +/** + * Check whether 'path' is a descendant of 'dir'. Both paths must be + * canonicalized. + */ bool isInDir(std::string_view path, std::string_view dir); -/* Check whether 'path' is equal to 'dir' or a descendant of - 'dir'. Both paths must be canonicalized. */ +/** + * Check whether 'path' is equal to 'dir' or a descendant of + * 'dir'. Both paths must be canonicalized. + */ bool isDirOrInDir(std::string_view path, std::string_view dir); -/* Get status of `path'. */ +/** + * Get status of `path`. + */ struct stat stat(const Path & path); struct stat lstat(const Path & path); -/* Return true iff the given path exists. */ +/** + * @return true iff the given path exists. + */ bool pathExists(const Path & path); -/* Read the contents (target) of a symbolic link. The result is not - in any way canonicalised. */ +/** + * Read the contents (target) of a symbolic link. The result is not + * in any way canonicalised. + */ Path readLink(const Path & path); bool isLink(const Path & path); -/* Read the contents of a directory. The entries `.' and `..' are - removed. */ +/** + * Read the contents of a directory. The entries `.` and `..` are + * removed. + */ struct DirEntry { std::string name; ino_t ino; - unsigned char type; // one of DT_* + /** + * one of DT_* + */ + unsigned char type; DirEntry(std::string name, ino_t ino, unsigned char type) : name(std::move(name)), ino(ino), type(type) { } }; @@ -113,74 +150,110 @@ DirEntries readDirectory(const Path & path); unsigned char getFileType(const Path & path); -/* Read the contents of a file into a string. */ +/** + * Read the contents of a file into a string. + */ std::string readFile(int fd); std::string readFile(const Path & path); void readFile(const Path & path, Sink & sink); -/* Write a string to a file. */ +/** + * Write a string to a file. + */ void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false); void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false); -/* Flush a file's parent directory to disk */ +/** + * Flush a file's parent directory to disk + */ void syncParent(const Path & path); -/* Read a line from a file descriptor. */ +/** + * Read a line from a file descriptor. + */ std::string readLine(int fd); -/* Write a line to a file descriptor. */ +/** + * Write a line to a file descriptor. + */ void writeLine(int fd, std::string s); -/* Delete a path; i.e., in the case of a directory, it is deleted - recursively. It's not an error if the path does not exist. The - second variant returns the number of bytes and blocks freed. */ +/** + * Delete a path; i.e., in the case of a directory, it is deleted + * recursively. It's not an error if the path does not exist. The + * second variant returns the number of bytes and blocks freed. + */ void deletePath(const Path & path); void deletePath(const Path & path, uint64_t & bytesFreed); std::string getUserName(); -/* Return the given user's home directory from /etc/passwd. */ +/** + * @return the given user's home directory from /etc/passwd. + */ Path getHomeOf(uid_t userId); -/* Return $HOME or the user's home directory from /etc/passwd. */ +/** + * @return $HOME or the user's home directory from /etc/passwd. + */ Path getHome(); -/* Return $XDG_CACHE_HOME or $HOME/.cache. */ +/** + * @return $XDG_CACHE_HOME or $HOME/.cache. + */ Path getCacheDir(); -/* Return $XDG_CONFIG_HOME or $HOME/.config. */ +/** + * @return $XDG_CONFIG_HOME or $HOME/.config. + */ Path getConfigDir(); -/* Return the directories to search for user configuration files */ +/** + * @return the directories to search for user configuration files + */ std::vector getConfigDirs(); -/* Return $XDG_DATA_HOME or $HOME/.local/share. */ +/** + * @return $XDG_DATA_HOME or $HOME/.local/share. + */ Path getDataDir(); -/* Return the path of the current executable. */ +/** + * @return the path of the current executable. + */ std::optional getSelfExe(); -/* Return $XDG_STATE_HOME or $HOME/.local/state. */ +/** + * @return $XDG_STATE_HOME or $HOME/.local/state. + */ Path getStateDir(); -/* Create the Nix state directory and return the path to it. */ +/** + * Create the Nix state directory and return the path to it. + */ Path createNixStateDir(); -/* Create a directory and all its parents, if necessary. Returns the - list of created directories, in order of creation. */ +/** + * Create a directory and all its parents, if necessary. Returns the + * list of created directories, in order of creation. + */ Paths createDirs(const Path & path); inline Paths createDirs(PathView path) { return createDirs(Path(path)); } -/* Create a symlink. */ +/** + * Create a symlink. + */ void createSymlink(const Path & target, const Path & link, std::optional mtime = {}); -/* Atomically create or replace a symlink. */ +/** + * Atomically create or replace a symlink. + */ void replaceSymlink(const Path & target, const Path & link, std::optional mtime = {}); @@ -196,24 +269,32 @@ void renameFile(const Path & src, const Path & dst); void moveFile(const Path & src, const Path & dst); -/* Wrappers arount read()/write() that read/write exactly the - requested number of bytes. */ +/** + * Wrappers arount read()/write() that read/write exactly the + * requested number of bytes. + */ void readFull(int fd, char * buf, size_t count); void writeFull(int fd, std::string_view s, bool allowInterrupts = true); MakeError(EndOfFile, Error); -/* Read a file descriptor until EOF occurs. */ +/** + * Read a file descriptor until EOF occurs. + */ std::string drainFD(int fd, bool block = true, const size_t reserveSize=0); void drainFD(int fd, Sink & sink, bool block = true); -/* If cgroups are active, attempt to calculate the number of CPUs available. - If cgroups are unavailable or if cpu.max is set to "max", return 0. */ +/** + * If cgroups are active, attempt to calculate the number of CPUs available. + * If cgroups are unavailable or if cpu.max is set to "max", return 0. + */ unsigned int getMaxCPU(); -/* Automatic cleanup of resources. */ +/** + * Automatic cleanup of resources. + */ class AutoDelete @@ -251,11 +332,15 @@ public: }; -/* Create a temporary directory. */ +/** + * Create a temporary directory. + */ Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); -/* Create a temporary file, returning a file handle and its path. */ +/** + * Create a temporary file, returning a file handle and its path. + */ std::pair createTempFile(const Path & prefix = "nix"); @@ -298,27 +383,36 @@ public: }; -/* Kill all processes running under the specified uid by sending them - a SIGKILL. */ +/** + * Kill all processes running under the specified uid by sending them + * a SIGKILL. + */ void killUser(uid_t uid); -/* Fork a process that runs the given function, and return the child - pid to the caller. */ +/** + * Fork a process that runs the given function, and return the child + * pid to the caller. + */ struct ProcessOptions { std::string errorPrefix = ""; bool dieWithParent = true; bool runExitHandlers = false; bool allowVfork = false; - int cloneFlags = 0; // use clone() with the specified flags (Linux only) + /** + * use clone() with the specified flags (Linux only) + */ + int cloneFlags = 0; }; pid_t startProcess(std::function fun, const ProcessOptions & options = ProcessOptions()); -/* Run a program and return its stdout in a string (i.e., like the - shell backtick operator). */ +/** + * Run a program and return its stdout in a string (i.e., like the + * shell backtick operator). + */ std::string runProgram(Path program, bool searchPath = false, const Strings & args = Strings(), const std::optional & input = {}); @@ -343,25 +437,37 @@ std::pair runProgram(RunOptions && options); void runProgram2(const RunOptions & options); -/* Change the stack size. */ +/** + * Change the stack size. + */ void setStackSize(size_t stackSize); -/* Restore the original inherited Unix process context (such as signal - masks, stack size). */ +/** + * Restore the original inherited Unix process context (such as signal + * masks, stack size). + + * See startSignalHandlerThread(), saveSignalMask(). + */ void restoreProcessContext(bool restoreMounts = true); -/* Save the current mount namespace. Ignored if called more than - once. */ +/** + * Save the current mount namespace. Ignored if called more than + * once. + */ void saveMountNamespace(); -/* Restore the mount namespace saved by saveMountNamespace(). Ignored - if saveMountNamespace() was never called. */ +/** + * Restore the mount namespace saved by saveMountNamespace(). Ignored + * if saveMountNamespace() was never called. + */ void restoreMountNamespace(); -/* Cause this thread to not share any FS attributes with the main - thread, because this causes setns() in restoreMountNamespace() to - fail. */ +/** + * Cause this thread to not share any FS attributes with the main + * thread, because this causes setns() in restoreMountNamespace() to + * fail. + */ void unshareFilesystem(); @@ -376,16 +482,22 @@ public: { } }; -/* Convert a list of strings to a null-terminated vector of char - *'s. The result must not be accessed beyond the lifetime of the - list of strings. */ +/** + * Convert a list of strings to a null-terminated vector of `char + * *`s. The result must not be accessed beyond the lifetime of the + * list of strings. + */ std::vector stringsToCharPtrs(const Strings & ss); -/* Close all file descriptors except those listed in the given set. - Good practice in child processes. */ +/** + * Close all file descriptors except those listed in the given set. + * Good practice in child processes. + */ void closeMostFDs(const std::set & exceptions); -/* Set the close-on-exec flag for the given file descriptor. */ +/** + * Set the close-on-exec flag for the given file descriptor. + */ void closeOnExec(int fd); @@ -411,12 +523,16 @@ MakeError(Interrupted, BaseError); MakeError(FormatError, Error); -/* String tokenizer. */ +/** + * String tokenizer. + */ template C tokenizeString(std::string_view s, std::string_view separators = " \t\n\r"); -/* Concatenate the given strings with a separator between the - elements. */ +/** + * Concatenate the given strings with a separator between the + * elements. + */ template std::string concatStringsSep(const std::string_view sep, const C & ss) { @@ -441,7 +557,9 @@ auto concatStrings(Parts && ... parts) } -/* Add quotes around a collection of strings. */ +/** + * Add quotes around a collection of strings. + */ template Strings quoteStrings(const C & c) { Strings res; @@ -450,16 +568,23 @@ template Strings quoteStrings(const C & c) return res; } -/* Remove trailing whitespace from a string. FIXME: return - std::string_view. */ +/** + * Remove trailing whitespace from a string. + * + * \todo return std::string_view. + */ std::string chomp(std::string_view s); -/* Remove whitespace from the start and end of a string. */ +/** + * Remove whitespace from the start and end of a string. + */ std::string trim(std::string_view s, std::string_view whitespace = " \n\r\t"); -/* Replace all occurrences of a string inside another string. */ +/** + * Replace all occurrences of a string inside another string. + */ std::string replaceStrings( std::string s, std::string_view from, @@ -469,14 +594,18 @@ std::string replaceStrings( std::string rewriteStrings(std::string s, const StringMap & rewrites); -/* Convert the exit status of a child as returned by wait() into an - error string. */ +/** + * Convert the exit status of a child as returned by wait() into an + * error string. + */ std::string statusToString(int status); bool statusOk(int status); -/* Parse a string into an integer. */ +/** + * Parse a string into an integer. + */ template std::optional string2Int(const std::string_view s) { @@ -489,8 +618,10 @@ std::optional string2Int(const std::string_view s) } } -/* Like string2Int(), but support an optional suffix 'K', 'M', 'G' or - 'T' denoting a binary unit prefix. */ +/** + * Like string2Int(), but support an optional suffix 'K', 'M', 'G' or + * 'T' denoting a binary unit prefix. + */ template N string2IntWithUnitPrefix(std::string_view s) { @@ -511,7 +642,9 @@ N string2IntWithUnitPrefix(std::string_view s) throw UsageError("'%s' is not an integer", s); } -/* Parse a string into a float. */ +/** + * Parse a string into a float. + */ template std::optional string2Float(const std::string_view s) { @@ -523,7 +656,9 @@ std::optional string2Float(const std::string_view s) } -/* Convert a little-endian integer to host order. */ +/** + * Convert a little-endian integer to host order. + */ template T readLittleEndian(unsigned char * p) { @@ -535,66 +670,90 @@ T readLittleEndian(unsigned char * p) } -/* Return true iff `s' starts with `prefix'. */ +/** + * @return true iff `s` starts with `prefix`. + */ bool hasPrefix(std::string_view s, std::string_view prefix); -/* Return true iff `s' ends in `suffix'. */ +/** + * @return true iff `s` ends in `suffix`. + */ bool hasSuffix(std::string_view s, std::string_view suffix); -/* Convert a string to lower case. */ +/** + * Convert a string to lower case. + */ std::string toLower(const std::string & s); -/* Escape a string as a shell word. */ +/** + * Escape a string as a shell word. + */ std::string shellEscape(const std::string_view s); -/* Exception handling in destructors: print an error message, then - ignore the exception. */ +/** + * Exception handling in destructors: print an error message, then + * ignore the exception. + */ void ignoreException(Verbosity lvl = lvlError); -/* Tree formatting. */ +/** + * Tree formatting. + */ constexpr char treeConn[] = "├───"; constexpr char treeLast[] = "└───"; constexpr char treeLine[] = "│ "; constexpr char treeNull[] = " "; -/* Determine whether ANSI escape sequences are appropriate for the - present output. */ +/** + * Determine whether ANSI escape sequences are appropriate for the + * present output. + */ bool shouldANSI(); -/* Truncate a string to 'width' printable characters. If 'filterAll' - is true, all ANSI escape sequences are filtered out. Otherwise, - some escape sequences (such as colour setting) are copied but not - included in the character count. Also, tabs are expanded to - spaces. */ +/** + * Truncate a string to 'width' printable characters. If 'filterAll' + * is true, all ANSI escape sequences are filtered out. Otherwise, + * some escape sequences (such as colour setting) are copied but not + * included in the character count. Also, tabs are expanded to + * spaces. + */ std::string filterANSIEscapes(std::string_view s, bool filterAll = false, unsigned int width = std::numeric_limits::max()); -/* Base64 encoding/decoding. */ +/** + * Base64 encoding/decoding. + */ std::string base64Encode(std::string_view s); std::string base64Decode(std::string_view s); -/* Remove common leading whitespace from the lines in the string - 's'. For example, if every line is indented by at least 3 spaces, - then we remove 3 spaces from the start of every line. */ +/** + * Remove common leading whitespace from the lines in the string + * 's'. For example, if every line is indented by at least 3 spaces, + * then we remove 3 spaces from the start of every line. + */ std::string stripIndentation(std::string_view s); -/* Get the prefix of 's' up to and excluding the next line break (LF - optionally preceded by CR), and the remainder following the line - break. */ +/** + * Get the prefix of 's' up to and excluding the next line break (LF + * optionally preceded by CR), and the remainder following the line + * break. + */ std::pair getLine(std::string_view s); -/* Get a value for the specified key from an associate container. */ +/** + * Get a value for the specified key from an associate container. + */ template const typename T::mapped_type * get(const T & map, const typename T::key_type & key) { @@ -611,7 +770,9 @@ typename T::mapped_type * get(T & map, const typename T::key_type & key) return &i->second; } -/* Get a value for the specified key from an associate container, or a default value if the key isn't present. */ +/** + * Get a value for the specified key from an associate container, or a default value if the key isn't present. + */ template const typename T::mapped_type & getOr(T & map, const typename T::key_type & key, @@ -622,7 +783,9 @@ const typename T::mapped_type & getOr(T & map, return i->second; } -/* Remove and return the first item from a container. */ +/** + * Remove and return the first item from a container. + */ template std::optional remove_begin(T & c) { @@ -634,7 +797,9 @@ std::optional remove_begin(T & c) } -/* Remove and return the first item from a container. */ +/** + * Remove and return the first item from a container. + */ template std::optional pop(T & c) { @@ -649,25 +814,48 @@ template class Callback; -/* Start a thread that handles various signals. Also block those signals - on the current thread (and thus any threads created by it). */ +/** + * Start a thread that handles various signals. Also block those signals + * on the current thread (and thus any threads created by it). + * Saves the signal mask before changing the mask to block those signals. + * See saveSignalMask(). + */ void startSignalHandlerThread(); +/** + * Saves the signal mask, which is the signal mask that nix will restore + * before creating child processes. + * See setChildSignalMask() to set an arbitrary signal mask instead of the + * current mask. + */ +void saveSignalMask(); + +/** + * Sets the signal mask. Like saveSignalMask() but for a signal set that doesn't + * necessarily match the current thread's mask. + * See saveSignalMask() to set the saved mask to the current mask. + */ +void setChildSignalMask(sigset_t *sigs); + struct InterruptCallback { virtual ~InterruptCallback() { }; }; -/* Register a function that gets called on SIGINT (in a non-signal - context). */ +/** + * Register a function that gets called on SIGINT (in a non-signal + * context). + */ std::unique_ptr createInterruptCallback( std::function callback); void triggerInterrupt(); -/* A RAII class that causes the current thread to receive SIGUSR1 when - the signal handler thread receives SIGINT. That is, this allows - SIGINT to be multiplexed to multiple threads. */ +/** + * A RAII class that causes the current thread to receive SIGUSR1 when + * the signal handler thread receives SIGINT. That is, this allows + * SIGINT to be multiplexed to multiple threads. + */ struct ReceiveInterrupts { pthread_t target; @@ -681,8 +869,10 @@ struct ReceiveInterrupts -/* A RAII helper that increments a counter on construction and - decrements it on destruction. */ +/** + * A RAII helper that increments a counter on construction and + * decrements it on destruction. + */ template struct MaintainCount { @@ -693,33 +883,50 @@ struct MaintainCount }; -/* Return the number of rows and columns of the terminal. */ +/** + * @return the number of rows and columns of the terminal. + */ std::pair getWindowSize(); -/* Used in various places. */ +/** + * Used in various places. + */ typedef std::function PathFilter; extern PathFilter defaultPathFilter; -/* Common initialisation performed in child processes. */ +/** + * Common initialisation performed in child processes. + */ void commonChildInit(); -/* Create a Unix domain socket. */ +/** + * Create a Unix domain socket. + */ AutoCloseFD createUnixDomainSocket(); -/* Create a Unix domain socket in listen mode. */ +/** + * Create a Unix domain socket in listen mode. + */ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); -/* Bind a Unix domain socket to a path. */ +/** + * Bind a Unix domain socket to a path. + */ void bind(int fd, const std::string & path); -/* Connect to a Unix domain socket. */ +/** + * Connect to a Unix domain socket. + */ void connect(int fd, const std::string & path); -// A Rust/Python-like enumerate() iterator adapter. -// Borrowed from http://reedbeta.com/blog/python-like-enumerate-in-cpp17. +/** + * A Rust/Python-like enumerate() iterator adapter. + * + * Borrowed from http://reedbeta.com/blog/python-like-enumerate-in-cpp17. + */ template ())), typename = decltype(std::end(std::declval()))> @@ -729,23 +936,25 @@ constexpr auto enumerate(T && iterable) { size_t i; TIter iter; - bool operator != (const iterator & other) const { return iter != other.iter; } - void operator ++ () { ++i; ++iter; } - auto operator * () const { return std::tie(i, *iter); } + constexpr bool operator != (const iterator & other) const { return iter != other.iter; } + constexpr void operator ++ () { ++i; ++iter; } + constexpr auto operator * () const { return std::tie(i, *iter); } }; struct iterable_wrapper { T iterable; - auto begin() { return iterator{ 0, std::begin(iterable) }; } - auto end() { return iterator{ 0, std::end(iterable) }; } + constexpr auto begin() { return iterator{ 0, std::begin(iterable) }; } + constexpr auto end() { return iterator{ 0, std::end(iterable) }; } }; return iterable_wrapper{ std::forward(iterable) }; } -// C++17 std::visit boilerplate +/** + * C++17 std::visit boilerplate + */ template struct overloaded : Ts... { using Ts::operator()...; }; template overloaded(Ts...) -> overloaded; @@ -753,8 +962,10 @@ template overloaded(Ts...) -> overloaded; std::string showBytes(uint64_t bytes); -/* Provide an addition operator between strings and string_views - inexplicably omitted from the standard library. */ +/** + * Provide an addition operator between strings and string_views + * inexplicably omitted from the standard library. + */ inline std::string operator + (const std::string & s1, std::string_view s2) { auto s = s1; diff --git a/src/libutil/xml-writer.hh b/src/libutil/xml-writer.hh index 4c91adee6..74f53b7ca 100644 --- a/src/libutil/xml-writer.hh +++ b/src/libutil/xml-writer.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include diff --git a/src/nix-env/user-env.hh b/src/nix-env/user-env.hh index 10646f713..af45d2d85 100644 --- a/src/nix-env/user-env.hh +++ b/src/nix-env/user-env.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "get-drvs.hh" diff --git a/src/nix-store/dotgraph.hh b/src/nix-store/dotgraph.hh index 73b8d06b9..4fd944080 100644 --- a/src/nix-store/dotgraph.hh +++ b/src/nix-store/dotgraph.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" diff --git a/src/nix-store/graphml.hh b/src/nix-store/graphml.hh index 78be8a367..bd3a4a37c 100644 --- a/src/nix-store/graphml.hh +++ b/src/nix-store/graphml.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 3d2dc49fd..03fc93962 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -283,17 +283,17 @@ static void printTree(const StorePath & path, static void opQuery(Strings opFlags, Strings opArgs) { enum QueryType - { qDefault, qOutputs, qRequisites, qReferences, qReferrers + { qOutputs, qRequisites, qReferences, qReferrers , qReferrersClosure, qDeriver, qBinding, qHash, qSize , qTree, qGraph, qGraphML, qResolve, qRoots }; - QueryType query = qDefault; + std::optional query; bool useOutput = false; bool includeOutputs = false; bool forceRealise = false; std::string bindingName; for (auto & i : opFlags) { - QueryType prev = query; + std::optional prev = query; if (i == "--outputs") query = qOutputs; else if (i == "--requisites" || i == "-R") query = qRequisites; else if (i == "--references") query = qReferences; @@ -318,15 +318,15 @@ static void opQuery(Strings opFlags, Strings opArgs) else if (i == "--force-realise" || i == "--force-realize" || i == "-f") forceRealise = true; else if (i == "--include-outputs") includeOutputs = true; else throw UsageError("unknown flag '%1%'", i); - if (prev != qDefault && prev != query) + if (prev && prev != query) throw UsageError("query type '%1%' conflicts with earlier flag", i); } - if (query == qDefault) query = qOutputs; + if (!query) query = qOutputs; RunPager pager; - switch (query) { + switch (*query) { case qOutputs: { for (auto & i : opArgs) { diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 7e4a7ba86..7ae7b4ea6 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -1,3 +1,5 @@ +///@file + #include "command.hh" #include "shared.hh" #include "local-store.hh" @@ -34,6 +36,19 @@ using namespace nix; using namespace nix::daemon; +/** + * Settings related to authenticating clients for the Nix daemon. + * + * For pipes we have little good information about the client side, but + * for Unix domain sockets we do. So currently these options implemented + * mandatory access control based on user names and group names (looked + * up and translated to UID/GIDs in the CLI process that runs the code + * in this file). + * + * No code outside of this file knows about these settings (this is not + * exposed in a header); all authentication and authorization happens in + * `daemon.cc`. + */ struct AuthorizationSettings : Config { Setting trustedUsers{ @@ -54,7 +69,9 @@ struct AuthorizationSettings : Config { > directories that are otherwise inacessible to them. )"}; - /* ?Who we trust to use the daemon in safe ways */ + /** + * Who we trust to use the daemon in safe ways + */ Setting allowedUsers{ this, {"*"}, "allowed-users", R"( @@ -112,8 +129,36 @@ static void setSigChldAction(bool autoReap) throw SysError("setting SIGCHLD handler"); } +/** + * @return Is the given user a member of this group? + * + * @param user User specified by username. + * + * @param group Group the user might be a member of. + */ +static bool matchUser(std::string_view user, const struct group & gr) +{ + for (char * * mem = gr.gr_mem; *mem; mem++) + if (user == std::string_view(*mem)) return true; + return false; +} -bool matchUser(const std::string & user, const std::string & group, const Strings & users) + +/** + * Does the given user (specified by user name and primary group name) + * match the given user/group whitelist? + * + * If the list allows all users: Yes. + * + * If the username is in the set: Yes. + * + * If the groupname is in the set: Yes. + * + * If the user is in another group which is in the set: yes. + * + * Otherwise: No. + */ +static bool matchUser(const std::string & user, const std::string & group, const Strings & users) { if (find(users.begin(), users.end(), "*") != users.end()) return true; @@ -126,8 +171,7 @@ bool matchUser(const std::string & user, const std::string & group, const String if (group == i.substr(1)) return true; struct group * gr = getgrnam(i.c_str() + 1); if (!gr) continue; - for (char * * mem = gr->gr_mem; *mem; mem++) - if (user == std::string(*mem)) return true; + if (matchUser(user, *gr)) return true; } return false; @@ -145,7 +189,9 @@ struct PeerInfo }; -// Get the identity of the caller, if possible. +/** + * Get the identity of the caller, if possible. + */ static PeerInfo getPeerInfo(int remote) { PeerInfo peer = { false, 0, false, 0, false, 0 }; @@ -179,6 +225,9 @@ static PeerInfo getPeerInfo(int remote) #define SD_LISTEN_FDS_START 3 +/** + * Open a store without a path info cache. + */ static ref openUncachedStore() { Store::Params params; // FIXME: get params from somewhere @@ -187,7 +236,44 @@ static ref openUncachedStore() return openStore(settings.storeUri, params); } +/** + * Authenticate a potential client + * + * @param peer Information about other end of the connection, the client which + * wants to communicate with us. + * + * @return A pair of a `TrustedFlag`, whether the potential client is trusted, + * and the name of the user (useful for printing messages). + * + * If the potential client is not allowed to talk to us, we throw an `Error`. + */ +static std::pair authPeer(const PeerInfo & peer) +{ + TrustedFlag trusted = NotTrusted; + struct passwd * pw = peer.uidKnown ? getpwuid(peer.uid) : 0; + std::string user = pw ? pw->pw_name : std::to_string(peer.uid); + + struct group * gr = peer.gidKnown ? getgrgid(peer.gid) : 0; + std::string group = gr ? gr->gr_name : std::to_string(peer.gid); + + const Strings & trustedUsers = authorizationSettings.trustedUsers; + const Strings & allowedUsers = authorizationSettings.allowedUsers; + + if (matchUser(user, group, trustedUsers)) + trusted = Trusted; + + if ((!trusted && !matchUser(user, group, allowedUsers)) || group == settings.buildUsersGroup) + throw Error("user '%1%' is not allowed to connect to the Nix daemon", user); + + return { trusted, std::move(user) }; +} + + +/** + * Run a server. The loop opens a socket and accepts new connections from that + * socket. + */ static void daemonLoop() { if (chdir("/") == -1) @@ -231,23 +317,9 @@ static void daemonLoop() closeOnExec(remote.get()); - TrustedFlag trusted = NotTrusted; PeerInfo peer = getPeerInfo(remote.get()); - - struct passwd * pw = peer.uidKnown ? getpwuid(peer.uid) : 0; - std::string user = pw ? pw->pw_name : std::to_string(peer.uid); - - struct group * gr = peer.gidKnown ? getgrgid(peer.gid) : 0; - std::string group = gr ? gr->gr_name : std::to_string(peer.gid); - - Strings trustedUsers = authorizationSettings.trustedUsers; - Strings allowedUsers = authorizationSettings.allowedUsers; - - if (matchUser(user, group, trustedUsers)) - trusted = Trusted; - - if ((!trusted && !matchUser(user, group, allowedUsers)) || group == settings.buildUsersGroup) - throw Error("user '%1%' is not allowed to connect to the Nix daemon", user); + auto [_trusted, user] = authPeer(peer); + auto trusted = _trusted; printInfo((std::string) "accepted connection from pid %1%, user %2%" + (trusted ? " (trusted)" : ""), peer.pidKnown ? std::to_string(peer.pid) : "", @@ -294,45 +366,74 @@ static void daemonLoop() } } +/** + * Forward a standard IO connection to the given remote store. + * + * We just act as a middleman blindly ferry output between the standard + * input/output and the remote store connection, not processing anything. + * + * Loops until standard input disconnects, or an error is encountered. + */ +static void forwardStdioConnection(RemoteStore & store) { + auto conn = store.openConnectionWrapper(); + int from = conn->from.fd; + int to = conn->to.fd; + + auto nfds = std::max(from, STDIN_FILENO) + 1; + while (true) { + fd_set fds; + FD_ZERO(&fds); + FD_SET(from, &fds); + FD_SET(STDIN_FILENO, &fds); + if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1) + throw SysError("waiting for data from client or server"); + if (FD_ISSET(from, &fds)) { + auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE); + if (res == -1) + throw SysError("splicing data from daemon socket to stdout"); + else if (res == 0) + throw EndOfFile("unexpected EOF from daemon socket"); + } + if (FD_ISSET(STDIN_FILENO, &fds)) { + auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE); + if (res == -1) + throw SysError("splicing data from stdin to daemon socket"); + else if (res == 0) + return; + } + } +} + +/** + * Process a client connecting to us via standard input/output + * + * Unlike `forwardStdioConnection()` we do process commands ourselves in + * this case, not delegating to another daemon. + * + * @note `Trusted` is unconditionally passed because in this mode we + * blindly trust the standard streams. Limiting access to those is + * explicitly not `nix-daemon`'s responsibility. + */ +static void processStdioConnection(ref store) +{ + FdSource from(STDIN_FILENO); + FdSink to(STDOUT_FILENO); + processConnection(store, from, to, Trusted, NotRecursive); +} + +/** + * Entry point shared between the new CLI `nix daemon` and old CLI + * `nix-daemon`. + */ static void runDaemon(bool stdio) { if (stdio) { - if (auto store = openUncachedStore().dynamic_pointer_cast()) { - auto conn = store->openConnectionWrapper(); - int from = conn->from.fd; - int to = conn->to.fd; + auto store = openUncachedStore(); - auto nfds = std::max(from, STDIN_FILENO) + 1; - while (true) { - fd_set fds; - FD_ZERO(&fds); - FD_SET(from, &fds); - FD_SET(STDIN_FILENO, &fds); - if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1) - throw SysError("waiting for data from client or server"); - if (FD_ISSET(from, &fds)) { - auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE); - if (res == -1) - throw SysError("splicing data from daemon socket to stdout"); - else if (res == 0) - throw EndOfFile("unexpected EOF from daemon socket"); - } - if (FD_ISSET(STDIN_FILENO, &fds)) { - auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE); - if (res == -1) - throw SysError("splicing data from stdin to daemon socket"); - else if (res == 0) - return; - } - } - } else { - FdSource from(STDIN_FILENO); - FdSink to(STDOUT_FILENO); - /* Auth hook is empty because in this mode we blindly trust the - standard streams. Limiting access to those is explicitly - not `nix-daemon`'s responsibility. */ - processConnection(openUncachedStore(), from, to, Trusted, NotRecursive); - } + if (auto remoteStore = store.dynamic_pointer_cast()) + forwardStdioConnection(*remoteStore); + else + processStdioConnection(store); } else daemonLoop(); } diff --git a/src/nix/derivation-add.cc b/src/nix/derivation-add.cc new file mode 100644 index 000000000..4d91d4538 --- /dev/null +++ b/src/nix/derivation-add.cc @@ -0,0 +1,45 @@ +// FIXME: rename to 'nix plan add' or 'nix derivation add'? + +#include "command.hh" +#include "common-args.hh" +#include "store-api.hh" +#include "archive.hh" +#include "derivations.hh" +#include + +using namespace nix; +using json = nlohmann::json; + +struct CmdAddDerivation : MixDryRun, StoreCommand +{ + std::string description() override + { + return "Add a store derivation"; + } + + std::string doc() override + { + return + #include "derivation-add.md" + ; + } + + Category category() override { return catUtility; } + + void run(ref store) override + { + auto json = nlohmann::json::parse(drainFD(STDIN_FILENO)); + + auto drv = Derivation::fromJSON(*store, json); + + auto drvPath = writeDerivation(*store, drv, NoRepair, /* read only */ dryRun); + + drv.checkInvariants(*store, drvPath); + + writeDerivation(*store, drv, NoRepair, dryRun); + + logger->cout("%s", store->printStorePath(drvPath)); + } +}; + +static auto rCmdAddDerivation = registerCommand2({"derivation", "add"}); diff --git a/src/nix/derivation-add.md b/src/nix/derivation-add.md new file mode 100644 index 000000000..f116681ab --- /dev/null +++ b/src/nix/derivation-add.md @@ -0,0 +1,18 @@ +R""( + +# Description + +This command reads from standard input a JSON representation of a +[store derivation] to which an [*installable*](./nix.md#installables) evaluates. + +Store derivations are used internally by Nix. They are store paths with +extension `.drv` that represent the build-time dependency graph to which +a Nix expression evaluates. + +[store derivation]: ../../glossary.md#gloss-store-derivation + +The JSON format is documented under the [`derivation show`] command. + +[`derivation show`]: ./nix3-derivation-show.md + +)"" diff --git a/src/nix/show-derivation.cc b/src/nix/derivation-show.cc similarity index 87% rename from src/nix/show-derivation.cc rename to src/nix/derivation-show.cc index 4a406ae08..bf637246d 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/derivation-show.cc @@ -1,5 +1,5 @@ // FIXME: integrate this with nix path-info? -// FIXME: rename to 'nix store show-derivation' or 'nix debug show-derivation'? +// FIXME: rename to 'nix store derivation show' or 'nix debug derivation show'? #include "command.hh" #include "common-args.hh" @@ -33,7 +33,7 @@ struct CmdShowDerivation : InstallablesCommand std::string doc() override { return - #include "show-derivation.md" + #include "derivation-show.md" ; } @@ -61,4 +61,4 @@ struct CmdShowDerivation : InstallablesCommand } }; -static auto rCmdShowDerivation = registerCommand("show-derivation"); +static auto rCmdShowDerivation = registerCommand2({"derivation", "show"}); diff --git a/src/nix/show-derivation.md b/src/nix/derivation-show.md similarity index 86% rename from src/nix/show-derivation.md rename to src/nix/derivation-show.md index 1d37c6f5a..1296e2885 100644 --- a/src/nix/show-derivation.md +++ b/src/nix/derivation-show.md @@ -8,7 +8,7 @@ R""( [store derivation]: ../../glossary.md#gloss-store-derivation ```console - # nix show-derivation nixpkgs#hello + # nix derivation show nixpkgs#hello { "/nix/store/s6rn4jz1sin56rf4qj5b5v8jxjm32hlk-hello-2.10.drv": { … @@ -20,14 +20,14 @@ R""( NixOS system: ```console - # nix show-derivation -r /run/current-system + # nix derivation show -r /run/current-system ``` * Print all files fetched using `fetchurl` by Firefox's dependency graph: ```console - # nix show-derivation -r nixpkgs#firefox \ + # nix derivation show -r nixpkgs#firefox \ | jq -r '.[] | select(.outputs.out.hash and .env.urls) | .env.urls' \ | uniq | sort ``` @@ -39,10 +39,11 @@ R""( # Description This command prints on standard output a JSON representation of the -[store derivation]s to which [*installables*](./nix.md#installables) evaluate. Store derivations -are used internally by Nix. They are store paths with extension `.drv` -that represent the build-time dependency graph to which a Nix -expression evaluates. +[store derivation]s to which [*installables*](./nix.md#installables) evaluate. + +Store derivations are used internally by Nix. They are store paths with +extension `.drv` that represent the build-time dependency graph to which +a Nix expression evaluates. By default, this command only shows top-level derivations, but with `--recursive`, it also shows their dependencies. @@ -51,6 +52,9 @@ The JSON output is a JSON object whose keys are the store paths of the derivations, and whose values are a JSON object with the following fields: +* `name`: The name of the derivation. This is used when calculating the + store paths of the derivation's outputs. + * `outputs`: Information about the output paths of the derivation. This is a JSON object with one member per output, where the key is the output name and the value is a JSON object with these diff --git a/src/nix/derivation.cc b/src/nix/derivation.cc new file mode 100644 index 000000000..cd3975a4f --- /dev/null +++ b/src/nix/derivation.cc @@ -0,0 +1,25 @@ +#include "command.hh" + +using namespace nix; + +struct CmdDerivation : virtual NixMultiCommand +{ + CmdDerivation() : MultiCommand(RegisterCommand::getCommandsFor({"derivation"})) + { } + + std::string description() override + { + return "Work with derivations, Nix's notion of a build plan."; + } + + Category category() override { return catUtility; } + + void run() override + { + if (!command) + throw UsageError("'nix derivation' requires a sub-command."); + command->second->run(); + } +}; + +static auto rCmdDerivation = registerCommand("derivation"); diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc index 7da4549a1..1aa6831d3 100644 --- a/src/nix/doctor.cc +++ b/src/nix/doctor.cc @@ -33,12 +33,24 @@ bool checkFail(const std::string & msg) { return false; } +void checkInfo(const std::string & msg) { + notice(ANSI_BLUE "[INFO] " ANSI_NORMAL + msg); +} + } struct CmdDoctor : StoreCommand { bool success = true; + /** + * This command is stable before the others + */ + std::optional experimentalFeature() override + { + return std::nullopt; + } + std::string description() override { return "check your system for potential problems and print a PASS or FAIL for each check"; @@ -55,6 +67,7 @@ struct CmdDoctor : StoreCommand success &= checkProfileRoots(store); } success &= checkStoreProtocol(store->getProtocol()); + checkTrustedUser(store); if (!success) throw Exit(2); @@ -130,6 +143,14 @@ struct CmdDoctor : StoreCommand return checkPass("Client protocol matches store protocol."); } + + void checkTrustedUser(ref store) + { + std::string_view trusted = store->isTrustedClient() + ? "trusted" + : "not trusted"; + checkInfo(fmt("You are %s by store uri: %s", trusted, store->getUri())); + } }; static auto rCmdDoctor = registerCommand("doctor"); diff --git a/src/nix/flake.md b/src/nix/flake.md index cd9f656e3..d70f34eeb 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -221,11 +221,46 @@ Currently the `type` attribute can be one of the following: commit hash (`rev`). Note that unlike Git, GitHub allows fetching by commit hash without specifying a branch or tag. + You can also specify `host` as a parameter, to point to a custom GitHub + Enterprise server. + Some examples: * `github:edolstra/dwarffs` * `github:edolstra/dwarffs/unstable` * `github:edolstra/dwarffs/d3f2baba8f425779026c6ec04021b2e927f61e31` + * `github:internal/project?host=company-github.example.org` + +* `gitlab`: Similar to `github`, is a more efficient way to fetch + GitLab repositories. The following attributes are required: + + * `owner`: The owner of the repository. + + * `repo`: The name of the repository. + + Like `github`, these are downloaded as tarball archives. + + The URL syntax for `gitlab` flakes is: + + `gitlab:/(/)?(\?)?` + + `` works the same as `github`. Either a branch or tag name + (`ref`), or a commit hash (`rev`) can be specified. + + Since GitLab allows for self-hosting, you can specify `host` as + a parameter, to point to any instances other than `gitlab.com`. + + Some examples: + + * `gitlab:veloren/veloren` + * `gitlab:veloren/veloren/master` + * `gitlab:veloren/veloren/80a4d7f13492d916e47d6195be23acae8001985a` + * `gitlab:openldap/openldap?host=git.openldap.org` + + When accessing a project in a (nested) subgroup, make sure to URL-encode any + slashes, i.e. replace `/` with `%2F`: + + * `gitlab:veloren%2Fdev/rfcs` * `sourcehut`: Similar to `github`, is a more efficient way to fetch SourceHut repositories. The following attributes are required: diff --git a/src/nix/main.cc b/src/nix/main.cc index 54c920b4e..705061d25 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -83,6 +83,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .description = "Print full build logs on standard error.", .category = loggingCategory, .handler = {[&]() { logger->setPrintBuildLogs(true); }}, + .experimentalFeature = Xp::NixCommand, }); addFlag({ @@ -98,6 +99,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .description = "Disable substituters and consider all previously downloaded files up-to-date.", .category = miscCategory, .handler = {[&]() { useNet = false; }}, + .experimentalFeature = Xp::NixCommand, }); addFlag({ @@ -105,6 +107,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .description = "Consider all previously downloaded files out-of-date.", .category = miscCategory, .handler = {[&]() { refresh = true; }}, + .experimentalFeature = Xp::NixCommand, }); } @@ -124,6 +127,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs {"optimise-store", {"store", "optimise"}}, {"ping-store", {"store", "ping"}}, {"sign-paths", {"store", "sign"}}, + {"show-derivation", {"derivation", "show"}}, {"to-base16", {"hash", "to-base16"}}, {"to-base32", {"hash", "to-base32"}}, {"to-base64", {"hash", "to-base64"}}, @@ -371,6 +375,11 @@ void mainWrapped(int argc, char * * argv) return; } + if (argc == 2 && std::string(argv[1]) == "__dump-xp-features") { + logger->cout(documentExperimentalFeatures().dump()); + return; + } + Finally printCompletions([&]() { if (completions) { @@ -420,10 +429,8 @@ void mainWrapped(int argc, char * * argv) if (!args.command) throw UsageError("no subcommand specified"); - if (args.command->first != "repl" - && args.command->first != "doctor" - && args.command->first != "upgrade-nix") - experimentalFeatureSettings.require(Xp::NixCommand); + experimentalFeatureSettings.require( + args.command->second->experimentalFeature()); if (args.useNet && !haveInternet()) { warn("you don't have Internet access; disabling some network-dependent features"); diff --git a/src/nix/nix.md b/src/nix/nix.md index e1865b31c..1ef6c7fcd 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -48,12 +48,17 @@ manual](https://nixos.org/manual/nix/stable/). # Installables +> **Warning** \ +> Installables are part of the unstable +> [`nix-command` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-nix-command), +> and subject to change without notice. + Many `nix` subcommands operate on one or more *installables*. These are command line arguments that represent something that can be realised in the Nix store. The following types of installable are supported by most commands: -- [Flake output attribute](#flake-output-attribute) +- [Flake output attribute](#flake-output-attribute) (experimental) - [Store path](#store-path) - [Nix file](#nix-file), optionally qualified by an attribute path - [Nix expression](#nix-expression), optionally qualified by an attribute path @@ -63,6 +68,13 @@ That is, Nix will operate on the default flake output attribute of the flake in ### Flake output attribute +> **Warning** \ +> Flake output attribute installables depend on both the +> [`flakes`](@docroot@/contributing/experimental-features.md#xp-feature-flakes) +> and +> [`nix-command`](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) +> experimental features, and subject to change without notice. + Example: `nixpkgs#hello` These have the form *flakeref*[`#`*attrpath*], where *flakeref* is a diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc index 5c44510ab..ec450e8e0 100644 --- a/src/nix/ping-store.cc +++ b/src/nix/ping-store.cc @@ -28,15 +28,20 @@ struct CmdPingStore : StoreCommand, MixJSON store->connect(); if (auto version = store->getVersion()) notice("Version: %s", *version); + if (auto trusted = store->isTrustedClient()) + notice("Trusted: %s", *trusted); } else { nlohmann::json res; Finally printRes([&]() { logger->cout("%s", res); }); + res["url"] = store->getUri(); store->connect(); if (auto version = store->getVersion()) res["version"] = *version; + if (auto trusted = store->isTrustedClient()) + res["trusted"] = *trusted; } } }; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 5aa87a313..fd63b3519 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -209,6 +209,7 @@ struct ProfileManifest }, .references = { .others = std::move(references), + // profiles never refer to themselves .self = false, }, }, diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 7aa8774e9..bb14f3f99 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -12,6 +12,14 @@ struct CmdRepl : RawInstallablesCommand evalSettings.pureEval = false; } + /** + * This command is stable before the others + */ + std::optional experimentalFeature() override + { + return std::nullopt; + } + std::vector files; Strings getDefaultFlakeAttrPaths() override diff --git a/src/nix/run.hh b/src/nix/run.hh index fed360158..97ddef19b 100644 --- a/src/nix/run.hh +++ b/src/nix/run.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 17796d6b8..2295d86d0 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -32,6 +32,14 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand }); } + /** + * This command is stable before the others + */ + std::optional experimentalFeature() override + { + return std::nullopt; + } + std::string description() override { return "upgrade Nix to the stable version declared in Nixpkgs"; diff --git a/tests/ca/build.sh b/tests/ca/build.sh index 98e1c5125..7754ad276 100644 --- a/tests/ca/build.sh +++ b/tests/ca/build.sh @@ -3,7 +3,7 @@ source common.sh drv=$(nix-instantiate ./content-addressed.nix -A rootCA --arg seed 1) -nix show-derivation "$drv" --arg seed 1 +nix derivation show "$drv" --arg seed 1 buildAttr () { local derivationPath=$1 diff --git a/tests/ca/derivation-json.sh b/tests/ca/derivation-json.sh new file mode 100644 index 000000000..3615177e9 --- /dev/null +++ b/tests/ca/derivation-json.sh @@ -0,0 +1,26 @@ +source common.sh + +export NIX_TESTS_CA_BY_DEFAULT=1 + +drvPath=$(nix-instantiate ../simple.nix) + +nix derivation show $drvPath | jq .[] > $TEST_HOME/simple.json + +drvPath2=$(nix derivation add < $TEST_HOME/simple.json) + +[[ "$drvPath" = "$drvPath2" ]] + +# Content-addressed derivations can be renamed. +jq '.name = "foo"' < $TEST_HOME/simple.json > $TEST_HOME/foo.json +drvPath3=$(nix derivation add --dry-run < $TEST_HOME/foo.json) +# With --dry-run nothing is actually written +[[ ! -e "$drvPath3" ]] + +# Without --dry-run it is actually written +drvPath4=$(nix derivation add < $TEST_HOME/foo.json) +[[ "$drvPath4" = "$drvPath3" ]] +[[ -e "$drvPath3" ]] + +# The modified derivation read back as JSON matches +nix derivation show $drvPath3 | jq .[] > $TEST_HOME/foo-read.json +diff $TEST_HOME/foo.json $TEST_HOME/foo-read.json diff --git a/tests/derivation-json.sh b/tests/derivation-json.sh new file mode 100644 index 000000000..b6be5d977 --- /dev/null +++ b/tests/derivation-json.sh @@ -0,0 +1,12 @@ +source common.sh + +drvPath=$(nix-instantiate simple.nix) + +nix derivation show $drvPath | jq .[] > $TEST_HOME/simple.json + +drvPath2=$(nix derivation add < $TEST_HOME/simple.json) + +[[ "$drvPath" = "$drvPath2" ]] + +# Input addressed derivations cannot be renamed. +jq '.name = "foo"' < $TEST_HOME/simple.json | expectStderr 1 nix derivation add | grepQuiet "has incorrect output" diff --git a/tests/experimental-features.sh b/tests/experimental-features.sh index 3be77d5cc..73554da8c 100644 --- a/tests/experimental-features.sh +++ b/tests/experimental-features.sh @@ -1,23 +1,42 @@ source common.sh -# Without flakes, flake options should not show up -# With flakes, flake options should show up - -function both_ways { - nix --experimental-features 'nix-command' "$@" | grepQuietInverse flake - nix --experimental-features 'nix-command flakes' "$@" | grepQuiet flake - - # Also, the order should not matter - nix "$@" --experimental-features 'nix-command' | grepQuietInverse flake - nix "$@" --experimental-features 'nix-command flakes' | grepQuiet flake -} - -# Simple case, the configuration effects the running command -both_ways show-config - -# Complicated case, earlier args effect later args - -both_ways store gc --help +# Skipping these two for now, because we actually *do* want flags and +# config settings to always show up in the manual, just be marked +# experimental. Will reenable once the manual generation takes advantage +# of the JSON metadata on this. +# +# # Without flakes, flake options should not show up +# # With flakes, flake options should show up +# +# function grep_both_ways { +# nix --experimental-features 'nix-command' "$@" | grepQuietInverse flake +# nix --experimental-features 'nix-command flakes' "$@" | grepQuiet flake +# +# # Also, the order should not matter +# nix "$@" --experimental-features 'nix-command' | grepQuietInverse flake +# nix "$@" --experimental-features 'nix-command flakes' | grepQuiet flake +# } +# +# # Simple case, the configuration effects the running command +# grep_both_ways show-config +# +# # Medium case, the configuration effects --help +# grep_both_ways store gc --help expect 1 nix --experimental-features 'nix-command' show-config --flake-registry 'https://no' nix --experimental-features 'nix-command flakes' show-config --flake-registry 'https://no' + +# Double check these are stable +nix --experimental-features '' --help +nix --experimental-features '' doctor --help +nix --experimental-features '' repl --help +nix --experimental-features '' upgrade-nix --help + +# These 3 arguments are currently given to all commands, which is wrong (as not +# all care). To deal with fixing later, we simply make them require the +# nix-command experimental features --- it so happens that the commands we wish +# stabilizing to do not need them anyways. +for arg in '--print-build-logs' '--offline' '--refresh'; do + nix --experimental-features 'nix-command' "$arg" --help + ! nix --experimental-features '' "$arg" --help +done diff --git a/tests/flakes/flakes.sh b/tests/flakes/flakes.sh index 5c922d7c5..f2e216435 100644 --- a/tests/flakes/flakes.sh +++ b/tests/flakes/flakes.sh @@ -96,7 +96,9 @@ json=$(nix flake metadata flake1 --json | jq .) hash1=$(echo "$json" | jq -r .revision) echo -n '# foo' >> $flake1Dir/flake.nix +flake1OriginalCommit=$(git -C $flake1Dir rev-parse HEAD) git -C $flake1Dir commit -a -m 'Foo' +flake1NewCommit=$(git -C $flake1Dir rev-parse HEAD) hash2=$(nix flake metadata flake1 --json --refresh | jq -r .revision) [[ $hash1 != $hash2 ]] @@ -491,3 +493,14 @@ nix store delete $(nix store add-path $badFlakeDir) [[ $(nix-instantiate --eval flake:git+file://$flake3Dir -A x) = 123 ]] [[ $(nix-instantiate -I flake3=flake:flake3 --eval '' -A x) = 123 ]] [[ $(NIX_PATH=flake3=flake:flake3 nix-instantiate --eval '' -A x) = 123 ]] + +# Test alternate lockfile paths. +nix flake lock $flake2Dir --output-lock-file $TEST_ROOT/flake2.lock +cmp $flake2Dir/flake.lock $TEST_ROOT/flake2.lock >/dev/null # lockfiles should be identical, since we're referencing flake2's original one + +nix flake lock $flake2Dir --output-lock-file $TEST_ROOT/flake2-overridden.lock --override-input flake1 git+file://$flake1Dir?rev=$flake1OriginalCommit +expectStderr 1 cmp $flake2Dir/flake.lock $TEST_ROOT/flake2-overridden.lock +nix flake metadata $flake2Dir --reference-lock-file $TEST_ROOT/flake2-overridden.lock | grepQuiet $flake1OriginalCommit + +# reference-lock-file can only be used if allow-dirty is set. +expectStderr 1 nix flake metadata $flake2Dir --no-allow-dirty --reference-lock-file $TEST_ROOT/flake2-overridden.lock diff --git a/tests/impure-derivations.sh b/tests/impure-derivations.sh index 7595fdd35..c7dadf397 100644 --- a/tests/impure-derivations.sh +++ b/tests/impure-derivations.sh @@ -37,8 +37,8 @@ path4=$(nix build -L --no-link --json --file ./impure-derivations.nix impureOnIm (! nix build -L --no-link --json --file ./impure-derivations.nix inputAddressed 2>&1) | grep 'depends on impure derivation' drvPath=$(nix eval --json --file ./impure-derivations.nix impure.drvPath | jq -r .) -[[ $(nix show-derivation $drvPath | jq ".[\"$drvPath\"].outputs.out.impure") = true ]] -[[ $(nix show-derivation $drvPath | jq ".[\"$drvPath\"].outputs.stuff.impure") = true ]] +[[ $(nix derivation show $drvPath | jq ".[\"$drvPath\"].outputs.out.impure") = true ]] +[[ $(nix derivation show $drvPath | jq ".[\"$drvPath\"].outputs.stuff.impure") = true ]] # Fixed-output derivations *can* depend on impure derivations. path5=$(nix build -L --no-link --json --file ./impure-derivations.nix contentAddressed | jq -r .[].outputs.out) diff --git a/tests/installer/default.nix b/tests/installer/default.nix index 8c9784eaf..49cfd2bcc 100644 --- a/tests/installer/default.nix +++ b/tests/installer/default.nix @@ -17,7 +17,7 @@ let script = '' tar -xf ./nix.tar.xz mv ./nix-* nix - ./nix/install --no-daemon + ./nix/install --no-daemon --no-channel-add ''; }; @@ -33,9 +33,9 @@ let mockChannel = pkgs: pkgs.runCommandNoCC "mock-channel" {} '' mkdir nixexprs - mkdir $out + mkdir -p $out/channel echo -n 'someContent' > nixexprs/someFile - tar cvf - nixexprs | bzip2 > $out/nixexprs.tar.bz2 + tar cvf - nixexprs | bzip2 > $out/channel/nixexprs.tar.bz2 ''; disableSELinux = "sudo setenforce 0"; @@ -198,7 +198,9 @@ let $ssh "set -eux; $installScript" echo "Copying the mock channel" - scp -r -P 20022 $ssh_opts ${mockChannel pkgs} vagrant@localhost:channel + # `scp -r` doesn't seem to work properly on some rhel instances, so let's + # use a plain tarpipe instead + tar -C ${mockChannel pkgs} -c channel | ssh -p 20022 $ssh_opts vagrant@localhost tar x -f- echo "Testing Nix installation..." $ssh <&1 | grepQuiet Hello +nix-instantiate --eval -E 'builtins.trace "Hello" 123' 2>/dev/null | grepQuiet 123 nix-instantiate --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 nix-instantiate --trace-verbose --eval -E 'builtins.traceVerbose "Hello" 123' 2>&1 | grepQuiet Hello nix-instantiate --eval -E 'builtins.traceVerbose "Hello" 123' 2>&1 | grepQuietInverse Hello nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 | grepQuietInverse Hello expectStderr 1 nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" (throw "Foo")' | grepQuiet Hello +nix-instantiate --eval -E 'let x = builtins.trace { x = x; } true; in x' \ + 2>&1 | grepQuiet -E 'trace: { x = «potential infinite recursion»; }' + +nix-instantiate --eval -E 'let x = { repeating = x; tracing = builtins.trace x true; }; in x.tracing'\ + 2>&1 | grepQuiet -F 'trace: { repeating = «repeated»; tracing = «potential infinite recursion»; }' + set +x fail=0 diff --git a/tests/legacy-ssh-store.sh b/tests/legacy-ssh-store.sh new file mode 100644 index 000000000..71b716b84 --- /dev/null +++ b/tests/legacy-ssh-store.sh @@ -0,0 +1,4 @@ +source common.sh + +# Check that store ping trusted doesn't yet work with ssh:// +nix --store ssh://localhost?remote-store=$TEST_ROOT/other-store store ping --json | jq -e 'has("trusted") | not' diff --git a/tests/local-store.sh b/tests/local-store.sh index 0247346f1..89502f864 100644 --- a/tests/local-store.sh +++ b/tests/local-store.sh @@ -17,3 +17,6 @@ PATH2=$(nix path-info --store "$PWD/x" $CORRECT_PATH) PATH3=$(nix path-info --store "local?root=$PWD/x" $CORRECT_PATH) [ $CORRECT_PATH == $PATH3 ] + +# Ensure store ping trusted works with local store +nix --store ./x store ping --json | jq -e '.trusted' diff --git a/tests/local.mk b/tests/local.mk index 5981177ed..4ed1d795f 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -17,6 +17,7 @@ nix_tests = \ ca/gc.sh \ gc.sh \ remote-store.sh \ + legacy-ssh-store.sh \ lang.sh \ experimental-features.sh \ fetchMercurial.sh \ @@ -101,6 +102,8 @@ nix_tests = \ eval-store.sh \ why-depends.sh \ ca/why-depends.sh \ + derivation-json.sh \ + ca/derivation-json.sh \ import-derivation.sh \ ca/import-derivation.sh \ ca/text-hashed-output.sh \ diff --git a/tests/nix-profile.sh b/tests/nix-profile.sh index 652e8a8f2..4ef5b484a 100644 --- a/tests/nix-profile.sh +++ b/tests/nix-profile.sh @@ -144,6 +144,7 @@ expect 1 nix profile install $flake2Dir diff -u <( nix --offline profile install $flake2Dir 2>&1 1> /dev/null \ | grep -vE "^warning: " \ + | grep -vE "^error \(ignored\): " \ || true ) <(cat << EOF error: An existing package already provides the following file: diff --git a/tests/nixos/nix-copy.nix b/tests/nixos/nix-copy.nix new file mode 100644 index 000000000..ee8b77100 --- /dev/null +++ b/tests/nixos/nix-copy.nix @@ -0,0 +1,85 @@ +# Test that ‘nix copy’ works over ssh. + +{ lib, config, nixpkgs, hostPkgs, ... }: + +let + pkgs = config.nodes.client.nixpkgs.pkgs; + + pkgA = pkgs.cowsay; + pkgB = pkgs.wget; + pkgC = pkgs.hello; + pkgD = pkgs.tmux; + +in { + name = "nix-copy"; + + enableOCR = true; + + nodes = + { client = + { config, lib, pkgs, ... }: + { virtualisation.writableStore = true; + virtualisation.additionalPaths = [ pkgA pkgD.drvPath ]; + nix.settings.substituters = lib.mkForce [ ]; + nix.settings.experimental-features = [ "nix-command" ]; + services.getty.autologinUser = "root"; + }; + + server = + { config, pkgs, ... }: + { services.openssh.enable = true; + services.openssh.permitRootLogin = "yes"; + users.users.root.password = "foobar"; + virtualisation.writableStore = true; + virtualisation.additionalPaths = [ pkgB pkgC ]; + }; + }; + + testScript = { nodes }: '' + # fmt: off + import subprocess + + # Create an SSH key on the client. + subprocess.run([ + "${pkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", "" + ], capture_output=True, check=True) + + start_all() + + server.wait_for_unit("sshd") + client.wait_for_unit("network.target") + client.wait_for_unit("getty@tty1.service") + client.wait_for_text("]#") + + # Copy the closure of package A from the client to the server using password authentication, + # and check that all prompts are visible + server.fail("nix-store --check-validity ${pkgA}") + client.send_chars("nix copy --to ssh://server ${pkgA} >&2; echo done\n") + client.wait_for_text("continue connecting") + client.send_chars("yes\n") + client.wait_for_text("Password:") + client.send_chars("foobar\n") + client.wait_for_text("done") + server.succeed("nix-store --check-validity ${pkgA}") + + client.copy_from_host("key", "/root/.ssh/id_ed25519") + client.succeed("chmod 600 /root/.ssh/id_ed25519") + + # Install the SSH key on the server. + server.copy_from_host("key.pub", "/root/.ssh/authorized_keys") + server.succeed("systemctl restart sshd") + client.succeed(f"ssh -o StrictHostKeyChecking=no {server.name} 'echo hello world'") + + # Copy the closure of package B from the server to the client, using ssh-ng. + client.fail("nix-store --check-validity ${pkgB}") + # Shouldn't download untrusted paths by default + client.fail("nix copy --from ssh-ng://server ${pkgB} >&2") + client.succeed("nix copy --no-check-sigs --from ssh-ng://server ${pkgB} >&2") + client.succeed("nix-store --check-validity ${pkgB}") + + # Copy the derivation of package D's derivation from the client to the server. + server.fail("nix-store --check-validity ${pkgD.drvPath}") + client.succeed("nix copy --derivation --to ssh://server ${pkgD.drvPath} >&2") + server.succeed("nix-store --check-validity ${pkgD.drvPath}") + ''; +} diff --git a/tests/remote-store.sh b/tests/remote-store.sh index 1ae126794..ea32a20d3 100644 --- a/tests/remote-store.sh +++ b/tests/remote-store.sh @@ -5,8 +5,19 @@ clearStore # Ensure "fake ssh" remote store works just as legacy fake ssh would. nix --store ssh-ng://localhost?remote-store=$TEST_ROOT/other-store doctor +# Ensure that store ping trusted works with ssh-ng:// +nix --store ssh-ng://localhost?remote-store=$TEST_ROOT/other-store store ping --json | jq -e '.trusted' + startDaemon +if isDaemonNewer "2.15pre0"; then + # Ensure that ping works trusted with new daemon + nix store ping --json | jq -e '.trusted' +else + # And the the field is absent with the old daemon + nix store ping --json | jq -e 'has("trusted") | not' +fi + # Test import-from-derivation through the daemon. [[ $(nix eval --impure --raw --expr ' with import ./config.nix; diff --git a/tests/repl.sh b/tests/repl.sh index be8adb742..2b3789521 100644 --- a/tests/repl.sh +++ b/tests/repl.sh @@ -79,6 +79,14 @@ testReplResponse ' "result: ${a}" ' "result: 2" +# check dollar escaping https://github.com/NixOS/nix/issues/4909 +# note the escaped \, +# \\ +# because the second argument is a regex +testReplResponse ' +"$" + "{hi}" +' '"\\${hi}"' + testReplResponse ' drvPath ' '".*-simple.drv"' \