diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d58577551..ab5908649 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -5,7 +5,7 @@ # For documentation on this mechanism, see https://help.github.com/articles/about-codeowners/ # Default reviewers if nothing else matches -* @edolstra @thufschmitt +* @edolstra # This file .github/CODEOWNERS @edolstra @@ -13,3 +13,6 @@ # Public documentation /doc @fricklerhandwerk *.md @fricklerhandwerk + +# Libstore layer +/src/libstore @thufschmitt diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 344f9405f..db69e51db 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -25,4 +25,4 @@ Maintainers: tick if completed or explain if not relevant - [ ] documentation in the manual - [ ] code and comments are self-explanatory - [ ] commit message explains why the change was made - - [ ] new feature or bug fix: updated release notes + - [ ] new feature or incompatible change: updated release notes diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index ca5af260f..b04723b95 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -21,12 +21,12 @@ jobs: fetch-depth: 0 - name: Create backport PRs # should be kept in sync with `version` - uses: zeebe-io/backport-action@v1.1.0 + uses: zeebe-io/backport-action@v1.2.0 with: # Config README: https://github.com/zeebe-io/backport-action#backport-action github_token: ${{ secrets.GITHUB_TOKEN }} github_workspace: ${{ github.workspace }} pull_description: |- - Bot-based backport to `${target_branch}`, triggered by a label in #${pull_number}. + Automatic backport to `${target_branch}`, triggered by a label in #${pull_number}. # should be kept in sync with `uses` version: v0.0.5 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dafba6d85..325579a5b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v18 + - uses: cachix/install-nix-action@v19 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - uses: cachix/cachix-action@v12 if: needs.check_secrets.outputs.cachix == 'true' @@ -58,7 +58,7 @@ jobs: with: fetch-depth: 0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v18 + - uses: cachix/install-nix-action@v19 - uses: cachix/cachix-action@v12 with: name: '${{ env.CACHIX_NAME }}' @@ -77,7 +77,7 @@ jobs: steps: - uses: actions/checkout@v3 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v18 + - uses: cachix/install-nix-action@v19 with: install_url: '${{needs.installer.outputs.installerURL}}' install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" @@ -102,7 +102,7 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v18 + - uses: cachix/install-nix-action@v19 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#default.version | tr -d \")" >> $GITHUB_ENV - uses: cachix/cachix-action@v12 diff --git a/.gitignore b/.gitignore index 47815fde6..e326966d6 100644 --- a/.gitignore +++ b/.gitignore @@ -75,7 +75,7 @@ perl/Makefile.config # /tests/ /tests/test-tmp -/tests/common.sh +/tests/common/vars-and-functions.sh /tests/result* /tests/restricted-innocent /tests/shell @@ -103,6 +103,7 @@ outputs/ *.a *.o +*.o.tmp *.so *.dylib *.dll diff --git a/Makefile b/Makefile index 42d11638b..8675c9925 100644 --- a/Makefile +++ b/Makefile @@ -36,4 +36,4 @@ endif include mk/lib.mk -GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++20 -I src +GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++2a -I src diff --git a/README.md b/README.md index 8a02c4c75..85b0902b1 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,8 @@ Information on additional installation methods is available on the [Nix download ## Building And Developing -See our [Hacking guide](https://nixos.org/manual/nix/stable/contributing/hacking.html) in our manual for instruction on how to -build nix from source with nix-build or how to get a development environment. +See our [Hacking guide](https://nixos.org/manual/nix/unstable/contributing/hacking.html) in our manual for instruction on how to +to set up a development environment and build Nix from source. ## Additional Resources diff --git a/doc/manual/custom.css b/doc/manual/custom.css index 69d48d4a7..b90f5423f 100644 --- a/doc/manual/custom.css +++ b/doc/manual/custom.css @@ -5,3 +5,7 @@ h1:not(:first-of-type) { h2 { margin-top: 1em; } + +.hljs-meta { + user-select: none; +} diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 190f0258a..f43510b6d 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -51,13 +51,13 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix @rm -rf $@ $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix { toplevel = builtins.readFile $<; }' - # @docroot@: https://nixos.org/manual/nix/unstable/contributing/hacking.html#docroot-variable + @# @docroot@: https://nixos.org/manual/nix/unstable/contributing/hacking.html#docroot-variable $(trace-gen) sed -i $@.tmp/*.md -e 's^@docroot@^../..^g' @mv $@.tmp $@ $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp - # @docroot@: https://nixos.org/manual/nix/unstable/contributing/hacking.html#docroot-variable + @# @docroot@: https://nixos.org/manual/nix/unstable/contributing/hacking.html#docroot-variable $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-options.nix (builtins.fromJSON (builtins.readFile $<))' \ | sed -e 's^@docroot@^..^g'>> $@.tmp @mv $@.tmp $@ @@ -72,7 +72,7 @@ $(d)/conf-file.json: $(bindir)/nix $(d)/src/language/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix @cat doc/manual/src/language/builtins-prefix.md > $@.tmp - # @docroot@: https://nixos.org/manual/nix/unstable/contributing/hacking.html#docroot-variable + @# @docroot@: https://nixos.org/manual/nix/unstable/contributing/hacking.html#docroot-variable $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' \ | sed -e 's^@docroot@^..^g' >> $@.tmp @cat doc/manual/src/language/builtins-suffix.md >> $@.tmp diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index b1c551969..964091285 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -67,6 +67,7 @@ - [CLI guideline](contributing/cli-guideline.md) - [Release Notes](release-notes/release-notes.md) - [Release X.Y (202?-??-??)](release-notes/rl-next.md) + - [Release 2.14 (2023-02-28)](release-notes/rl-2.14.md) - [Release 2.13 (2023-01-17)](release-notes/rl-2.13.md) - [Release 2.12 (2022-12-06)](release-notes/rl-2.12.md) - [Release 2.11 (2022-08-25)](release-notes/rl-2.11.md) diff --git a/doc/manual/src/command-ref/env-common.md b/doc/manual/src/command-ref/env-common.md index bb85a6b07..c5d38db47 100644 --- a/doc/manual/src/command-ref/env-common.md +++ b/doc/manual/src/command-ref/env-common.md @@ -91,3 +91,16 @@ Most Nix commands interpret the following environment variables: variable sets the initial size of the heap in bytes. It defaults to 384 MiB. Setting it to a low value reduces memory consumption, but will increase runtime due to the overhead of garbage collection. + +## XDG Base Directory + +New Nix commands conform to the [XDG Base Directory Specification], and use the following environment variables to determine locations of various state and configuration files: + +- [`XDG_CONFIG_HOME`]{#env-XDG_CONFIG_HOME} (default `~/.config`) +- [`XDG_STATE_HOME`]{#env-XDG_STATE_HOME} (default `~/.local/state`) +- [`XDG_CACHE_HOME`]{#env-XDG_CACHE_HOME} (default `~/.cache`) + +Classic Nix commands can also be made to follow this standard using the [`use-xdg-base-directories`] configuration option. + +[XDG Base Directory Specification]: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html +[`use-xdg-base-directories`]: ../command-ref/conf-file.md#conf-use-xdg-base-directories \ No newline at end of file diff --git a/doc/manual/src/contributing/cli-guideline.md b/doc/manual/src/contributing/cli-guideline.md index 01a1b1e73..e53d2d178 100644 --- a/doc/manual/src/contributing/cli-guideline.md +++ b/doc/manual/src/contributing/cli-guideline.md @@ -389,6 +389,88 @@ colors, no emojis and using ASCII instead of Unicode symbols). The same should happen when TTY is not detected on STDERR. We should not display progress / status section, but only print warnings and errors. +## Returning future proof JSON + +The schema of JSON output should allow for backwards compatible extension. This section explains how to achieve this. + +Two definitions are helpful here, because while JSON only defines one "key-value" +object type, we use it to cover two use cases: + + - **dictionary**: a map from names to value that all have the same type. In + C++ this would be a `std::map` with string keys. + - **record**: a fixed set of attributes each with their own type. In C++, this + would be represented by a `struct`. + +It is best not to mix these use cases, as that may lead to incompatibilities when the schema changes. For example, adding a record field to a dictionary breaks consumers that assume all JSON object fields to have the same meaning and type. + +This leads to the following guidelines: + + - The top-level (root) value must be a record. + + Otherwise, one can not change the structure of a command's output. + + - The value of a dictionary item must be a record. + + Otherwise, the item type can not be extended. + + - List items should be records. + + Otherwise, one can not change the structure of the list items. + + If the order of the items does not matter, and each item has a unique key that is a string, consider representing the list as a dictionary instead. If the order of the items needs to be preserved, return a list of records. + + - Streaming JSON should return records. + + An example of a streaming JSON format is [JSON lines](https://jsonlines.org/), where each line represents a JSON value. These JSON values can be considered top-level values or list items, and they must be records. + +### Examples + + +This is bad, because all keys must be assumed to be store implementations: + +```json +{ + "local": { ... }, + "remote": { ... }, + "http": { ... } +} +``` + +This is good, because the it is extensible at the root, and is somewhat self-documenting: + +```json +{ + "storeTypes": { "local": { ... }, ... }, + "pluginSupport": true +} +``` + +While the dictionary of store types seems like a very complete response at first, a use case may arise that warrants returning additional information. +For example, the presence of plugin support may be crucial information for a client to proceed when their desired store type is missing. + + + +The following representation is bad because it is not extensible: + +```json +{ "outputs": [ "out" "bin" ] } +``` + +However, simply converting everything to records is not enough, because the order of outputs must be preserved: + +```json +{ "outputs": { "bin": {}, "out": {} } } +``` + +The first item is the default output. Deriving this information from the outputs ordering is not great, but this is how Nix currently happens to work. +While it is possible for a JSON parser to preserve the order of fields, we can not rely on this capability to be present in all JSON libraries. + +This representation is extensible and preserves the ordering: + +```json +{ "outputs": [ { "outputName": "out" }, { "outputName": "bin" } ] } +``` + ## Dialog with the user CLIs don't always make it clear when an action has taken place. For every diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 9dbafcc0a..3869c37a4 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -8,25 +8,64 @@ $ git clone https://github.com/NixOS/nix.git $ cd nix ``` -To build Nix for the current operating system/architecture use +The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. If you don't have it installed, follow the [installation instructions]. + +[installation instructions]: ../installation/installation.md + +## Nix with flakes + +This section assumes you are using Nix with [flakes] enabled. See the [next section](#classic-nix) for equivalent instructions which don't require flakes. + +[flakes]: ../command-ref/new-cli/nix3-flake.md#description + +To build all dependencies and start a shell in which all environment +variables are set up so that those dependencies can be found: ```console -$ nix-build +$ nix develop ``` -or if you have a flake-enabled nix: +This shell also adds `./outputs/bin/nix` to your `$PATH` so you can run `nix` immediately after building it. + +To get a shell with one of the other [supported compilation environments](#compilation-environments): + +```console +$ nix develop .#native-clang11StdenvPackages +``` + +> **Note** +> +> Use `ccacheStdenv` to drastically improve rebuild time. +> By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`. + +To build Nix itself in this shell: + +```console +[nix-shell]$ ./bootstrap.sh +[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out +[nix-shell]$ make -j $NIX_BUILD_CORES +``` + +To install it in `$(pwd)/outputs` and test it: + +```console +[nix-shell]$ make install +[nix-shell]$ make installcheck -j $NIX_BUILD_CORES +[nix-shell]$ nix --version +nix (Nix) 2.12 +``` + +To build a release version of Nix: ```console $ nix build ``` -This will build `defaultPackage` attribute defined in the `flake.nix` -file. To build for other platforms add one of the following suffixes to -it: aarch64-linux, i686-linux, x86\_64-darwin, x86\_64-linux. i.e. +You can also build Nix for one of the [supported target platforms](#target-platforms). -```console -$ nix-build -A defaultPackage.x86_64-linux -``` +## Classic Nix + +This section is for Nix without [flakes]. To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: @@ -35,27 +74,16 @@ variables are set up so that those dependencies can be found: $ nix-shell ``` -or if you have a flake-enabled nix: +To get a shell with one of the other [supported compilation environments](#compilation-environments): ```console -$ nix develop +$ nix-shell -A devShells.x86_64-linux.native-clang11StdenvPackages ``` -To get a shell with a different compilation environment (e.g. stdenv, -gccStdenv, clangStdenv, clang11Stdenv, ccacheStdenv): - -```console -$ nix-shell -A devShells.x86_64-linux.clang11StdenvPackages -``` - -or if you have a flake-enabled nix: - -```console -$ nix develop .#clang11StdenvPackages -``` - -Note: you can use `ccacheStdenv` to drastically improve rebuild -time. By default, ccache keeps artifacts in `~/.cache/ccache/`. +> **Note** +> +> You can use `native-ccacheStdenvPackages` to drastically improve rebuild time. +> By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`. To build Nix itself in this shell: @@ -71,21 +99,99 @@ To install it in `$(pwd)/outputs` and test it: [nix-shell]$ make install [nix-shell]$ make installcheck -j $NIX_BUILD_CORES [nix-shell]$ ./outputs/out/bin/nix --version -nix (Nix) 3.0 +nix (Nix) 2.12 ``` -If you have a flakes-enabled Nix you can replace: +To build Nix for the current operating system and CPU architecture use ```console -$ nix-shell +$ nix-build ``` -by: +You can also build Nix for one of the [supported target platforms](#target-platforms). + +## Platforms + +As specified in [`flake.nix`], Nix can be built for various platforms: + +- `aarch64-linux` +- `i686-linux` +- `x86_64-darwin` +- `x86_64-linux` + +[`flake.nix`]: https://github.com/nixos/nix/blob/master/flake.nix + +In order to build Nix for a different platform than the one you're currently +on, you need to have some way for your system Nix to build code for that +platform. Common solutions include [remote builders] and [binfmt emulation] +(only supported on NixOS). + +[remote builders]: ../advanced-topics/distributed-builds.md +[binfmt emulation]: https://nixos.org/manual/nixos/stable/options.html#opt-boot.binfmt.emulatedSystems + +These solutions let Nix perform builds as if you're on the native platform, so +executing the build is as simple as ```console -$ nix develop +$ nix build .#packages.aarch64-linux.default ``` +for flake-enabled Nix, or + +```console +$ nix-build -A packages.aarch64-linux.default +``` + +for classic Nix. + +You can use any of the other supported platforms in place of `aarch64-linux`. + +Cross-compiled builds are available for ARMv6 and ARMv7, and Nix on unsupported platforms can be bootstrapped by adding more `crossSystems` in `flake.nix`. + +## Compilation environments + +Nix can be compiled using multiple environments: + +- `stdenv`: default; +- `gccStdenv`: force the use of `gcc` compiler; +- `clangStdenv`: force the use of `clang` compiler; +- `ccacheStdenv`: enable [ccache], a compiler cache to speed up compilation. + +To build with one of those environments, you can use + +```console +$ nix build .#nix-ccacheStdenv +``` + +for flake-enabled Nix, or + +```console +$ nix-build -A nix-ccacheStdenv +``` + +for classic Nix. + +You can use any of the other supported environments in place of `nix-ccacheStdenv`. + +## Editor integration + +The `clangd` LSP server is installed by default on the `clang`-based `devShell`s. +See [supported compilation environments](#compilation-environments) and instructions how to set up a shell [with flakes](#nix-with-flakes) or in [classic Nix](#classic-nix). + +To use the LSP with your editor, you first need to [set up `clangd`](https://clangd.llvm.org/installation#project-setup) by running: + +```console +make clean && bear -- make -j$NIX_BUILD_CORES install +``` + +Configure your editor to use the `clangd` from the shell, either by running it inside the development shell, or by using [nix-direnv](https://github.com/nix-community/nix-direnv) and [the appropriate editor plugin](https://github.com/direnv/direnv/wiki#editor-integration). + +> **Note** +> +> For some editors (e.g. Visual Studio Code), you may need to install a [special extension](https://open-vsx.org/extension/llvm-vs-code-extensions/vscode-clangd) for the editor to interact with `clangd`. +> Some other editors (e.g. Emacs, Vim) need a plugin to support LSP servers in general (e.g. [lsp-mode](https://github.com/emacs-lsp/lsp-mode) for Emacs and [vim-lsp](https://github.com/prabirshrestha/vim-lsp) for vim). +> Editor-specific setup is typically opinionated, so we will not cover it here in more detail. + ## Running tests ### Unit-tests @@ -219,7 +325,7 @@ After the CI run completes, you can check the output to extract the installer UR 5. To generate an install command, plug this `install_url` and your GitHub username into this template: ```console - sh <(curl -L ) --tarball-url-prefix https://-nix-install-tests.cachix.org/serve + curl -L | sh -s -- --tarball-url-prefix https://-nix-install-tests.cachix.org/serve ```