Merge remote-tracking branch 'upstream/master' into path-info

Makefile

@@ -36,4 +36,4 @@ endif
 
 include mk/lib.mk
 
-GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++17 -I src
+GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++20 -I src

configure.ac

@@ -276,8 +276,11 @@ PKG_CHECK_MODULES([GTEST], [gtest_main])
 
 # Look for rapidcheck.
 # No pkg-config yet, https://github.com/emil-e/rapidcheck/issues/302
+AC_LANG_PUSH(C++)
 AC_CHECK_HEADERS([rapidcheck/gtest.h], [], [], [#include <gtest/gtest.h>])
-AC_CHECK_LIB([rapidcheck], [])
+dnl No good for C++ libs with mangled symbols
+dnl AC_CHECK_LIB([rapidcheck], [])
+AC_LANG_POP(C++)
 
 
 # Look for nlohmann/json.

doc/manual/src/command-ref/nix-store.md

@@ -633,7 +633,7 @@ written to standard output.
 
 A NAR archive is like a TAR or Zip archive, but it contains only the
 information that Nix considers important. For instance, timestamps are
-elided because all files in the Nix store have their timestamp set to 0
+elided because all files in the Nix store have their timestamp set to 1
 anyway. Likewise, all permissions are left out except for the execute
 bit, because all files in the Nix store have 444 or 555 permission.
 

doc/manual/src/language/advanced-attributes.md

@@ -255,3 +255,67 @@ Derivations can declare some infrequently used optional attributes.
     > substituted. Thus it is usually a good idea to align `system` with
     > `builtins.currentSystem` when setting `allowSubstitutes` to
     > `false`. For most trivial derivations this should be the case.
+
+  - [`__structuredAttrs`]{#adv-attr-structuredAttrs}\
+    If the special attribute `__structuredAttrs` is set to `true`, the other derivation
+    attributes are serialised in JSON format and made available to the
+    builder via the file `.attrs.json` in the builder’s temporary
+    directory. This obviates the need for `passAsFile` since JSON files
+    have no size restrictions, unlike process environments.
+
+    It also makes it possible to tweak derivation settings in a structured way; see
+    [`outputChecks`](#adv-attr-outputChecks) for example.
+
+    As a convenience to Bash builders,
+    Nix writes a script named `.attrs.sh` to the builder’s directory
+    that initialises shell variables corresponding to all attributes
+    that are representable in Bash. This includes non-nested
+    (associative) arrays. For example, the attribute `hardening.format = true`
+    ends up as the Bash associative array element `${hardening[format]}`.
+
+  - [`outputChecks`]{#adv-attr-outputChecks}\
+    When using [structured attributes](#adv-attr-structuredAttrs), the `outputChecks`
+    attribute allows defining checks per-output.
+
+    In addition to
+    [`allowedReferences`](#adv-attr-allowedReferences), [`allowedRequisites`](#adv-attr-allowedRequisites),
+    [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites),
+    the following attributes are available:
+
+    - `maxSize` defines the maximum size of the output path.
+    - `maxClosureSize` defines the maximum size of the output's closure.
+    - `ignoreSelfRefs` controls whether self-references should be considered when
+      checking for allowed references/requisites.
+
+    ```nix
+    __structuredAttrs = true;
+
+    outputChecks.out = {
+      # The closure of 'out' must not be larger than 256 MiB.
+      maxClosureSize = 256 * 1024 * 1024;
+
+      # It must not refer to the C compiler or to the 'dev' output.
+      disallowedRequisites = [ stdenv.cc "dev" ];
+    };
+
+    outputChecks.dev = {
+      # The 'dev' output must not be larger than 128 KiB.
+      maxSize = 128 * 1024;
+    };
+    ```
+
+  - [`unsafeDiscardReferences`]{#adv-attr-unsafeDiscardReferences}\
+    When using [structured attributes](#adv-attr-structuredAttrs), the **experimental**
+    attribute `unsafeDiscardReferences` is a per-output boolean which, if set to `true`,
+    disables scanning the build output for runtime dependencies altogether.
+
+    ```nix
+    __structuredAttrs = true;
+    unsafeDiscardReferences.out = true;
+    ```
+
+    This is useful, for example, when generating self-contained filesystem images with
+    their own embedded Nix store: hashes found inside such an image refer
+    to the embedded store and not to the host's Nix store.
+
+    This is only allowed if the `discard-references` experimental feature is enabled.

perl/Makefile

@@ -1,6 +1,6 @@
 makefiles = local.mk
 
-GLOBAL_CXXFLAGS += -g -Wall -std=c++17 -I ../src
+GLOBAL_CXXFLAGS += -g -Wall -std=c++20 -I ../src
 
 -include Makefile.config
 

src/libcmd/installables.cc

@@ -379,10 +379,9 @@ Installable::getCursors(EvalState & state)
 ref<eval_cache::AttrCursor>
 Installable::getCursor(EvalState & state)
 {
-    auto cursors = getCursors(state);
+    /* Although getCursors should return at least one element, in case it doesn't,
-    if (cursors.empty())
+       bound check to avoid an undefined behavior for vector[0] */
-        throw Error("cannot find flake attribute '%s'", what());
+    return getCursors(state).at(0);
-    return cursors[0];
 }
 
 static StorePath getDeriver(
@@ -696,46 +695,28 @@ InstallableFlake::getCursors(EvalState & state)
 
     std::vector<ref<eval_cache::AttrCursor>> res;
 
-    for (auto & attrPath : getActualAttrPaths()) {
-        auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath));
-        if (attr) res.push_back(ref(*attr));
-    }
-
-    return res;
-}
-
-ref<eval_cache::AttrCursor> InstallableFlake::getCursor(EvalState & state)
-{
-    auto lockedFlake = getLockedFlake();
-
-    auto cache = openEvalCache(state, lockedFlake);
-    auto root = cache->getRoot();
-
     Suggestions suggestions;
-
     auto attrPaths = getActualAttrPaths();
 
     for (auto & attrPath : attrPaths) {
         debug("trying flake output attribute '%s'", attrPath);
 
-        auto attrOrSuggestions = root->findAlongAttrPath(
+        auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath));
-            parseAttrPath(state, attrPath),
+        if (attr) {
-            true
+            res.push_back(ref(*attr));
-        );
+        } else {
-
+            suggestions += attr.getSuggestions();
-        if (!attrOrSuggestions) {
+        }
-            suggestions += attrOrSuggestions.getSuggestions();
-            continue;
-        }
-
-        return *attrOrSuggestions;
     }
 
+    if (res.size() == 0)
         throw Error(
             suggestions,
             "flake '%s' does not provide attribute %s",
             flakeRef,
             showAttrPaths(attrPaths));
+
+    return res;
 }
 
 std::shared_ptr<flake::LockedFlake> InstallableFlake::getLockedFlake() const

src/libcmd/installables.hh

@@ -103,9 +103,13 @@ struct Installable
         return {};
     }
 
+    /* Get a cursor to each value this Installable could refer to. However
+       if none exists, throw exception instead of returning empty vector. */
     virtual std::vector<ref<eval_cache::AttrCursor>>
     getCursors(EvalState & state);
 
+    /* Get the first and most preferred cursor this Installable could refer
+       to, or throw an exception if none exists. */
     virtual ref<eval_cache::AttrCursor>
     getCursor(EvalState & state);
 
@@ -193,15 +197,11 @@ struct InstallableFlake : InstallableValue
 
     std::pair<Value *, PosIdx> toValue(EvalState & state) override;
 
-    /* Get a cursor to every attrpath in getActualAttrPaths() that
+    /* Get a cursor to every attrpath in getActualAttrPaths()
-       exists. */
+       that exists. However if none exists, throw an exception. */
     std::vector<ref<eval_cache::AttrCursor>>
     getCursors(EvalState & state) override;
 
-    /* Get a cursor to the first attrpath in getActualAttrPaths() that
-       exists, or throw an exception with suggestions if none exists. */
-    ref<eval_cache::AttrCursor> getCursor(EvalState & state) override;
-
     std::shared_ptr<flake::LockedFlake> getLockedFlake() const;
 
     FlakeRef nixpkgsFlakeRef() const override;

src/libcmd/nix-cmd.pc.in

@@ -6,4 +6,4 @@ Name: Nix
 Description: Nix Package Manager
 Version: @PACKAGE_VERSION@
 Libs: -L${libdir} -lnixcmd
-Cflags: -I${includedir}/nix -std=c++17
+Cflags: -I${includedir}/nix -std=c++20

src/libexpr/nix-expr.pc.in

@@ -7,4 +7,4 @@ Description: Nix Package Manager
 Version: @PACKAGE_VERSION@
 Requires: nix-store bdw-gc
 Libs: -L${libdir} -lnixexpr
-Cflags: -I${includedir}/nix -std=c++17
+Cflags: -I${includedir}/nix -std=c++20

src/libmain/nix-main.pc.in

@@ -6,4 +6,4 @@ Name: Nix
 Description: Nix Package Manager
 Version: @PACKAGE_VERSION@
 Libs: -L${libdir} -lnixmain
-Cflags: -I${includedir}/nix -std=c++17
+Cflags: -I${includedir}/nix -std=c++20

src/libstore/binary-cache-store.cc

@@ -380,7 +380,7 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
     auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
 
     getFile(narInfoFile,
-        {[=](std::future<std::optional<std::string>> fut) {
+        {[=,this](std::future<std::optional<std::string>> fut) {
             try {
                 auto data = fut.get();
 

src/libstore/build/worker.cc

@@ -276,7 +276,7 @@ void Worker::run(const Goals & _topGoals)
         if (!children.empty() || !waitingForAWhile.empty())
             waitForInput();
         else {
-            if (awake.empty() && 0 == settings.maxBuildJobs)
+            if (awake.empty() && 0U == settings.maxBuildJobs)
             {
                 if (getMachines().empty())
                    throw Error("unable to start any build; either increase '--max-jobs' "

src/libstore/globals.cc

@@ -222,19 +222,19 @@ template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::s
         .longName = name,
         .description = "Enable sandboxing.",
         .category = category,
-        .handler = {[=]() { override(smEnabled); }}
+        .handler = {[this]() { override(smEnabled); }}
     });
     args.addFlag({
         .longName = "no-" + name,
         .description = "Disable sandboxing.",
         .category = category,
-        .handler = {[=]() { override(smDisabled); }}
+        .handler = {[this]() { override(smDisabled); }}
     });
     args.addFlag({
         .longName = "relaxed-" + name,
         .description = "Enable sandboxing, but allow builds to disable it.",
         .category = category,
-        .handler = {[=]() { override(smRelaxed); }}
+        .handler = {[this]() { override(smRelaxed); }}
     });
 }
 

src/libstore/nix-store.pc.in

@@ -6,4 +6,4 @@ Name: Nix
 Description: Nix Package Manager
 Version: @PACKAGE_VERSION@
 Libs: -L${libdir} -lnixstore -lnixutil
-Cflags: -I${includedir}/nix -std=c++17
+Cflags: -I${includedir}/nix -std=c++20

src/libutil/args.cc

@@ -324,7 +324,7 @@ MultiCommand::MultiCommand(const Commands & commands_)
     expectArgs({
         .label = "subcommand",
         .optional = true,
-        .handler = {[=](std::string s) {
+        .handler = {[=,this](std::string s) {
             assert(!command);
             auto i = commands.find(s);
             if (i == commands.end()) {

src/libutil/config.cc

@@ -209,7 +209,7 @@ void BaseSetting<T>::convertToArg(Args & args, const std::string & category)
         .description = fmt("Set the `%s` setting.", name),
         .category = category,
         .labels = {"value"},
-        .handler = {[=](std::string s) { overridden = true; set(s); }},
+        .handler = {[this](std::string s) { overridden = true; set(s); }},
     });
 
     if (isAppendable())
@@ -218,7 +218,7 @@ void BaseSetting<T>::convertToArg(Args & args, const std::string & category)
             .description = fmt("Append to the `%s` setting.", name),
             .category = category,
             .labels = {"value"},
-            .handler = {[=](std::string s) { overridden = true; set(s, true); }},
+            .handler = {[this](std::string s) { overridden = true; set(s, true); }},
         });
 }
 
@@ -270,13 +270,13 @@ template<> void BaseSetting<bool>::convertToArg(Args & args, const std::string &
         .longName = name,
         .description = fmt("Enable the `%s` setting.", name),
         .category = category,
-        .handler = {[=]() { override(true); }}
+        .handler = {[this]() { override(true); }}
     });
     args.addFlag({
         .longName = "no-" + name,
         .description = fmt("Disable the `%s` setting.", name),
         .category = category,
-        .handler = {[=]() { override(false); }}
+        .handler = {[this]() { override(false); }}
     });
 }
 

src/libutil/config.hh

@@ -250,11 +250,15 @@ public:
     operator const T &() const { return value; }
     operator T &() { return value; }
     const T & get() const { return value; }
-    bool operator ==(const T & v2) const { return value == v2; }
+    template<typename U>
-    bool operator !=(const T & v2) const { return value != v2; }
+    bool operator ==(const U & v2) const { return value == v2; }
-    void operator =(const T & v) { assign(v); }
+    template<typename U>
+    bool operator !=(const U & v2) const { return value != v2; }
+    template<typename U>
+    void operator =(const U & v) { assign(v); }
     virtual void assign(const T & v) { value = v; }
-    void setDefault(const T & v) { if (!overridden) value = v; }
+    template<typename U>
+    void setDefault(const U & v) { if (!overridden) value = v; }
 
     void set(const std::string & str, bool append = false) override;
 

src/nix/flake.cc

@@ -1002,6 +1002,61 @@ struct CmdFlakeShow : FlakeCommand, MixJSON
         auto flake = std::make_shared<LockedFlake>(lockFlake());
         auto localSystem = std::string(settings.thisSystem.get());
 
+        std::function<bool(
+            eval_cache::AttrCursor & visitor,
+            const std::vector<Symbol> &attrPath,
+            const Symbol &attr)> hasContent;
+
+        // For frameworks it's important that structures are as lazy as possible
+        // to prevent infinite recursions, performance issues and errors that
+        // aren't related to the thing to evaluate. As a consequence, they have
+        // to emit more attributes than strictly (sic) necessary.
+        // However, these attributes with empty values are not useful to the user
+        // so we omit them.
+        hasContent = [&](
+            eval_cache::AttrCursor & visitor,
+            const std::vector<Symbol> &attrPath,
+            const Symbol &attr) -> bool
+        {
+            auto attrPath2(attrPath);
+            attrPath2.push_back(attr);
+            auto attrPathS = state->symbols.resolve(attrPath2);
+            const auto & attrName = state->symbols[attr];
+
+            auto visitor2 = visitor.getAttr(attrName);
+
+            if ((attrPathS[0] == "apps"
+                    || attrPathS[0] == "checks"
+                    || attrPathS[0] == "devShells"
+                    || attrPathS[0] == "legacyPackages"
+                    || attrPathS[0] == "packages")
+                && (attrPathS.size() == 1 || attrPathS.size() == 2)) {
+                for (const auto &subAttr : visitor2->getAttrs()) {
+                    if (hasContent(*visitor2, attrPath2, subAttr)) {
+                        return true;
+                    }
+                }
+                return false;
+            }
+
+            if ((attrPathS.size() == 1)
+                && (attrPathS[0] == "formatter"
+                    || attrPathS[0] == "nixosConfigurations"
+                    || attrPathS[0] == "nixosModules"
+                    || attrPathS[0] == "overlays"
+                    )) {
+                for (const auto &subAttr : visitor2->getAttrs()) {
+                    if (hasContent(*visitor2, attrPath2, subAttr)) {
+                        return true;
+                    }
+                }
+                return false;
+            }
+
+            // If we don't recognize it, it's probably content
+            return true;
+        };
+
         std::function<nlohmann::json(
             eval_cache::AttrCursor & visitor,
             const std::vector<Symbol> & attrPath,
@@ -1027,7 +1082,12 @@ struct CmdFlakeShow : FlakeCommand, MixJSON
                 {
                     if (!json)
                         logger->cout("%s", headerPrefix);
-                    auto attrs = visitor.getAttrs();
+                    std::vector<Symbol> attrs;
+                    for (const auto &attr : visitor.getAttrs()) {
+                        if (hasContent(visitor, attrPath, attr))
+                            attrs.push_back(attr);
+                    }
+
                     for (const auto & [i, attr] : enumerate(attrs)) {
                         const auto & attrName = state->symbols[attr];
                         bool last = i + 1 == attrs.size();

src/nix/search.cc

@@ -56,8 +56,8 @@ struct CmdSearch : InstallableCommand, MixJSON
     Strings getDefaultFlakeAttrPaths() override
     {
         return {
-            "packages." + settings.thisSystem.get() + ".",
+            "packages." + settings.thisSystem.get(),
-            "legacyPackages." + settings.thisSystem.get() + "."
+            "legacyPackages." + settings.thisSystem.get()
         };
     }
 

tests/flakes/show.sh

@@ -37,3 +37,30 @@ in
 assert show_output.legacyPackages.${builtins.currentSystem}.hello.name == "simple";
 true
 '
+
+# Test that attributes are only reported when they have actual content
+cat >flake.nix <<EOF
+{
+  description = "Bla bla";
+
+  outputs = inputs: rec {
+    apps.$system = { };
+    checks.$system = { };
+    devShells.$system = { };
+    legacyPackages.$system = { };
+    packages.$system = { };
+    packages.someOtherSystem = { };
+
+    formatter = { };
+    nixosConfigurations = { };
+    nixosModules = { };
+  };
+}
+EOF
+nix flake show --json --all-systems > show-output.json
+nix eval --impure --expr '
+let show_output = builtins.fromJSON (builtins.readFile ./show-output.json);
+in
+assert show_output == { };
+true
+'