Merge pull request #6426 from edolstra/respect-outputs-to-install

nix: Respect meta.outputsToInstall, and use all outputs by default

src/libcmd/installables.cc

@@ -440,9 +440,7 @@ DerivedPaths InstallableValue::toDerivedPaths()
 
     // Group by derivation, helps with .all in particular
     for (auto & drv : toDerivations()) {
-        auto outputName = drv.outputName;
+        for (auto & outputName : drv.outputsToInstall)
-        if (outputName == "")
-            throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(drv.drvPath));
             drvsToOutputs[drv.drvPath].insert(outputName);
         drvsToCopy.insert(drv.drvPath);
     }
@@ -497,7 +495,13 @@ std::vector<InstallableValue::DerivationInfo> InstallableAttrPath::toDerivations
         auto drvPath = drvInfo.queryDrvPath();
         if (!drvPath)
             throw Error("'%s' is not a derivation", what());
-        res.push_back({ *drvPath, drvInfo.queryOutputName() });
+        std::set<std::string> outputsToInstall;
+        for (auto & output : drvInfo.queryOutputs(false, true))
+            outputsToInstall.insert(output.first);
+        res.push_back(DerivationInfo {
+            .drvPath = *drvPath,
+            .outputsToInstall = std::move(outputsToInstall)
+        });
     }
 
     return res;
@@ -598,9 +602,24 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
 
     auto drvPath = attr->forceDerivation();
 
+    std::set<std::string> outputsToInstall;
+
+    if (auto aMeta = attr->maybeGetAttr(state->sMeta))
+        if (auto aOutputsToInstall = aMeta->maybeGetAttr("outputsToInstall"))
+            for (auto & s : aOutputsToInstall->getListOfStrings())
+                outputsToInstall.insert(s);
+
+    if (outputsToInstall.empty())
+        if (auto aOutputs = attr->maybeGetAttr(state->sOutputs))
+            for (auto & s : aOutputs->getListOfStrings())
+                outputsToInstall.insert(s);
+
+    if (outputsToInstall.empty())
+        outputsToInstall.insert("out");
+
     auto drvInfo = DerivationInfo {
-        std::move(drvPath),
+        .drvPath = std::move(drvPath),
-        attr->getAttr(state->sOutputName)->getString()
+        .outputsToInstall = std::move(outputsToInstall),
     };
 
     return {attrPath, getLockedFlake()->flake.lockedRef, std::move(drvInfo)};

src/libcmd/installables.hh

@@ -141,7 +141,7 @@ struct InstallableValue : Installable
     struct DerivationInfo
     {
         StorePath drvPath;
-        std::string outputName;
+        std::set<std::string> outputsToInstall;
     };
 
     virtual std::vector<DerivationInfo> toDerivations() = 0;

src/libexpr/eval-cache.cc

@@ -47,7 +47,7 @@ struct AttrDb
     {
         auto state(_state->lock());
 
-        Path cacheDir = getCacheDir() + "/nix/eval-cache-v2";
+        Path cacheDir = getCacheDir() + "/nix/eval-cache-v3";
         createDirs(cacheDir);
 
         Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
@@ -175,6 +175,24 @@ struct AttrDb
         });
     }
 
+    AttrId setListOfStrings(
+        AttrKey key,
+        const std::vector<std::string> & l)
+    {
+        return doSQLite([&]()
+        {
+            auto state(_state->lock());
+
+            state->insertAttribute.use()
+                (key.first)
+                (symbols[key.second])
+                (AttrType::ListOfStrings)
+                (concatStringsSep("\t", l)).exec();
+
+            return state->db.getLastInsertedRowId();
+        });
+    }
+
     AttrId setPlaceholder(AttrKey key)
     {
         return doSQLite([&]()
@@ -269,6 +287,8 @@ struct AttrDb
             }
             case AttrType::Bool:
                 return {{rowId, queryAttribute.getInt(2) != 0}};
+            case AttrType::ListOfStrings:
+                return {{rowId, tokenizeString<std::vector<std::string>>(queryAttribute.getStr(2), "\t")}};
             case AttrType::Missing:
                 return {{rowId, missing_t()}};
             case AttrType::Misc:
@@ -385,7 +405,7 @@ std::string AttrCursor::getAttrPathStr(Symbol name) const
 
 Value & AttrCursor::forceValue()
 {
-    debug("evaluating uncached attribute %s", getAttrPathStr());
+    debug("evaluating uncached attribute '%s'", getAttrPathStr());
 
     auto & v = getValue();
 
@@ -601,6 +621,38 @@ bool AttrCursor::getBool()
     return v.boolean;
 }
 
+std::vector<std::string> AttrCursor::getListOfStrings()
+{
+    if (root->db) {
+        if (!cachedValue)
+            cachedValue = root->db->getAttr(getKey());
+        if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
+            if (auto l = std::get_if<std::vector<std::string>>(&cachedValue->second)) {
+                debug("using cached list of strings attribute '%s'", getAttrPathStr());
+                return *l;
+            } else
+                throw TypeError("'%s' is not a list of strings", getAttrPathStr());
+        }
+    }
+
+    debug("evaluating uncached attribute '%s'", getAttrPathStr());
+
+    auto & v = getValue();
+    root->state.forceValue(v, noPos);
+
+    if (v.type() != nList)
+        throw TypeError("'%s' is not a list", getAttrPathStr());
+
+    std::vector<std::string> res;
+
+    for (auto & elem : v.listItems())
+        res.push_back(std::string(root->state.forceStringNoCtx(*elem)));
+
+    cachedValue = {root->db->setListOfStrings(getKey(), res), res};
+
+    return res;
+}
+
 std::vector<Symbol> AttrCursor::getAttrs()
 {
     if (root->db) {

src/libexpr/eval-cache.hh

@@ -44,6 +44,7 @@ enum AttrType {
     Misc = 4,
     Failed = 5,
     Bool = 6,
+    ListOfStrings = 7,
 };
 
 struct placeholder_t {};
@@ -61,7 +62,8 @@ typedef std::variant<
     missing_t,
     misc_t,
     failed_t,
-    bool
+    bool,
+    std::vector<std::string>
     > AttrValue;
 
 class AttrCursor : public std::enable_shared_from_this<AttrCursor>
@@ -114,6 +116,8 @@ public:
 
     bool getBool();
 
+    std::vector<std::string> getListOfStrings();
+
     std::vector<Symbol> getAttrs();
 
     bool isDerivation();

src/nix/app.cc

@@ -89,7 +89,7 @@ UnresolvedApp Installable::toApp(EvalState & state)
         auto outputName = cursor->getAttr(state.sOutputName)->getString();
         auto name = cursor->getAttr(state.sName)->getString();
         auto aPname = cursor->maybeGetAttr("pname");
-        auto aMeta = cursor->maybeGetAttr("meta");
+        auto aMeta = cursor->maybeGetAttr(state.sMeta);
         auto aMainProgram = aMeta ? aMeta->maybeGetAttr("mainProgram") : nullptr;
         auto mainProgram =
             aMainProgram

src/nix/flake.cc

@@ -1015,8 +1015,8 @@ struct CmdFlakeShow : FlakeCommand, MixJSON
                     auto name = visitor.getAttr(state->sName)->getString();
                     if (json) {
                         std::optional<std::string> description;
-                        if (auto aMeta = visitor.maybeGetAttr("meta")) {
+                        if (auto aMeta = visitor.maybeGetAttr(state->sMeta)) {
-                            if (auto aDescription = aMeta->maybeGetAttr("description"))
+                            if (auto aDescription = aMeta->maybeGetAttr(state->sDescription))
                                 description = aDescription->getString();
                         }
                         j.emplace("type", "derivation");

src/nix/profile.cc

@@ -67,7 +67,6 @@ struct ProfileElement
         ref<Store> store,
         const BuiltPaths & builtPaths)
     {
-        // FIXME: respect meta.outputsToInstall
         storePaths.clear();
         for (auto & buildable : builtPaths) {
             std::visit(overloaded {

src/nix/search.cc

@@ -93,10 +93,10 @@ struct CmdSearch : InstallableCommand, MixJSON
                 };
 
                 if (cursor.isDerivation()) {
-                    DrvName name(cursor.getAttr("name")->getString());
+                    DrvName name(cursor.getAttr(state->sName)->getString());
 
-                    auto aMeta = cursor.maybeGetAttr("meta");
+                    auto aMeta = cursor.maybeGetAttr(state->sMeta);
-                    auto aDescription = aMeta ? aMeta->maybeGetAttr("description") : nullptr;
+                    auto aDescription = aMeta ? aMeta->maybeGetAttr(state->sDescription) : nullptr;
                     auto description = aDescription ? aDescription->getString() : "";
                     std::replace(description.begin(), description.end(), '\n', ' ');
                     auto attrPath2 = concatStringsSep(".", attrPathS);

tests/build.sh

@@ -2,15 +2,8 @@ source common.sh
 
 clearStore
 
-# Make sure that 'nix build' only returns the outputs we asked for.
+# Make sure that 'nix build' returns all outputs by default.
-nix build -f multiple-outputs.nix --json a --no-link | jq --exit-status '
+nix build -f multiple-outputs.nix --json a b --no-link | jq --exit-status '
-  (.[0] |
-    (.drvPath | match(".*multiple-outputs-a.drv")) and
-    (.outputs | keys | length == 1) and
-    (.outputs.first | match(".*multiple-outputs-a-first")))
-'
-
-nix build -f multiple-outputs.nix --json a.all b.all --no-link | jq --exit-status '
   (.[0] |
     (.drvPath | match(".*multiple-outputs-a.drv")) and
     (.outputs | keys | length == 2) and

tests/ca/substitute.sh

@@ -25,7 +25,8 @@ buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0 transi
 # Check that the thing we’ve just substituted has its realisation stored
 nix realisation info --file ./content-addressed.nix transitivelyDependentCA
 # Check that its dependencies have it too
-nix realisation info --file ./content-addressed.nix dependentCA rootCA
+nix realisation info --file ./content-addressed.nix dependentCA
+# nix realisation info --file ./content-addressed.nix rootCA --outputs out
 
 # Same thing, but
 # 1. With non-ca derivations

tests/nix-profile.sh

@@ -17,6 +17,7 @@ cat > $flake1Dir/flake.nix <<EOF
   outputs = { self }: with import ./config.nix; rec {
     packages.$system.default = mkDerivation {
       name = "profile-test-\${builtins.readFile ./version}";
+      outputs = [ "out" "man" "dev" ];
       builder = builtins.toFile "builder.sh"
         ''
           mkdir -p \$out/bin
@@ -26,10 +27,13 @@ cat > $flake1Dir/flake.nix <<EOF
           EOF
           chmod +x \$out/bin/hello
           echo DONE
+          mkdir -p \$man/share/man
+          mkdir -p \$dev/include
         '';
       __contentAddressed = import ./ca.nix;
       outputHashMode = "recursive";
       outputHashAlgo = "sha256";
+      meta.outputsToInstall = [ "out" "man" ];
     };
   };
 }
@@ -46,6 +50,8 @@ nix-env -f ./user-envs.nix -i foo-1.0
 nix profile list | grep '0 - - .*-foo-1.0'
 nix profile install $flake1Dir -L
 [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]]
+[ -e $TEST_HOME/.nix-profile/share/man ]
+(! [ -e $TEST_HOME/.nix-profile/include ])
 nix profile history
 nix profile history | grep "packages.$system.default: ∅ -> 1.0"
 nix profile diff-closures | grep 'env-manifest.nix: ε → ∅'
@@ -55,7 +61,7 @@ printf NixOS > $flake1Dir/who
 printf 2.0 > $flake1Dir/version
 nix profile upgrade 1
 [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello NixOS" ]]
-nix profile history | grep "packages.$system.default: 1.0 -> 2.0"
+nix profile history | grep "packages.$system.default: 1.0, 1.0-man -> 2.0, 2.0-man"
 
 # Test 'history', 'diff-closures'.
 nix profile diff-closures
@@ -86,7 +92,7 @@ nix profile wipe-history
 printf true > $flake1Dir/ca.nix
 printf 3.0 > $flake1Dir/version
 nix profile upgrade 0
-nix profile history | grep "packages.$system.default: 1.0 -> 3.0"
+nix profile history | grep "packages.$system.default: 1.0, 1.0-man -> 3.0, 3.0-man"
 
 # Test new install of CA package.
 nix profile remove 0