Merge pull request #8813 from obsidiansystems/outputOf

Create (experimental) `outputOf` primop.

doc/manual/src/release-notes/rl-next.md

@@ -19,3 +19,6 @@
 
 - The JSON output for derived paths with are store paths is now a string, not an object with a single `path` field.
   This only affects `nix-build --json` when "building" non-derivation things like fetched sources, which is a no-op.
+
+- Introduce a new [`outputOf`](@docroot@/language/builtins.md#builtins-outputOf) builtin.
+  It is part of the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature.

src/libexpr/eval.cc

@@ -1027,24 +1027,67 @@ void EvalState::mkStorePathString(const StorePath & p, Value & v)
 }
 
 
+std::string EvalState::mkOutputStringRaw(
+    const SingleDerivedPath::Built & b,
+    std::optional<StorePath> optStaticOutputPath,
+    const ExperimentalFeatureSettings & xpSettings)
+{
+    /* In practice, this is testing for the case of CA derivations, or
+       dynamic derivations. */
+    return optStaticOutputPath
+        ? store->printStorePath(*std::move(optStaticOutputPath))
+        /* Downstream we would substitute this for an actual path once
+           we build the floating CA derivation */
+        : DownstreamPlaceholder::fromSingleDerivedPathBuilt(b, xpSettings).render();
+}
+
+
 void EvalState::mkOutputString(
     Value & value,
-    const StorePath & drvPath,
+    const SingleDerivedPath::Built & b,
-    const std::string outputName,
+    std::optional<StorePath> optStaticOutputPath,
-    std::optional<StorePath> optOutputPath,
     const ExperimentalFeatureSettings & xpSettings)
 {
     value.mkString(
-        optOutputPath
+        mkOutputStringRaw(b, optStaticOutputPath, xpSettings),
-            ? store->printStorePath(*std::move(optOutputPath))
+        NixStringContext { b });
-            /* Downstream we would substitute this for an actual path once
+}
-               we build the floating CA derivation */
+
-            : DownstreamPlaceholder::unknownCaOutput(drvPath, outputName, xpSettings).render(),
+
+std::string EvalState::mkSingleDerivedPathStringRaw(
+    const SingleDerivedPath & p)
+{
+    return std::visit(overloaded {
+        [&](const SingleDerivedPath::Opaque & o) {
+            return store->printStorePath(o.path);
+        },
+        [&](const SingleDerivedPath::Built & b) {
+            auto optStaticOutputPath = std::visit(overloaded {
+                [&](const SingleDerivedPath::Opaque & o) {
+                    auto drv = store->readDerivation(o.path);
+                    auto i = drv.outputs.find(b.output);
+                    if (i == drv.outputs.end())
+                        throw Error("derivation '%s' does not have output '%s'", b.drvPath->to_string(*store), b.output);
+                    return i->second.path(*store, drv.name, b.output);
+                },
+                [&](const SingleDerivedPath::Built & o) -> std::optional<StorePath> {
+                    return std::nullopt;
+                },
+            }, b.drvPath->raw());
+            return mkOutputStringRaw(b, optStaticOutputPath);
+        }
+    }, p.raw());
+}
+
+
+void EvalState::mkSingleDerivedPathString(
+    const SingleDerivedPath & p,
+    Value & v)
+{
+    v.mkString(
+        mkSingleDerivedPathStringRaw(p),
         NixStringContext {
-            NixStringContextElem::Built {
+            std::visit([](auto && v) -> NixStringContextElem { return v; }, p),
-                .drvPath = makeConstantStorePathRef(drvPath),
-                .output = outputName,
-            }
         });
 }
 
@@ -2333,39 +2376,25 @@ SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value &
 {
     auto [derivedPath, s_] = coerceToSingleDerivedPathUnchecked(pos, v, errorCtx);
     auto s = s_;
-    std::visit(overloaded {
+    auto sExpected = mkSingleDerivedPathStringRaw(derivedPath);
-        [&](const SingleDerivedPath::Opaque & o) {
+    if (s != sExpected) {
-            auto sExpected = store->printStorePath(o.path);
+        /* `std::visit` is used here just to provide a more precise
-            if (s != sExpected)
+           error message. */
+        std::visit(overloaded {
+            [&](const SingleDerivedPath::Opaque & o) {
                 error(
                     "path string '%s' has context with the different path '%s'",
                     s, sExpected)
                     .withTrace(pos, errorCtx).debugThrow<EvalError>();
-        },
+            },
-        [&](const SingleDerivedPath::Built & b) {
+            [&](const SingleDerivedPath::Built & b) {
-            auto sExpected = std::visit(overloaded {
-                [&](const SingleDerivedPath::Opaque & o) {
-                    auto drv = store->readDerivation(o.path);
-                    auto i = drv.outputs.find(b.output);
-                    if (i == drv.outputs.end())
-                        throw Error("derivation '%s' does not have output '%s'", b.drvPath->to_string(*store), b.output);
-                    auto optOutputPath = i->second.path(*store, drv.name, b.output);
-                    // This is testing for the case of CA derivations
-                    return optOutputPath
-                        ? store->printStorePath(*optOutputPath)
-                        : DownstreamPlaceholder::fromSingleDerivedPathBuilt(b).render();
-                },
-                [&](const SingleDerivedPath::Built & o) {
-                    return DownstreamPlaceholder::fromSingleDerivedPathBuilt(b).render();
-                },
-            }, b.drvPath->raw());
-            if (s != sExpected)
                 error(
                     "string '%s' has context with the output '%s' from derivation '%s', but the string is not the right placeholder for this derivation output. It should be '%s'",
                     s, b.output, b.drvPath->to_string(*store), sExpected)
                     .withTrace(pos, errorCtx).debugThrow<EvalError>();
-        }
+            }
-    }, derivedPath.raw());
+        }, derivedPath.raw());
+    }
     return derivedPath;
 }
 

src/libexpr/eval.hh

@@ -668,37 +668,46 @@ public:
     /**
      * Create a string representing a store path.
      *
-     * The string is the printed store path with a context containing a single
+     * The string is the printed store path with a context containing a
-     * `NixStringContextElem::Opaque` element of that store path.
+     * single `NixStringContextElem::Opaque` element of that store path.
      */
     void mkStorePathString(const StorePath & storePath, Value & v);
 
     /**
-     * Create a string representing a `DerivedPath::Built`.
+     * Create a string representing a `SingleDerivedPath::Built`.
      *
-     * The string is the printed store path with a context containing a single
+     * The string is the printed store path with a context containing a
-     * `NixStringContextElem::Built` element of the drv path and output name.
+     * single `NixStringContextElem::Built` element of the drv path and
+     * output name.
      *
      * @param value Value we are settings
      *
-     * @param drvPath Path the drv whose output we are making a string for
+     * @param b the drv whose output we are making a string for, and the
+     * output
      *
-     * @param outputName Name of the output
+     * @param optStaticOutputPath Optional output path for that string.
-     *
+     * Must be passed if and only if output store object is
-     * @param optOutputPath Optional output path for that string. Must
+     * input-addressed or fixed output. Will be printed to form string
-     * be passed if and only if output store object is input-addressed.
+     * if passed, otherwise a placeholder will be used (see
-     * Will be printed to form string if passed, otherwise a placeholder
+     * `DownstreamPlaceholder`).
-     * will be used (see `DownstreamPlaceholder`).
      *
      * @param xpSettings Stop-gap to avoid globals during unit tests.
      */
     void mkOutputString(
         Value & value,
-        const StorePath & drvPath,
+        const SingleDerivedPath::Built & b,
-        const std::string outputName,
+        std::optional<StorePath> optStaticOutputPath,
-        std::optional<StorePath> optOutputPath,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 
+    /**
+     * Create a string representing a `SingleDerivedPath`.
+     *
+     * A combination of `mkStorePathString` and `mkOutputString`.
+     */
+    void mkSingleDerivedPathString(
+        const SingleDerivedPath & p,
+        Value & v);
+
     void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx);
 
     /**
@@ -714,6 +723,22 @@ public:
 
 private:
 
+    /**
+     * Like `mkOutputString` but just creates a raw string, not an
+     * string Value, which would also have a string context.
+     */
+    std::string mkOutputStringRaw(
+        const SingleDerivedPath::Built & b,
+        std::optional<StorePath> optStaticOutputPath,
+        const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
+
+    /**
+     * Like `mkSingleDerivedPathStringRaw` but just creates a raw string
+     * Value, which would also have a string context.
+     */
+    std::string mkSingleDerivedPathStringRaw(
+        const SingleDerivedPath & p);
+
     unsigned long nrEnvs = 0;
     unsigned long nrValuesInEnvs = 0;
     unsigned long nrValues = 0;

src/libexpr/primops.cc

@@ -156,8 +156,10 @@ static void mkOutputString(
 {
     state.mkOutputString(
         attrs.alloc(o.first),
-        drvPath,
+        SingleDerivedPath::Built {
-        o.first,
+            .drvPath = makeConstantStorePathRef(drvPath),
+            .output = o.first,
+        },
         o.second.path(*state.store, Derivation::nameFromPath(drvPath), o.first));
 }
 
@@ -1836,6 +1838,45 @@ static RegisterPrimOp primop_readDir({
     .fun = prim_readDir,
 });
 
+/* Extend single element string context with another output. */
+static void prim_outputOf(EvalState & state, const PosIdx pos, Value * * args, Value & v)
+{
+    SingleDerivedPath drvPath = state.coerceToSingleDerivedPath(pos, *args[0], "while evaluating the first argument to builtins.outputOf");
+
+    std::string_view outputName = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument to builtins.outputOf");
+
+    state.mkSingleDerivedPathString(
+        SingleDerivedPath::Built {
+            .drvPath = make_ref<SingleDerivedPath>(drvPath),
+            .output = std::string { outputName },
+        },
+        v);
+}
+
+static RegisterPrimOp primop_outputOf({
+    .name = "__outputOf",
+    .args = {"derivation-reference", "output-name"},
+    .doc = R"(
+      Return the output path of a derivation, literally or using a placeholder if needed.
+
+      If the derivation has a statically-known output path (i.e. the derivation output is input-addressed, or fixed content-addresed), the output path will just be returned.
+      But if the derivation is content-addressed or if the derivation is itself not-statically produced (i.e. is the output of another derivation), a placeholder will be returned instead.
+
+      *`derivation reference`* must be a string that may contain a regular store path to a derivation, or may be a placeholder reference. If the derivation is produced by a derivation, you must explicitly select `drv.outPath`.
+      This primop can be chained arbitrarily deeply.
+      For instance,
+      ```nix
+      builtins.outputOf
+        (builtins.outputOf myDrv "out)
+        "out"
+      ```
+      will return a placeholder for the output of the output of `myDrv`.
+
+      This primop corresponds to the `^` sigil for derivable paths, e.g. as part of installable syntax on the command line.
+    )",
+    .fun = prim_outputOf,
+    .experimentalFeature = Xp::DynamicDerivations,
+});
 
 /*************************************************************
  * Creating files

src/libexpr/tests/derived-path.cc

@@ -34,8 +34,8 @@ RC_GTEST_FIXTURE_PROP(
 
 RC_GTEST_FIXTURE_PROP(
     DerivedPathExpressionTest,
-    prop_built_path_placeholder_round_trip,
+    prop_derived_path_built_placeholder_round_trip,
-    (const StorePath & drvPath, const StorePathName & outputName))
+    (const SingleDerivedPath::Built & b))
 {
     /**
      * We set these in tests rather than the regular globals so we don't have
@@ -45,27 +45,19 @@ RC_GTEST_FIXTURE_PROP(
     mockXpSettings.set("experimental-features", "ca-derivations");
 
     auto * v = state.allocValue();
-    state.mkOutputString(*v, drvPath, outputName.name, std::nullopt, mockXpSettings);
+    state.mkOutputString(*v, b, std::nullopt, mockXpSettings);
     auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, "");
-    SingleDerivedPath::Built b {
-        .drvPath = makeConstantStorePathRef(drvPath),
-        .output = outputName.name,
-    };
     RC_ASSERT(SingleDerivedPath { b } == d);
 }
 
 RC_GTEST_FIXTURE_PROP(
     DerivedPathExpressionTest,
-    prop_built_path_out_path_round_trip,
+    prop_derived_path_built_out_path_round_trip,
-    (const StorePath & drvPath, const StorePathName & outputName, const StorePath & outPath))
+    (const SingleDerivedPath::Built & b, const StorePath & outPath))
 {
     auto * v = state.allocValue();
-    state.mkOutputString(*v, drvPath, outputName.name, outPath);
+    state.mkOutputString(*v, b, outPath);
     auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, "");
-    SingleDerivedPath::Built b {
-        .drvPath = makeConstantStorePathRef(drvPath),
-        .output = outputName.name,
-    };
     RC_ASSERT(SingleDerivedPath { b } == d);
 }
 

src/libstore/downstream-placeholder.cc

@@ -39,16 +39,18 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation(
 }
 
 DownstreamPlaceholder DownstreamPlaceholder::fromSingleDerivedPathBuilt(
-    const SingleDerivedPath::Built & b)
+    const SingleDerivedPath::Built & b,
+    const ExperimentalFeatureSettings & xpSettings)
 {
     return std::visit(overloaded {
         [&](const SingleDerivedPath::Opaque & o) {
-            return DownstreamPlaceholder::unknownCaOutput(o.path, b.output);
+            return DownstreamPlaceholder::unknownCaOutput(o.path, b.output, xpSettings);
         },
         [&](const SingleDerivedPath::Built & b2) {
             return DownstreamPlaceholder::unknownDerivation(
-                DownstreamPlaceholder::fromSingleDerivedPathBuilt(b2),
+                DownstreamPlaceholder::fromSingleDerivedPathBuilt(b2, xpSettings),
-                b.output);
+                b.output,
+                xpSettings);
         },
     }, b.drvPath->raw());
 }

src/libstore/downstream-placeholder.hh

@@ -84,7 +84,8 @@ public:
      * `SingleDerivedPath::Built.drvPath` chain.
      */
     static DownstreamPlaceholder fromSingleDerivedPathBuilt(
-        const SingleDerivedPath::Built & built);
+        const SingleDerivedPath::Built & built,
+        const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 };
 
 }

src/nix/main.cc

@@ -359,6 +359,7 @@ void mainWrapped(int argc, char * * argv)
         experimentalFeatureSettings.experimentalFeatures = {
             Xp::Flakes,
             Xp::FetchClosure,
+            Xp::DynamicDerivations,
         };
         evalSettings.pureEval = false;
         EvalState state({}, openStore("dummy://"));

tests/dyn-drv/dep-built-drv.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+source common.sh
+
+out1=$(nix-build ./text-hashed-output.nix -A hello --no-out-link)
+
+clearStore
+
+expectStderr 1 nix-build ./text-hashed-output.nix -A wrapper --no-out-link | grepQuiet "Dependencies on the outputs of dynamic derivations are not yet supported"

tests/dyn-drv/eval-outputOf.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+
+source ./common.sh
+
+# Without the dynamic-derivations XP feature, we don't have the builtin.
+nix --experimental-features 'nix-command' eval --impure  --expr \
+    'assert ! (builtins ? outputOf); ""'
+
+# Test that a string is required.
+#
+# We currently require a string to be passed, rather than a derivation
+# object that could be coerced to a string. We might liberalise this in
+# the future so it does work, but there are some design questions to
+# resolve first. Adding a test so we don't liberalise it by accident.
+expectStderr 1 nix --experimental-features 'nix-command dynamic-derivations' eval --impure --expr \
+    'builtins.outputOf (import ../dependencies.nix) "out"' \
+    | grepQuiet "value is a set while a string was expected"
+
+# Test that "DrvDeep" string contexts are not supported at this time
+#
+# Like the above, this is a restriction we could relax later.
+expectStderr 1 nix --experimental-features 'nix-command dynamic-derivations' eval --impure --expr \
+    'builtins.outputOf (import ../dependencies.nix).drvPath "out"' \
+    | grepQuiet "has a context which refers to a complete source and binary closure. This is not supported at this time"
+
+# Test using `builtins.outputOf` with static derivations
+testStaticHello () {
+    nix eval --impure --expr \
+        'with (import ./text-hashed-output.nix); let
+           a = hello.outPath;
+           b = builtins.outputOf (builtins.unsafeDiscardOutputDependency hello.drvPath) "out";
+         in builtins.trace a
+           (builtins.trace b
+             (assert a == b; null))'
+}
+
+# Test with a regular old input-addresed derivation
+#
+# `builtins.outputOf` works without ca-derivations and doesn't create a
+# placeholder but just returns the output path.
+testStaticHello
+
+# Test with content addressed derivation.
+NIX_TESTS_CA_BY_DEFAULT=1 testStaticHello
+
+# Test with derivation-producing derivation
+#
+# This is hardly different from the preceding cases, except that we're
+# only taking 1 outputOf out of 2 possible outputOfs. Note that
+# `.outPath` could be defined as `outputOf drvPath`, which is what we're
+# testing here. The other `outputOf` that we're not testing here is the
+# use of _dynamic_ derivations.
+nix eval --impure --expr \
+    'with (import ./text-hashed-output.nix); let
+       a = producingDrv.outPath;
+       b = builtins.outputOf (builtins.builtins.unsafeDiscardOutputDependency producingDrv.drvPath) "out";
+     in builtins.trace a
+       (builtins.trace b
+         (assert a == b; null))'
+
+# Test with unbuilt output of derivation-producing derivation.
+#
+# This function similar to `testStaticHello` used above, but instead of
+# checking the property on a constant derivation, we check it on a
+# derivation that's from another derivation's output (outPath).
+testDynamicHello () {
+    nix eval --impure --expr \
+        'with (import ./text-hashed-output.nix); let
+           a = builtins.outputOf producingDrv.outPath "out";
+           b = builtins.outputOf (builtins.outputOf (builtins.unsafeDiscardOutputDependency producingDrv.drvPath) "out") "out";
+         in builtins.trace a
+           (builtins.trace b
+             (assert a == b; null))'
+}
+
+# inner dynamic derivation is input-addressed
+testDynamicHello
+
+# inner dynamic derivation is content-addressed
+NIX_TESTS_CA_BY_DEFAULT=1 testDynamicHello

tests/dyn-drv/local.mk

@@ -1,7 +1,9 @@
 dyn-drv-tests := \
   $(d)/text-hashed-output.sh \
   $(d)/recursive-mod-json.sh \
-  $(d)/build-built-drv.sh
+  $(d)/build-built-drv.sh \
+  $(d)/eval-outputOf.sh \
+  $(d)/dep-built-drv.sh
 
 install-tests-groups += dyn-drv
 

tests/dyn-drv/recursive-mod-json.sh

@@ -3,6 +3,8 @@ source common.sh
 # FIXME
 if [[ $(uname) != Linux ]]; then skipTest "Not running Linux"; fi
 
+export NIX_TESTS_CA_BY_DEFAULT=1
+
 enableFeatures 'recursive-nix'
 restartDaemon
 

tests/dyn-drv/text-hashed-output.nix

@@ -12,9 +12,6 @@ rec {
       mkdir -p $out
       echo "Hello World" > $out/hello
     '';
-    __contentAddressed = true;
-    outputHashMode = "recursive";
-    outputHashAlgo = "sha256";
   };
   producingDrv = mkDerivation {
     name = "hello.drv";
@@ -26,4 +23,11 @@ rec {
     outputHashMode = "text";
     outputHashAlgo = "sha256";
   };
+  wrapper = mkDerivation {
+    name = "use-dynamic-drv-in-non-dynamic-drv";
+    buildCommand = ''
+      echo "Copying the output of the dynamic derivation"
+      cp -r ${builtins.outputOf producingDrv.outPath "out"} $out
+    '';
+  };
 }