Trustless remote building

Co-authored-by: Matthew Bauer <mjbauer95@gmail.com>

src/build-remote/build-remote.cc

@@ -247,6 +247,9 @@ static int _main(int argc, char * * argv)
 connected:
         close(5);
 
+        assert(sshStore);
+        auto sshStore2 = ref<Store>(sshStore);
+
         std::cerr << "# accept\n" << storeUri << "\n";
 
         auto inputs = readStrings<PathSet>(source);
@@ -269,18 +272,23 @@ connected:
 
         {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri));
-            copyPaths(store, ref<Store>(sshStore), store->parseStorePathSet(inputs), NoRepair, NoCheckSigs, substitute);
+            copyPaths(store, sshStore2, store->parseStorePathSet(inputs), NoRepair, NoCheckSigs, substitute);
         }
 
         uploadLock = -1;
 
-        auto drv = store->readDerivation(*drvPath);
+        BasicDerivation drv = store->readDerivation(*drvPath);
+
+        if (sshStore2->isTrusting || derivationIsCA(drv.type())) {
             drv.inputSrcs = store->parseStorePathSet(inputs);
-
+            auto result = sshStore2->buildDerivation(*drvPath, drv);
-        auto result = sshStore->buildDerivation(*drvPath, drv);
-
             if (!result.success())
                 throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg);
+        } else {
+            copyPaths(store, sshStore2, {*drvPath}, NoRepair, NoCheckSigs, substitute);
+            sshStore2->buildPaths({{*drvPath}});
+        }
+
 
         StorePathSet missing;
         for (auto & path : outputs)
@@ -290,7 +298,7 @@ connected:
             Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri));
             for (auto & i : missing)
                 store->locksHeld.insert(store->printStorePath(i)); /* FIXME: ugly */
-            copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, NoSubstitute);
+            copyPaths(sshStore2, store, missing, NoRepair, NoCheckSigs, NoSubstitute);
         }
 
         return 0;

src/libstore/daemon.cc

@@ -914,6 +914,8 @@ void processConnection(
 
             opCount++;
 
+            debug("performing daemon worker op: %d", op);
+
             try {
                 performOp(tunnelLogger, store, trusted, recursive, clientVersion, from, to, op);
             } catch (Error & e) {

src/libstore/store-api.cc

@@ -834,7 +834,23 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
                 MaintainCount<decltype(nrRunning)> mc(nrRunning);
                 showProgress();
                 try {
+                    if (dstStore->isTrusting || info->ca) {
                         copyStorePath(srcStore, dstStore, storePath, repair, checkSigs);
+                    } else if (info->deriver && dstStore->storeDir == srcStore->storeDir) {
+                        auto drvPath = *info->deriver;
+                        auto outputMap = srcStore->queryDerivationOutputMap(drvPath);
+                        auto p = std::find_if(outputMap.begin(), outputMap.end(), [&](auto & i) {
+                            return i.second == storePath;
+                        });
+                        // drv file is always CA
+                        copyStorePath(srcStore, dstStore, drvPath, repair, checkSigs);
+                        dstStore->buildPaths({{
+                            drvPath,
+                            p != outputMap.end() ? StringSet { p->first } : StringSet {},
+                        }});
+                    } else {
+                        dstStore->ensurePath(storePath);
+                    }
                 } catch (Error &e) {
                     nrFailed++;
                     if (!settings.keepGoing)

src/libstore/store-api.hh

@@ -158,7 +158,9 @@ public:
 
     const Setting<int> pathInfoCacheSize{this, 65536, "path-info-cache-size", "size of the in-memory store path information cache"};
 
-    const Setting<bool> isTrusted{this, false, "trusted", "whether paths from this store can be used as substitutes even when they lack trusted signatures"};
+    const Setting<bool> isTrusted{this, false, "trusted", "whether paths from this store can be used as substitutes even when they lack trusted signatures. Compare \"trusting\""};
+
+    Setting<bool> isTrusting{this, true, "trusting", "whether (we think) paths can be added to this store even when they lack trusted signatures. Compare \"trusted\""};
 
     Setting<int> priority{this, 0, "priority", "priority of this substituter (lower value means higher priority)"};
 

src/nix-daemon/nix-daemon.cc

@@ -268,6 +268,7 @@ static int _main(int argc, char * * argv)
 {
     {
         auto stdio = false;
+        std::optional<TrustedFlag> isTrustedOpt;
 
         parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
             if (*arg == "--daemon")
@@ -278,14 +279,26 @@ static int _main(int argc, char * * argv)
                 printVersion("nix-daemon");
             else if (*arg == "--stdio")
                 stdio = true;
-            else return false;
+            else if (*arg == "--trust") {
+                settings.requireExperimentalFeature("nix-testing");
+                isTrustedOpt = Trusted;
+            } else if (*arg == "--no-trust") {
+                settings.requireExperimentalFeature("nix-testing");
+                isTrustedOpt = NotTrusted;
+            } else return false;
             return true;
         });
 
         initPlugins();
 
+        auto ensureNoTrustedFlag = [&]() {
+            if (isTrustedOpt)
+                throw Error("--trust and --no-trust flags are only for use with --stdio when this nix-daemon process is not proxying another");
+        };
+
         if (stdio) {
             if (getStoreType() == tDaemon) {
+                ensureNoTrustedFlag();
                 //  Forward on this connection to the real daemon
                 auto socketPath = settings.nixDaemonSocketFile;
                 auto s = socket(PF_UNIX, SOCK_STREAM, 0);
@@ -335,9 +348,11 @@ static int _main(int argc, char * * argv)
                 /* Auth hook is empty because in this mode we blindly trust the
                    standard streams. Limitting access to thoses is explicitly
                    not `nix-daemon`'s responsibility. */
-                processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, [&](Store & _){});
+                auto isTrusted = isTrustedOpt.value_or(Trusted);
+                processConnection(openUncachedStore(), from, to, isTrusted, NotRecursive, [&](Store & _){});
             }
         } else {
+            ensureNoTrustedFlag();
             daemonLoop(argv);
         }
 

tests/build-hook-ca.nix

@@ -0,0 +1,52 @@
+{ busybox }:
+
+with import ./config.nix;
+
+let
+
+  mkDerivation = args:
+    derivation ({
+      inherit system;
+      builder = busybox;
+      args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
+      outputHashMode = "recursive";
+      outputHashAlgo = "sha256";
+    } // removeAttrs args ["builder" "meta"])
+    // { meta = args.meta or {}; };
+
+  input1 = mkDerivation {
+    shell = busybox;
+    name = "build-remote-input-1";
+    buildCommand = "echo FOO > $out";
+    outputHash = "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=";
+  };
+
+  input2 = mkDerivation {
+    shell = busybox;
+    name = "build-remote-input-2";
+    buildCommand = "echo BAR > $out";
+    outputHash = "sha256-XArauVH91AVwP9hBBQNlkX9ccuPpSYx9o0zeIHb6e+Q=";
+  };
+
+  input3 = mkDerivation {
+    shell = busybox;
+    name = "build-remote-input-3";
+    buildCommand = ''
+      read x < ${input2}
+      echo $x BAZ > $out
+    '';
+    outputHash = "sha256-daKAcPp/+BYMQsVi/YYMlCKoNAxCNDsaivwSHgQqD2s=";
+  };
+
+in
+
+  mkDerivation {
+    shell = busybox;
+    name = "build-remote";
+    buildCommand = ''
+      read x < ${input1}
+      read y < ${input3}
+      echo "$x $y" > $out
+    '';
+    outputHash = "sha256-5SxbkUw6xe2l9TE1uwCvTtTDysD1vhRor38OtDF0LqQ=";
+  }

tests/build-remote-trustless-should-fail-0.sh

@@ -0,0 +1,11 @@
+source common.sh
+
+# We act as if remote trusts us, but it doesn't. This fails since we are
+# building input-addressed derivations with `buildDerivation`, which
+# depends on trust.
+file=build-hook.nix
+prog=$(readlink -e ./nix-daemon-untrusting.sh)
+proto=ssh-ng
+trusting=true
+
+! source build-remote-trustless.sh

tests/build-remote-trustless-should-pass-0.sh

@@ -0,0 +1,9 @@
+source common.sh
+
+# Remote trusts us but we pretend it doesn't.
+file=build-hook.nix
+prog=nix-store
+proto=ssh
+trusting=false
+
+source build-remote-trustless.sh

tests/build-remote-trustless-should-pass-1.sh

@@ -0,0 +1,9 @@
+source common.sh
+
+# Remote trusts us but we pretend it doesn't.
+file=build-hook.nix
+prog=nix-daemon
+proto=ssh-ng
+trusting=false
+
+source build-remote-trustless.sh

tests/build-remote-trustless-should-pass-2.sh

@@ -0,0 +1,9 @@
+source common.sh
+
+# Remote doesn't trust us nor do we think it does
+file=build-hook.nix
+prog=$(readlink -e ./nix-daemon-untrusting.sh)
+proto=ssh-ng
+trusting=false
+
+source build-remote-trustless.sh

tests/build-remote-trustless-should-pass-3.sh

@@ -0,0 +1,10 @@
+source common.sh
+
+# We act as if remote trusts us, but it doesn't. This is fine because we
+# are only building (fixed) CA derivations.
+file=build-hook-ca.nix
+prog=$(readlink -e ./nix-daemon-untrusting.sh)
+proto=ssh-ng
+trusting=true
+
+source build-remote-trustless.sh

tests/build-remote-trustless.sh

@@ -0,0 +1,18 @@
+if ! canUseSandbox; then exit; fi
+if ! [[ $busybox =~ busybox ]]; then exit; fi
+
+unset NIX_STORE_DIR
+unset NIX_STATE_DIR
+
+# Note: ssh://localhost bypasses ssh, directly invoking nix-store as a
+# child process. This allows us to test LegacySSHStore::buildDerivation().
+# ssh-ng://... likewise allows us to test RemoteStore::buildDerivation().
+
+nix build -L -v -f $file -o $TEST_ROOT/result --max-jobs 0 \
+  --arg busybox $busybox \
+  --store $TEST_ROOT/local \
+  --builders "$proto://localhost?remote-program=$prog&trusting=$trusting&remote-store=$TEST_ROOT/remote%3Fsystem-features=foo%20bar%20baz - - 1 1 foo,bar,baz"
+
+outPath=$(readlink -f $TEST_ROOT/result)
+
+grep 'FOO BAR BAZ' $TEST_ROOT/${subDir}/local${outPath}

tests/init.sh

@@ -17,7 +17,7 @@ cat > "$NIX_CONF_DIR"/nix.conf <<EOF
 build-users-group =
 keep-derivations = false
 sandbox = false
-experimental-features = nix-command flakes
+experimental-features = nix-command flakes nix-testing
 gc-reserved-space = 0
 flake-registry = $TEST_ROOT/registry.json
 include nix.conf.extra

tests/local.mk

@@ -15,6 +15,10 @@ nix_tests = \
   linux-sandbox.sh \
   build-dry.sh \
   build-remote.sh \
+  build-remote-trustless-should-pass-1.sh \
+  build-remote-trustless-should-pass-2.sh \
+  build-remote-trustless-should-pass-3.sh \
+  build-remote-trustless-should-fail-0.sh \
   nar-access.sh \
   structured-attrs.sh \
   fetchGit.sh \
@@ -34,6 +38,7 @@ nix_tests = \
   recursive.sh \
   flakes.sh
   # parallel.sh
+  # build-remote-trustless-should-pass-0.sh # problem with legacy ssh-store only
 
 install-tests += $(foreach x, $(nix_tests), tests/$(x))
 

tests/nix-daemon-untrusting.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+exec nix-daemon --no-trust "$@"