From 04b113f6cb5356d44992f98928a595cdf93d561a Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 8 Mar 2022 06:02:25 +0100 Subject: [PATCH 1/3] Fix `nix log` with CA derivations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix #6209 When trying to run `nix log `, try first to resolve the derivation pointed to by `` as it is the resolved one that holds the build log. This has a couple of shortcomings: 1. It’s expensive as it requires re-reading the derivation 2. It’s brittle because if the derivation doesn’t exist anymore or can’t be resolved (which is the case if any one of its build inputs is missing), then we can’t access the log anymore However, I don’t think we can do better (at least not right now). The alternatives I see are: 1. Copy the build log for the un-resolved derivation. But that means a lot of duplication 2. Store the results of the resolving in the db. Which might be the best long-term solution, but leads to a whole new class of potential issues. --- src/libstore/binary-cache-store.cc | 16 +++------ src/libstore/local-fs-store.cc | 15 +++----- src/libstore/store-api.cc | 28 +++++++++++++++ src/libstore/store-api.hh | 7 ++++ tests/build-hook-ca-floating.nix | 55 +++--------------------------- tests/build-hook.nix | 11 ++++-- tests/build-remote.sh | 9 ++--- 7 files changed, 59 insertions(+), 82 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 12d0c32fb..14584f0a2 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -504,18 +504,10 @@ void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSe std::optional BinaryCacheStore::getBuildLog(const StorePath & path) { - auto drvPath = path; - - if (!path.isDerivation()) { - try { - auto info = queryPathInfo(path); - // FIXME: add a "Log" field to .narinfo - if (!info->deriver) return std::nullopt; - drvPath = *info->deriver; - } catch (InvalidPath &) { - return std::nullopt; - } - } + auto maybePath = getBuildDerivationPath(path); + if (!maybePath) + return std::nullopt; + auto drvPath = maybePath.value(); auto logPath = "log/" + std::string(baseNameOf(printStorePath(drvPath))); diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index c5ae7536f..3a0585b15 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -89,17 +89,10 @@ const std::string LocalFSStore::drvsLogDir = "drvs"; std::optional LocalFSStore::getBuildLog(const StorePath & path_) { - auto path = path_; - - if (!path.isDerivation()) { - try { - auto info = queryPathInfo(path); - if (!info->deriver) return std::nullopt; - path = *info->deriver; - } catch (InvalidPath &) { - return std::nullopt; - } - } + auto maybePath = getBuildDerivationPath(path_); + if (!maybePath) + return std::nullopt; + auto path = maybePath.value(); auto baseName = path.to_string(); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 8811ab578..ebd641aa4 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1300,6 +1300,34 @@ Derivation readDerivationCommon(Store& store, const StorePath& drvPath, bool req } } +std::optional Store::getBuildDerivationPath(const StorePath & path) +{ + + if (!path.isDerivation()) { + try { + auto info = queryPathInfo(path); + if (!info->deriver) return std::nullopt; + return *info->deriver; + } catch (InvalidPath &) { + return std::nullopt; + } + } + + if (!settings.isExperimentalFeatureEnabled(Xp::CaDerivations) || !isValidPath(path)) + return path; + + auto drv = readDerivation(path); + if (!derivationHasKnownOutputPaths(drv.type())) { + // The build log is actually attached to the corresponding + // resolved derivation, so we need to get it first + auto resolvedDrv = drv.tryResolve(*this); + if (resolvedDrv) + return writeDerivation(*this, *resolvedDrv, NoRepair, true); + } + + return path; +} + Derivation Store::readDerivation(const StorePath & drvPath) { return readDerivationCommon(*this, drvPath, true); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 151ec10d6..88a11d953 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -618,6 +618,13 @@ public: */ StorePathSet exportReferences(const StorePathSet & storePaths, const StorePathSet & inputPaths); + /** + * Given a store path, return the realisation actually used in the realisation of this path: + * - If the path is a content-addressed derivation, try to resolve it + * - Otherwise, find one of its derivers + */ + std::optional getBuildDerivationPath(const StorePath &); + /* Hack to allow long-running processes like hydra-queue-runner to occasionally flush their path info cache. */ void clearPathInfoCache() diff --git a/tests/build-hook-ca-floating.nix b/tests/build-hook-ca-floating.nix index 67295985f..dfdbb82da 100644 --- a/tests/build-hook-ca-floating.nix +++ b/tests/build-hook-ca-floating.nix @@ -1,53 +1,6 @@ { busybox }: -with import ./config.nix; - -let - - mkDerivation = args: - derivation ({ - inherit system; - builder = busybox; - args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; - outputHashMode = "recursive"; - outputHashAlgo = "sha256"; - __contentAddressed = true; - } // removeAttrs args ["builder" "meta"]) - // { meta = args.meta or {}; }; - - input1 = mkDerivation { - shell = busybox; - name = "build-remote-input-1"; - buildCommand = "echo FOO > $out"; - requiredSystemFeatures = ["foo"]; - }; - - input2 = mkDerivation { - shell = busybox; - name = "build-remote-input-2"; - buildCommand = "echo BAR > $out"; - requiredSystemFeatures = ["bar"]; - }; - - input3 = mkDerivation { - shell = busybox; - name = "build-remote-input-3"; - buildCommand = '' - read x < ${input2} - echo $x BAZ > $out - ''; - requiredSystemFeatures = ["baz"]; - }; - -in - - mkDerivation { - shell = busybox; - name = "build-remote"; - buildCommand = - '' - read x < ${input1} - read y < ${input3} - echo "$x $y" > $out - ''; - } +import ./build-hook.nix { + inherit busybox; + contentAddressed = true; +} diff --git a/tests/build-hook.nix b/tests/build-hook.nix index 643334caa..7effd7903 100644 --- a/tests/build-hook.nix +++ b/tests/build-hook.nix @@ -1,15 +1,22 @@ -{ busybox }: +{ busybox, contentAddressed ? false }: with import ./config.nix; let + caArgs = if contentAddressed then { + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + __contentAddressed = true; + } else {}; + mkDerivation = args: derivation ({ inherit system; builder = busybox; args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; - } // removeAttrs args ["builder" "meta" "passthru"]) + } // removeAttrs args ["builder" "meta" "passthru"] + // caArgs) // { meta = args.meta or {}; passthru = args.passthru or {}; }; input1 = mkDerivation { diff --git a/tests/build-remote.sh b/tests/build-remote.sh index e73c37ea4..25a482003 100644 --- a/tests/build-remote.sh +++ b/tests/build-remote.sh @@ -63,12 +63,9 @@ nix path-info --store $TEST_ROOT/machine3 --all \ | grep builder-build-remote-input-3.sh -# Temporarily disabled because of https://github.com/NixOS/nix/issues/6209 -if [[ -z "$CONTENT_ADDRESSED" ]]; then - for i in input1 input3; do - nix log --store $TEST_ROOT/machine0 --file "$file" --arg busybox $busybox passthru."$i" | grep hi-$i - done -fi +for i in input1 input3; do +nix log --store $TEST_ROOT/machine0 --file "$file" --arg busybox $busybox passthru."$i" | grep hi-$i +done # Behavior of keep-failed out="$(nix-build 2>&1 failing.nix \ From 3b27181ee54176aba2cdff98028586048e08e74d Mon Sep 17 00:00:00 2001 From: Taeer Bar-Yam Date: Thu, 8 Dec 2022 16:59:21 -0500 Subject: [PATCH 2/3] fix missing function after rebase --- src/libstore/store-api.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index ebd641aa4..6c0261446 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1317,7 +1317,7 @@ std::optional Store::getBuildDerivationPath(const StorePath & path) return path; auto drv = readDerivation(path); - if (!derivationHasKnownOutputPaths(drv.type())) { + if (!drv.type().hasKnownOutputPaths()) { // The build log is actually attached to the corresponding // resolved derivation, so we need to get it first auto resolvedDrv = drv.tryResolve(*this); From e5eb05c5990d837e0bbc1529e3b5b167f7015be0 Mon Sep 17 00:00:00 2001 From: Taeer Bar-Yam Date: Thu, 15 Dec 2022 15:58:54 -0500 Subject: [PATCH 3/3] getBuildLog: factor out resolving derivations --- src/libstore/binary-cache-store.cc | 9 ++------- src/libstore/binary-cache-store.hh | 2 +- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/local-fs-store.cc | 7 +------ src/libstore/local-fs-store.hh | 2 +- src/libstore/log-store.hh | 9 ++++++++- src/libstore/ssh-store.cc | 4 ++-- 7 files changed, 16 insertions(+), 19 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 14584f0a2..0fef2d23a 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -502,14 +502,9 @@ void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSe writeNarInfo(narInfo); } -std::optional BinaryCacheStore::getBuildLog(const StorePath & path) +std::optional BinaryCacheStore::getBuildLogExact(const StorePath & path) { - auto maybePath = getBuildDerivationPath(path); - if (!maybePath) - return std::nullopt; - auto drvPath = maybePath.value(); - - auto logPath = "log/" + std::string(baseNameOf(printStorePath(drvPath))); + auto logPath = "log/" + std::string(baseNameOf(printStorePath(path))); debug("fetching build log from binary cache '%s/%s'", getUri(), logPath); diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 8c82e2387..abd92a83c 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -129,7 +129,7 @@ public: void addSignatures(const StorePath & storePath, const StringSet & sigs) override; - std::optional getBuildLog(const StorePath & path) override; + std::optional getBuildLogExact(const StorePath & path) override; void addBuildLog(const StorePath & drvPath, std::string_view log) override; diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 6fe3bc49c..c867c0f1d 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1460,7 +1460,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo unknown, downloadSize, narSize); } - virtual std::optional getBuildLog(const StorePath & path) override + virtual std::optional getBuildLogExact(const StorePath & path) override { return std::nullopt; } virtual void addBuildLog(const StorePath & path, std::string_view log) override diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 3a0585b15..b224fc3e9 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -87,13 +87,8 @@ void LocalFSStore::narFromPath(const StorePath & path, Sink & sink) const std::string LocalFSStore::drvsLogDir = "drvs"; -std::optional LocalFSStore::getBuildLog(const StorePath & path_) +std::optional LocalFSStore::getBuildLogExact(const StorePath & path) { - auto maybePath = getBuildDerivationPath(path_); - if (!maybePath) - return std::nullopt; - auto path = maybePath.value(); - auto baseName = path.to_string(); for (int j = 0; j < 2; j++) { diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index e6fb3201a..947707341 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -50,7 +50,7 @@ public: return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1); } - std::optional getBuildLog(const StorePath & path) override; + std::optional getBuildLogExact(const StorePath & path) override; }; diff --git a/src/libstore/log-store.hh b/src/libstore/log-store.hh index ff1b92e17..b807e3e71 100644 --- a/src/libstore/log-store.hh +++ b/src/libstore/log-store.hh @@ -11,7 +11,14 @@ struct LogStore : public virtual Store /* Return the build log of the specified store path, if available, or null otherwise. */ - virtual std::optional getBuildLog(const StorePath & path) = 0; + std::optional getBuildLog(const StorePath & path) { + auto maybePath = getBuildDerivationPath(path); + if (!maybePath) + return std::nullopt; + return getBuildLogExact(maybePath.value()); + } + + virtual std::optional getBuildLogExact(const StorePath & path) = 0; virtual void addBuildLog(const StorePath & path, std::string_view log) = 0; diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 62daa838c..a1d4daafd 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -53,8 +53,8 @@ public: { return false; } // FIXME extend daemon protocol, move implementation to RemoteStore - std::optional getBuildLog(const StorePath & path) override - { unsupported("getBuildLog"); } + std::optional getBuildLogExact(const StorePath & path) override + { unsupported("getBuildLogExact"); } private: