Merge pull request #7430 from tweag/ca/fix-nix-log

Ca/fix nix log
This commit is contained in:
Théophane Hufschmitt 2023-01-13 11:00:56 +01:00 committed by GitHub
commit bdeb6de889
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 67 additions and 93 deletions

View file

@ -502,22 +502,9 @@ void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSe
writeNarInfo(narInfo); writeNarInfo(narInfo);
} }
std::optional<std::string> BinaryCacheStore::getBuildLog(const StorePath & path) std::optional<std::string> BinaryCacheStore::getBuildLogExact(const StorePath & path)
{ {
auto drvPath = path; auto logPath = "log/" + std::string(baseNameOf(printStorePath(path)));
if (!path.isDerivation()) {
try {
auto info = queryPathInfo(path);
// FIXME: add a "Log" field to .narinfo
if (!info->deriver) return std::nullopt;
drvPath = *info->deriver;
} catch (InvalidPath &) {
return std::nullopt;
}
}
auto logPath = "log/" + std::string(baseNameOf(printStorePath(drvPath)));
debug("fetching build log from binary cache '%s/%s'", getUri(), logPath); debug("fetching build log from binary cache '%s/%s'", getUri(), logPath);

View file

@ -129,7 +129,7 @@ public:
void addSignatures(const StorePath & storePath, const StringSet & sigs) override; void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
std::optional<std::string> getBuildLog(const StorePath & path) override; std::optional<std::string> getBuildLogExact(const StorePath & path) override;
void addBuildLog(const StorePath & drvPath, std::string_view log) override; void addBuildLog(const StorePath & drvPath, std::string_view log) override;

View file

@ -1459,7 +1459,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
unknown, downloadSize, narSize); unknown, downloadSize, narSize);
} }
virtual std::optional<std::string> getBuildLog(const StorePath & path) override virtual std::optional<std::string> getBuildLogExact(const StorePath & path) override
{ return std::nullopt; } { return std::nullopt; }
virtual void addBuildLog(const StorePath & path, std::string_view log) override virtual void addBuildLog(const StorePath & path, std::string_view log) override

View file

@ -87,20 +87,8 @@ void LocalFSStore::narFromPath(const StorePath & path, Sink & sink)
const std::string LocalFSStore::drvsLogDir = "drvs"; const std::string LocalFSStore::drvsLogDir = "drvs";
std::optional<std::string> LocalFSStore::getBuildLog(const StorePath & path_) std::optional<std::string> LocalFSStore::getBuildLogExact(const StorePath & path)
{ {
auto path = path_;
if (!path.isDerivation()) {
try {
auto info = queryPathInfo(path);
if (!info->deriver) return std::nullopt;
path = *info->deriver;
} catch (InvalidPath &) {
return std::nullopt;
}
}
auto baseName = path.to_string(); auto baseName = path.to_string();
for (int j = 0; j < 2; j++) { for (int j = 0; j < 2; j++) {

View file

@ -50,7 +50,7 @@ public:
return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1); return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1);
} }
std::optional<std::string> getBuildLog(const StorePath & path) override; std::optional<std::string> getBuildLogExact(const StorePath & path) override;
}; };

View file

@ -11,7 +11,14 @@ struct LogStore : public virtual Store
/* Return the build log of the specified store path, if available, /* Return the build log of the specified store path, if available,
or null otherwise. */ or null otherwise. */
virtual std::optional<std::string> getBuildLog(const StorePath & path) = 0; std::optional<std::string> getBuildLog(const StorePath & path) {
auto maybePath = getBuildDerivationPath(path);
if (!maybePath)
return std::nullopt;
return getBuildLogExact(maybePath.value());
}
virtual std::optional<std::string> getBuildLogExact(const StorePath & path) = 0;
virtual void addBuildLog(const StorePath & path, std::string_view log) = 0; virtual void addBuildLog(const StorePath & path, std::string_view log) = 0;

View file

@ -53,8 +53,8 @@ public:
{ return false; } { return false; }
// FIXME extend daemon protocol, move implementation to RemoteStore // FIXME extend daemon protocol, move implementation to RemoteStore
std::optional<std::string> getBuildLog(const StorePath & path) override std::optional<std::string> getBuildLogExact(const StorePath & path) override
{ unsupported("getBuildLog"); } { unsupported("getBuildLogExact"); }
private: private:

View file

@ -1301,6 +1301,34 @@ Derivation readDerivationCommon(Store& store, const StorePath& drvPath, bool req
} }
} }
std::optional<StorePath> Store::getBuildDerivationPath(const StorePath & path)
{
if (!path.isDerivation()) {
try {
auto info = queryPathInfo(path);
if (!info->deriver) return std::nullopt;
return *info->deriver;
} catch (InvalidPath &) {
return std::nullopt;
}
}
if (!settings.isExperimentalFeatureEnabled(Xp::CaDerivations) || !isValidPath(path))
return path;
auto drv = readDerivation(path);
if (!drv.type().hasKnownOutputPaths()) {
// The build log is actually attached to the corresponding
// resolved derivation, so we need to get it first
auto resolvedDrv = drv.tryResolve(*this);
if (resolvedDrv)
return writeDerivation(*this, *resolvedDrv, NoRepair, true);
}
return path;
}
Derivation Store::readDerivation(const StorePath & drvPath) Derivation Store::readDerivation(const StorePath & drvPath)
{ return readDerivationCommon(*this, drvPath, true); } { return readDerivationCommon(*this, drvPath, true); }

View file

@ -618,6 +618,13 @@ public:
*/ */
StorePathSet exportReferences(const StorePathSet & storePaths, const StorePathSet & inputPaths); StorePathSet exportReferences(const StorePathSet & storePaths, const StorePathSet & inputPaths);
/**
* Given a store path, return the realisation actually used in the realisation of this path:
* - If the path is a content-addressed derivation, try to resolve it
* - Otherwise, find one of its derivers
*/
std::optional<StorePath> getBuildDerivationPath(const StorePath &);
/* Hack to allow long-running processes like hydra-queue-runner to /* Hack to allow long-running processes like hydra-queue-runner to
occasionally flush their path info cache. */ occasionally flush their path info cache. */
void clearPathInfoCache() void clearPathInfoCache()

View file

@ -1,53 +1,6 @@
{ busybox }: { busybox }:
with import ./config.nix; import ./build-hook.nix {
inherit busybox;
let contentAddressed = true;
}
mkDerivation = args:
derivation ({
inherit system;
builder = busybox;
args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
outputHashMode = "recursive";
outputHashAlgo = "sha256";
__contentAddressed = true;
} // removeAttrs args ["builder" "meta"])
// { meta = args.meta or {}; };
input1 = mkDerivation {
shell = busybox;
name = "build-remote-input-1";
buildCommand = "echo FOO > $out";
requiredSystemFeatures = ["foo"];
};
input2 = mkDerivation {
shell = busybox;
name = "build-remote-input-2";
buildCommand = "echo BAR > $out";
requiredSystemFeatures = ["bar"];
};
input3 = mkDerivation {
shell = busybox;
name = "build-remote-input-3";
buildCommand = ''
read x < ${input2}
echo $x BAZ > $out
'';
requiredSystemFeatures = ["baz"];
};
in
mkDerivation {
shell = busybox;
name = "build-remote";
buildCommand =
''
read x < ${input1}
read y < ${input3}
echo "$x $y" > $out
'';
}

View file

@ -1,15 +1,22 @@
{ busybox }: { busybox, contentAddressed ? false }:
with import ./config.nix; with import ./config.nix;
let let
caArgs = if contentAddressed then {
outputHashMode = "recursive";
outputHashAlgo = "sha256";
__contentAddressed = true;
} else {};
mkDerivation = args: mkDerivation = args:
derivation ({ derivation ({
inherit system; inherit system;
builder = busybox; builder = busybox;
args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
} // removeAttrs args ["builder" "meta" "passthru"]) } // removeAttrs args ["builder" "meta" "passthru"]
// caArgs)
// { meta = args.meta or {}; passthru = args.passthru or {}; }; // { meta = args.meta or {}; passthru = args.passthru or {}; };
input1 = mkDerivation { input1 = mkDerivation {

View file

@ -63,12 +63,9 @@ nix path-info --store $TEST_ROOT/machine3 --all \
| grep builder-build-remote-input-3.sh | grep builder-build-remote-input-3.sh
# Temporarily disabled because of https://github.com/NixOS/nix/issues/6209 for i in input1 input3; do
if [[ -z "$CONTENT_ADDRESSED" ]]; then nix log --store $TEST_ROOT/machine0 --file "$file" --arg busybox $busybox passthru."$i" | grep hi-$i
for i in input1 input3; do done
nix log --store $TEST_ROOT/machine0 --file "$file" --arg busybox $busybox passthru."$i" | grep hi-$i
done
fi
# Behavior of keep-failed # Behavior of keep-failed
out="$(nix-build 2>&1 failing.nix \ out="$(nix-build 2>&1 failing.nix \