Merge pull request #5929 from edolstra/copy-logs

Add command to copy build logs between stores
This commit is contained in:
Eelco Dolstra 2022-01-18 19:41:10 +01:00 committed by GitHub
commit d6e3fbc6f5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
20 changed files with 266 additions and 56 deletions

View file

@ -7,3 +7,5 @@
set or toggle display of error traces. set or toggle display of error traces.
* New builtin function `builtins.zipAttrsWith` with same functionality * New builtin function `builtins.zipAttrsWith` with same functionality
as `lib.zipAttrsWith` from nixpkgs, but much more efficient. as `lib.zipAttrsWith` from nixpkgs, but much more efficient.
* New command `nix store copy-log` to copy build logs from one store
to another.

View file

@ -54,6 +54,36 @@ void StoreCommand::run()
run(getStore()); run(getStore());
} }
CopyCommand::CopyCommand()
{
addFlag({
.longName = "from",
.description = "URL of the source Nix store.",
.labels = {"store-uri"},
.handler = {&srcUri},
});
addFlag({
.longName = "to",
.description = "URL of the destination Nix store.",
.labels = {"store-uri"},
.handler = {&dstUri},
});
}
ref<Store> CopyCommand::createStore()
{
return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri);
}
ref<Store> CopyCommand::getDstStore()
{
if (srcUri.empty() && dstUri.empty())
throw UsageError("you must pass '--from' and/or '--to'");
return dstUri.empty() ? openStore() : openStore(dstUri);
}
EvalCommand::EvalCommand() EvalCommand::EvalCommand()
{ {
} }
@ -73,13 +103,16 @@ ref<Store> EvalCommand::getEvalStore()
ref<EvalState> EvalCommand::getEvalState() ref<EvalState> EvalCommand::getEvalState()
{ {
if (!evalState) evalState = if (!evalState)
#if HAVE_BOEHMGC evalState =
std::allocate_shared<EvalState>(traceable_allocator<EvalState>(), #if HAVE_BOEHMGC
#else std::allocate_shared<EvalState>(traceable_allocator<EvalState>(),
std::make_shared<EvalState>( searchPath, getEvalStore(), getStore())
#endif #else
searchPath, getEvalStore(), getStore()); std::make_shared<EvalState>(
searchPath, getEvalStore(), getStore())
#endif
;
return ref<EvalState>(evalState); return ref<EvalState>(evalState);
} }

View file

@ -43,6 +43,19 @@ private:
std::shared_ptr<Store> _store; std::shared_ptr<Store> _store;
}; };
/* A command that copies something between `--from` and `--to`
stores. */
struct CopyCommand : virtual StoreCommand
{
std::string srcUri, dstUri;
CopyCommand();
ref<Store> createStore() override;
ref<Store> getDstStore();
};
struct EvalCommand : virtual StoreCommand, MixEvalArgs struct EvalCommand : virtual StoreCommand, MixEvalArgs
{ {
EvalCommand(); EvalCommand();

View file

@ -345,6 +345,18 @@ Installable::getCursor(EvalState & state)
return cursors[0]; return cursors[0];
} }
static StorePath getDeriver(
ref<Store> store,
const Installable & i,
const StorePath & drvPath)
{
auto derivers = store->queryValidDerivers(drvPath);
if (derivers.empty())
throw Error("'%s' does not have a known deriver", i.what());
// FIXME: use all derivers?
return *derivers.begin();
}
struct InstallableStorePath : Installable struct InstallableStorePath : Installable
{ {
ref<Store> store; ref<Store> store;
@ -353,7 +365,7 @@ struct InstallableStorePath : Installable
InstallableStorePath(ref<Store> store, StorePath && storePath) InstallableStorePath(ref<Store> store, StorePath && storePath)
: store(store), storePath(std::move(storePath)) { } : store(store), storePath(std::move(storePath)) { }
std::string what() override { return store->printStorePath(storePath); } std::string what() const override { return store->printStorePath(storePath); }
DerivedPaths toDerivedPaths() override DerivedPaths toDerivedPaths() override
{ {
@ -374,6 +386,15 @@ struct InstallableStorePath : Installable
} }
} }
StorePathSet toDrvPaths(ref<Store> store) override
{
if (storePath.isDerivation()) {
return {storePath};
} else {
return {getDeriver(store, *this, storePath)};
}
}
std::optional<StorePath> getStorePath() override std::optional<StorePath> getStorePath() override
{ {
return storePath; return storePath;
@ -402,6 +423,14 @@ DerivedPaths InstallableValue::toDerivedPaths()
return res; return res;
} }
StorePathSet InstallableValue::toDrvPaths(ref<Store> store)
{
StorePathSet res;
for (auto & drv : toDerivations())
res.insert(drv.drvPath);
return res;
}
struct InstallableAttrPath : InstallableValue struct InstallableAttrPath : InstallableValue
{ {
SourceExprCommand & cmd; SourceExprCommand & cmd;
@ -412,7 +441,7 @@ struct InstallableAttrPath : InstallableValue
: InstallableValue(state), cmd(cmd), v(allocRootValue(v)), attrPath(attrPath) : InstallableValue(state), cmd(cmd), v(allocRootValue(v)), attrPath(attrPath)
{ } { }
std::string what() override { return attrPath; } std::string what() const override { return attrPath; }
std::pair<Value *, Pos> toValue(EvalState & state) override std::pair<Value *, Pos> toValue(EvalState & state) override
{ {
@ -836,11 +865,7 @@ StorePathSet toDerivations(
[&](const DerivedPath::Opaque & bo) { [&](const DerivedPath::Opaque & bo) {
if (!useDeriver) if (!useDeriver)
throw Error("argument '%s' did not evaluate to a derivation", i->what()); throw Error("argument '%s' did not evaluate to a derivation", i->what());
auto derivers = store->queryValidDerivers(bo.path); drvPaths.insert(getDeriver(store, *i, bo.path));
if (derivers.empty())
throw Error("'%s' does not have a known deriver", i->what());
// FIXME: use all derivers?
drvPaths.insert(*derivers.begin());
}, },
[&](const DerivedPath::Built & bfd) { [&](const DerivedPath::Built & bfd) {
drvPaths.insert(bfd.drvPath); drvPaths.insert(bfd.drvPath);

View file

@ -33,10 +33,15 @@ struct Installable
{ {
virtual ~Installable() { } virtual ~Installable() { }
virtual std::string what() = 0; virtual std::string what() const = 0;
virtual DerivedPaths toDerivedPaths() = 0; virtual DerivedPaths toDerivedPaths() = 0;
virtual StorePathSet toDrvPaths(ref<Store> store)
{
throw Error("'%s' cannot be converted to a derivation path", what());
}
DerivedPath toDerivedPath(); DerivedPath toDerivedPath();
UnresolvedApp toApp(EvalState & state); UnresolvedApp toApp(EvalState & state);
@ -81,6 +86,8 @@ struct InstallableValue : Installable
virtual std::vector<DerivationInfo> toDerivations() = 0; virtual std::vector<DerivationInfo> toDerivations() = 0;
DerivedPaths toDerivedPaths() override; DerivedPaths toDerivedPaths() override;
StorePathSet toDrvPaths(ref<Store> store) override;
}; };
struct InstallableFlake : InstallableValue struct InstallableFlake : InstallableValue
@ -99,7 +106,7 @@ struct InstallableFlake : InstallableValue
Strings && prefixes, Strings && prefixes,
const flake::LockFlags & lockFlags); const flake::LockFlags & lockFlags);
std::string what() override { return flakeRef.to_string() + "#" + *attrPaths.begin(); } std::string what() const override { return flakeRef.to_string() + "#" + *attrPaths.begin(); }
std::vector<std::string> getActualAttrPaths(); std::vector<std::string> getActualAttrPaths();

View file

@ -512,4 +512,14 @@ std::optional<std::string> BinaryCacheStore::getBuildLog(const StorePath & path)
return getFile(logPath); return getFile(logPath);
} }
void BinaryCacheStore::addBuildLog(const StorePath & drvPath, std::string_view log)
{
assert(drvPath.isDerivation());
upsertFile(
"log/" + std::string(drvPath.to_string()),
(std::string) log, // FIXME: don't copy
"text/plain; charset=utf-8");
}
} }

View file

@ -51,6 +51,7 @@ public:
const std::string & mimeType) = 0; const std::string & mimeType) = 0;
void upsertFile(const std::string & path, void upsertFile(const std::string & path,
// FIXME: use std::string_view
std::string && data, std::string && data,
const std::string & mimeType); const std::string & mimeType);
@ -120,6 +121,8 @@ public:
std::optional<std::string> getBuildLog(const StorePath & path) override; std::optional<std::string> getBuildLog(const StorePath & path) override;
void addBuildLog(const StorePath & drvPath, std::string_view log) override;
}; };
MakeError(NoSuchBinaryCacheFile, Error); MakeError(NoSuchBinaryCacheFile, Error);

View file

@ -468,10 +468,12 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
dontCheckSigs = false; dontCheckSigs = false;
logger->startWork(); logger->startWork();
FramedSource source(from); {
store->addMultipleToStore(source, FramedSource source(from);
RepairFlag{repair}, store->addMultipleToStore(source,
dontCheckSigs ? NoCheckSigs : CheckSigs); RepairFlag{repair},
dontCheckSigs ? NoCheckSigs : CheckSigs);
}
logger->stopWork(); logger->stopWork();
break; break;
} }
@ -920,6 +922,22 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
break; break;
} }
case wopAddBuildLog: {
StorePath path{readString(from)};
logger->startWork();
if (!trusted)
throw Error("you are not privileged to add logs");
{
FramedSource source(from);
StringSink sink;
source.drainInto(sink);
store->addBuildLog(path, sink.s);
}
logger->stopWork();
to << 1;
break;
}
default: default:
throw Error("invalid operation %1%", op); throw Error("invalid operation %1%", op);
} }

View file

@ -96,6 +96,7 @@ void LocalBinaryCacheStore::init()
createDirs(binaryCacheDir + "/" + realisationsPrefix); createDirs(binaryCacheDir + "/" + realisationsPrefix);
if (writeDebugInfo) if (writeDebugInfo)
createDirs(binaryCacheDir + "/debuginfo"); createDirs(binaryCacheDir + "/debuginfo");
createDirs(binaryCacheDir + "/log");
BinaryCacheStore::init(); BinaryCacheStore::init();
} }

View file

@ -9,6 +9,7 @@
#include "callback.hh" #include "callback.hh"
#include "topo-sort.hh" #include "topo-sort.hh"
#include "finally.hh" #include "finally.hh"
#include "compression.hh"
#include <iostream> #include <iostream>
#include <algorithm> #include <algorithm>
@ -1898,4 +1899,24 @@ FixedOutputHash LocalStore::hashCAPath(
}; };
} }
void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log)
{
assert(drvPath.isDerivation());
auto baseName = drvPath.to_string();
auto logPath = fmt("%s/%s/%s/%s.bz2", logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2));
if (pathExists(logPath)) return;
createDirs(dirOf(logPath));
auto tmpFile = fmt("%s.tmp.%d", logPath, getpid());
writeFile(tmpFile, compress("bzip2", log));
if (rename(tmpFile.c_str(), logPath.c_str()) != 0)
throw SysError("renaming '%1%' to '%2%'", tmpFile, logPath);
}
} // namespace nix } // namespace nix

View file

@ -280,6 +280,8 @@ private:
const std::string_view pathHash const std::string_view pathHash
); );
void addBuildLog(const StorePath & drvPath, std::string_view log) override;
friend struct LocalDerivationGoal; friend struct LocalDerivationGoal;
friend struct PathSubstitutionGoal; friend struct PathSubstitutionGoal;
friend struct SubstitutionGoal; friend struct SubstitutionGoal;

View file

@ -908,6 +908,18 @@ void RemoteStore::queryMissing(const std::vector<DerivedPath> & targets,
} }
void RemoteStore::addBuildLog(const StorePath & drvPath, std::string_view log)
{
auto conn(getConnection());
conn->to << wopAddBuildLog << drvPath.to_string();
StringSource source(log);
conn.withFramedSink([&](Sink & sink) {
source.drainInto(sink);
});
readInt(conn->from);
}
void RemoteStore::connect() void RemoteStore::connect()
{ {
auto conn(getConnection()); auto conn(getConnection());

View file

@ -116,6 +116,8 @@ public:
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
uint64_t & downloadSize, uint64_t & narSize) override; uint64_t & downloadSize, uint64_t & narSize) override;
void addBuildLog(const StorePath & drvPath, std::string_view log) override;
void connect() override; void connect() override;
unsigned int getProtocol() override; unsigned int getProtocol() override;

View file

@ -727,6 +727,9 @@ public:
virtual std::optional<std::string> getBuildLog(const StorePath & path) virtual std::optional<std::string> getBuildLog(const StorePath & path)
{ return std::nullopt; } { return std::nullopt; }
virtual void addBuildLog(const StorePath & path, std::string_view log)
{ unsupported("addBuildLog"); }
/* Hack to allow long-running processes like hydra-queue-runner to /* Hack to allow long-running processes like hydra-queue-runner to
occasionally flush their path info cache. */ occasionally flush their path info cache. */
void clearPathInfoCache() void clearPathInfoCache()

View file

@ -56,6 +56,7 @@ typedef enum {
wopRegisterDrvOutput = 42, wopRegisterDrvOutput = 42,
wopQueryRealisation = 43, wopQueryRealisation = 43,
wopAddMultipleToStore = 44, wopAddMultipleToStore = 44,
wopAddBuildLog = 45,
} WorkerOp; } WorkerOp;

View file

@ -19,7 +19,7 @@ struct InstallableDerivedPath : Installable
} }
std::string what() override { return derivedPath.to_string(*store); } std::string what() const override { return derivedPath.to_string(*store); }
DerivedPaths toDerivedPaths() override DerivedPaths toDerivedPaths() override
{ {

View file

@ -1,17 +1,11 @@
#include "command.hh" #include "command.hh"
#include "shared.hh" #include "shared.hh"
#include "store-api.hh" #include "store-api.hh"
#include "sync.hh"
#include "thread-pool.hh"
#include <atomic>
using namespace nix; using namespace nix;
struct CmdCopy : BuiltPathsCommand struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand
{ {
std::string srcUri, dstUri;
CheckSigsFlag checkSigs = CheckSigs; CheckSigsFlag checkSigs = CheckSigs;
SubstituteFlag substitute = NoSubstitute; SubstituteFlag substitute = NoSubstitute;
@ -21,20 +15,6 @@ struct CmdCopy : BuiltPathsCommand
CmdCopy() CmdCopy()
: BuiltPathsCommand(true) : BuiltPathsCommand(true)
{ {
addFlag({
.longName = "from",
.description = "URL of the source Nix store.",
.labels = {"store-uri"},
.handler = {&srcUri},
});
addFlag({
.longName = "to",
.description = "URL of the destination Nix store.",
.labels = {"store-uri"},
.handler = {&dstUri},
});
addFlag({ addFlag({
.longName = "no-check-sigs", .longName = "no-check-sigs",
.description = "Do not require that paths are signed by trusted keys.", .description = "Do not require that paths are signed by trusted keys.",
@ -65,22 +45,9 @@ struct CmdCopy : BuiltPathsCommand
Category category() override { return catSecondary; } Category category() override { return catSecondary; }
ref<Store> createStore() override
{
return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri);
}
void run(ref<Store> store) override
{
if (srcUri.empty() && dstUri.empty())
throw UsageError("you must pass '--from' and/or '--to'");
BuiltPathsCommand::run(store);
}
void run(ref<Store> srcStore, BuiltPaths && paths) override void run(ref<Store> srcStore, BuiltPaths && paths) override
{ {
ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri); auto dstStore = getDstStore();
RealisedPath::Set stuffToCopy; RealisedPath::Set stuffToCopy;

46
src/nix/store-copy-log.cc Normal file
View file

@ -0,0 +1,46 @@
#include "command.hh"
#include "shared.hh"
#include "store-api.hh"
#include "sync.hh"
#include "thread-pool.hh"
#include <atomic>
using namespace nix;
struct CmdCopyLog : virtual CopyCommand, virtual InstallablesCommand
{
std::string description() override
{
return "copy build logs between Nix stores";
}
std::string doc() override
{
return
#include "store-copy-log.md"
;
}
Category category() override { return catUtility; }
void run(ref<Store> srcStore) override
{
auto dstStore = getDstStore();
StorePathSet drvPaths;
for (auto & i : installables)
for (auto & drvPath : i->toDrvPaths(getEvalStore()))
drvPaths.insert(drvPath);
for (auto & drvPath : drvPaths) {
if (auto log = srcStore->getBuildLog(drvPath))
dstStore->addBuildLog(drvPath, *log);
else
throw Error("build log for '%s' is not available", srcStore->printStorePath(drvPath));
}
}
};
static auto rCmdCopyLog = registerCommand2<CmdCopyLog>({"store", "copy-log"});

33
src/nix/store-copy-log.md Normal file
View file

@ -0,0 +1,33 @@
R""(
# Examples
* To copy the build log of the `hello` package from
https://cache.nixos.org to the local store:
```console
# nix store copy-log --from https://cache.nixos.org --eval-store auto nixpkgs#hello
```
You can verify that the log is available locally:
```console
# nix log --substituters '' nixpkgs#hello
```
(The flag `--substituters ''` avoids querying
`https://cache.nixos.org` for the log.)
* To copy the log for a specific store derivation via SSH:
```console
# nix store copy-log --to ssh-ng://machine /nix/store/ilgm50plpmcgjhcp33z6n4qbnpqfhxym-glibc-2.33-59.drv
```
# Description
`nix store copy-log` copies build logs between two Nix stores. The
source store is specified using `--from` and the destination using
`--to`. If one of these is omitted, it defaults to the local store.
)""

View file

@ -14,6 +14,17 @@ outPath=$(nix-build dependencies.nix --no-out-link)
nix copy --to file://$cacheDir $outPath nix copy --to file://$cacheDir $outPath
# Test copying build logs to the binary cache.
nix log --store file://$cacheDir $outPath 2>&1 | grep 'is not available'
nix store copy-log --to file://$cacheDir $outPath
nix log --store file://$cacheDir $outPath | grep FOO
rm -rf $TEST_ROOT/var/log/nix
nix log $outPath 2>&1 | grep 'is not available'
nix log --substituters file://$cacheDir $outPath | grep FOO
# Test copying build logs from the binary cache.
nix store copy-log --from file://$cacheDir $(nix-store -qd $outPath)
nix log $outPath | grep FOO
basicDownloadTests() { basicDownloadTests() {
# No uploading tests bcause upload with force HTTP doesn't work. # No uploading tests bcause upload with force HTTP doesn't work.