forked from lix-project/lix
Merge pull request #5929 from edolstra/copy-logs
Add command to copy build logs between stores
This commit is contained in:
commit
d6e3fbc6f5
|
@ -7,3 +7,5 @@
|
|||
set or toggle display of error traces.
|
||||
* New builtin function `builtins.zipAttrsWith` with same functionality
|
||||
as `lib.zipAttrsWith` from nixpkgs, but much more efficient.
|
||||
* New command `nix store copy-log` to copy build logs from one store
|
||||
to another.
|
||||
|
|
|
@ -54,6 +54,36 @@ void StoreCommand::run()
|
|||
run(getStore());
|
||||
}
|
||||
|
||||
CopyCommand::CopyCommand()
|
||||
{
|
||||
addFlag({
|
||||
.longName = "from",
|
||||
.description = "URL of the source Nix store.",
|
||||
.labels = {"store-uri"},
|
||||
.handler = {&srcUri},
|
||||
});
|
||||
|
||||
addFlag({
|
||||
.longName = "to",
|
||||
.description = "URL of the destination Nix store.",
|
||||
.labels = {"store-uri"},
|
||||
.handler = {&dstUri},
|
||||
});
|
||||
}
|
||||
|
||||
ref<Store> CopyCommand::createStore()
|
||||
{
|
||||
return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri);
|
||||
}
|
||||
|
||||
ref<Store> CopyCommand::getDstStore()
|
||||
{
|
||||
if (srcUri.empty() && dstUri.empty())
|
||||
throw UsageError("you must pass '--from' and/or '--to'");
|
||||
|
||||
return dstUri.empty() ? openStore() : openStore(dstUri);
|
||||
}
|
||||
|
||||
EvalCommand::EvalCommand()
|
||||
{
|
||||
}
|
||||
|
@ -73,13 +103,16 @@ ref<Store> EvalCommand::getEvalStore()
|
|||
|
||||
ref<EvalState> EvalCommand::getEvalState()
|
||||
{
|
||||
if (!evalState) evalState =
|
||||
#if HAVE_BOEHMGC
|
||||
std::allocate_shared<EvalState>(traceable_allocator<EvalState>(),
|
||||
#else
|
||||
std::make_shared<EvalState>(
|
||||
#endif
|
||||
searchPath, getEvalStore(), getStore());
|
||||
if (!evalState)
|
||||
evalState =
|
||||
#if HAVE_BOEHMGC
|
||||
std::allocate_shared<EvalState>(traceable_allocator<EvalState>(),
|
||||
searchPath, getEvalStore(), getStore())
|
||||
#else
|
||||
std::make_shared<EvalState>(
|
||||
searchPath, getEvalStore(), getStore())
|
||||
#endif
|
||||
;
|
||||
return ref<EvalState>(evalState);
|
||||
}
|
||||
|
||||
|
|
|
@ -43,6 +43,19 @@ private:
|
|||
std::shared_ptr<Store> _store;
|
||||
};
|
||||
|
||||
/* A command that copies something between `--from` and `--to`
|
||||
stores. */
|
||||
struct CopyCommand : virtual StoreCommand
|
||||
{
|
||||
std::string srcUri, dstUri;
|
||||
|
||||
CopyCommand();
|
||||
|
||||
ref<Store> createStore() override;
|
||||
|
||||
ref<Store> getDstStore();
|
||||
};
|
||||
|
||||
struct EvalCommand : virtual StoreCommand, MixEvalArgs
|
||||
{
|
||||
EvalCommand();
|
||||
|
|
|
@ -345,6 +345,18 @@ Installable::getCursor(EvalState & state)
|
|||
return cursors[0];
|
||||
}
|
||||
|
||||
static StorePath getDeriver(
|
||||
ref<Store> store,
|
||||
const Installable & i,
|
||||
const StorePath & drvPath)
|
||||
{
|
||||
auto derivers = store->queryValidDerivers(drvPath);
|
||||
if (derivers.empty())
|
||||
throw Error("'%s' does not have a known deriver", i.what());
|
||||
// FIXME: use all derivers?
|
||||
return *derivers.begin();
|
||||
}
|
||||
|
||||
struct InstallableStorePath : Installable
|
||||
{
|
||||
ref<Store> store;
|
||||
|
@ -353,7 +365,7 @@ struct InstallableStorePath : Installable
|
|||
InstallableStorePath(ref<Store> store, StorePath && storePath)
|
||||
: store(store), storePath(std::move(storePath)) { }
|
||||
|
||||
std::string what() override { return store->printStorePath(storePath); }
|
||||
std::string what() const override { return store->printStorePath(storePath); }
|
||||
|
||||
DerivedPaths toDerivedPaths() override
|
||||
{
|
||||
|
@ -374,6 +386,15 @@ struct InstallableStorePath : Installable
|
|||
}
|
||||
}
|
||||
|
||||
StorePathSet toDrvPaths(ref<Store> store) override
|
||||
{
|
||||
if (storePath.isDerivation()) {
|
||||
return {storePath};
|
||||
} else {
|
||||
return {getDeriver(store, *this, storePath)};
|
||||
}
|
||||
}
|
||||
|
||||
std::optional<StorePath> getStorePath() override
|
||||
{
|
||||
return storePath;
|
||||
|
@ -402,6 +423,14 @@ DerivedPaths InstallableValue::toDerivedPaths()
|
|||
return res;
|
||||
}
|
||||
|
||||
StorePathSet InstallableValue::toDrvPaths(ref<Store> store)
|
||||
{
|
||||
StorePathSet res;
|
||||
for (auto & drv : toDerivations())
|
||||
res.insert(drv.drvPath);
|
||||
return res;
|
||||
}
|
||||
|
||||
struct InstallableAttrPath : InstallableValue
|
||||
{
|
||||
SourceExprCommand & cmd;
|
||||
|
@ -412,7 +441,7 @@ struct InstallableAttrPath : InstallableValue
|
|||
: InstallableValue(state), cmd(cmd), v(allocRootValue(v)), attrPath(attrPath)
|
||||
{ }
|
||||
|
||||
std::string what() override { return attrPath; }
|
||||
std::string what() const override { return attrPath; }
|
||||
|
||||
std::pair<Value *, Pos> toValue(EvalState & state) override
|
||||
{
|
||||
|
@ -836,11 +865,7 @@ StorePathSet toDerivations(
|
|||
[&](const DerivedPath::Opaque & bo) {
|
||||
if (!useDeriver)
|
||||
throw Error("argument '%s' did not evaluate to a derivation", i->what());
|
||||
auto derivers = store->queryValidDerivers(bo.path);
|
||||
if (derivers.empty())
|
||||
throw Error("'%s' does not have a known deriver", i->what());
|
||||
// FIXME: use all derivers?
|
||||
drvPaths.insert(*derivers.begin());
|
||||
drvPaths.insert(getDeriver(store, *i, bo.path));
|
||||
},
|
||||
[&](const DerivedPath::Built & bfd) {
|
||||
drvPaths.insert(bfd.drvPath);
|
||||
|
|
|
@ -33,10 +33,15 @@ struct Installable
|
|||
{
|
||||
virtual ~Installable() { }
|
||||
|
||||
virtual std::string what() = 0;
|
||||
virtual std::string what() const = 0;
|
||||
|
||||
virtual DerivedPaths toDerivedPaths() = 0;
|
||||
|
||||
virtual StorePathSet toDrvPaths(ref<Store> store)
|
||||
{
|
||||
throw Error("'%s' cannot be converted to a derivation path", what());
|
||||
}
|
||||
|
||||
DerivedPath toDerivedPath();
|
||||
|
||||
UnresolvedApp toApp(EvalState & state);
|
||||
|
@ -81,6 +86,8 @@ struct InstallableValue : Installable
|
|||
virtual std::vector<DerivationInfo> toDerivations() = 0;
|
||||
|
||||
DerivedPaths toDerivedPaths() override;
|
||||
|
||||
StorePathSet toDrvPaths(ref<Store> store) override;
|
||||
};
|
||||
|
||||
struct InstallableFlake : InstallableValue
|
||||
|
@ -99,7 +106,7 @@ struct InstallableFlake : InstallableValue
|
|||
Strings && prefixes,
|
||||
const flake::LockFlags & lockFlags);
|
||||
|
||||
std::string what() override { return flakeRef.to_string() + "#" + *attrPaths.begin(); }
|
||||
std::string what() const override { return flakeRef.to_string() + "#" + *attrPaths.begin(); }
|
||||
|
||||
std::vector<std::string> getActualAttrPaths();
|
||||
|
||||
|
|
|
@ -512,4 +512,14 @@ std::optional<std::string> BinaryCacheStore::getBuildLog(const StorePath & path)
|
|||
return getFile(logPath);
|
||||
}
|
||||
|
||||
void BinaryCacheStore::addBuildLog(const StorePath & drvPath, std::string_view log)
|
||||
{
|
||||
assert(drvPath.isDerivation());
|
||||
|
||||
upsertFile(
|
||||
"log/" + std::string(drvPath.to_string()),
|
||||
(std::string) log, // FIXME: don't copy
|
||||
"text/plain; charset=utf-8");
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -51,6 +51,7 @@ public:
|
|||
const std::string & mimeType) = 0;
|
||||
|
||||
void upsertFile(const std::string & path,
|
||||
// FIXME: use std::string_view
|
||||
std::string && data,
|
||||
const std::string & mimeType);
|
||||
|
||||
|
@ -120,6 +121,8 @@ public:
|
|||
|
||||
std::optional<std::string> getBuildLog(const StorePath & path) override;
|
||||
|
||||
void addBuildLog(const StorePath & drvPath, std::string_view log) override;
|
||||
|
||||
};
|
||||
|
||||
MakeError(NoSuchBinaryCacheFile, Error);
|
||||
|
|
|
@ -468,10 +468,12 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
dontCheckSigs = false;
|
||||
|
||||
logger->startWork();
|
||||
FramedSource source(from);
|
||||
store->addMultipleToStore(source,
|
||||
RepairFlag{repair},
|
||||
dontCheckSigs ? NoCheckSigs : CheckSigs);
|
||||
{
|
||||
FramedSource source(from);
|
||||
store->addMultipleToStore(source,
|
||||
RepairFlag{repair},
|
||||
dontCheckSigs ? NoCheckSigs : CheckSigs);
|
||||
}
|
||||
logger->stopWork();
|
||||
break;
|
||||
}
|
||||
|
@ -920,6 +922,22 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
break;
|
||||
}
|
||||
|
||||
case wopAddBuildLog: {
|
||||
StorePath path{readString(from)};
|
||||
logger->startWork();
|
||||
if (!trusted)
|
||||
throw Error("you are not privileged to add logs");
|
||||
{
|
||||
FramedSource source(from);
|
||||
StringSink sink;
|
||||
source.drainInto(sink);
|
||||
store->addBuildLog(path, sink.s);
|
||||
}
|
||||
logger->stopWork();
|
||||
to << 1;
|
||||
break;
|
||||
}
|
||||
|
||||
default:
|
||||
throw Error("invalid operation %1%", op);
|
||||
}
|
||||
|
|
|
@ -96,6 +96,7 @@ void LocalBinaryCacheStore::init()
|
|||
createDirs(binaryCacheDir + "/" + realisationsPrefix);
|
||||
if (writeDebugInfo)
|
||||
createDirs(binaryCacheDir + "/debuginfo");
|
||||
createDirs(binaryCacheDir + "/log");
|
||||
BinaryCacheStore::init();
|
||||
}
|
||||
|
||||
|
|
|
@ -9,6 +9,7 @@
|
|||
#include "callback.hh"
|
||||
#include "topo-sort.hh"
|
||||
#include "finally.hh"
|
||||
#include "compression.hh"
|
||||
|
||||
#include <iostream>
|
||||
#include <algorithm>
|
||||
|
@ -1898,4 +1899,24 @@ FixedOutputHash LocalStore::hashCAPath(
|
|||
};
|
||||
}
|
||||
|
||||
void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log)
|
||||
{
|
||||
assert(drvPath.isDerivation());
|
||||
|
||||
auto baseName = drvPath.to_string();
|
||||
|
||||
auto logPath = fmt("%s/%s/%s/%s.bz2", logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2));
|
||||
|
||||
if (pathExists(logPath)) return;
|
||||
|
||||
createDirs(dirOf(logPath));
|
||||
|
||||
auto tmpFile = fmt("%s.tmp.%d", logPath, getpid());
|
||||
|
||||
writeFile(tmpFile, compress("bzip2", log));
|
||||
|
||||
if (rename(tmpFile.c_str(), logPath.c_str()) != 0)
|
||||
throw SysError("renaming '%1%' to '%2%'", tmpFile, logPath);
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
|
|
|
@ -280,6 +280,8 @@ private:
|
|||
const std::string_view pathHash
|
||||
);
|
||||
|
||||
void addBuildLog(const StorePath & drvPath, std::string_view log) override;
|
||||
|
||||
friend struct LocalDerivationGoal;
|
||||
friend struct PathSubstitutionGoal;
|
||||
friend struct SubstitutionGoal;
|
||||
|
|
|
@ -908,6 +908,18 @@ void RemoteStore::queryMissing(const std::vector<DerivedPath> & targets,
|
|||
}
|
||||
|
||||
|
||||
void RemoteStore::addBuildLog(const StorePath & drvPath, std::string_view log)
|
||||
{
|
||||
auto conn(getConnection());
|
||||
conn->to << wopAddBuildLog << drvPath.to_string();
|
||||
StringSource source(log);
|
||||
conn.withFramedSink([&](Sink & sink) {
|
||||
source.drainInto(sink);
|
||||
});
|
||||
readInt(conn->from);
|
||||
}
|
||||
|
||||
|
||||
void RemoteStore::connect()
|
||||
{
|
||||
auto conn(getConnection());
|
||||
|
|
|
@ -116,6 +116,8 @@ public:
|
|||
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
|
||||
uint64_t & downloadSize, uint64_t & narSize) override;
|
||||
|
||||
void addBuildLog(const StorePath & drvPath, std::string_view log) override;
|
||||
|
||||
void connect() override;
|
||||
|
||||
unsigned int getProtocol() override;
|
||||
|
|
|
@ -727,6 +727,9 @@ public:
|
|||
virtual std::optional<std::string> getBuildLog(const StorePath & path)
|
||||
{ return std::nullopt; }
|
||||
|
||||
virtual void addBuildLog(const StorePath & path, std::string_view log)
|
||||
{ unsupported("addBuildLog"); }
|
||||
|
||||
/* Hack to allow long-running processes like hydra-queue-runner to
|
||||
occasionally flush their path info cache. */
|
||||
void clearPathInfoCache()
|
||||
|
|
|
@ -56,6 +56,7 @@ typedef enum {
|
|||
wopRegisterDrvOutput = 42,
|
||||
wopQueryRealisation = 43,
|
||||
wopAddMultipleToStore = 44,
|
||||
wopAddBuildLog = 45,
|
||||
} WorkerOp;
|
||||
|
||||
|
||||
|
|
|
@ -19,7 +19,7 @@ struct InstallableDerivedPath : Installable
|
|||
}
|
||||
|
||||
|
||||
std::string what() override { return derivedPath.to_string(*store); }
|
||||
std::string what() const override { return derivedPath.to_string(*store); }
|
||||
|
||||
DerivedPaths toDerivedPaths() override
|
||||
{
|
||||
|
|
|
@ -1,17 +1,11 @@
|
|||
#include "command.hh"
|
||||
#include "shared.hh"
|
||||
#include "store-api.hh"
|
||||
#include "sync.hh"
|
||||
#include "thread-pool.hh"
|
||||
|
||||
#include <atomic>
|
||||
|
||||
using namespace nix;
|
||||
|
||||
struct CmdCopy : BuiltPathsCommand
|
||||
struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand
|
||||
{
|
||||
std::string srcUri, dstUri;
|
||||
|
||||
CheckSigsFlag checkSigs = CheckSigs;
|
||||
|
||||
SubstituteFlag substitute = NoSubstitute;
|
||||
|
@ -21,20 +15,6 @@ struct CmdCopy : BuiltPathsCommand
|
|||
CmdCopy()
|
||||
: BuiltPathsCommand(true)
|
||||
{
|
||||
addFlag({
|
||||
.longName = "from",
|
||||
.description = "URL of the source Nix store.",
|
||||
.labels = {"store-uri"},
|
||||
.handler = {&srcUri},
|
||||
});
|
||||
|
||||
addFlag({
|
||||
.longName = "to",
|
||||
.description = "URL of the destination Nix store.",
|
||||
.labels = {"store-uri"},
|
||||
.handler = {&dstUri},
|
||||
});
|
||||
|
||||
addFlag({
|
||||
.longName = "no-check-sigs",
|
||||
.description = "Do not require that paths are signed by trusted keys.",
|
||||
|
@ -65,22 +45,9 @@ struct CmdCopy : BuiltPathsCommand
|
|||
|
||||
Category category() override { return catSecondary; }
|
||||
|
||||
ref<Store> createStore() override
|
||||
{
|
||||
return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri);
|
||||
}
|
||||
|
||||
void run(ref<Store> store) override
|
||||
{
|
||||
if (srcUri.empty() && dstUri.empty())
|
||||
throw UsageError("you must pass '--from' and/or '--to'");
|
||||
|
||||
BuiltPathsCommand::run(store);
|
||||
}
|
||||
|
||||
void run(ref<Store> srcStore, BuiltPaths && paths) override
|
||||
{
|
||||
ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri);
|
||||
auto dstStore = getDstStore();
|
||||
|
||||
RealisedPath::Set stuffToCopy;
|
||||
|
||||
|
|
46
src/nix/store-copy-log.cc
Normal file
46
src/nix/store-copy-log.cc
Normal file
|
@ -0,0 +1,46 @@
|
|||
#include "command.hh"
|
||||
#include "shared.hh"
|
||||
#include "store-api.hh"
|
||||
#include "sync.hh"
|
||||
#include "thread-pool.hh"
|
||||
|
||||
#include <atomic>
|
||||
|
||||
using namespace nix;
|
||||
|
||||
struct CmdCopyLog : virtual CopyCommand, virtual InstallablesCommand
|
||||
{
|
||||
std::string description() override
|
||||
{
|
||||
return "copy build logs between Nix stores";
|
||||
}
|
||||
|
||||
std::string doc() override
|
||||
{
|
||||
return
|
||||
#include "store-copy-log.md"
|
||||
;
|
||||
}
|
||||
|
||||
Category category() override { return catUtility; }
|
||||
|
||||
void run(ref<Store> srcStore) override
|
||||
{
|
||||
auto dstStore = getDstStore();
|
||||
|
||||
StorePathSet drvPaths;
|
||||
|
||||
for (auto & i : installables)
|
||||
for (auto & drvPath : i->toDrvPaths(getEvalStore()))
|
||||
drvPaths.insert(drvPath);
|
||||
|
||||
for (auto & drvPath : drvPaths) {
|
||||
if (auto log = srcStore->getBuildLog(drvPath))
|
||||
dstStore->addBuildLog(drvPath, *log);
|
||||
else
|
||||
throw Error("build log for '%s' is not available", srcStore->printStorePath(drvPath));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
static auto rCmdCopyLog = registerCommand2<CmdCopyLog>({"store", "copy-log"});
|
33
src/nix/store-copy-log.md
Normal file
33
src/nix/store-copy-log.md
Normal file
|
@ -0,0 +1,33 @@
|
|||
R""(
|
||||
|
||||
# Examples
|
||||
|
||||
* To copy the build log of the `hello` package from
|
||||
https://cache.nixos.org to the local store:
|
||||
|
||||
```console
|
||||
# nix store copy-log --from https://cache.nixos.org --eval-store auto nixpkgs#hello
|
||||
```
|
||||
|
||||
You can verify that the log is available locally:
|
||||
|
||||
```console
|
||||
# nix log --substituters '' nixpkgs#hello
|
||||
```
|
||||
|
||||
(The flag `--substituters ''` avoids querying
|
||||
`https://cache.nixos.org` for the log.)
|
||||
|
||||
* To copy the log for a specific store derivation via SSH:
|
||||
|
||||
```console
|
||||
# nix store copy-log --to ssh-ng://machine /nix/store/ilgm50plpmcgjhcp33z6n4qbnpqfhxym-glibc-2.33-59.drv
|
||||
```
|
||||
|
||||
# Description
|
||||
|
||||
`nix store copy-log` copies build logs between two Nix stores. The
|
||||
source store is specified using `--from` and the destination using
|
||||
`--to`. If one of these is omitted, it defaults to the local store.
|
||||
|
||||
)""
|
|
@ -14,6 +14,17 @@ outPath=$(nix-build dependencies.nix --no-out-link)
|
|||
|
||||
nix copy --to file://$cacheDir $outPath
|
||||
|
||||
# Test copying build logs to the binary cache.
|
||||
nix log --store file://$cacheDir $outPath 2>&1 | grep 'is not available'
|
||||
nix store copy-log --to file://$cacheDir $outPath
|
||||
nix log --store file://$cacheDir $outPath | grep FOO
|
||||
rm -rf $TEST_ROOT/var/log/nix
|
||||
nix log $outPath 2>&1 | grep 'is not available'
|
||||
nix log --substituters file://$cacheDir $outPath | grep FOO
|
||||
|
||||
# Test copying build logs from the binary cache.
|
||||
nix store copy-log --from file://$cacheDir $(nix-store -qd $outPath)
|
||||
nix log $outPath | grep FOO
|
||||
|
||||
basicDownloadTests() {
|
||||
# No uploading tests bcause upload with force HTTP doesn't work.
|
||||
|
|
Loading…
Reference in a new issue