forked from lix-project/lix
Compare commits
25 commits
fe13a447f0
...
f4dd67e436
Author | SHA1 | Date | |
---|---|---|---|
alois31 | f4dd67e436 | ||
alois31 | 865a2f6448 | ||
kloenk | da4e46dd1f | ||
Qyriad | 21865ccce0 | ||
jade | 375f4c0337 | ||
Qyriad | fd250c51ed | ||
Qyriad | e44dcd63c4 | ||
Qyriad | 6515b1a495 | ||
Qyriad | 50be55ffca | ||
Qyriad | 079eeb1de7 | ||
Qyriad | b9e9235ac0 | ||
Qyriad | 1e5f134560 | ||
Qyriad | 8ba1939540 | ||
Qyriad | f9594b592b | ||
Qyriad | 3a4c21fc9e | ||
Ilya K | 697ef65c14 | ||
Ilya K | 7d52d74bbe | ||
jade | 6c29a2a6fc | ||
jade | 50472aa5be | ||
fb7d315411 | |||
eldritch horrors | c55dcc6c13 | ||
eldritch horrors | 11f4a5bc7e | ||
eldritch horrors | 67f778670c | ||
eldritch horrors | 3425e90d76 | ||
jade | 66a9fbb7ff |
|
@ -87,6 +87,7 @@ Most commands in Lix accept the following command-line options:
|
||||||
|
|
||||||
Displayes the raw logs, with a progress bar and activities each in a new line at the bottom.
|
Displayes the raw logs, with a progress bar and activities each in a new line at the bottom.
|
||||||
|
|
||||||
|
|
||||||
- <span id="opt-no-build-output">[`--no-build-output`](#opt-no-build-output)</span> / `-Q`
|
- <span id="opt-no-build-output">[`--no-build-output`](#opt-no-build-output)</span> / `-Q`
|
||||||
|
|
||||||
By default, output written by builders to standard output and standard error is echoed to the Lix command's standard error.
|
By default, output written by builders to standard output and standard error is echoed to the Lix command's standard error.
|
||||||
|
|
|
@ -1,11 +0,0 @@
|
||||||
#include "command-installable-value.hh"
|
|
||||||
|
|
||||||
namespace nix {
|
|
||||||
|
|
||||||
void InstallableValueCommand::run(ref<Store> store, ref<Installable> installable)
|
|
||||||
{
|
|
||||||
auto installableValue = InstallableValue::require(installable);
|
|
||||||
run(store, installableValue);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
|
@ -1,23 +0,0 @@
|
||||||
#pragma once
|
|
||||||
///@file
|
|
||||||
|
|
||||||
#include "installable-value.hh"
|
|
||||||
#include "command.hh"
|
|
||||||
|
|
||||||
namespace nix {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* An InstallableCommand where the single positional argument must be an
|
|
||||||
* InstallableValue in particular.
|
|
||||||
*/
|
|
||||||
struct InstallableValueCommand : InstallableCommand
|
|
||||||
{
|
|
||||||
/**
|
|
||||||
* Entry point to this command
|
|
||||||
*/
|
|
||||||
virtual void run(ref<Store> store, ref<InstallableValue> installable) = 0;
|
|
||||||
|
|
||||||
void run(ref<Store> store, ref<Installable> installable) override;
|
|
||||||
};
|
|
||||||
|
|
||||||
}
|
|
|
@ -393,13 +393,10 @@ ref<eval_cache::EvalCache> openEvalCache(
|
||||||
EvalState & state,
|
EvalState & state,
|
||||||
std::shared_ptr<flake::LockedFlake> lockedFlake)
|
std::shared_ptr<flake::LockedFlake> lockedFlake)
|
||||||
{
|
{
|
||||||
auto fingerprint = lockedFlake->getFingerprint();
|
auto fingerprint = evalSettings.useEvalCache && evalSettings.pureEval
|
||||||
return make_ref<nix::eval_cache::EvalCache>(
|
? std::make_optional(lockedFlake->getFingerprint())
|
||||||
evalSettings.useEvalCache && evalSettings.pureEval
|
: std::nullopt;
|
||||||
? std::optional { std::cref(fingerprint) }
|
auto rootLoader = [&state, lockedFlake]()
|
||||||
: std::nullopt,
|
|
||||||
state,
|
|
||||||
[&state, lockedFlake]()
|
|
||||||
{
|
{
|
||||||
/* For testing whether the evaluation cache is
|
/* For testing whether the evaluation cache is
|
||||||
complete. */
|
complete. */
|
||||||
|
@ -415,7 +412,17 @@ ref<eval_cache::EvalCache> openEvalCache(
|
||||||
assert(aOutputs);
|
assert(aOutputs);
|
||||||
|
|
||||||
return aOutputs->value;
|
return aOutputs->value;
|
||||||
});
|
};
|
||||||
|
|
||||||
|
if (fingerprint) {
|
||||||
|
auto search = state.evalCaches.find(fingerprint.value());
|
||||||
|
if (search == state.evalCaches.end()) {
|
||||||
|
search = state.evalCaches.emplace(fingerprint.value(), make_ref<nix::eval_cache::EvalCache>(fingerprint, state, rootLoader)).first;
|
||||||
|
}
|
||||||
|
return search->second;
|
||||||
|
} else {
|
||||||
|
return make_ref<nix::eval_cache::EvalCache>(std::nullopt, state, rootLoader);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Installables SourceExprCommand::parseInstallables(
|
Installables SourceExprCommand::parseInstallables(
|
||||||
|
|
|
@ -1,6 +1,5 @@
|
||||||
libcmd_sources = files(
|
libcmd_sources = files(
|
||||||
'built-path.cc',
|
'built-path.cc',
|
||||||
'command-installable-value.cc',
|
|
||||||
'cmd-profiles.cc',
|
'cmd-profiles.cc',
|
||||||
'command.cc',
|
'command.cc',
|
||||||
'common-eval-args.cc',
|
'common-eval-args.cc',
|
||||||
|
@ -18,7 +17,6 @@ libcmd_sources = files(
|
||||||
|
|
||||||
libcmd_headers = files(
|
libcmd_headers = files(
|
||||||
'built-path.hh',
|
'built-path.hh',
|
||||||
'command-installable-value.hh',
|
|
||||||
'cmd-profiles.hh',
|
'cmd-profiles.hh',
|
||||||
'command.hh',
|
'command.hh',
|
||||||
'common-eval-args.hh',
|
'common-eval-args.hh',
|
||||||
|
|
|
@ -33,6 +33,10 @@ class EvalState;
|
||||||
class StorePath;
|
class StorePath;
|
||||||
struct SingleDerivedPath;
|
struct SingleDerivedPath;
|
||||||
enum RepairFlag : bool;
|
enum RepairFlag : bool;
|
||||||
|
struct MemoryInputAccessor;
|
||||||
|
namespace eval_cache {
|
||||||
|
class EvalCache;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -234,6 +238,11 @@ public:
|
||||||
return *new EvalErrorBuilder<T>(*this, args...);
|
return *new EvalErrorBuilder<T>(*this, args...);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A cache for evaluation caches, so as to reuse the same root value if possible
|
||||||
|
*/
|
||||||
|
std::map<const Hash, ref<eval_cache::EvalCache>> evalCaches;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|
||||||
/* Cache for calls to addToStore(); maps source paths to the store
|
/* Cache for calls to addToStore(); maps source paths to the store
|
||||||
|
|
|
@ -36,89 +36,14 @@ static std::string_view storePathToName(std::string_view path)
|
||||||
return i == std::string::npos ? base.substr(0, 0) : base.substr(i + 1);
|
return i == std::string::npos ? base.substr(0, 0) : base.substr(i + 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 100 years ought to be enough for anyone (yet sufficiently smaller than max() to not cause signed integer overflow).
|
|
||||||
constexpr const auto A_LONG_TIME = std::chrono::duration_cast<std::chrono::milliseconds>(100 * 365 * 86400s);
|
|
||||||
|
|
||||||
class ProgressBar : public Logger
|
ProgressBar::~ProgressBar()
|
||||||
{
|
|
||||||
private:
|
|
||||||
|
|
||||||
struct ActInfo
|
|
||||||
{
|
|
||||||
std::string s, lastLine, phase;
|
|
||||||
ActivityType type = actUnknown;
|
|
||||||
uint64_t done = 0;
|
|
||||||
uint64_t expected = 0;
|
|
||||||
uint64_t running = 0;
|
|
||||||
uint64_t failed = 0;
|
|
||||||
std::map<ActivityType, uint64_t> expectedByType;
|
|
||||||
bool visible = true;
|
|
||||||
ActivityId parent;
|
|
||||||
std::optional<std::string> name;
|
|
||||||
std::chrono::time_point<std::chrono::steady_clock> startTime;
|
|
||||||
};
|
|
||||||
|
|
||||||
struct ActivitiesByType
|
|
||||||
{
|
|
||||||
std::map<ActivityId, std::list<ActInfo>::iterator> its;
|
|
||||||
uint64_t done = 0;
|
|
||||||
uint64_t expected = 0;
|
|
||||||
uint64_t failed = 0;
|
|
||||||
};
|
|
||||||
|
|
||||||
struct State
|
|
||||||
{
|
|
||||||
std::list<ActInfo> activities;
|
|
||||||
std::map<ActivityId, std::list<ActInfo>::iterator> its;
|
|
||||||
|
|
||||||
std::map<ActivityType, ActivitiesByType> activitiesByType;
|
|
||||||
|
|
||||||
int lastLines = 0;
|
|
||||||
|
|
||||||
uint64_t filesLinked = 0, bytesLinked = 0;
|
|
||||||
|
|
||||||
uint64_t corruptedPaths = 0, untrustedPaths = 0;
|
|
||||||
|
|
||||||
bool active = true;
|
|
||||||
bool paused = false;
|
|
||||||
bool haveUpdate = true;
|
|
||||||
};
|
|
||||||
|
|
||||||
Sync<State> state_;
|
|
||||||
|
|
||||||
std::thread updateThread;
|
|
||||||
|
|
||||||
std::condition_variable quitCV, updateCV;
|
|
||||||
|
|
||||||
bool printBuildLogs = false;
|
|
||||||
bool printMultiline = false;
|
|
||||||
bool isTTY;
|
|
||||||
|
|
||||||
public:
|
|
||||||
|
|
||||||
ProgressBar(bool isTTY)
|
|
||||||
: isTTY(isTTY)
|
|
||||||
{
|
|
||||||
state_.lock()->active = isTTY;
|
|
||||||
updateThread = std::thread([&]() {
|
|
||||||
auto state(state_.lock());
|
|
||||||
auto nextWakeup = A_LONG_TIME;
|
|
||||||
while (state->active) {
|
|
||||||
if (!state->haveUpdate)
|
|
||||||
state.wait_for(updateCV, nextWakeup);
|
|
||||||
nextWakeup = draw(*state, {});
|
|
||||||
state.wait_for(quitCV, std::chrono::milliseconds(50));
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
~ProgressBar()
|
|
||||||
{
|
{
|
||||||
stop();
|
stop();
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Called by destructor, can't be overridden */
|
/* Called by destructor, can't be overridden */
|
||||||
void stop() override final
|
void ProgressBar::stop()
|
||||||
{
|
{
|
||||||
{
|
{
|
||||||
auto state(state_.lock());
|
auto state(state_.lock());
|
||||||
|
@ -131,31 +56,33 @@ public:
|
||||||
updateThread.join();
|
updateThread.join();
|
||||||
}
|
}
|
||||||
|
|
||||||
void pause() override {
|
void ProgressBar::pause()
|
||||||
|
{
|
||||||
state_.lock()->paused = true;
|
state_.lock()->paused = true;
|
||||||
writeToStderr("\r\e[K");
|
writeToStderr("\r\e[K");
|
||||||
}
|
}
|
||||||
|
|
||||||
void resume() override {
|
void ProgressBar::resume()
|
||||||
|
{
|
||||||
state_.lock()->paused = false;
|
state_.lock()->paused = false;
|
||||||
writeToStderr("\r\e[K");
|
writeToStderr("\r\e[K");
|
||||||
state_.lock()->haveUpdate = true;
|
state_.lock()->haveUpdate = true;
|
||||||
updateCV.notify_one();
|
updateCV.notify_one();
|
||||||
}
|
}
|
||||||
|
|
||||||
bool isVerbose() override
|
bool ProgressBar::isVerbose()
|
||||||
{
|
{
|
||||||
return printBuildLogs;
|
return printBuildLogs;
|
||||||
}
|
}
|
||||||
|
|
||||||
void log(Verbosity lvl, std::string_view s) override
|
void ProgressBar::log(Verbosity lvl, std::string_view s)
|
||||||
{
|
{
|
||||||
if (lvl > verbosity) return;
|
if (lvl > verbosity) return;
|
||||||
auto state(state_.lock());
|
auto state(state_.lock());
|
||||||
log(*state, lvl, s);
|
log(*state, lvl, s);
|
||||||
}
|
}
|
||||||
|
|
||||||
void logEI(const ErrorInfo & ei) override
|
void ProgressBar::logEI(const ErrorInfo & ei)
|
||||||
{
|
{
|
||||||
auto state(state_.lock());
|
auto state(state_.lock());
|
||||||
|
|
||||||
|
@ -165,7 +92,7 @@ public:
|
||||||
log(*state, ei.level, oss.str());
|
log(*state, ei.level, oss.str());
|
||||||
}
|
}
|
||||||
|
|
||||||
void log(State & state, Verbosity lvl, std::string_view s)
|
void ProgressBar::log(State & state, Verbosity lvl, std::string_view s)
|
||||||
{
|
{
|
||||||
if (state.active) {
|
if (state.active) {
|
||||||
draw(state, s);
|
draw(state, s);
|
||||||
|
@ -176,8 +103,14 @@ public:
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void startActivity(ActivityId act, Verbosity lvl, ActivityType type,
|
void ProgressBar::startActivity(
|
||||||
const std::string & s, const Fields & fields, ActivityId parent) override
|
ActivityId act,
|
||||||
|
Verbosity lvl,
|
||||||
|
ActivityType type,
|
||||||
|
const std::string & s,
|
||||||
|
const Fields & fields,
|
||||||
|
ActivityId parent
|
||||||
|
)
|
||||||
{
|
{
|
||||||
auto state(state_.lock());
|
auto state(state_.lock());
|
||||||
|
|
||||||
|
@ -244,7 +177,7 @@ public:
|
||||||
|
|
||||||
/* Check whether an activity has an ancestore with the specified
|
/* Check whether an activity has an ancestore with the specified
|
||||||
type. */
|
type. */
|
||||||
bool hasAncestor(State & state, ActivityType type, ActivityId act)
|
bool ProgressBar::hasAncestor(State & state, ActivityType type, ActivityId act)
|
||||||
{
|
{
|
||||||
while (act != 0) {
|
while (act != 0) {
|
||||||
auto i = state.its.find(act);
|
auto i = state.its.find(act);
|
||||||
|
@ -255,7 +188,7 @@ public:
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
void stopActivity(ActivityId act) override
|
void ProgressBar::stopActivity(ActivityId act)
|
||||||
{
|
{
|
||||||
auto state(state_.lock());
|
auto state(state_.lock());
|
||||||
|
|
||||||
|
@ -277,7 +210,7 @@ public:
|
||||||
update(*state);
|
update(*state);
|
||||||
}
|
}
|
||||||
|
|
||||||
void result(ActivityId act, ResultType type, const std::vector<Field> & fields) override
|
void ProgressBar::result(ActivityId act, ResultType type, const std::vector<Field> & fields)
|
||||||
{
|
{
|
||||||
auto state(state_.lock());
|
auto state(state_.lock());
|
||||||
|
|
||||||
|
@ -354,13 +287,13 @@ public:
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void update(State & state)
|
void ProgressBar::update(State & state)
|
||||||
{
|
{
|
||||||
state.haveUpdate = true;
|
state.haveUpdate = true;
|
||||||
updateCV.notify_one();
|
updateCV.notify_one();
|
||||||
}
|
}
|
||||||
|
|
||||||
std::chrono::milliseconds draw(State & state, const std::optional<std::string_view> & s)
|
std::chrono::milliseconds ProgressBar::draw(State & state, const std::optional<std::string_view> & s)
|
||||||
{
|
{
|
||||||
auto nextWakeup = A_LONG_TIME;
|
auto nextWakeup = A_LONG_TIME;
|
||||||
|
|
||||||
|
@ -396,7 +329,7 @@ public:
|
||||||
}
|
}
|
||||||
|
|
||||||
auto height = windowSize.first > 0 ? windowSize.first : 25;
|
auto height = windowSize.first > 0 ? windowSize.first : 25;
|
||||||
auto moreBuilds = 0;
|
auto moreActivities = 0;
|
||||||
auto now = std::chrono::steady_clock::now();
|
auto now = std::chrono::steady_clock::now();
|
||||||
|
|
||||||
std::string activity_line;
|
std::string activity_line;
|
||||||
|
@ -405,7 +338,6 @@ public:
|
||||||
if (!(i->visible && (!i->s.empty() || !i->lastLine.empty()))) {
|
if (!(i->visible && (!i->s.empty() || !i->lastLine.empty()))) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Don't show activities until some time has
|
/* Don't show activities until some time has
|
||||||
passed, to avoid displaying very short
|
passed, to avoid displaying very short
|
||||||
activities. */
|
activities. */
|
||||||
|
@ -427,25 +359,22 @@ public:
|
||||||
activity_line += ")";
|
activity_line += ")";
|
||||||
}
|
}
|
||||||
if (!i->lastLine.empty()) {
|
if (!i->lastLine.empty()) {
|
||||||
if (!i->s.empty()) {
|
if (!i->s.empty())
|
||||||
activity_line += ": ";
|
activity_line += ": ";
|
||||||
}
|
|
||||||
activity_line += i->lastLine;
|
activity_line += i->lastLine;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (printMultiline) {
|
if (printMultiline) {
|
||||||
if (state.lastLines < (height -1)) {
|
if (state.lastLines < (height -1)) {
|
||||||
writeToStderr(filterANSIEscapes(activity_line, false, width) + "\n");
|
writeToStderr(filterANSIEscapes(activity_line, false, width) + "\n");
|
||||||
state.lastLines++;
|
state.lastLines++;
|
||||||
} else {
|
} else moreActivities++;
|
||||||
moreBuilds++;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (printMultiline && moreBuilds) {
|
if (printMultiline && moreActivities)
|
||||||
writeToStderr(fmt("And %d more...", moreBuilds));
|
writeToStderr(fmt("And %d more...", moreActivities));
|
||||||
}
|
|
||||||
|
|
||||||
if (!printMultiline) {
|
if (!printMultiline) {
|
||||||
line += " " + activity_line;
|
line += " " + activity_line;
|
||||||
|
@ -455,20 +384,20 @@ public:
|
||||||
return nextWakeup;
|
return nextWakeup;
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string getStatus(State & state)
|
std::string ProgressBar::getStatus(State & state)
|
||||||
{
|
{
|
||||||
auto MiB = 1024.0 * 1024.0;
|
constexpr auto MiB = 1024.0 * 1024.0;
|
||||||
|
|
||||||
std::string res;
|
std::string res;
|
||||||
|
|
||||||
auto renderActivity = [&](ActivityType type, const std::string & itemFmt, const std::string & numberFmt = "%d", double unit = 1) {
|
auto renderActivity = [&](ActivityType type, const std::string & itemFmt, const std::string & numberFmt = "%d", double unit = 1) {
|
||||||
auto & act = state.activitiesByType[type];
|
auto & act = state.activitiesByType[type];
|
||||||
uint64_t done = act.done, expected = act.done, running = 0, failed = act.failed;
|
uint64_t done = act.done, expected = act.done, running = 0, failed = act.failed;
|
||||||
for (auto & j : act.its) {
|
for (auto & [actId, infoIt] : act.its) {
|
||||||
done += j.second->done;
|
done += infoIt->done;
|
||||||
expected += j.second->expected;
|
expected += infoIt->expected;
|
||||||
running += j.second->running;
|
running += infoIt->running;
|
||||||
failed += j.second->failed;
|
failed += infoIt->failed;
|
||||||
}
|
}
|
||||||
|
|
||||||
expected = std::max(expected, act.expected);
|
expected = std::max(expected, act.expected);
|
||||||
|
@ -545,7 +474,7 @@ public:
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
void writeToStdout(std::string_view s) override
|
void ProgressBar::writeToStdout(std::string_view s)
|
||||||
{
|
{
|
||||||
auto state(state_.lock());
|
auto state(state_.lock());
|
||||||
if (state->active) {
|
if (state->active) {
|
||||||
|
@ -556,7 +485,7 @@ public:
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
std::optional<char> ask(std::string_view msg) override
|
std::optional<char> ProgressBar::ask(std::string_view msg)
|
||||||
{
|
{
|
||||||
auto state(state_.lock());
|
auto state(state_.lock());
|
||||||
if (!state->active || !isatty(STDIN_FILENO)) return {};
|
if (!state->active || !isatty(STDIN_FILENO)) return {};
|
||||||
|
@ -567,16 +496,15 @@ public:
|
||||||
return s[0];
|
return s[0];
|
||||||
}
|
}
|
||||||
|
|
||||||
void setPrintBuildLogs(bool printBuildLogs) override
|
void ProgressBar::setPrintBuildLogs(bool printBuildLogs)
|
||||||
{
|
{
|
||||||
this->printBuildLogs = printBuildLogs;
|
this->printBuildLogs = printBuildLogs;
|
||||||
}
|
}
|
||||||
|
|
||||||
void setPrintMultiline(bool printMultiline) override
|
void ProgressBar::setPrintMultiline(bool printMultiline)
|
||||||
{
|
{
|
||||||
this->printMultiline = printMultiline;
|
this->printMultiline = printMultiline;
|
||||||
}
|
}
|
||||||
};
|
|
||||||
|
|
||||||
Logger * makeProgressBar()
|
Logger * makeProgressBar()
|
||||||
{
|
{
|
||||||
|
|
|
@ -1,10 +1,135 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
///@file
|
///@file
|
||||||
|
|
||||||
|
#include <chrono>
|
||||||
|
|
||||||
#include "logging.hh"
|
#include "logging.hh"
|
||||||
|
#include "sync.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
// 100 years ought to be enough for anyone (yet sufficiently smaller than max() to not cause signed integer overflow).
|
||||||
|
constexpr const auto A_LONG_TIME = std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||||
|
100 * 365 * std::chrono::seconds(86400)
|
||||||
|
);
|
||||||
|
|
||||||
|
struct ProgressBar : public Logger
|
||||||
|
{
|
||||||
|
struct ActInfo
|
||||||
|
{
|
||||||
|
using TimePoint = std::chrono::time_point<std::chrono::steady_clock>;
|
||||||
|
|
||||||
|
std::string s, lastLine, phase;
|
||||||
|
ActivityType type = actUnknown;
|
||||||
|
uint64_t done = 0;
|
||||||
|
uint64_t expected = 0;
|
||||||
|
uint64_t running = 0;
|
||||||
|
uint64_t failed = 0;
|
||||||
|
std::map<ActivityType, uint64_t> expectedByType;
|
||||||
|
bool visible = true;
|
||||||
|
ActivityId parent;
|
||||||
|
std::optional<std::string> name;
|
||||||
|
TimePoint startTime;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct ActivitiesByType
|
||||||
|
{
|
||||||
|
std::map<ActivityId, std::list<ActInfo>::iterator> its;
|
||||||
|
uint64_t done = 0;
|
||||||
|
uint64_t expected = 0;
|
||||||
|
uint64_t failed = 0;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct State
|
||||||
|
{
|
||||||
|
std::list<ActInfo> activities;
|
||||||
|
std::map<ActivityId, std::list<ActInfo>::iterator> its;
|
||||||
|
|
||||||
|
std::map<ActivityType, ActivitiesByType> activitiesByType;
|
||||||
|
|
||||||
|
int lastLines = 0;
|
||||||
|
|
||||||
|
uint64_t filesLinked = 0, bytesLinked = 0;
|
||||||
|
|
||||||
|
uint64_t corruptedPaths = 0, untrustedPaths = 0;
|
||||||
|
|
||||||
|
bool active = true;
|
||||||
|
bool paused = false;
|
||||||
|
bool haveUpdate = true;
|
||||||
|
};
|
||||||
|
|
||||||
|
Sync<State> state_;
|
||||||
|
|
||||||
|
std::thread updateThread;
|
||||||
|
|
||||||
|
std::condition_variable quitCV, updateCV;
|
||||||
|
|
||||||
|
bool printBuildLogs = false;
|
||||||
|
bool printMultiline = false;
|
||||||
|
bool isTTY;
|
||||||
|
|
||||||
|
ProgressBar(bool isTTY)
|
||||||
|
: isTTY(isTTY)
|
||||||
|
{
|
||||||
|
state_.lock()->active = isTTY;
|
||||||
|
updateThread = std::thread([&]() {
|
||||||
|
auto state(state_.lock());
|
||||||
|
auto nextWakeup = A_LONG_TIME;
|
||||||
|
while (state->active) {
|
||||||
|
if (!state->haveUpdate)
|
||||||
|
state.wait_for(updateCV, nextWakeup);
|
||||||
|
nextWakeup = draw(*state, {});
|
||||||
|
state.wait_for(quitCV, std::chrono::milliseconds(50));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
~ProgressBar();
|
||||||
|
|
||||||
|
void stop() override final;
|
||||||
|
|
||||||
|
void pause() override;
|
||||||
|
|
||||||
|
void resume() override;
|
||||||
|
|
||||||
|
bool isVerbose() override;
|
||||||
|
|
||||||
|
void log(Verbosity lvl, std::string_view s) override;
|
||||||
|
|
||||||
|
void logEI(const ErrorInfo & ei) override;
|
||||||
|
|
||||||
|
void log(State & state, Verbosity lvl, std::string_view s);
|
||||||
|
|
||||||
|
void startActivity(
|
||||||
|
ActivityId act,
|
||||||
|
Verbosity lvl,
|
||||||
|
ActivityType type,
|
||||||
|
const std::string & s,
|
||||||
|
const Fields & fields,
|
||||||
|
ActivityId parent
|
||||||
|
) override;
|
||||||
|
|
||||||
|
bool hasAncestor(State & state, ActivityType type, ActivityId act);
|
||||||
|
|
||||||
|
void stopActivity(ActivityId act) override;
|
||||||
|
|
||||||
|
void result(ActivityId act, ResultType type, const std::vector<Field> & fields) override;
|
||||||
|
|
||||||
|
void update(State & state);
|
||||||
|
|
||||||
|
std::chrono::milliseconds draw(State & state, const std::optional<std::string_view> & s);
|
||||||
|
|
||||||
|
std::string getStatus(State & state);
|
||||||
|
|
||||||
|
void writeToStdout(std::string_view s) override;
|
||||||
|
|
||||||
|
std::optional<char> ask(std::string_view msg) override;
|
||||||
|
|
||||||
|
void setPrintBuildLogs(bool printBuildLogs) override;
|
||||||
|
|
||||||
|
void setPrintMultiline(bool printMultiline) override;
|
||||||
|
};
|
||||||
|
|
||||||
Logger * makeProgressBar();
|
Logger * makeProgressBar();
|
||||||
|
|
||||||
void startProgressBar();
|
void startProgressBar();
|
||||||
|
|
|
@ -38,7 +38,7 @@ void BinaryCacheStore::init()
|
||||||
{
|
{
|
||||||
std::string cacheInfoFile = "nix-cache-info";
|
std::string cacheInfoFile = "nix-cache-info";
|
||||||
|
|
||||||
auto cacheInfo = getFile(cacheInfoFile);
|
auto cacheInfo = getFileContents(cacheInfoFile);
|
||||||
if (!cacheInfo) {
|
if (!cacheInfo) {
|
||||||
upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info");
|
upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info");
|
||||||
} else {
|
} else {
|
||||||
|
@ -69,10 +69,10 @@ void BinaryCacheStore::upsertFile(const std::string & path,
|
||||||
|
|
||||||
void BinaryCacheStore::getFile(const std::string & path, Sink & sink)
|
void BinaryCacheStore::getFile(const std::string & path, Sink & sink)
|
||||||
{
|
{
|
||||||
sink(*getFile(path));
|
sink(*getFileContents(path));
|
||||||
}
|
}
|
||||||
|
|
||||||
std::optional<std::string> BinaryCacheStore::getFile(const std::string & path)
|
std::optional<std::string> BinaryCacheStore::getFileContents(const std::string & path)
|
||||||
{
|
{
|
||||||
StringSink sink;
|
StringSink sink;
|
||||||
try {
|
try {
|
||||||
|
@ -359,7 +359,7 @@ std::shared_ptr<const ValidPathInfo> BinaryCacheStore::queryPathInfoUncached(con
|
||||||
|
|
||||||
auto narInfoFile = narInfoFileFor(storePath);
|
auto narInfoFile = narInfoFileFor(storePath);
|
||||||
|
|
||||||
auto data = getFile(narInfoFile);
|
auto data = getFileContents(narInfoFile);
|
||||||
|
|
||||||
if (!data) return nullptr;
|
if (!data) return nullptr;
|
||||||
|
|
||||||
|
@ -385,7 +385,7 @@ StorePath BinaryCacheStore::addToStore(
|
||||||
if (method == FileIngestionMethod::Recursive) {
|
if (method == FileIngestionMethod::Recursive) {
|
||||||
dumpPath(srcPath, sink, filter);
|
dumpPath(srcPath, sink, filter);
|
||||||
} else {
|
} else {
|
||||||
readFile(srcPath, sink);
|
readFileSource(srcPath)->drainInto(sink);
|
||||||
}
|
}
|
||||||
auto h = sink.finish().first;
|
auto h = sink.finish().first;
|
||||||
|
|
||||||
|
@ -446,7 +446,7 @@ std::shared_ptr<const Realisation> BinaryCacheStore::queryRealisationUncached(co
|
||||||
{
|
{
|
||||||
auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi";
|
auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi";
|
||||||
|
|
||||||
auto data = getFile(outputInfoFilePath);
|
auto data = getFileContents(outputInfoFilePath);
|
||||||
if (!data) return {};
|
if (!data) return {};
|
||||||
|
|
||||||
auto realisation = Realisation::fromJSON(
|
auto realisation = Realisation::fromJSON(
|
||||||
|
@ -486,7 +486,7 @@ std::optional<std::string> BinaryCacheStore::getBuildLogExact(const StorePath &
|
||||||
|
|
||||||
debug("fetching build log from binary cache '%s/%s'", getUri(), logPath);
|
debug("fetching build log from binary cache '%s/%s'", getUri(), logPath);
|
||||||
|
|
||||||
return getFile(logPath);
|
return getFileContents(logPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
void BinaryCacheStore::addBuildLog(const StorePath & drvPath, std::string_view log)
|
void BinaryCacheStore::addBuildLog(const StorePath & drvPath, std::string_view log)
|
||||||
|
|
|
@ -85,7 +85,7 @@ public:
|
||||||
*/
|
*/
|
||||||
virtual void getFile(const std::string & path, Sink & sink);
|
virtual void getFile(const std::string & path, Sink & sink);
|
||||||
|
|
||||||
virtual std::optional<std::string> getFile(const std::string & path);
|
virtual std::optional<std::string> getFileContents(const std::string & path);
|
||||||
|
|
||||||
public:
|
public:
|
||||||
|
|
||||||
|
|
|
@ -2981,7 +2981,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
|
||||||
HashModuloSink caSink { outputHash.hashType, oldHashPart };
|
HashModuloSink caSink { outputHash.hashType, oldHashPart };
|
||||||
std::visit(overloaded {
|
std::visit(overloaded {
|
||||||
[&](const TextIngestionMethod &) {
|
[&](const TextIngestionMethod &) {
|
||||||
readFile(actualPath, caSink);
|
readFileSource(actualPath)->drainInto(caSink);
|
||||||
},
|
},
|
||||||
[&](const FileIngestionMethod & m2) {
|
[&](const FileIngestionMethod & m2) {
|
||||||
switch (m2) {
|
switch (m2) {
|
||||||
|
@ -2989,7 +2989,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
|
||||||
dumpPath(actualPath, caSink);
|
dumpPath(actualPath, caSink);
|
||||||
break;
|
break;
|
||||||
case FileIngestionMethod::Flat:
|
case FileIngestionMethod::Flat:
|
||||||
readFile(actualPath, caSink);
|
readFileSource(actualPath)->drainInto(caSink);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
|
@ -41,7 +41,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
|
||||||
|
|
||||||
auto decompressor = makeDecompressionSink(
|
auto decompressor = makeDecompressionSink(
|
||||||
unpack && mainUrl.ends_with(".xz") ? "xz" : "none", sink);
|
unpack && mainUrl.ends_with(".xz") ? "xz" : "none", sink);
|
||||||
fileTransfer->download(std::move(request), *decompressor);
|
fileTransfer->download(std::move(request))->drainInto(*decompressor);
|
||||||
decompressor->finish();
|
decompressor->finish();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
|
@ -296,17 +296,6 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
case WorkerProto::Op::HasSubstitutes: {
|
|
||||||
auto path = store->parseStorePath(readString(from));
|
|
||||||
logger->startWork();
|
|
||||||
StorePathSet paths; // FIXME
|
|
||||||
paths.insert(path);
|
|
||||||
auto res = store->querySubstitutablePaths(paths);
|
|
||||||
logger->stopWork();
|
|
||||||
to << (res.count(path) != 0);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
case WorkerProto::Op::QuerySubstitutablePaths: {
|
case WorkerProto::Op::QuerySubstitutablePaths: {
|
||||||
auto paths = WorkerProto::Serialise<StorePathSet>::read(*store, rconn);
|
auto paths = WorkerProto::Serialise<StorePathSet>::read(*store, rconn);
|
||||||
logger->startWork();
|
logger->startWork();
|
||||||
|
@ -316,36 +305,74 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
case WorkerProto::Op::QueryPathHash: {
|
case WorkerProto::Op::HasSubstitutes: {
|
||||||
auto path = store->parseStorePath(readString(from));
|
throw UnimplementedError("HasSubstitutes is not supported in Lix. This is not used if the declared server protocol is > 1.12 (Nix 1.0, 2012)");
|
||||||
logger->startWork();
|
break;
|
||||||
auto hash = store->queryPathInfo(path)->narHash;
|
}
|
||||||
logger->stopWork();
|
|
||||||
to << hash.to_string(Base16, false);
|
case WorkerProto::Op::QueryPathHash: {
|
||||||
|
throw UnimplementedError("QueryPathHash is not supported in Lix, client usages were removed in 2016 in e0204f8d462041387651af388074491fd0bf36d6");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
case WorkerProto::Op::QueryReferences: {
|
||||||
|
throw UnimplementedError("QueryReferences is not supported in Lix, client usages were removed in 2016 in e0204f8d462041387651af388074491fd0bf36d6");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
case WorkerProto::Op::QueryDeriver: {
|
||||||
|
throw UnimplementedError("QueryDeriver is not supported in Lix, client usages were removed in 2016 in e0204f8d462041387651af388074491fd0bf36d6");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
case WorkerProto::Op::ExportPath: {
|
||||||
|
throw UnimplementedError("ExportPath is not supported in Lix, client usage were removed in 2017 in 27dc76c1a5dbe654465245ff5f6bc22e2c8902da");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
case WorkerProto::Op::ImportPaths: {
|
||||||
|
throw UnimplementedError("ImportPaths is not supported in Lix. This is not used if the declared server protocol is >= 1.18 (Nix 2.0, 2016)");
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
case WorkerProto::Op::QueryReferences:
|
|
||||||
case WorkerProto::Op::QueryReferrers:
|
case WorkerProto::Op::QueryReferrers:
|
||||||
case WorkerProto::Op::QueryValidDerivers:
|
case WorkerProto::Op::QueryValidDerivers:
|
||||||
case WorkerProto::Op::QueryDerivationOutputs: {
|
case WorkerProto::Op::QueryDerivationOutputs: {
|
||||||
auto path = store->parseStorePath(readString(from));
|
auto path = store->parseStorePath(readString(from));
|
||||||
logger->startWork();
|
logger->startWork();
|
||||||
StorePathSet paths;
|
StorePathSet paths;
|
||||||
if (op == WorkerProto::Op::QueryReferences)
|
|
||||||
for (auto & i : store->queryPathInfo(path)->references)
|
#pragma GCC diagnostic push
|
||||||
paths.insert(i);
|
#pragma GCC diagnostic ignored "-Wswitch-enum"
|
||||||
else if (op == WorkerProto::Op::QueryReferrers)
|
switch (op) {
|
||||||
|
case WorkerProto::Op::QueryReferrers: {
|
||||||
store->queryReferrers(path, paths);
|
store->queryReferrers(path, paths);
|
||||||
else if (op == WorkerProto::Op::QueryValidDerivers)
|
break;
|
||||||
|
}
|
||||||
|
case WorkerProto::Op::QueryValidDerivers: {
|
||||||
paths = store->queryValidDerivers(path);
|
paths = store->queryValidDerivers(path);
|
||||||
else paths = store->queryDerivationOutputs(path);
|
break;
|
||||||
|
}
|
||||||
|
case WorkerProto::Op::QueryDerivationOutputs: {
|
||||||
|
// Only sent if server presents proto version <= 1.21
|
||||||
|
REMOVE_AFTER_DROPPING_PROTO_MINOR(21);
|
||||||
|
paths = store->queryDerivationOutputs(path);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
abort();
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
#pragma GCC diagnostic pop
|
||||||
|
|
||||||
logger->stopWork();
|
logger->stopWork();
|
||||||
WorkerProto::write(*store, wconn, paths);
|
WorkerProto::write(*store, wconn, paths);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
case WorkerProto::Op::QueryDerivationOutputNames: {
|
case WorkerProto::Op::QueryDerivationOutputNames: {
|
||||||
|
// Unused in CppNix >= 2.4 (removed in 045b07200c77bf1fe19c0a986aafb531e7e1ba54)
|
||||||
|
REMOVE_AFTER_DROPPING_PROTO_MINOR(31);
|
||||||
auto path = store->parseStorePath(readString(from));
|
auto path = store->parseStorePath(readString(from));
|
||||||
logger->startWork();
|
logger->startWork();
|
||||||
auto names = store->readDerivation(path).outputNames();
|
auto names = store->readDerivation(path).outputNames();
|
||||||
|
@ -363,15 +390,6 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
case WorkerProto::Op::QueryDeriver: {
|
|
||||||
auto path = store->parseStorePath(readString(from));
|
|
||||||
logger->startWork();
|
|
||||||
auto info = store->queryPathInfo(path);
|
|
||||||
logger->stopWork();
|
|
||||||
to << (info->deriver ? store->printStorePath(*info->deriver) : "");
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
case WorkerProto::Op::QueryPathFromHashPart: {
|
case WorkerProto::Op::QueryPathFromHashPart: {
|
||||||
auto hashPart = readString(from);
|
auto hashPart = readString(from);
|
||||||
logger->startWork();
|
logger->startWork();
|
||||||
|
@ -493,29 +511,6 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
case WorkerProto::Op::ExportPath: {
|
|
||||||
auto path = store->parseStorePath(readString(from));
|
|
||||||
readInt(from); // obsolete
|
|
||||||
logger->startWork();
|
|
||||||
TunnelSink sink(to);
|
|
||||||
store->exportPath(path, sink);
|
|
||||||
logger->stopWork();
|
|
||||||
to << 1;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
case WorkerProto::Op::ImportPaths: {
|
|
||||||
logger->startWork();
|
|
||||||
TunnelSource source(from, to);
|
|
||||||
auto paths = store->importPaths(source,
|
|
||||||
trusted ? NoCheckSigs : CheckSigs);
|
|
||||||
logger->stopWork();
|
|
||||||
Strings paths2;
|
|
||||||
for (auto & i : paths) paths2.push_back(store->printStorePath(i));
|
|
||||||
to << paths2;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
case WorkerProto::Op::BuildPaths: {
|
case WorkerProto::Op::BuildPaths: {
|
||||||
auto drvs = WorkerProto::Serialise<DerivedPaths>::read(*store, rconn);
|
auto drvs = WorkerProto::Serialise<DerivedPaths>::read(*store, rconn);
|
||||||
BuildMode mode = buildModeFromInteger(readInt(from));
|
BuildMode mode = buildModeFromInteger(readInt(from));
|
||||||
|
@ -666,8 +661,10 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Obsolete.
|
// Obsolete since 9947f1646a26b339fff2e02b77798e9841fac7f0 (included in CppNix 2.5.0).
|
||||||
case WorkerProto::Op::SyncWithGC: {
|
case WorkerProto::Op::SyncWithGC: {
|
||||||
|
// CppNix 2.5.0 is 32
|
||||||
|
REMOVE_AFTER_DROPPING_PROTO_MINOR(31);
|
||||||
logger->startWork();
|
logger->startWork();
|
||||||
logger->stopWork();
|
logger->stopWork();
|
||||||
to << 1;
|
to << 1;
|
||||||
|
|
|
@ -686,16 +686,8 @@ struct curlFileTransfer : public FileTransfer
|
||||||
->callback.get_future();
|
->callback.get_future();
|
||||||
}
|
}
|
||||||
|
|
||||||
void download(FileTransferRequest && request, Sink & sink) override
|
box_ptr<Source> download(FileTransferRequest && request) override
|
||||||
{
|
{
|
||||||
/* Note: we can't call 'sink' via request.dataCallback, because
|
|
||||||
that would cause the sink to execute on the fileTransfer
|
|
||||||
thread. If 'sink' is a coroutine, this will fail. Also, if the
|
|
||||||
sink is expensive (e.g. one that does decompression and writing
|
|
||||||
to the Nix store), it would stall the download thread too much.
|
|
||||||
Therefore we use a buffer to communicate data between the
|
|
||||||
download thread and the calling thread. */
|
|
||||||
|
|
||||||
struct State {
|
struct State {
|
||||||
bool done = false, failed = false;
|
bool done = false, failed = false;
|
||||||
std::exception_ptr exc;
|
std::exception_ptr exc;
|
||||||
|
@ -705,13 +697,6 @@ struct curlFileTransfer : public FileTransfer
|
||||||
|
|
||||||
auto _state = std::make_shared<Sync<State>>();
|
auto _state = std::make_shared<Sync<State>>();
|
||||||
|
|
||||||
/* In case of an exception, wake up the download thread. */
|
|
||||||
Finally finally([&]() {
|
|
||||||
auto state(_state->lock());
|
|
||||||
state->failed |= std::uncaught_exceptions() != 0;
|
|
||||||
state->request.notify_one();
|
|
||||||
});
|
|
||||||
|
|
||||||
enqueueFileTransfer(
|
enqueueFileTransfer(
|
||||||
request,
|
request,
|
||||||
[_state](std::exception_ptr ex) {
|
[_state](std::exception_ptr ex) {
|
||||||
|
@ -750,50 +735,99 @@ struct curlFileTransfer : public FileTransfer
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
||||||
std::unique_ptr<FinishSink> decompressor;
|
struct InnerSource : Source
|
||||||
|
{
|
||||||
while (true) {
|
const std::shared_ptr<Sync<State>> _state;
|
||||||
checkInterrupt();
|
|
||||||
|
|
||||||
std::string chunk;
|
std::string chunk;
|
||||||
|
std::string_view buffered;
|
||||||
|
|
||||||
|
explicit InnerSource(const std::shared_ptr<Sync<State>> & state) : _state(state) {}
|
||||||
|
|
||||||
|
~InnerSource()
|
||||||
|
{
|
||||||
|
// wake up the download thread if it's still going and have it abort
|
||||||
|
auto state(_state->lock());
|
||||||
|
state->failed |= !state->done;
|
||||||
|
state->request.notify_one();
|
||||||
|
}
|
||||||
|
|
||||||
|
void awaitData(Sync<State>::Lock & state)
|
||||||
|
{
|
||||||
/* Grab data if available, otherwise wait for the download
|
/* Grab data if available, otherwise wait for the download
|
||||||
thread to wake us up. */
|
thread to wake us up. */
|
||||||
{
|
while (buffered.empty()) {
|
||||||
auto state(_state->lock());
|
|
||||||
|
|
||||||
if (state->data.empty()) {
|
if (state->data.empty()) {
|
||||||
|
|
||||||
if (state->done) {
|
if (state->done) {
|
||||||
if (state->exc) std::rethrow_exception(state->exc);
|
if (state->exc) {
|
||||||
if (decompressor) {
|
std::rethrow_exception(state->exc);
|
||||||
decompressor->finish();
|
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
state.wait(state->avail);
|
state.wait(state->avail);
|
||||||
|
|
||||||
if (state->data.empty()) continue;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
chunk = std::move(state->data);
|
chunk = std::move(state->data);
|
||||||
/* Reset state->data after the move, since we check data.empty() */
|
buffered = chunk;
|
||||||
state->data = "";
|
|
||||||
|
|
||||||
if (!decompressor) {
|
|
||||||
decompressor = makeDecompressionSink(state->encoding, sink);
|
|
||||||
}
|
|
||||||
|
|
||||||
state->request.notify_one();
|
state->request.notify_one();
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Flush the data to the sink and wake up the download thread
|
|
||||||
if it's blocked on a full buffer. We don't hold the state
|
|
||||||
lock while doing this to prevent blocking the download
|
|
||||||
thread if sink() takes a long time. */
|
|
||||||
(*decompressor)(chunk);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t read(char * data, size_t len) override
|
||||||
|
{
|
||||||
|
auto readPartial = [this](char * data, size_t len) {
|
||||||
|
const auto available = std::min(len, buffered.size());
|
||||||
|
memcpy(data, buffered.data(), available);
|
||||||
|
buffered.remove_prefix(available);
|
||||||
|
return available;
|
||||||
|
};
|
||||||
|
size_t total = readPartial(data, len);
|
||||||
|
|
||||||
|
while (total < len) {
|
||||||
|
{
|
||||||
|
auto state(_state->lock());
|
||||||
|
awaitData(state);
|
||||||
|
}
|
||||||
|
const auto current = readPartial(data + total, len - total);
|
||||||
|
total += current;
|
||||||
|
if (total == 0 || current == 0) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (total == 0) {
|
||||||
|
throw EndOfFile("download finished");
|
||||||
|
}
|
||||||
|
|
||||||
|
return total;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
struct DownloadSource : Source
|
||||||
|
{
|
||||||
|
InnerSource inner;
|
||||||
|
std::unique_ptr<Source> decompressor;
|
||||||
|
|
||||||
|
explicit DownloadSource(const std::shared_ptr<Sync<State>> & state) : inner(state) {}
|
||||||
|
|
||||||
|
size_t read(char * data, size_t len) override
|
||||||
|
{
|
||||||
|
checkInterrupt();
|
||||||
|
|
||||||
|
if (!decompressor) {
|
||||||
|
auto state(inner._state->lock());
|
||||||
|
inner.awaitData(state);
|
||||||
|
decompressor = makeDecompressionSource(state->encoding, inner);
|
||||||
|
}
|
||||||
|
|
||||||
|
return decompressor->read(data, len);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
auto source = make_box_ptr<DownloadSource>(_state);
|
||||||
|
auto lock(_state->lock());
|
||||||
|
source->inner.awaitData(lock);
|
||||||
|
return source;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
///@file
|
///@file
|
||||||
|
|
||||||
|
#include "box_ptr.hh"
|
||||||
#include "logging.hh"
|
#include "logging.hh"
|
||||||
#include "serialise.hh"
|
#include "serialise.hh"
|
||||||
#include "types.hh"
|
#include "types.hh"
|
||||||
|
@ -104,10 +105,13 @@ struct FileTransfer
|
||||||
FileTransferResult transfer(const FileTransferRequest & request);
|
FileTransferResult transfer(const FileTransferRequest & request);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Download a file, writing its data to a sink. The sink will be
|
* Download a file, returning its contents through a source. Will not return
|
||||||
* invoked on the thread of the caller.
|
* before the transfer has fully started, ensuring that any errors thrown by
|
||||||
|
* the setup phase (e.g. HTTP 404 or similar errors) are not postponed to be
|
||||||
|
* thrown by the returned source. The source will only throw errors detected
|
||||||
|
* during the transfer itself (decompression errors, connection drops, etc).
|
||||||
*/
|
*/
|
||||||
virtual void download(FileTransferRequest && request, Sink & sink) = 0;
|
virtual box_ptr<Source> download(FileTransferRequest && request) = 0;
|
||||||
|
|
||||||
enum Error { NotFound, Forbidden, Misc, Transient, Interrupted };
|
enum Error { NotFound, Forbidden, Misc, Transient, Interrupted };
|
||||||
};
|
};
|
||||||
|
|
|
@ -155,7 +155,7 @@ protected:
|
||||||
checkEnabled();
|
checkEnabled();
|
||||||
auto request(makeRequest(path));
|
auto request(makeRequest(path));
|
||||||
try {
|
try {
|
||||||
getFileTransfer()->download(std::move(request), sink);
|
getFileTransfer()->download(std::move(request))->drainInto(sink);
|
||||||
} catch (FileTransferError & e) {
|
} catch (FileTransferError & e) {
|
||||||
if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
|
if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
|
||||||
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
|
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
|
||||||
|
@ -164,7 +164,7 @@ protected:
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
std::optional<std::string> getFile(const std::string & path) override
|
std::optional<std::string> getFileContents(const std::string & path) override
|
||||||
{
|
{
|
||||||
checkEnabled();
|
checkEnabled();
|
||||||
|
|
||||||
|
|
|
@ -71,7 +71,7 @@ protected:
|
||||||
void getFile(const std::string & path, Sink & sink) override
|
void getFile(const std::string & path, Sink & sink) override
|
||||||
{
|
{
|
||||||
try {
|
try {
|
||||||
readFile(binaryCacheDir + "/" + path, sink);
|
readFileSource(binaryCacheDir + "/" + path)->drainInto(sink);
|
||||||
} catch (SysError & e) {
|
} catch (SysError & e) {
|
||||||
if (e.errNo == ENOENT)
|
if (e.errNo == ENOENT)
|
||||||
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache", path);
|
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache", path);
|
||||||
|
|
|
@ -1890,7 +1890,7 @@ ContentAddress LocalStore::hashCAPath(
|
||||||
HashModuloSink caSink ( hashType, std::string(pathHash) );
|
HashModuloSink caSink ( hashType, std::string(pathHash) );
|
||||||
std::visit(overloaded {
|
std::visit(overloaded {
|
||||||
[&](const TextIngestionMethod &) {
|
[&](const TextIngestionMethod &) {
|
||||||
readFile(path, caSink);
|
readFileSource(path)->drainInto(caSink);
|
||||||
},
|
},
|
||||||
[&](const FileIngestionMethod & m2) {
|
[&](const FileIngestionMethod & m2) {
|
||||||
switch (m2) {
|
switch (m2) {
|
||||||
|
@ -1898,7 +1898,7 @@ ContentAddress LocalStore::hashCAPath(
|
||||||
dumpPath(path, caSink);
|
dumpPath(path, caSink);
|
||||||
break;
|
break;
|
||||||
case FileIngestionMethod::Flat:
|
case FileIngestionMethod::Flat:
|
||||||
readFile(path, caSink);
|
readFileSource(path)->drainInto(caSink);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
|
@ -307,6 +307,7 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
|
||||||
if (GET_PROTOCOL_MINOR(getProtocol()) >= 22) {
|
if (GET_PROTOCOL_MINOR(getProtocol()) >= 22) {
|
||||||
return Store::queryDerivationOutputs(path);
|
return Store::queryDerivationOutputs(path);
|
||||||
}
|
}
|
||||||
|
REMOVE_AFTER_DROPPING_PROTO_MINOR(21);
|
||||||
auto conn(getConnection());
|
auto conn(getConnection());
|
||||||
conn->to << WorkerProto::Op::QueryDerivationOutputs << printStorePath(path);
|
conn->to << WorkerProto::Op::QueryDerivationOutputs << printStorePath(path);
|
||||||
conn.processStderr();
|
conn.processStderr();
|
||||||
|
@ -336,6 +337,7 @@ std::map<std::string, std::optional<StorePath>> RemoteStore::queryPartialDerivat
|
||||||
return outputs;
|
return outputs;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
REMOVE_AFTER_DROPPING_PROTO_MINOR(21);
|
||||||
auto & evalStore = evalStore_ ? *evalStore_ : *this;
|
auto & evalStore = evalStore_ ? *evalStore_ : *this;
|
||||||
// Fallback for old daemon versions.
|
// Fallback for old daemon versions.
|
||||||
// For floating-CA derivations (and their co-dependencies) this is an
|
// For floating-CA derivations (and their co-dependencies) this is an
|
||||||
|
@ -530,6 +532,7 @@ void RemoteStore::registerDrvOutput(const Realisation & info)
|
||||||
auto conn(getConnection());
|
auto conn(getConnection());
|
||||||
conn->to << WorkerProto::Op::RegisterDrvOutput;
|
conn->to << WorkerProto::Op::RegisterDrvOutput;
|
||||||
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) {
|
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) {
|
||||||
|
REMOVE_AFTER_DROPPING_PROTO_MINOR(30);
|
||||||
conn->to << info.id.to_string();
|
conn->to << info.id.to_string();
|
||||||
conn->to << std::string(info.outPath.to_string());
|
conn->to << std::string(info.outPath.to_string());
|
||||||
} else {
|
} else {
|
||||||
|
@ -617,6 +620,7 @@ std::vector<KeyedBuildResult> RemoteStore::buildPathsWithResults(
|
||||||
conn.processStderr();
|
conn.processStderr();
|
||||||
return WorkerProto::Serialise<std::vector<KeyedBuildResult>>::read(*this, *conn);
|
return WorkerProto::Serialise<std::vector<KeyedBuildResult>>::read(*this, *conn);
|
||||||
} else {
|
} else {
|
||||||
|
REMOVE_AFTER_DROPPING_PROTO_MINOR(33);
|
||||||
// Avoid deadlock.
|
// Avoid deadlock.
|
||||||
conn_.reset();
|
conn_.reset();
|
||||||
|
|
||||||
|
|
|
@ -8,6 +8,11 @@ namespace nix {
|
||||||
#define SERVE_MAGIC_1 0x390c9deb
|
#define SERVE_MAGIC_1 0x390c9deb
|
||||||
#define SERVE_MAGIC_2 0x5452eecb
|
#define SERVE_MAGIC_2 0x5452eecb
|
||||||
|
|
||||||
|
// This must remain at 2.7 (Nix 2.18) forever in Lix, since the protocol
|
||||||
|
// versioning is monotonic, so if we ever change it in the future, it will
|
||||||
|
// break compatibility with any potential CppNix-originated protocol changes.
|
||||||
|
//
|
||||||
|
// Lix intends to replace this protocol entirely.
|
||||||
#define SERVE_PROTOCOL_VERSION (2 << 8 | 7)
|
#define SERVE_PROTOCOL_VERSION (2 << 8 | 7)
|
||||||
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
|
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
|
||||||
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
|
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
|
||||||
|
|
|
@ -279,7 +279,7 @@ StorePath Store::addToStore(
|
||||||
if (method == FileIngestionMethod::Recursive)
|
if (method == FileIngestionMethod::Recursive)
|
||||||
dumpPath(srcPath, sink, filter);
|
dumpPath(srcPath, sink, filter);
|
||||||
else
|
else
|
||||||
readFile(srcPath, sink);
|
readFileSource(srcPath)->drainInto(sink);
|
||||||
});
|
});
|
||||||
return addToStoreFromDump(*source, name, method, hashAlgo, repair, references);
|
return addToStoreFromDump(*source, name, method, hashAlgo, repair, references);
|
||||||
}
|
}
|
||||||
|
@ -803,17 +803,31 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m
|
||||||
|
|
||||||
auto doQuery = [&](const StorePath & path) {
|
auto doQuery = [&](const StorePath & path) {
|
||||||
checkInterrupt();
|
checkInterrupt();
|
||||||
auto state(state_.lock());
|
|
||||||
|
bool exists = false;
|
||||||
|
std::exception_ptr newExc{};
|
||||||
|
|
||||||
try {
|
try {
|
||||||
auto info = queryPathInfo(path);
|
queryPathInfo(path);
|
||||||
state->valid.insert(path);
|
exists = true;
|
||||||
} catch (InvalidPath &) {
|
} catch (InvalidPath &) {
|
||||||
} catch (...) {
|
} catch (...) {
|
||||||
state->exc = std::current_exception();
|
newExc = std::current_exception();
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
auto state(state_.lock());
|
||||||
|
|
||||||
|
if (exists) {
|
||||||
|
state->valid.insert(path);
|
||||||
|
}
|
||||||
|
if (newExc != nullptr) {
|
||||||
|
state->exc = newExc;
|
||||||
}
|
}
|
||||||
assert(state->left);
|
assert(state->left);
|
||||||
if (!--state->left)
|
if (!--state->left)
|
||||||
wakeup.notify_one();
|
wakeup.notify_one();
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
for (auto & path : paths)
|
for (auto & path : paths)
|
||||||
|
|
|
@ -23,6 +23,10 @@ namespace nix {
|
||||||
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
|
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
|
||||||
|
|
||||||
|
|
||||||
|
#define REMOVE_AFTER_DROPPING_PROTO_MINOR(protoMinor) \
|
||||||
|
static_assert(MIN_SUPPORTED_MINOR_WORKER_PROTO_VERSION <= (protoMinor))
|
||||||
|
|
||||||
|
|
||||||
#define STDERR_NEXT 0x6f6c6d67
|
#define STDERR_NEXT 0x6f6c6d67
|
||||||
#define STDERR_READ 0x64617461 // data needed from source
|
#define STDERR_READ 0x64617461 // data needed from source
|
||||||
#define STDERR_WRITE 0x64617416 // data for sink
|
#define STDERR_WRITE 0x64617416 // data for sink
|
||||||
|
@ -136,30 +140,30 @@ struct WorkerProto
|
||||||
enum struct WorkerProto::Op : uint64_t
|
enum struct WorkerProto::Op : uint64_t
|
||||||
{
|
{
|
||||||
IsValidPath = 1,
|
IsValidPath = 1,
|
||||||
HasSubstitutes = 3,
|
HasSubstitutes = 3, // obsolete since 2012, stubbed to error
|
||||||
QueryPathHash = 4, // obsolete
|
QueryPathHash = 4, // obsolete since 2016, stubbed to error
|
||||||
QueryReferences = 5, // obsolete
|
QueryReferences = 5, // obsolete since 2016, stubbed to error
|
||||||
QueryReferrers = 6,
|
QueryReferrers = 6,
|
||||||
AddToStore = 7,
|
AddToStore = 7,
|
||||||
AddTextToStore = 8, // obsolete since 1.25, Nix 3.0. Use WorkerProto::Op::AddToStore
|
AddTextToStore = 8, // obsolete since protocol 1.25, CppNix 2.4. Use WorkerProto::Op::AddToStore
|
||||||
BuildPaths = 9,
|
BuildPaths = 9,
|
||||||
EnsurePath = 10,
|
EnsurePath = 10,
|
||||||
AddTempRoot = 11,
|
AddTempRoot = 11,
|
||||||
AddIndirectRoot = 12,
|
AddIndirectRoot = 12,
|
||||||
SyncWithGC = 13,
|
SyncWithGC = 13, // obsolete since CppNix 2.5.0
|
||||||
FindRoots = 14,
|
FindRoots = 14,
|
||||||
ExportPath = 16, // obsolete
|
ExportPath = 16, // obsolete since 2017, stubbed to error
|
||||||
QueryDeriver = 18, // obsolete
|
QueryDeriver = 18, // obsolete since 2016, stubbed to error
|
||||||
SetOptions = 19,
|
SetOptions = 19,
|
||||||
CollectGarbage = 20,
|
CollectGarbage = 20,
|
||||||
QuerySubstitutablePathInfo = 21,
|
QuerySubstitutablePathInfo = 21,
|
||||||
QueryDerivationOutputs = 22, // obsolete
|
QueryDerivationOutputs = 22, // obsolete since protocol 1.21, CppNix 2.4
|
||||||
QueryAllValidPaths = 23,
|
QueryAllValidPaths = 23,
|
||||||
QueryFailedPaths = 24,
|
QueryFailedPaths = 24, // obsolete, removed
|
||||||
ClearFailedPaths = 25,
|
ClearFailedPaths = 25, // obsolete, removed
|
||||||
QueryPathInfo = 26,
|
QueryPathInfo = 26,
|
||||||
ImportPaths = 27, // obsolete
|
ImportPaths = 27, // obsolete since 2016
|
||||||
QueryDerivationOutputNames = 28, // obsolete
|
QueryDerivationOutputNames = 28, // obsolete since CppNix 2.4
|
||||||
QueryPathFromHashPart = 29,
|
QueryPathFromHashPart = 29,
|
||||||
QuerySubstitutablePathInfos = 30,
|
QuerySubstitutablePathInfos = 30,
|
||||||
QueryValidPaths = 31,
|
QueryValidPaths = 31,
|
||||||
|
|
|
@ -163,12 +163,15 @@ struct BrotliDecompressionSource : Source
|
||||||
uint8_t * out = (uint8_t *) data;
|
uint8_t * out = (uint8_t *) data;
|
||||||
const auto * begin = out;
|
const auto * begin = out;
|
||||||
|
|
||||||
try {
|
|
||||||
while (len && !BrotliDecoderIsFinished(state.get())) {
|
while (len && !BrotliDecoderIsFinished(state.get())) {
|
||||||
checkInterrupt();
|
checkInterrupt();
|
||||||
|
|
||||||
while (avail_in == 0) {
|
while (avail_in == 0) {
|
||||||
|
try {
|
||||||
avail_in = inner->read(buf.get(), BUF_SIZE);
|
avail_in = inner->read(buf.get(), BUF_SIZE);
|
||||||
|
} catch (EndOfFile &) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
next_in = (const uint8_t *) buf.get();
|
next_in = (const uint8_t *) buf.get();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -179,8 +182,6 @@ struct BrotliDecompressionSource : Source
|
||||||
throw CompressionError("error while decompressing brotli file");
|
throw CompressionError("error while decompressing brotli file");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch (EndOfFile &) {
|
|
||||||
}
|
|
||||||
|
|
||||||
if (begin != out) {
|
if (begin != out) {
|
||||||
return out - begin;
|
return out - begin;
|
||||||
|
@ -192,11 +193,9 @@ struct BrotliDecompressionSource : Source
|
||||||
|
|
||||||
std::string decompress(const std::string & method, std::string_view in)
|
std::string decompress(const std::string & method, std::string_view in)
|
||||||
{
|
{
|
||||||
StringSink ssink;
|
StringSource src{in};
|
||||||
auto sink = makeDecompressionSink(method, ssink);
|
auto filter = makeDecompressionSource(method, src);
|
||||||
(*sink)(in);
|
return filter->drain();
|
||||||
sink->finish();
|
|
||||||
return std::move(ssink.s);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Sink & nextSink)
|
std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Sink & nextSink)
|
||||||
|
@ -224,6 +223,19 @@ std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Si
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
std::unique_ptr<Source> makeDecompressionSource(const std::string & method, Source & inner)
|
||||||
|
{
|
||||||
|
if (method == "none" || method == "") {
|
||||||
|
return std::make_unique<LambdaSource>([&](char * data, size_t len) {
|
||||||
|
return inner.read(data, len);
|
||||||
|
});
|
||||||
|
} else if (method == "br") {
|
||||||
|
return std::make_unique<BrotliDecompressionSource>(inner);
|
||||||
|
} else {
|
||||||
|
return std::make_unique<ArchiveDecompressionSource>(inner);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
struct BrotliCompressionSink : ChunkedCompressionSink
|
struct BrotliCompressionSink : ChunkedCompressionSink
|
||||||
{
|
{
|
||||||
Sink & nextSink;
|
Sink & nextSink;
|
||||||
|
|
|
@ -19,6 +19,7 @@ struct CompressionSink : BufferedSink, FinishSink
|
||||||
std::string decompress(const std::string & method, std::string_view in);
|
std::string decompress(const std::string & method, std::string_view in);
|
||||||
|
|
||||||
std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Sink & nextSink);
|
std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Sink & nextSink);
|
||||||
|
std::unique_ptr<Source> makeDecompressionSource(const std::string & method, Source & inner);
|
||||||
|
|
||||||
std::string compress(const std::string & method, std::string_view in, const bool parallel = false, int level = -1);
|
std::string compress(const std::string & method, std::string_view in, const bool parallel = false, int level = -1);
|
||||||
|
|
||||||
|
|
|
@ -289,12 +289,17 @@ std::string readFile(const Path & path)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void readFile(const Path & path, Sink & sink)
|
box_ptr<Source> readFileSource(const Path & path)
|
||||||
{
|
{
|
||||||
AutoCloseFD fd{open(path.c_str(), O_RDONLY | O_CLOEXEC)};
|
AutoCloseFD fd{open(path.c_str(), O_RDONLY | O_CLOEXEC)};
|
||||||
if (!fd)
|
if (!fd)
|
||||||
throw SysError("opening file '%s'", path);
|
throw SysError("opening file '%s'", path);
|
||||||
drainFD(fd.get(), sink);
|
|
||||||
|
struct FileSource : FdSource {
|
||||||
|
AutoCloseFD fd;
|
||||||
|
explicit FileSource(AutoCloseFD fd) : FdSource(fd.get()), fd(std::move(fd)) {}
|
||||||
|
};
|
||||||
|
return make_box_ptr<FileSource>(std::move(fd));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -5,6 +5,7 @@
|
||||||
* Utiltities for working with the file sytem and file paths.
|
* Utiltities for working with the file sytem and file paths.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
#include "box_ptr.hh"
|
||||||
#include "types.hh"
|
#include "types.hh"
|
||||||
#include "file-descriptor.hh"
|
#include "file-descriptor.hh"
|
||||||
|
|
||||||
|
@ -142,7 +143,7 @@ unsigned char getFileType(const Path & path);
|
||||||
* Read the contents of a file into a string.
|
* Read the contents of a file into a string.
|
||||||
*/
|
*/
|
||||||
std::string readFile(const Path & path);
|
std::string readFile(const Path & path);
|
||||||
void readFile(const Path & path, Sink & sink);
|
box_ptr<Source> readFileSource(const Path & path);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Write a string to a file.
|
* Write a string to a file.
|
||||||
|
|
|
@ -324,7 +324,7 @@ Hash hashString(HashType ht, std::string_view s)
|
||||||
Hash hashFile(HashType ht, const Path & path)
|
Hash hashFile(HashType ht, const Path & path)
|
||||||
{
|
{
|
||||||
HashSink sink(ht);
|
HashSink sink(ht);
|
||||||
readFile(path, sink);
|
readFileSource(path)->drainInto(sink);
|
||||||
return sink.finish().first;
|
return sink.finish().first;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -37,7 +37,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand
|
||||||
Hash hash = narHash;
|
Hash hash = narHash;
|
||||||
if (ingestionMethod == FileIngestionMethod::Flat) {
|
if (ingestionMethod == FileIngestionMethod::Flat) {
|
||||||
HashSink hsink(htSHA256);
|
HashSink hsink(htSHA256);
|
||||||
readFile(path, hsink);
|
readFileSource(path)->drainInto(hsink);
|
||||||
hash = hsink.finish().first;
|
hash = hsink.finish().first;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
#include "installable-flake.hh"
|
#include "installable-flake.hh"
|
||||||
#include "command-installable-value.hh"
|
#include "command.hh"
|
||||||
#include "common-args.hh"
|
#include "common-args.hh"
|
||||||
#include "shared.hh"
|
#include "shared.hh"
|
||||||
#include "store-api.hh"
|
#include "store-api.hh"
|
||||||
|
@ -9,7 +9,7 @@
|
||||||
|
|
||||||
using namespace nix;
|
using namespace nix;
|
||||||
|
|
||||||
struct CmdBundle : InstallableValueCommand
|
struct CmdBundle : InstallableCommand
|
||||||
{
|
{
|
||||||
std::string bundler = "github:NixOS/bundlers";
|
std::string bundler = "github:NixOS/bundlers";
|
||||||
std::optional<Path> outLink;
|
std::optional<Path> outLink;
|
||||||
|
@ -71,11 +71,13 @@ struct CmdBundle : InstallableValueCommand
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
void run(ref<Store> store, ref<InstallableValue> installable) override
|
void run(ref<Store> store, ref<Installable> installable) override
|
||||||
{
|
{
|
||||||
auto evalState = getEvalState();
|
auto evalState = getEvalState();
|
||||||
|
|
||||||
auto val = installable->toValue(*evalState).first;
|
auto const installableValue = InstallableValue::require(installable);
|
||||||
|
|
||||||
|
auto val = installableValue->toValue(*evalState).first;
|
||||||
|
|
||||||
auto [bundlerFlakeRef, bundlerName, extendedOutputsSpec] = parseFlakeRefWithFragmentAndExtendedOutputsSpec(bundler, absPath("."));
|
auto [bundlerFlakeRef, bundlerName, extendedOutputsSpec] = parseFlakeRefWithFragmentAndExtendedOutputsSpec(bundler, absPath("."));
|
||||||
const flake::LockFlags lockFlags{ .writeLockFile = false };
|
const flake::LockFlags lockFlags{ .writeLockFile = false };
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
#include "eval.hh"
|
#include "eval.hh"
|
||||||
#include "installable-flake.hh"
|
#include "installable-flake.hh"
|
||||||
#include "command-installable-value.hh"
|
#include "command.hh"
|
||||||
#include "common-args.hh"
|
#include "common-args.hh"
|
||||||
#include "shared.hh"
|
#include "shared.hh"
|
||||||
#include "store-api.hh"
|
#include "store-api.hh"
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
#include "command-installable-value.hh"
|
#include "command.hh"
|
||||||
#include "shared.hh"
|
#include "shared.hh"
|
||||||
#include "eval.hh"
|
#include "eval.hh"
|
||||||
#include "attr-path.hh"
|
#include "attr-path.hh"
|
||||||
|
@ -10,7 +10,7 @@
|
||||||
|
|
||||||
using namespace nix;
|
using namespace nix;
|
||||||
|
|
||||||
struct CmdEdit : InstallableValueCommand
|
struct CmdEdit : InstallableCommand
|
||||||
{
|
{
|
||||||
std::string description() override
|
std::string description() override
|
||||||
{
|
{
|
||||||
|
@ -26,17 +26,19 @@ struct CmdEdit : InstallableValueCommand
|
||||||
|
|
||||||
Category category() override { return catSecondary; }
|
Category category() override { return catSecondary; }
|
||||||
|
|
||||||
void run(ref<Store> store, ref<InstallableValue> installable) override
|
void run(ref<Store> store, ref<Installable> installable) override
|
||||||
{
|
{
|
||||||
auto state = getEvalState();
|
auto state = getEvalState();
|
||||||
|
|
||||||
|
auto const installableValue = InstallableValue::require(installable);
|
||||||
|
|
||||||
const auto [file, line] = [&] {
|
const auto [file, line] = [&] {
|
||||||
auto [v, pos] = installable->toValue(*state);
|
auto [v, pos] = installableValue->toValue(*state);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
return findPackageFilename(*state, *v, installable->what());
|
return findPackageFilename(*state, *v, installable->what());
|
||||||
} catch (NoPositionInfo &) {
|
} catch (NoPositionInfo &) {
|
||||||
throw Error("cannot find position information for '%s", installable->what());
|
throw Error("cannot find position information for '%s", installableValue->what());
|
||||||
}
|
}
|
||||||
}();
|
}();
|
||||||
|
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
#include "command-installable-value.hh"
|
#include "command.hh"
|
||||||
#include "common-args.hh"
|
#include "common-args.hh"
|
||||||
#include "print-options.hh"
|
#include "print-options.hh"
|
||||||
#include "shared.hh"
|
#include "shared.hh"
|
||||||
|
@ -12,13 +12,13 @@
|
||||||
|
|
||||||
using namespace nix;
|
using namespace nix;
|
||||||
|
|
||||||
struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption
|
struct CmdEval : MixJSON, InstallableCommand, MixReadOnlyOption
|
||||||
{
|
{
|
||||||
bool raw = false;
|
bool raw = false;
|
||||||
std::optional<std::string> apply;
|
std::optional<std::string> apply;
|
||||||
std::optional<Path> writeTo;
|
std::optional<Path> writeTo;
|
||||||
|
|
||||||
CmdEval() : InstallableValueCommand()
|
CmdEval() : InstallableCommand()
|
||||||
{
|
{
|
||||||
addFlag({
|
addFlag({
|
||||||
.longName = "raw",
|
.longName = "raw",
|
||||||
|
@ -55,14 +55,16 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption
|
||||||
|
|
||||||
Category category() override { return catSecondary; }
|
Category category() override { return catSecondary; }
|
||||||
|
|
||||||
void run(ref<Store> store, ref<InstallableValue> installable) override
|
void run(ref<Store> store, ref<Installable> installable) override
|
||||||
{
|
{
|
||||||
if (raw && json)
|
if (raw && json)
|
||||||
throw UsageError("--raw and --json are mutually exclusive");
|
throw UsageError("--raw and --json are mutually exclusive");
|
||||||
|
|
||||||
|
auto const installableValue = InstallableValue::require(installable);
|
||||||
|
|
||||||
auto state = getEvalState();
|
auto state = getEvalState();
|
||||||
|
|
||||||
auto [v, pos] = installable->toValue(*state);
|
auto [v, pos] = installableValue->toValue(*state);
|
||||||
NixStringContext context;
|
NixStringContext context;
|
||||||
|
|
||||||
if (apply) {
|
if (apply) {
|
||||||
|
|
|
@ -85,7 +85,7 @@ struct CmdHashBase : Command
|
||||||
|
|
||||||
switch (mode) {
|
switch (mode) {
|
||||||
case FileIngestionMethod::Flat:
|
case FileIngestionMethod::Flat:
|
||||||
readFile(path, *hashSink);
|
readFileSource(path)->drainInto(*hashSink);
|
||||||
break;
|
break;
|
||||||
case FileIngestionMethod::Recursive:
|
case FileIngestionMethod::Recursive:
|
||||||
dumpPath(path, *hashSink);
|
dumpPath(path, *hashSink);
|
||||||
|
|
|
@ -98,7 +98,7 @@ std::tuple<StorePath, Hash> prefetchFile(
|
||||||
FdSink sink(fd.get());
|
FdSink sink(fd.get());
|
||||||
|
|
||||||
FileTransferRequest req(url);
|
FileTransferRequest req(url);
|
||||||
getFileTransfer()->download(std::move(req), sink);
|
getFileTransfer()->download(std::move(req))->drainInto(sink);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Optionally unpack the file. */
|
/* Optionally unpack the file. */
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
#include "run.hh"
|
#include "run.hh"
|
||||||
#include "command-installable-value.hh"
|
#include "command.hh"
|
||||||
#include "common-args.hh"
|
#include "common-args.hh"
|
||||||
|
#include "installables.hh"
|
||||||
#include "shared.hh"
|
#include "shared.hh"
|
||||||
#include "store-api.hh"
|
#include "store-api.hh"
|
||||||
#include "derivations.hh"
|
#include "derivations.hh"
|
||||||
|
@ -145,7 +146,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment
|
||||||
|
|
||||||
static auto rCmdShell = registerCommand<CmdShell>("shell");
|
static auto rCmdShell = registerCommand<CmdShell>("shell");
|
||||||
|
|
||||||
struct CmdRun : InstallableValueCommand
|
struct CmdRun : InstallableCommand
|
||||||
{
|
{
|
||||||
using InstallableCommand::run;
|
using InstallableCommand::run;
|
||||||
|
|
||||||
|
@ -191,12 +192,14 @@ struct CmdRun : InstallableValueCommand
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
void run(ref<Store> store, ref<InstallableValue> installable) override
|
void run(ref<Store> store, ref<Installable> installable) override
|
||||||
{
|
{
|
||||||
auto state = getEvalState();
|
auto state = getEvalState();
|
||||||
|
|
||||||
|
auto installableValue = InstallableValue::require(installable);
|
||||||
|
|
||||||
lockFlags.applyNixConfig = true;
|
lockFlags.applyNixConfig = true;
|
||||||
auto app = installable->toApp(*state).resolve(getEvalStore(), store);
|
auto app = installableValue->toApp(*state).resolve(getEvalStore(), store);
|
||||||
|
|
||||||
Strings allArgs{app.program};
|
Strings allArgs{app.program};
|
||||||
for (auto & i : args) allArgs.push_back(i);
|
for (auto & i : args) allArgs.push_back(i);
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
#include "command-installable-value.hh"
|
#include "command.hh"
|
||||||
#include "globals.hh"
|
#include "globals.hh"
|
||||||
#include "eval.hh"
|
#include "eval.hh"
|
||||||
#include "eval-inline.hh"
|
#include "eval-inline.hh"
|
||||||
|
@ -23,7 +23,7 @@ std::string wrap(std::string prefix, std::string s)
|
||||||
return concatStrings(prefix, s, ANSI_NORMAL);
|
return concatStrings(prefix, s, ANSI_NORMAL);
|
||||||
}
|
}
|
||||||
|
|
||||||
struct CmdSearch : InstallableValueCommand, MixJSON
|
struct CmdSearch : InstallableCommand, MixJSON
|
||||||
{
|
{
|
||||||
std::vector<std::string> res;
|
std::vector<std::string> res;
|
||||||
std::vector<std::string> excludeRes;
|
std::vector<std::string> excludeRes;
|
||||||
|
@ -62,8 +62,10 @@ struct CmdSearch : InstallableValueCommand, MixJSON
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
void run(ref<Store> store, ref<InstallableValue> installable) override
|
void run(ref<Store> store, ref<Installable> installable) override
|
||||||
{
|
{
|
||||||
|
auto const installableValue = InstallableValue::require(installable);
|
||||||
|
|
||||||
settings.readOnlyMode = true;
|
settings.readOnlyMode = true;
|
||||||
evalSettings.enableImportFromDerivation.setDefault(false);
|
evalSettings.enableImportFromDerivation.setDefault(false);
|
||||||
|
|
||||||
|
@ -192,7 +194,7 @@ struct CmdSearch : InstallableValueCommand, MixJSON
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
for (auto & cursor : installable->getCursors(*state))
|
for (auto & cursor : installableValue->getCursors(*state))
|
||||||
visit(*cursor, cursor->getAttrPath(), true);
|
visit(*cursor, cursor->getAttrPath(), true);
|
||||||
|
|
||||||
if (json)
|
if (json)
|
||||||
|
|
43
tests/unit/libmain/progress-bar.cc
Normal file
43
tests/unit/libmain/progress-bar.cc
Normal file
|
@ -0,0 +1,43 @@
|
||||||
|
#include <gtest/gtest.h>
|
||||||
|
|
||||||
|
#include "eval.hh"
|
||||||
|
#include "progress-bar.hh"
|
||||||
|
#include "logging.hh"
|
||||||
|
#include "shared.hh"
|
||||||
|
|
||||||
|
constexpr std::string_view TEST_URL = "https://github.com/NixOS/nixpkgs/archive/master.tar.gz";
|
||||||
|
// Arbitrary number. We picked the size of a Nixpkgs tarball that we downloaded.
|
||||||
|
constexpr uint64_t TEST_EXPECTED = 43'370'307;
|
||||||
|
// Arbitrary number. We picked the progress made on a Nixpkgs tarball download we interrupted.
|
||||||
|
constexpr uint64_t TEST_DONE = 1'787'251;
|
||||||
|
|
||||||
|
constexpr std::string_view EXPECTED = ANSI_GREEN "1.7" ANSI_NORMAL "/41.4 MiB DL";
|
||||||
|
// Mostly here for informational purposes, but also if we change the way the escape codes
|
||||||
|
// are defined this test might break in some annoying to debug way.
|
||||||
|
constexpr std::string_view EXPECTED_RAW = "\x1b[32;1m1.7\x1b[0m/41.4 MiB DL";
|
||||||
|
static_assert(EXPECTED == EXPECTED_RAW, "Hey, hey, the ANSI escape code definitions prolly changed");
|
||||||
|
|
||||||
|
namespace nix
|
||||||
|
{
|
||||||
|
TEST(ProgressBar, basicStatusRender) {
|
||||||
|
initNix();
|
||||||
|
initGC();
|
||||||
|
|
||||||
|
startProgressBar();
|
||||||
|
ASSERT_NE(dynamic_cast<ProgressBar *>(logger), nullptr);
|
||||||
|
ProgressBar & progressBar = dynamic_cast<ProgressBar &>(*logger);
|
||||||
|
|
||||||
|
Activity act(
|
||||||
|
progressBar,
|
||||||
|
lvlDebug,
|
||||||
|
actFileTransfer,
|
||||||
|
fmt("downloading '%s'", TEST_URL),
|
||||||
|
{ "https://github.com/NixOS/nixpkgs/archive/master.tar.gz" }
|
||||||
|
);
|
||||||
|
act.progress(TEST_DONE, TEST_EXPECTED);
|
||||||
|
auto state = progressBar.state_.lock();
|
||||||
|
std::string const renderedStatus = progressBar.getStatus(*state);
|
||||||
|
|
||||||
|
ASSERT_EQ(renderedStatus, EXPECTED);
|
||||||
|
}
|
||||||
|
}
|
|
@ -7,6 +7,7 @@
|
||||||
#include <gtest/gtest.h>
|
#include <gtest/gtest.h>
|
||||||
#include <netinet/in.h>
|
#include <netinet/in.h>
|
||||||
#include <stdexcept>
|
#include <stdexcept>
|
||||||
|
#include <string>
|
||||||
#include <string_view>
|
#include <string_view>
|
||||||
#include <sys/poll.h>
|
#include <sys/poll.h>
|
||||||
#include <sys/socket.h>
|
#include <sys/socket.h>
|
||||||
|
@ -136,7 +137,7 @@ TEST(FileTransfer, exceptionAbortsDownload)
|
||||||
|
|
||||||
LambdaSink broken([](auto block) { throw Done(); });
|
LambdaSink broken([](auto block) { throw Done(); });
|
||||||
|
|
||||||
ASSERT_THROW(ft->download(FileTransferRequest("file:///dev/zero"), broken), Done);
|
ASSERT_THROW(ft->download(FileTransferRequest("file:///dev/zero"))->drainInto(broken), Done);
|
||||||
|
|
||||||
// makeFileTransfer returns a ref<>, which cannot be cleared. since we also
|
// makeFileTransfer returns a ref<>, which cannot be cleared. since we also
|
||||||
// can't default-construct it we'll have to overwrite it instead, but we'll
|
// can't default-construct it we'll have to overwrite it instead, but we'll
|
||||||
|
@ -159,16 +160,21 @@ TEST(FileTransfer, NOT_ON_DARWIN(reportsSetupErrors))
|
||||||
FileTransferError);
|
FileTransferError);
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST(FileTransfer, NOT_ON_DARWIN(reportsTransferError))
|
TEST(FileTransfer, NOT_ON_DARWIN(defersFailures))
|
||||||
{
|
{
|
||||||
auto [port, srv] = serveHTTP("200 ok", "content-length: 100\r\n", [] {
|
auto [port, srv] = serveHTTP("200 ok", "content-length: 100000000\r\n", [] {
|
||||||
std::this_thread::sleep_for(10ms);
|
std::this_thread::sleep_for(10ms);
|
||||||
return "";
|
// just a bunch of data to fill the curl wrapper buffer, otherwise the
|
||||||
|
// initial wait for header data will also wait for the the response to
|
||||||
|
// complete (the source is only woken when curl returns data, and curl
|
||||||
|
// might only do so once its internal buffer has already been filled.)
|
||||||
|
return std::string(1024 * 1024, ' ');
|
||||||
});
|
});
|
||||||
auto ft = makeFileTransfer();
|
auto ft = makeFileTransfer();
|
||||||
FileTransferRequest req(fmt("http://[::1]:%d/index", port));
|
FileTransferRequest req(fmt("http://[::1]:%d/index", port));
|
||||||
req.baseRetryTimeMs = 0;
|
req.baseRetryTimeMs = 0;
|
||||||
ASSERT_THROW(ft->transfer(req), FileTransferError);
|
auto src = ft->download(std::move(req));
|
||||||
|
ASSERT_THROW(src->drain(), FileTransferError);
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST(FileTransfer, NOT_ON_DARWIN(handlesContentEncoding))
|
TEST(FileTransfer, NOT_ON_DARWIN(handlesContentEncoding))
|
||||||
|
@ -180,7 +186,7 @@ TEST(FileTransfer, NOT_ON_DARWIN(handlesContentEncoding))
|
||||||
auto ft = makeFileTransfer();
|
auto ft = makeFileTransfer();
|
||||||
|
|
||||||
StringSink sink;
|
StringSink sink;
|
||||||
ft->download(FileTransferRequest(fmt("http://[::1]:%d/index", port)), sink);
|
ft->download(FileTransferRequest(fmt("http://[::1]:%d/index", port)))->drainInto(sink);
|
||||||
EXPECT_EQ(sink.s, original);
|
EXPECT_EQ(sink.s, original);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -66,6 +66,16 @@ namespace nix {
|
||||||
ASSERT_THROW(decompress(method, str), CompressionError);
|
ASSERT_THROW(decompress(method, str), CompressionError);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
TEST(decompress, veryLongBrotli) {
|
||||||
|
auto method = "br";
|
||||||
|
auto str = std::string(65536, 'a');
|
||||||
|
auto o = decompress(method, compress(method, str));
|
||||||
|
|
||||||
|
// This is just to not print 64k of "a" for most failures
|
||||||
|
ASSERT_EQ(o.length(), str.length());
|
||||||
|
ASSERT_EQ(o, str);
|
||||||
|
}
|
||||||
|
|
||||||
/* ----------------------------------------------------------------------------
|
/* ----------------------------------------------------------------------------
|
||||||
* compression sinks
|
* compression sinks
|
||||||
* --------------------------------------------------------------------------*/
|
* --------------------------------------------------------------------------*/
|
||||||
|
@ -81,16 +91,17 @@ namespace nix {
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST(makeCompressionSink, compressAndDecompress) {
|
TEST(makeCompressionSink, compressAndDecompress) {
|
||||||
StringSink strSink;
|
|
||||||
auto inputString = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
|
auto inputString = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
|
||||||
auto decompressionSink = makeDecompressionSink("bzip2", strSink);
|
|
||||||
auto sink = makeCompressionSink("bzip2", *decompressionSink);
|
|
||||||
|
|
||||||
|
StringSink strSink;
|
||||||
|
auto sink = makeCompressionSink("bzip2", strSink);
|
||||||
(*sink)(inputString);
|
(*sink)(inputString);
|
||||||
sink->finish();
|
sink->finish();
|
||||||
decompressionSink->finish();
|
|
||||||
|
|
||||||
ASSERT_STREQ(strSink.s.c_str(), inputString);
|
StringSource strSource{strSink.s};
|
||||||
|
auto decompressionSource = makeDecompressionSource("bzip2", strSource);
|
||||||
|
|
||||||
|
ASSERT_STREQ(decompressionSource->drain().c_str(), inputString);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -249,3 +249,25 @@ test(
|
||||||
suite : 'check',
|
suite : 'check',
|
||||||
protocol : 'gtest',
|
protocol : 'gtest',
|
||||||
)
|
)
|
||||||
|
|
||||||
|
libmain_tester = executable(
|
||||||
|
'liblixmain-tests',
|
||||||
|
files('libmain/progress-bar.cc'),
|
||||||
|
dependencies : [
|
||||||
|
liblixmain,
|
||||||
|
liblixexpr,
|
||||||
|
liblixutil,
|
||||||
|
liblixstore,
|
||||||
|
gtest,
|
||||||
|
boost,
|
||||||
|
],
|
||||||
|
cpp_pch : cpp_pch,
|
||||||
|
)
|
||||||
|
|
||||||
|
test(
|
||||||
|
'libmain-unit-tests',
|
||||||
|
libmain_tester,
|
||||||
|
args : tests_args,
|
||||||
|
suite : 'check',
|
||||||
|
protocol : 'gtest',
|
||||||
|
)
|
||||||
|
|
Loading…
Reference in a new issue