forked from lix-project/lix
Compare commits
3 commits
f4dd67e436
...
fe13a447f0
Author | SHA1 | Date | |
---|---|---|---|
|
fe13a447f0 | ||
|
6c726aa23b | ||
|
49399e481a |
|
@ -87,7 +87,6 @@ Most commands in Lix accept the following command-line options:
|
||||||
|
|
||||||
Displayes the raw logs, with a progress bar and activities each in a new line at the bottom.
|
Displayes the raw logs, with a progress bar and activities each in a new line at the bottom.
|
||||||
|
|
||||||
|
|
||||||
- <span id="opt-no-build-output">[`--no-build-output`](#opt-no-build-output)</span> / `-Q`
|
- <span id="opt-no-build-output">[`--no-build-output`](#opt-no-build-output)</span> / `-Q`
|
||||||
|
|
||||||
By default, output written by builders to standard output and standard error is echoed to the Lix command's standard error.
|
By default, output written by builders to standard output and standard error is echoed to the Lix command's standard error.
|
||||||
|
|
11
src/libcmd/command-installable-value.cc
Normal file
11
src/libcmd/command-installable-value.cc
Normal file
|
@ -0,0 +1,11 @@
|
||||||
|
#include "command-installable-value.hh"
|
||||||
|
|
||||||
|
namespace nix {
|
||||||
|
|
||||||
|
void InstallableValueCommand::run(ref<Store> store, ref<Installable> installable)
|
||||||
|
{
|
||||||
|
auto installableValue = InstallableValue::require(installable);
|
||||||
|
run(store, installableValue);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
23
src/libcmd/command-installable-value.hh
Normal file
23
src/libcmd/command-installable-value.hh
Normal file
|
@ -0,0 +1,23 @@
|
||||||
|
#pragma once
|
||||||
|
///@file
|
||||||
|
|
||||||
|
#include "installable-value.hh"
|
||||||
|
#include "command.hh"
|
||||||
|
|
||||||
|
namespace nix {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* An InstallableCommand where the single positional argument must be an
|
||||||
|
* InstallableValue in particular.
|
||||||
|
*/
|
||||||
|
struct InstallableValueCommand : InstallableCommand
|
||||||
|
{
|
||||||
|
/**
|
||||||
|
* Entry point to this command
|
||||||
|
*/
|
||||||
|
virtual void run(ref<Store> store, ref<InstallableValue> installable) = 0;
|
||||||
|
|
||||||
|
void run(ref<Store> store, ref<Installable> installable) override;
|
||||||
|
};
|
||||||
|
|
||||||
|
}
|
|
@ -393,10 +393,13 @@ ref<eval_cache::EvalCache> openEvalCache(
|
||||||
EvalState & state,
|
EvalState & state,
|
||||||
std::shared_ptr<flake::LockedFlake> lockedFlake)
|
std::shared_ptr<flake::LockedFlake> lockedFlake)
|
||||||
{
|
{
|
||||||
auto fingerprint = evalSettings.useEvalCache && evalSettings.pureEval
|
auto fingerprint = lockedFlake->getFingerprint();
|
||||||
? std::make_optional(lockedFlake->getFingerprint())
|
return make_ref<nix::eval_cache::EvalCache>(
|
||||||
: std::nullopt;
|
evalSettings.useEvalCache && evalSettings.pureEval
|
||||||
auto rootLoader = [&state, lockedFlake]()
|
? std::optional { std::cref(fingerprint) }
|
||||||
|
: std::nullopt,
|
||||||
|
state,
|
||||||
|
[&state, lockedFlake]()
|
||||||
{
|
{
|
||||||
/* For testing whether the evaluation cache is
|
/* For testing whether the evaluation cache is
|
||||||
complete. */
|
complete. */
|
||||||
|
@ -412,17 +415,7 @@ ref<eval_cache::EvalCache> openEvalCache(
|
||||||
assert(aOutputs);
|
assert(aOutputs);
|
||||||
|
|
||||||
return aOutputs->value;
|
return aOutputs->value;
|
||||||
};
|
});
|
||||||
|
|
||||||
if (fingerprint) {
|
|
||||||
auto search = state.evalCaches.find(fingerprint.value());
|
|
||||||
if (search == state.evalCaches.end()) {
|
|
||||||
search = state.evalCaches.emplace(fingerprint.value(), make_ref<nix::eval_cache::EvalCache>(fingerprint, state, rootLoader)).first;
|
|
||||||
}
|
|
||||||
return search->second;
|
|
||||||
} else {
|
|
||||||
return make_ref<nix::eval_cache::EvalCache>(std::nullopt, state, rootLoader);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Installables SourceExprCommand::parseInstallables(
|
Installables SourceExprCommand::parseInstallables(
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
libcmd_sources = files(
|
libcmd_sources = files(
|
||||||
'built-path.cc',
|
'built-path.cc',
|
||||||
|
'command-installable-value.cc',
|
||||||
'cmd-profiles.cc',
|
'cmd-profiles.cc',
|
||||||
'command.cc',
|
'command.cc',
|
||||||
'common-eval-args.cc',
|
'common-eval-args.cc',
|
||||||
|
@ -17,6 +18,7 @@ libcmd_sources = files(
|
||||||
|
|
||||||
libcmd_headers = files(
|
libcmd_headers = files(
|
||||||
'built-path.hh',
|
'built-path.hh',
|
||||||
|
'command-installable-value.hh',
|
||||||
'cmd-profiles.hh',
|
'cmd-profiles.hh',
|
||||||
'command.hh',
|
'command.hh',
|
||||||
'common-eval-args.hh',
|
'common-eval-args.hh',
|
||||||
|
|
|
@ -33,10 +33,6 @@ class EvalState;
|
||||||
class StorePath;
|
class StorePath;
|
||||||
struct SingleDerivedPath;
|
struct SingleDerivedPath;
|
||||||
enum RepairFlag : bool;
|
enum RepairFlag : bool;
|
||||||
struct MemoryInputAccessor;
|
|
||||||
namespace eval_cache {
|
|
||||||
class EvalCache;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -238,11 +234,6 @@ public:
|
||||||
return *new EvalErrorBuilder<T>(*this, args...);
|
return *new EvalErrorBuilder<T>(*this, args...);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* A cache for evaluation caches, so as to reuse the same root value if possible
|
|
||||||
*/
|
|
||||||
std::map<const Hash, ref<eval_cache::EvalCache>> evalCaches;
|
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|
||||||
/* Cache for calls to addToStore(); maps source paths to the store
|
/* Cache for calls to addToStore(); maps source paths to the store
|
||||||
|
|
|
@ -36,14 +36,89 @@ static std::string_view storePathToName(std::string_view path)
|
||||||
return i == std::string::npos ? base.substr(0, 0) : base.substr(i + 1);
|
return i == std::string::npos ? base.substr(0, 0) : base.substr(i + 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 100 years ought to be enough for anyone (yet sufficiently smaller than max() to not cause signed integer overflow).
|
||||||
|
constexpr const auto A_LONG_TIME = std::chrono::duration_cast<std::chrono::milliseconds>(100 * 365 * 86400s);
|
||||||
|
|
||||||
ProgressBar::~ProgressBar()
|
class ProgressBar : public Logger
|
||||||
|
{
|
||||||
|
private:
|
||||||
|
|
||||||
|
struct ActInfo
|
||||||
|
{
|
||||||
|
std::string s, lastLine, phase;
|
||||||
|
ActivityType type = actUnknown;
|
||||||
|
uint64_t done = 0;
|
||||||
|
uint64_t expected = 0;
|
||||||
|
uint64_t running = 0;
|
||||||
|
uint64_t failed = 0;
|
||||||
|
std::map<ActivityType, uint64_t> expectedByType;
|
||||||
|
bool visible = true;
|
||||||
|
ActivityId parent;
|
||||||
|
std::optional<std::string> name;
|
||||||
|
std::chrono::time_point<std::chrono::steady_clock> startTime;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct ActivitiesByType
|
||||||
|
{
|
||||||
|
std::map<ActivityId, std::list<ActInfo>::iterator> its;
|
||||||
|
uint64_t done = 0;
|
||||||
|
uint64_t expected = 0;
|
||||||
|
uint64_t failed = 0;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct State
|
||||||
|
{
|
||||||
|
std::list<ActInfo> activities;
|
||||||
|
std::map<ActivityId, std::list<ActInfo>::iterator> its;
|
||||||
|
|
||||||
|
std::map<ActivityType, ActivitiesByType> activitiesByType;
|
||||||
|
|
||||||
|
int lastLines = 0;
|
||||||
|
|
||||||
|
uint64_t filesLinked = 0, bytesLinked = 0;
|
||||||
|
|
||||||
|
uint64_t corruptedPaths = 0, untrustedPaths = 0;
|
||||||
|
|
||||||
|
bool active = true;
|
||||||
|
bool paused = false;
|
||||||
|
bool haveUpdate = true;
|
||||||
|
};
|
||||||
|
|
||||||
|
Sync<State> state_;
|
||||||
|
|
||||||
|
std::thread updateThread;
|
||||||
|
|
||||||
|
std::condition_variable quitCV, updateCV;
|
||||||
|
|
||||||
|
bool printBuildLogs = false;
|
||||||
|
bool printMultiline = false;
|
||||||
|
bool isTTY;
|
||||||
|
|
||||||
|
public:
|
||||||
|
|
||||||
|
ProgressBar(bool isTTY)
|
||||||
|
: isTTY(isTTY)
|
||||||
|
{
|
||||||
|
state_.lock()->active = isTTY;
|
||||||
|
updateThread = std::thread([&]() {
|
||||||
|
auto state(state_.lock());
|
||||||
|
auto nextWakeup = A_LONG_TIME;
|
||||||
|
while (state->active) {
|
||||||
|
if (!state->haveUpdate)
|
||||||
|
state.wait_for(updateCV, nextWakeup);
|
||||||
|
nextWakeup = draw(*state, {});
|
||||||
|
state.wait_for(quitCV, std::chrono::milliseconds(50));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
~ProgressBar()
|
||||||
{
|
{
|
||||||
stop();
|
stop();
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Called by destructor, can't be overridden */
|
/* Called by destructor, can't be overridden */
|
||||||
void ProgressBar::stop()
|
void stop() override final
|
||||||
{
|
{
|
||||||
{
|
{
|
||||||
auto state(state_.lock());
|
auto state(state_.lock());
|
||||||
|
@ -56,33 +131,31 @@ void ProgressBar::stop()
|
||||||
updateThread.join();
|
updateThread.join();
|
||||||
}
|
}
|
||||||
|
|
||||||
void ProgressBar::pause()
|
void pause() override {
|
||||||
{
|
|
||||||
state_.lock()->paused = true;
|
state_.lock()->paused = true;
|
||||||
writeToStderr("\r\e[K");
|
writeToStderr("\r\e[K");
|
||||||
}
|
}
|
||||||
|
|
||||||
void ProgressBar::resume()
|
void resume() override {
|
||||||
{
|
|
||||||
state_.lock()->paused = false;
|
state_.lock()->paused = false;
|
||||||
writeToStderr("\r\e[K");
|
writeToStderr("\r\e[K");
|
||||||
state_.lock()->haveUpdate = true;
|
state_.lock()->haveUpdate = true;
|
||||||
updateCV.notify_one();
|
updateCV.notify_one();
|
||||||
}
|
}
|
||||||
|
|
||||||
bool ProgressBar::isVerbose()
|
bool isVerbose() override
|
||||||
{
|
{
|
||||||
return printBuildLogs;
|
return printBuildLogs;
|
||||||
}
|
}
|
||||||
|
|
||||||
void ProgressBar::log(Verbosity lvl, std::string_view s)
|
void log(Verbosity lvl, std::string_view s) override
|
||||||
{
|
{
|
||||||
if (lvl > verbosity) return;
|
if (lvl > verbosity) return;
|
||||||
auto state(state_.lock());
|
auto state(state_.lock());
|
||||||
log(*state, lvl, s);
|
log(*state, lvl, s);
|
||||||
}
|
}
|
||||||
|
|
||||||
void ProgressBar::logEI(const ErrorInfo & ei)
|
void logEI(const ErrorInfo & ei) override
|
||||||
{
|
{
|
||||||
auto state(state_.lock());
|
auto state(state_.lock());
|
||||||
|
|
||||||
|
@ -92,7 +165,7 @@ void ProgressBar::logEI(const ErrorInfo & ei)
|
||||||
log(*state, ei.level, oss.str());
|
log(*state, ei.level, oss.str());
|
||||||
}
|
}
|
||||||
|
|
||||||
void ProgressBar::log(State & state, Verbosity lvl, std::string_view s)
|
void log(State & state, Verbosity lvl, std::string_view s)
|
||||||
{
|
{
|
||||||
if (state.active) {
|
if (state.active) {
|
||||||
draw(state, s);
|
draw(state, s);
|
||||||
|
@ -103,14 +176,8 @@ void ProgressBar::log(State & state, Verbosity lvl, std::string_view s)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void ProgressBar::startActivity(
|
void startActivity(ActivityId act, Verbosity lvl, ActivityType type,
|
||||||
ActivityId act,
|
const std::string & s, const Fields & fields, ActivityId parent) override
|
||||||
Verbosity lvl,
|
|
||||||
ActivityType type,
|
|
||||||
const std::string & s,
|
|
||||||
const Fields & fields,
|
|
||||||
ActivityId parent
|
|
||||||
)
|
|
||||||
{
|
{
|
||||||
auto state(state_.lock());
|
auto state(state_.lock());
|
||||||
|
|
||||||
|
@ -177,7 +244,7 @@ void ProgressBar::startActivity(
|
||||||
|
|
||||||
/* Check whether an activity has an ancestore with the specified
|
/* Check whether an activity has an ancestore with the specified
|
||||||
type. */
|
type. */
|
||||||
bool ProgressBar::hasAncestor(State & state, ActivityType type, ActivityId act)
|
bool hasAncestor(State & state, ActivityType type, ActivityId act)
|
||||||
{
|
{
|
||||||
while (act != 0) {
|
while (act != 0) {
|
||||||
auto i = state.its.find(act);
|
auto i = state.its.find(act);
|
||||||
|
@ -188,7 +255,7 @@ bool ProgressBar::hasAncestor(State & state, ActivityType type, ActivityId act)
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
void ProgressBar::stopActivity(ActivityId act)
|
void stopActivity(ActivityId act) override
|
||||||
{
|
{
|
||||||
auto state(state_.lock());
|
auto state(state_.lock());
|
||||||
|
|
||||||
|
@ -210,7 +277,7 @@ void ProgressBar::stopActivity(ActivityId act)
|
||||||
update(*state);
|
update(*state);
|
||||||
}
|
}
|
||||||
|
|
||||||
void ProgressBar::result(ActivityId act, ResultType type, const std::vector<Field> & fields)
|
void result(ActivityId act, ResultType type, const std::vector<Field> & fields) override
|
||||||
{
|
{
|
||||||
auto state(state_.lock());
|
auto state(state_.lock());
|
||||||
|
|
||||||
|
@ -287,13 +354,13 @@ void ProgressBar::result(ActivityId act, ResultType type, const std::vector<Fiel
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void ProgressBar::update(State & state)
|
void update(State & state)
|
||||||
{
|
{
|
||||||
state.haveUpdate = true;
|
state.haveUpdate = true;
|
||||||
updateCV.notify_one();
|
updateCV.notify_one();
|
||||||
}
|
}
|
||||||
|
|
||||||
std::chrono::milliseconds ProgressBar::draw(State & state, const std::optional<std::string_view> & s)
|
std::chrono::milliseconds draw(State & state, const std::optional<std::string_view> & s)
|
||||||
{
|
{
|
||||||
auto nextWakeup = A_LONG_TIME;
|
auto nextWakeup = A_LONG_TIME;
|
||||||
|
|
||||||
|
@ -329,7 +396,7 @@ std::chrono::milliseconds ProgressBar::draw(State & state, const std::optional<s
|
||||||
}
|
}
|
||||||
|
|
||||||
auto height = windowSize.first > 0 ? windowSize.first : 25;
|
auto height = windowSize.first > 0 ? windowSize.first : 25;
|
||||||
auto moreActivities = 0;
|
auto moreBuilds = 0;
|
||||||
auto now = std::chrono::steady_clock::now();
|
auto now = std::chrono::steady_clock::now();
|
||||||
|
|
||||||
std::string activity_line;
|
std::string activity_line;
|
||||||
|
@ -338,6 +405,7 @@ std::chrono::milliseconds ProgressBar::draw(State & state, const std::optional<s
|
||||||
if (!(i->visible && (!i->s.empty() || !i->lastLine.empty()))) {
|
if (!(i->visible && (!i->s.empty() || !i->lastLine.empty()))) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Don't show activities until some time has
|
/* Don't show activities until some time has
|
||||||
passed, to avoid displaying very short
|
passed, to avoid displaying very short
|
||||||
activities. */
|
activities. */
|
||||||
|
@ -359,22 +427,25 @@ std::chrono::milliseconds ProgressBar::draw(State & state, const std::optional<s
|
||||||
activity_line += ")";
|
activity_line += ")";
|
||||||
}
|
}
|
||||||
if (!i->lastLine.empty()) {
|
if (!i->lastLine.empty()) {
|
||||||
if (!i->s.empty())
|
if (!i->s.empty()) {
|
||||||
activity_line += ": ";
|
activity_line += ": ";
|
||||||
|
}
|
||||||
activity_line += i->lastLine;
|
activity_line += i->lastLine;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (printMultiline) {
|
if (printMultiline) {
|
||||||
if (state.lastLines < (height - 1)) {
|
if (state.lastLines < (height - 1)) {
|
||||||
writeToStderr(filterANSIEscapes(activity_line, false, width) + "\n");
|
writeToStderr(filterANSIEscapes(activity_line, false, width) + "\n");
|
||||||
state.lastLines++;
|
state.lastLines++;
|
||||||
} else moreActivities++;
|
} else {
|
||||||
|
moreBuilds++;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (printMultiline && moreActivities)
|
if (printMultiline && moreBuilds) {
|
||||||
writeToStderr(fmt("And %d more...", moreActivities));
|
writeToStderr(fmt("And %d more...", moreBuilds));
|
||||||
|
}
|
||||||
|
|
||||||
if (!printMultiline) {
|
if (!printMultiline) {
|
||||||
line += " " + activity_line;
|
line += " " + activity_line;
|
||||||
|
@ -384,20 +455,20 @@ std::chrono::milliseconds ProgressBar::draw(State & state, const std::optional<s
|
||||||
return nextWakeup;
|
return nextWakeup;
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string ProgressBar::getStatus(State & state)
|
std::string getStatus(State & state)
|
||||||
{
|
{
|
||||||
constexpr auto MiB = 1024.0 * 1024.0;
|
auto MiB = 1024.0 * 1024.0;
|
||||||
|
|
||||||
std::string res;
|
std::string res;
|
||||||
|
|
||||||
auto renderActivity = [&](ActivityType type, const std::string & itemFmt, const std::string & numberFmt = "%d", double unit = 1) {
|
auto renderActivity = [&](ActivityType type, const std::string & itemFmt, const std::string & numberFmt = "%d", double unit = 1) {
|
||||||
auto & act = state.activitiesByType[type];
|
auto & act = state.activitiesByType[type];
|
||||||
uint64_t done = act.done, expected = act.done, running = 0, failed = act.failed;
|
uint64_t done = act.done, expected = act.done, running = 0, failed = act.failed;
|
||||||
for (auto & [actId, infoIt] : act.its) {
|
for (auto & j : act.its) {
|
||||||
done += infoIt->done;
|
done += j.second->done;
|
||||||
expected += infoIt->expected;
|
expected += j.second->expected;
|
||||||
running += infoIt->running;
|
running += j.second->running;
|
||||||
failed += infoIt->failed;
|
failed += j.second->failed;
|
||||||
}
|
}
|
||||||
|
|
||||||
expected = std::max(expected, act.expected);
|
expected = std::max(expected, act.expected);
|
||||||
|
@ -474,7 +545,7 @@ std::string ProgressBar::getStatus(State & state)
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
void ProgressBar::writeToStdout(std::string_view s)
|
void writeToStdout(std::string_view s) override
|
||||||
{
|
{
|
||||||
auto state(state_.lock());
|
auto state(state_.lock());
|
||||||
if (state->active) {
|
if (state->active) {
|
||||||
|
@ -485,7 +556,7 @@ void ProgressBar::writeToStdout(std::string_view s)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
std::optional<char> ProgressBar::ask(std::string_view msg)
|
std::optional<char> ask(std::string_view msg) override
|
||||||
{
|
{
|
||||||
auto state(state_.lock());
|
auto state(state_.lock());
|
||||||
if (!state->active || !isatty(STDIN_FILENO)) return {};
|
if (!state->active || !isatty(STDIN_FILENO)) return {};
|
||||||
|
@ -496,15 +567,16 @@ std::optional<char> ProgressBar::ask(std::string_view msg)
|
||||||
return s[0];
|
return s[0];
|
||||||
}
|
}
|
||||||
|
|
||||||
void ProgressBar::setPrintBuildLogs(bool printBuildLogs)
|
void setPrintBuildLogs(bool printBuildLogs) override
|
||||||
{
|
{
|
||||||
this->printBuildLogs = printBuildLogs;
|
this->printBuildLogs = printBuildLogs;
|
||||||
}
|
}
|
||||||
|
|
||||||
void ProgressBar::setPrintMultiline(bool printMultiline)
|
void setPrintMultiline(bool printMultiline) override
|
||||||
{
|
{
|
||||||
this->printMultiline = printMultiline;
|
this->printMultiline = printMultiline;
|
||||||
}
|
}
|
||||||
|
};
|
||||||
|
|
||||||
Logger * makeProgressBar()
|
Logger * makeProgressBar()
|
||||||
{
|
{
|
||||||
|
|
|
@ -1,135 +1,10 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
///@file
|
///@file
|
||||||
|
|
||||||
#include <chrono>
|
|
||||||
|
|
||||||
#include "logging.hh"
|
#include "logging.hh"
|
||||||
#include "sync.hh"
|
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
// 100 years ought to be enough for anyone (yet sufficiently smaller than max() to not cause signed integer overflow).
|
|
||||||
constexpr const auto A_LONG_TIME = std::chrono::duration_cast<std::chrono::milliseconds>(
|
|
||||||
100 * 365 * std::chrono::seconds(86400)
|
|
||||||
);
|
|
||||||
|
|
||||||
struct ProgressBar : public Logger
|
|
||||||
{
|
|
||||||
struct ActInfo
|
|
||||||
{
|
|
||||||
using TimePoint = std::chrono::time_point<std::chrono::steady_clock>;
|
|
||||||
|
|
||||||
std::string s, lastLine, phase;
|
|
||||||
ActivityType type = actUnknown;
|
|
||||||
uint64_t done = 0;
|
|
||||||
uint64_t expected = 0;
|
|
||||||
uint64_t running = 0;
|
|
||||||
uint64_t failed = 0;
|
|
||||||
std::map<ActivityType, uint64_t> expectedByType;
|
|
||||||
bool visible = true;
|
|
||||||
ActivityId parent;
|
|
||||||
std::optional<std::string> name;
|
|
||||||
TimePoint startTime;
|
|
||||||
};
|
|
||||||
|
|
||||||
struct ActivitiesByType
|
|
||||||
{
|
|
||||||
std::map<ActivityId, std::list<ActInfo>::iterator> its;
|
|
||||||
uint64_t done = 0;
|
|
||||||
uint64_t expected = 0;
|
|
||||||
uint64_t failed = 0;
|
|
||||||
};
|
|
||||||
|
|
||||||
struct State
|
|
||||||
{
|
|
||||||
std::list<ActInfo> activities;
|
|
||||||
std::map<ActivityId, std::list<ActInfo>::iterator> its;
|
|
||||||
|
|
||||||
std::map<ActivityType, ActivitiesByType> activitiesByType;
|
|
||||||
|
|
||||||
int lastLines = 0;
|
|
||||||
|
|
||||||
uint64_t filesLinked = 0, bytesLinked = 0;
|
|
||||||
|
|
||||||
uint64_t corruptedPaths = 0, untrustedPaths = 0;
|
|
||||||
|
|
||||||
bool active = true;
|
|
||||||
bool paused = false;
|
|
||||||
bool haveUpdate = true;
|
|
||||||
};
|
|
||||||
|
|
||||||
Sync<State> state_;
|
|
||||||
|
|
||||||
std::thread updateThread;
|
|
||||||
|
|
||||||
std::condition_variable quitCV, updateCV;
|
|
||||||
|
|
||||||
bool printBuildLogs = false;
|
|
||||||
bool printMultiline = false;
|
|
||||||
bool isTTY;
|
|
||||||
|
|
||||||
ProgressBar(bool isTTY)
|
|
||||||
: isTTY(isTTY)
|
|
||||||
{
|
|
||||||
state_.lock()->active = isTTY;
|
|
||||||
updateThread = std::thread([&]() {
|
|
||||||
auto state(state_.lock());
|
|
||||||
auto nextWakeup = A_LONG_TIME;
|
|
||||||
while (state->active) {
|
|
||||||
if (!state->haveUpdate)
|
|
||||||
state.wait_for(updateCV, nextWakeup);
|
|
||||||
nextWakeup = draw(*state, {});
|
|
||||||
state.wait_for(quitCV, std::chrono::milliseconds(50));
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
~ProgressBar();
|
|
||||||
|
|
||||||
void stop() override final;
|
|
||||||
|
|
||||||
void pause() override;
|
|
||||||
|
|
||||||
void resume() override;
|
|
||||||
|
|
||||||
bool isVerbose() override;
|
|
||||||
|
|
||||||
void log(Verbosity lvl, std::string_view s) override;
|
|
||||||
|
|
||||||
void logEI(const ErrorInfo & ei) override;
|
|
||||||
|
|
||||||
void log(State & state, Verbosity lvl, std::string_view s);
|
|
||||||
|
|
||||||
void startActivity(
|
|
||||||
ActivityId act,
|
|
||||||
Verbosity lvl,
|
|
||||||
ActivityType type,
|
|
||||||
const std::string & s,
|
|
||||||
const Fields & fields,
|
|
||||||
ActivityId parent
|
|
||||||
) override;
|
|
||||||
|
|
||||||
bool hasAncestor(State & state, ActivityType type, ActivityId act);
|
|
||||||
|
|
||||||
void stopActivity(ActivityId act) override;
|
|
||||||
|
|
||||||
void result(ActivityId act, ResultType type, const std::vector<Field> & fields) override;
|
|
||||||
|
|
||||||
void update(State & state);
|
|
||||||
|
|
||||||
std::chrono::milliseconds draw(State & state, const std::optional<std::string_view> & s);
|
|
||||||
|
|
||||||
std::string getStatus(State & state);
|
|
||||||
|
|
||||||
void writeToStdout(std::string_view s) override;
|
|
||||||
|
|
||||||
std::optional<char> ask(std::string_view msg) override;
|
|
||||||
|
|
||||||
void setPrintBuildLogs(bool printBuildLogs) override;
|
|
||||||
|
|
||||||
void setPrintMultiline(bool printMultiline) override;
|
|
||||||
};
|
|
||||||
|
|
||||||
Logger * makeProgressBar();
|
Logger * makeProgressBar();
|
||||||
|
|
||||||
void startProgressBar();
|
void startProgressBar();
|
||||||
|
|
|
@ -38,7 +38,7 @@ void BinaryCacheStore::init()
|
||||||
{
|
{
|
||||||
std::string cacheInfoFile = "nix-cache-info";
|
std::string cacheInfoFile = "nix-cache-info";
|
||||||
|
|
||||||
auto cacheInfo = getFileContents(cacheInfoFile);
|
auto cacheInfo = getFile(cacheInfoFile);
|
||||||
if (!cacheInfo) {
|
if (!cacheInfo) {
|
||||||
upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info");
|
upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info");
|
||||||
} else {
|
} else {
|
||||||
|
@ -69,10 +69,10 @@ void BinaryCacheStore::upsertFile(const std::string & path,
|
||||||
|
|
||||||
void BinaryCacheStore::getFile(const std::string & path, Sink & sink)
|
void BinaryCacheStore::getFile(const std::string & path, Sink & sink)
|
||||||
{
|
{
|
||||||
sink(*getFileContents(path));
|
sink(*getFile(path));
|
||||||
}
|
}
|
||||||
|
|
||||||
std::optional<std::string> BinaryCacheStore::getFileContents(const std::string & path)
|
std::optional<std::string> BinaryCacheStore::getFile(const std::string & path)
|
||||||
{
|
{
|
||||||
StringSink sink;
|
StringSink sink;
|
||||||
try {
|
try {
|
||||||
|
@ -359,7 +359,7 @@ std::shared_ptr<const ValidPathInfo> BinaryCacheStore::queryPathInfoUncached(con
|
||||||
|
|
||||||
auto narInfoFile = narInfoFileFor(storePath);
|
auto narInfoFile = narInfoFileFor(storePath);
|
||||||
|
|
||||||
auto data = getFileContents(narInfoFile);
|
auto data = getFile(narInfoFile);
|
||||||
|
|
||||||
if (!data) return nullptr;
|
if (!data) return nullptr;
|
||||||
|
|
||||||
|
@ -385,7 +385,7 @@ StorePath BinaryCacheStore::addToStore(
|
||||||
if (method == FileIngestionMethod::Recursive) {
|
if (method == FileIngestionMethod::Recursive) {
|
||||||
dumpPath(srcPath, sink, filter);
|
dumpPath(srcPath, sink, filter);
|
||||||
} else {
|
} else {
|
||||||
readFileSource(srcPath)->drainInto(sink);
|
readFile(srcPath, sink);
|
||||||
}
|
}
|
||||||
auto h = sink.finish().first;
|
auto h = sink.finish().first;
|
||||||
|
|
||||||
|
@ -446,7 +446,7 @@ std::shared_ptr<const Realisation> BinaryCacheStore::queryRealisationUncached(co
|
||||||
{
|
{
|
||||||
auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi";
|
auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi";
|
||||||
|
|
||||||
auto data = getFileContents(outputInfoFilePath);
|
auto data = getFile(outputInfoFilePath);
|
||||||
if (!data) return {};
|
if (!data) return {};
|
||||||
|
|
||||||
auto realisation = Realisation::fromJSON(
|
auto realisation = Realisation::fromJSON(
|
||||||
|
@ -486,7 +486,7 @@ std::optional<std::string> BinaryCacheStore::getBuildLogExact(const StorePath &
|
||||||
|
|
||||||
debug("fetching build log from binary cache '%s/%s'", getUri(), logPath);
|
debug("fetching build log from binary cache '%s/%s'", getUri(), logPath);
|
||||||
|
|
||||||
return getFileContents(logPath);
|
return getFile(logPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
void BinaryCacheStore::addBuildLog(const StorePath & drvPath, std::string_view log)
|
void BinaryCacheStore::addBuildLog(const StorePath & drvPath, std::string_view log)
|
||||||
|
|
|
@ -85,7 +85,7 @@ public:
|
||||||
*/
|
*/
|
||||||
virtual void getFile(const std::string & path, Sink & sink);
|
virtual void getFile(const std::string & path, Sink & sink);
|
||||||
|
|
||||||
virtual std::optional<std::string> getFileContents(const std::string & path);
|
virtual std::optional<std::string> getFile(const std::string & path);
|
||||||
|
|
||||||
public:
|
public:
|
||||||
|
|
||||||
|
|
|
@ -2981,7 +2981,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
|
||||||
HashModuloSink caSink { outputHash.hashType, oldHashPart };
|
HashModuloSink caSink { outputHash.hashType, oldHashPart };
|
||||||
std::visit(overloaded {
|
std::visit(overloaded {
|
||||||
[&](const TextIngestionMethod &) {
|
[&](const TextIngestionMethod &) {
|
||||||
readFileSource(actualPath)->drainInto(caSink);
|
readFile(actualPath, caSink);
|
||||||
},
|
},
|
||||||
[&](const FileIngestionMethod & m2) {
|
[&](const FileIngestionMethod & m2) {
|
||||||
switch (m2) {
|
switch (m2) {
|
||||||
|
@ -2989,7 +2989,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
|
||||||
dumpPath(actualPath, caSink);
|
dumpPath(actualPath, caSink);
|
||||||
break;
|
break;
|
||||||
case FileIngestionMethod::Flat:
|
case FileIngestionMethod::Flat:
|
||||||
readFileSource(actualPath)->drainInto(caSink);
|
readFile(actualPath, caSink);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
|
@ -41,7 +41,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
|
||||||
|
|
||||||
auto decompressor = makeDecompressionSink(
|
auto decompressor = makeDecompressionSink(
|
||||||
unpack && mainUrl.ends_with(".xz") ? "xz" : "none", sink);
|
unpack && mainUrl.ends_with(".xz") ? "xz" : "none", sink);
|
||||||
fileTransfer->download(std::move(request))->drainInto(*decompressor);
|
fileTransfer->download(std::move(request), *decompressor);
|
||||||
decompressor->finish();
|
decompressor->finish();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
|
@ -296,6 +296,17 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
case WorkerProto::Op::HasSubstitutes: {
|
||||||
|
auto path = store->parseStorePath(readString(from));
|
||||||
|
logger->startWork();
|
||||||
|
StorePathSet paths; // FIXME
|
||||||
|
paths.insert(path);
|
||||||
|
auto res = store->querySubstitutablePaths(paths);
|
||||||
|
logger->stopWork();
|
||||||
|
to << (res.count(path) != 0);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
case WorkerProto::Op::QuerySubstitutablePaths: {
|
case WorkerProto::Op::QuerySubstitutablePaths: {
|
||||||
auto paths = WorkerProto::Serialise<StorePathSet>::read(*store, rconn);
|
auto paths = WorkerProto::Serialise<StorePathSet>::read(*store, rconn);
|
||||||
logger->startWork();
|
logger->startWork();
|
||||||
|
@ -305,74 +316,36 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
case WorkerProto::Op::HasSubstitutes: {
|
|
||||||
throw UnimplementedError("HasSubstitutes is not supported in Lix. This is not used if the declared server protocol is > 1.12 (Nix 1.0, 2012)");
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
case WorkerProto::Op::QueryPathHash: {
|
case WorkerProto::Op::QueryPathHash: {
|
||||||
throw UnimplementedError("QueryPathHash is not supported in Lix, client usages were removed in 2016 in e0204f8d462041387651af388074491fd0bf36d6");
|
auto path = store->parseStorePath(readString(from));
|
||||||
break;
|
logger->startWork();
|
||||||
}
|
auto hash = store->queryPathInfo(path)->narHash;
|
||||||
|
logger->stopWork();
|
||||||
case WorkerProto::Op::QueryReferences: {
|
to << hash.to_string(Base16, false);
|
||||||
throw UnimplementedError("QueryReferences is not supported in Lix, client usages were removed in 2016 in e0204f8d462041387651af388074491fd0bf36d6");
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
case WorkerProto::Op::QueryDeriver: {
|
|
||||||
throw UnimplementedError("QueryDeriver is not supported in Lix, client usages were removed in 2016 in e0204f8d462041387651af388074491fd0bf36d6");
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
case WorkerProto::Op::ExportPath: {
|
|
||||||
throw UnimplementedError("ExportPath is not supported in Lix, client usage were removed in 2017 in 27dc76c1a5dbe654465245ff5f6bc22e2c8902da");
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
case WorkerProto::Op::ImportPaths: {
|
|
||||||
throw UnimplementedError("ImportPaths is not supported in Lix. This is not used if the declared server protocol is >= 1.18 (Nix 2.0, 2016)");
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
case WorkerProto::Op::QueryReferences:
|
||||||
case WorkerProto::Op::QueryReferrers:
|
case WorkerProto::Op::QueryReferrers:
|
||||||
case WorkerProto::Op::QueryValidDerivers:
|
case WorkerProto::Op::QueryValidDerivers:
|
||||||
case WorkerProto::Op::QueryDerivationOutputs: {
|
case WorkerProto::Op::QueryDerivationOutputs: {
|
||||||
auto path = store->parseStorePath(readString(from));
|
auto path = store->parseStorePath(readString(from));
|
||||||
logger->startWork();
|
logger->startWork();
|
||||||
StorePathSet paths;
|
StorePathSet paths;
|
||||||
|
if (op == WorkerProto::Op::QueryReferences)
|
||||||
#pragma GCC diagnostic push
|
for (auto & i : store->queryPathInfo(path)->references)
|
||||||
#pragma GCC diagnostic ignored "-Wswitch-enum"
|
paths.insert(i);
|
||||||
switch (op) {
|
else if (op == WorkerProto::Op::QueryReferrers)
|
||||||
case WorkerProto::Op::QueryReferrers: {
|
|
||||||
store->queryReferrers(path, paths);
|
store->queryReferrers(path, paths);
|
||||||
break;
|
else if (op == WorkerProto::Op::QueryValidDerivers)
|
||||||
}
|
|
||||||
case WorkerProto::Op::QueryValidDerivers: {
|
|
||||||
paths = store->queryValidDerivers(path);
|
paths = store->queryValidDerivers(path);
|
||||||
break;
|
else paths = store->queryDerivationOutputs(path);
|
||||||
}
|
|
||||||
case WorkerProto::Op::QueryDerivationOutputs: {
|
|
||||||
// Only sent if server presents proto version <= 1.21
|
|
||||||
REMOVE_AFTER_DROPPING_PROTO_MINOR(21);
|
|
||||||
paths = store->queryDerivationOutputs(path);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
abort();
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
#pragma GCC diagnostic pop
|
|
||||||
|
|
||||||
logger->stopWork();
|
logger->stopWork();
|
||||||
WorkerProto::write(*store, wconn, paths);
|
WorkerProto::write(*store, wconn, paths);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
case WorkerProto::Op::QueryDerivationOutputNames: {
|
case WorkerProto::Op::QueryDerivationOutputNames: {
|
||||||
// Unused in CppNix >= 2.4 (removed in 045b07200c77bf1fe19c0a986aafb531e7e1ba54)
|
|
||||||
REMOVE_AFTER_DROPPING_PROTO_MINOR(31);
|
|
||||||
auto path = store->parseStorePath(readString(from));
|
auto path = store->parseStorePath(readString(from));
|
||||||
logger->startWork();
|
logger->startWork();
|
||||||
auto names = store->readDerivation(path).outputNames();
|
auto names = store->readDerivation(path).outputNames();
|
||||||
|
@ -390,6 +363,15 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
case WorkerProto::Op::QueryDeriver: {
|
||||||
|
auto path = store->parseStorePath(readString(from));
|
||||||
|
logger->startWork();
|
||||||
|
auto info = store->queryPathInfo(path);
|
||||||
|
logger->stopWork();
|
||||||
|
to << (info->deriver ? store->printStorePath(*info->deriver) : "");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
case WorkerProto::Op::QueryPathFromHashPart: {
|
case WorkerProto::Op::QueryPathFromHashPart: {
|
||||||
auto hashPart = readString(from);
|
auto hashPart = readString(from);
|
||||||
logger->startWork();
|
logger->startWork();
|
||||||
|
@ -511,6 +493,29 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
case WorkerProto::Op::ExportPath: {
|
||||||
|
auto path = store->parseStorePath(readString(from));
|
||||||
|
readInt(from); // obsolete
|
||||||
|
logger->startWork();
|
||||||
|
TunnelSink sink(to);
|
||||||
|
store->exportPath(path, sink);
|
||||||
|
logger->stopWork();
|
||||||
|
to << 1;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
case WorkerProto::Op::ImportPaths: {
|
||||||
|
logger->startWork();
|
||||||
|
TunnelSource source(from, to);
|
||||||
|
auto paths = store->importPaths(source,
|
||||||
|
trusted ? NoCheckSigs : CheckSigs);
|
||||||
|
logger->stopWork();
|
||||||
|
Strings paths2;
|
||||||
|
for (auto & i : paths) paths2.push_back(store->printStorePath(i));
|
||||||
|
to << paths2;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
case WorkerProto::Op::BuildPaths: {
|
case WorkerProto::Op::BuildPaths: {
|
||||||
auto drvs = WorkerProto::Serialise<DerivedPaths>::read(*store, rconn);
|
auto drvs = WorkerProto::Serialise<DerivedPaths>::read(*store, rconn);
|
||||||
BuildMode mode = buildModeFromInteger(readInt(from));
|
BuildMode mode = buildModeFromInteger(readInt(from));
|
||||||
|
@ -661,10 +666,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Obsolete since 9947f1646a26b339fff2e02b77798e9841fac7f0 (included in CppNix 2.5.0).
|
// Obsolete.
|
||||||
case WorkerProto::Op::SyncWithGC: {
|
case WorkerProto::Op::SyncWithGC: {
|
||||||
// CppNix 2.5.0 is 32
|
|
||||||
REMOVE_AFTER_DROPPING_PROTO_MINOR(31);
|
|
||||||
logger->startWork();
|
logger->startWork();
|
||||||
logger->stopWork();
|
logger->stopWork();
|
||||||
to << 1;
|
to << 1;
|
||||||
|
|
|
@ -686,8 +686,16 @@ struct curlFileTransfer : public FileTransfer
|
||||||
->callback.get_future();
|
->callback.get_future();
|
||||||
}
|
}
|
||||||
|
|
||||||
box_ptr<Source> download(FileTransferRequest && request) override
|
void download(FileTransferRequest && request, Sink & sink) override
|
||||||
{
|
{
|
||||||
|
/* Note: we can't call 'sink' via request.dataCallback, because
|
||||||
|
that would cause the sink to execute on the fileTransfer
|
||||||
|
thread. If 'sink' is a coroutine, this will fail. Also, if the
|
||||||
|
sink is expensive (e.g. one that does decompression and writing
|
||||||
|
to the Nix store), it would stall the download thread too much.
|
||||||
|
Therefore we use a buffer to communicate data between the
|
||||||
|
download thread and the calling thread. */
|
||||||
|
|
||||||
struct State {
|
struct State {
|
||||||
bool done = false, failed = false;
|
bool done = false, failed = false;
|
||||||
std::exception_ptr exc;
|
std::exception_ptr exc;
|
||||||
|
@ -697,6 +705,13 @@ struct curlFileTransfer : public FileTransfer
|
||||||
|
|
||||||
auto _state = std::make_shared<Sync<State>>();
|
auto _state = std::make_shared<Sync<State>>();
|
||||||
|
|
||||||
|
/* In case of an exception, wake up the download thread. */
|
||||||
|
Finally finally([&]() {
|
||||||
|
auto state(_state->lock());
|
||||||
|
state->failed |= std::uncaught_exceptions() != 0;
|
||||||
|
state->request.notify_one();
|
||||||
|
});
|
||||||
|
|
||||||
enqueueFileTransfer(
|
enqueueFileTransfer(
|
||||||
request,
|
request,
|
||||||
[_state](std::exception_ptr ex) {
|
[_state](std::exception_ptr ex) {
|
||||||
|
@ -735,99 +750,50 @@ struct curlFileTransfer : public FileTransfer
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
||||||
struct InnerSource : Source
|
std::unique_ptr<FinishSink> decompressor;
|
||||||
{
|
|
||||||
const std::shared_ptr<Sync<State>> _state;
|
while (true) {
|
||||||
|
checkInterrupt();
|
||||||
|
|
||||||
std::string chunk;
|
std::string chunk;
|
||||||
std::string_view buffered;
|
|
||||||
|
|
||||||
explicit InnerSource(const std::shared_ptr<Sync<State>> & state) : _state(state) {}
|
|
||||||
|
|
||||||
~InnerSource()
|
|
||||||
{
|
|
||||||
// wake up the download thread if it's still going and have it abort
|
|
||||||
auto state(_state->lock());
|
|
||||||
state->failed |= !state->done;
|
|
||||||
state->request.notify_one();
|
|
||||||
}
|
|
||||||
|
|
||||||
void awaitData(Sync<State>::Lock & state)
|
|
||||||
{
|
|
||||||
/* Grab data if available, otherwise wait for the download
|
/* Grab data if available, otherwise wait for the download
|
||||||
thread to wake us up. */
|
thread to wake us up. */
|
||||||
while (buffered.empty()) {
|
{
|
||||||
|
auto state(_state->lock());
|
||||||
|
|
||||||
if (state->data.empty()) {
|
if (state->data.empty()) {
|
||||||
|
|
||||||
if (state->done) {
|
if (state->done) {
|
||||||
if (state->exc) {
|
if (state->exc) std::rethrow_exception(state->exc);
|
||||||
std::rethrow_exception(state->exc);
|
if (decompressor) {
|
||||||
|
decompressor->finish();
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
state.wait(state->avail);
|
state.wait(state->avail);
|
||||||
|
|
||||||
|
if (state->data.empty()) continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
chunk = std::move(state->data);
|
chunk = std::move(state->data);
|
||||||
buffered = chunk;
|
/* Reset state->data after the move, since we check data.empty() */
|
||||||
state->request.notify_one();
|
state->data = "";
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
size_t read(char * data, size_t len) override
|
|
||||||
{
|
|
||||||
auto readPartial = [this](char * data, size_t len) {
|
|
||||||
const auto available = std::min(len, buffered.size());
|
|
||||||
memcpy(data, buffered.data(), available);
|
|
||||||
buffered.remove_prefix(available);
|
|
||||||
return available;
|
|
||||||
};
|
|
||||||
size_t total = readPartial(data, len);
|
|
||||||
|
|
||||||
while (total < len) {
|
|
||||||
{
|
|
||||||
auto state(_state->lock());
|
|
||||||
awaitData(state);
|
|
||||||
}
|
|
||||||
const auto current = readPartial(data + total, len - total);
|
|
||||||
total += current;
|
|
||||||
if (total == 0 || current == 0) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (total == 0) {
|
|
||||||
throw EndOfFile("download finished");
|
|
||||||
}
|
|
||||||
|
|
||||||
return total;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
struct DownloadSource : Source
|
|
||||||
{
|
|
||||||
InnerSource inner;
|
|
||||||
std::unique_ptr<Source> decompressor;
|
|
||||||
|
|
||||||
explicit DownloadSource(const std::shared_ptr<Sync<State>> & state) : inner(state) {}
|
|
||||||
|
|
||||||
size_t read(char * data, size_t len) override
|
|
||||||
{
|
|
||||||
checkInterrupt();
|
|
||||||
|
|
||||||
if (!decompressor) {
|
if (!decompressor) {
|
||||||
auto state(inner._state->lock());
|
decompressor = makeDecompressionSink(state->encoding, sink);
|
||||||
inner.awaitData(state);
|
|
||||||
decompressor = makeDecompressionSource(state->encoding, inner);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return decompressor->read(data, len);
|
state->request.notify_one();
|
||||||
}
|
}
|
||||||
};
|
|
||||||
|
|
||||||
auto source = make_box_ptr<DownloadSource>(_state);
|
/* Flush the data to the sink and wake up the download thread
|
||||||
auto lock(_state->lock());
|
if it's blocked on a full buffer. We don't hold the state
|
||||||
source->inner.awaitData(lock);
|
lock while doing this to prevent blocking the download
|
||||||
return source;
|
thread if sink() takes a long time. */
|
||||||
|
(*decompressor)(chunk);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -1,7 +1,6 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
///@file
|
///@file
|
||||||
|
|
||||||
#include "box_ptr.hh"
|
|
||||||
#include "logging.hh"
|
#include "logging.hh"
|
||||||
#include "serialise.hh"
|
#include "serialise.hh"
|
||||||
#include "types.hh"
|
#include "types.hh"
|
||||||
|
@ -105,13 +104,10 @@ struct FileTransfer
|
||||||
FileTransferResult transfer(const FileTransferRequest & request);
|
FileTransferResult transfer(const FileTransferRequest & request);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Download a file, returning its contents through a source. Will not return
|
* Download a file, writing its data to a sink. The sink will be
|
||||||
* before the transfer has fully started, ensuring that any errors thrown by
|
* invoked on the thread of the caller.
|
||||||
* the setup phase (e.g. HTTP 404 or similar errors) are not postponed to be
|
|
||||||
* thrown by the returned source. The source will only throw errors detected
|
|
||||||
* during the transfer itself (decompression errors, connection drops, etc).
|
|
||||||
*/
|
*/
|
||||||
virtual box_ptr<Source> download(FileTransferRequest && request) = 0;
|
virtual void download(FileTransferRequest && request, Sink & sink) = 0;
|
||||||
|
|
||||||
enum Error { NotFound, Forbidden, Misc, Transient, Interrupted };
|
enum Error { NotFound, Forbidden, Misc, Transient, Interrupted };
|
||||||
};
|
};
|
||||||
|
|
|
@ -155,7 +155,7 @@ protected:
|
||||||
checkEnabled();
|
checkEnabled();
|
||||||
auto request(makeRequest(path));
|
auto request(makeRequest(path));
|
||||||
try {
|
try {
|
||||||
getFileTransfer()->download(std::move(request))->drainInto(sink);
|
getFileTransfer()->download(std::move(request), sink);
|
||||||
} catch (FileTransferError & e) {
|
} catch (FileTransferError & e) {
|
||||||
if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
|
if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
|
||||||
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
|
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
|
||||||
|
@ -164,7 +164,7 @@ protected:
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
std::optional<std::string> getFileContents(const std::string & path) override
|
std::optional<std::string> getFile(const std::string & path) override
|
||||||
{
|
{
|
||||||
checkEnabled();
|
checkEnabled();
|
||||||
|
|
||||||
|
|
|
@ -71,7 +71,7 @@ protected:
|
||||||
void getFile(const std::string & path, Sink & sink) override
|
void getFile(const std::string & path, Sink & sink) override
|
||||||
{
|
{
|
||||||
try {
|
try {
|
||||||
readFileSource(binaryCacheDir + "/" + path)->drainInto(sink);
|
readFile(binaryCacheDir + "/" + path, sink);
|
||||||
} catch (SysError & e) {
|
} catch (SysError & e) {
|
||||||
if (e.errNo == ENOENT)
|
if (e.errNo == ENOENT)
|
||||||
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache", path);
|
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache", path);
|
||||||
|
|
|
@ -1890,7 +1890,7 @@ ContentAddress LocalStore::hashCAPath(
|
||||||
HashModuloSink caSink ( hashType, std::string(pathHash) );
|
HashModuloSink caSink ( hashType, std::string(pathHash) );
|
||||||
std::visit(overloaded {
|
std::visit(overloaded {
|
||||||
[&](const TextIngestionMethod &) {
|
[&](const TextIngestionMethod &) {
|
||||||
readFileSource(path)->drainInto(caSink);
|
readFile(path, caSink);
|
||||||
},
|
},
|
||||||
[&](const FileIngestionMethod & m2) {
|
[&](const FileIngestionMethod & m2) {
|
||||||
switch (m2) {
|
switch (m2) {
|
||||||
|
@ -1898,7 +1898,7 @@ ContentAddress LocalStore::hashCAPath(
|
||||||
dumpPath(path, caSink);
|
dumpPath(path, caSink);
|
||||||
break;
|
break;
|
||||||
case FileIngestionMethod::Flat:
|
case FileIngestionMethod::Flat:
|
||||||
readFileSource(path)->drainInto(caSink);
|
readFile(path, caSink);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
|
@ -307,7 +307,6 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
|
||||||
if (GET_PROTOCOL_MINOR(getProtocol()) >= 22) {
|
if (GET_PROTOCOL_MINOR(getProtocol()) >= 22) {
|
||||||
return Store::queryDerivationOutputs(path);
|
return Store::queryDerivationOutputs(path);
|
||||||
}
|
}
|
||||||
REMOVE_AFTER_DROPPING_PROTO_MINOR(21);
|
|
||||||
auto conn(getConnection());
|
auto conn(getConnection());
|
||||||
conn->to << WorkerProto::Op::QueryDerivationOutputs << printStorePath(path);
|
conn->to << WorkerProto::Op::QueryDerivationOutputs << printStorePath(path);
|
||||||
conn.processStderr();
|
conn.processStderr();
|
||||||
|
@ -337,7 +336,6 @@ std::map<std::string, std::optional<StorePath>> RemoteStore::queryPartialDerivat
|
||||||
return outputs;
|
return outputs;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
REMOVE_AFTER_DROPPING_PROTO_MINOR(21);
|
|
||||||
auto & evalStore = evalStore_ ? *evalStore_ : *this;
|
auto & evalStore = evalStore_ ? *evalStore_ : *this;
|
||||||
// Fallback for old daemon versions.
|
// Fallback for old daemon versions.
|
||||||
// For floating-CA derivations (and their co-dependencies) this is an
|
// For floating-CA derivations (and their co-dependencies) this is an
|
||||||
|
@ -532,7 +530,6 @@ void RemoteStore::registerDrvOutput(const Realisation & info)
|
||||||
auto conn(getConnection());
|
auto conn(getConnection());
|
||||||
conn->to << WorkerProto::Op::RegisterDrvOutput;
|
conn->to << WorkerProto::Op::RegisterDrvOutput;
|
||||||
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) {
|
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) {
|
||||||
REMOVE_AFTER_DROPPING_PROTO_MINOR(30);
|
|
||||||
conn->to << info.id.to_string();
|
conn->to << info.id.to_string();
|
||||||
conn->to << std::string(info.outPath.to_string());
|
conn->to << std::string(info.outPath.to_string());
|
||||||
} else {
|
} else {
|
||||||
|
@ -620,7 +617,6 @@ std::vector<KeyedBuildResult> RemoteStore::buildPathsWithResults(
|
||||||
conn.processStderr();
|
conn.processStderr();
|
||||||
return WorkerProto::Serialise<std::vector<KeyedBuildResult>>::read(*this, *conn);
|
return WorkerProto::Serialise<std::vector<KeyedBuildResult>>::read(*this, *conn);
|
||||||
} else {
|
} else {
|
||||||
REMOVE_AFTER_DROPPING_PROTO_MINOR(33);
|
|
||||||
// Avoid deadlock.
|
// Avoid deadlock.
|
||||||
conn_.reset();
|
conn_.reset();
|
||||||
|
|
||||||
|
|
|
@ -8,11 +8,6 @@ namespace nix {
|
||||||
#define SERVE_MAGIC_1 0x390c9deb
|
#define SERVE_MAGIC_1 0x390c9deb
|
||||||
#define SERVE_MAGIC_2 0x5452eecb
|
#define SERVE_MAGIC_2 0x5452eecb
|
||||||
|
|
||||||
// This must remain at 2.7 (Nix 2.18) forever in Lix, since the protocol
|
|
||||||
// versioning is monotonic, so if we ever change it in the future, it will
|
|
||||||
// break compatibility with any potential CppNix-originated protocol changes.
|
|
||||||
//
|
|
||||||
// Lix intends to replace this protocol entirely.
|
|
||||||
#define SERVE_PROTOCOL_VERSION (2 << 8 | 7)
|
#define SERVE_PROTOCOL_VERSION (2 << 8 | 7)
|
||||||
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
|
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
|
||||||
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
|
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
|
||||||
|
|
|
@ -279,7 +279,7 @@ StorePath Store::addToStore(
|
||||||
if (method == FileIngestionMethod::Recursive)
|
if (method == FileIngestionMethod::Recursive)
|
||||||
dumpPath(srcPath, sink, filter);
|
dumpPath(srcPath, sink, filter);
|
||||||
else
|
else
|
||||||
readFileSource(srcPath)->drainInto(sink);
|
readFile(srcPath, sink);
|
||||||
});
|
});
|
||||||
return addToStoreFromDump(*source, name, method, hashAlgo, repair, references);
|
return addToStoreFromDump(*source, name, method, hashAlgo, repair, references);
|
||||||
}
|
}
|
||||||
|
@ -803,31 +803,17 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m
|
||||||
|
|
||||||
auto doQuery = [&](const StorePath & path) {
|
auto doQuery = [&](const StorePath & path) {
|
||||||
checkInterrupt();
|
checkInterrupt();
|
||||||
|
auto state(state_.lock());
|
||||||
bool exists = false;
|
|
||||||
std::exception_ptr newExc{};
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
queryPathInfo(path);
|
auto info = queryPathInfo(path);
|
||||||
exists = true;
|
state->valid.insert(path);
|
||||||
} catch (InvalidPath &) {
|
} catch (InvalidPath &) {
|
||||||
} catch (...) {
|
} catch (...) {
|
||||||
newExc = std::current_exception();
|
state->exc = std::current_exception();
|
||||||
}
|
|
||||||
|
|
||||||
{
|
|
||||||
auto state(state_.lock());
|
|
||||||
|
|
||||||
if (exists) {
|
|
||||||
state->valid.insert(path);
|
|
||||||
}
|
|
||||||
if (newExc != nullptr) {
|
|
||||||
state->exc = newExc;
|
|
||||||
}
|
}
|
||||||
assert(state->left);
|
assert(state->left);
|
||||||
if (!--state->left)
|
if (!--state->left)
|
||||||
wakeup.notify_one();
|
wakeup.notify_one();
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
for (auto & path : paths)
|
for (auto & path : paths)
|
||||||
|
|
|
@ -23,10 +23,6 @@ namespace nix {
|
||||||
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
|
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
|
||||||
|
|
||||||
|
|
||||||
#define REMOVE_AFTER_DROPPING_PROTO_MINOR(protoMinor) \
|
|
||||||
static_assert(MIN_SUPPORTED_MINOR_WORKER_PROTO_VERSION <= (protoMinor))
|
|
||||||
|
|
||||||
|
|
||||||
#define STDERR_NEXT 0x6f6c6d67
|
#define STDERR_NEXT 0x6f6c6d67
|
||||||
#define STDERR_READ 0x64617461 // data needed from source
|
#define STDERR_READ 0x64617461 // data needed from source
|
||||||
#define STDERR_WRITE 0x64617416 // data for sink
|
#define STDERR_WRITE 0x64617416 // data for sink
|
||||||
|
@ -140,30 +136,30 @@ struct WorkerProto
|
||||||
enum struct WorkerProto::Op : uint64_t
|
enum struct WorkerProto::Op : uint64_t
|
||||||
{
|
{
|
||||||
IsValidPath = 1,
|
IsValidPath = 1,
|
||||||
HasSubstitutes = 3, // obsolete since 2012, stubbed to error
|
HasSubstitutes = 3,
|
||||||
QueryPathHash = 4, // obsolete since 2016, stubbed to error
|
QueryPathHash = 4, // obsolete
|
||||||
QueryReferences = 5, // obsolete since 2016, stubbed to error
|
QueryReferences = 5, // obsolete
|
||||||
QueryReferrers = 6,
|
QueryReferrers = 6,
|
||||||
AddToStore = 7,
|
AddToStore = 7,
|
||||||
AddTextToStore = 8, // obsolete since protocol 1.25, CppNix 2.4. Use WorkerProto::Op::AddToStore
|
AddTextToStore = 8, // obsolete since 1.25, Nix 3.0. Use WorkerProto::Op::AddToStore
|
||||||
BuildPaths = 9,
|
BuildPaths = 9,
|
||||||
EnsurePath = 10,
|
EnsurePath = 10,
|
||||||
AddTempRoot = 11,
|
AddTempRoot = 11,
|
||||||
AddIndirectRoot = 12,
|
AddIndirectRoot = 12,
|
||||||
SyncWithGC = 13, // obsolete since CppNix 2.5.0
|
SyncWithGC = 13,
|
||||||
FindRoots = 14,
|
FindRoots = 14,
|
||||||
ExportPath = 16, // obsolete since 2017, stubbed to error
|
ExportPath = 16, // obsolete
|
||||||
QueryDeriver = 18, // obsolete since 2016, stubbed to error
|
QueryDeriver = 18, // obsolete
|
||||||
SetOptions = 19,
|
SetOptions = 19,
|
||||||
CollectGarbage = 20,
|
CollectGarbage = 20,
|
||||||
QuerySubstitutablePathInfo = 21,
|
QuerySubstitutablePathInfo = 21,
|
||||||
QueryDerivationOutputs = 22, // obsolete since protocol 1.21, CppNix 2.4
|
QueryDerivationOutputs = 22, // obsolete
|
||||||
QueryAllValidPaths = 23,
|
QueryAllValidPaths = 23,
|
||||||
QueryFailedPaths = 24, // obsolete, removed
|
QueryFailedPaths = 24,
|
||||||
ClearFailedPaths = 25, // obsolete, removed
|
ClearFailedPaths = 25,
|
||||||
QueryPathInfo = 26,
|
QueryPathInfo = 26,
|
||||||
ImportPaths = 27, // obsolete since 2016
|
ImportPaths = 27, // obsolete
|
||||||
QueryDerivationOutputNames = 28, // obsolete since CppNix 2.4
|
QueryDerivationOutputNames = 28, // obsolete
|
||||||
QueryPathFromHashPart = 29,
|
QueryPathFromHashPart = 29,
|
||||||
QuerySubstitutablePathInfos = 30,
|
QuerySubstitutablePathInfos = 30,
|
||||||
QueryValidPaths = 31,
|
QueryValidPaths = 31,
|
||||||
|
|
|
@ -163,15 +163,12 @@ struct BrotliDecompressionSource : Source
|
||||||
uint8_t * out = (uint8_t *) data;
|
uint8_t * out = (uint8_t *) data;
|
||||||
const auto * begin = out;
|
const auto * begin = out;
|
||||||
|
|
||||||
|
try {
|
||||||
while (len && !BrotliDecoderIsFinished(state.get())) {
|
while (len && !BrotliDecoderIsFinished(state.get())) {
|
||||||
checkInterrupt();
|
checkInterrupt();
|
||||||
|
|
||||||
while (avail_in == 0) {
|
while (avail_in == 0) {
|
||||||
try {
|
|
||||||
avail_in = inner->read(buf.get(), BUF_SIZE);
|
avail_in = inner->read(buf.get(), BUF_SIZE);
|
||||||
} catch (EndOfFile &) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
next_in = (const uint8_t *) buf.get();
|
next_in = (const uint8_t *) buf.get();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -182,6 +179,8 @@ struct BrotliDecompressionSource : Source
|
||||||
throw CompressionError("error while decompressing brotli file");
|
throw CompressionError("error while decompressing brotli file");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
} catch (EndOfFile &) {
|
||||||
|
}
|
||||||
|
|
||||||
if (begin != out) {
|
if (begin != out) {
|
||||||
return out - begin;
|
return out - begin;
|
||||||
|
@ -193,9 +192,11 @@ struct BrotliDecompressionSource : Source
|
||||||
|
|
||||||
std::string decompress(const std::string & method, std::string_view in)
|
std::string decompress(const std::string & method, std::string_view in)
|
||||||
{
|
{
|
||||||
StringSource src{in};
|
StringSink ssink;
|
||||||
auto filter = makeDecompressionSource(method, src);
|
auto sink = makeDecompressionSink(method, ssink);
|
||||||
return filter->drain();
|
(*sink)(in);
|
||||||
|
sink->finish();
|
||||||
|
return std::move(ssink.s);
|
||||||
}
|
}
|
||||||
|
|
||||||
std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Sink & nextSink)
|
std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Sink & nextSink)
|
||||||
|
@ -223,19 +224,6 @@ std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Si
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
std::unique_ptr<Source> makeDecompressionSource(const std::string & method, Source & inner)
|
|
||||||
{
|
|
||||||
if (method == "none" || method == "") {
|
|
||||||
return std::make_unique<LambdaSource>([&](char * data, size_t len) {
|
|
||||||
return inner.read(data, len);
|
|
||||||
});
|
|
||||||
} else if (method == "br") {
|
|
||||||
return std::make_unique<BrotliDecompressionSource>(inner);
|
|
||||||
} else {
|
|
||||||
return std::make_unique<ArchiveDecompressionSource>(inner);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
struct BrotliCompressionSink : ChunkedCompressionSink
|
struct BrotliCompressionSink : ChunkedCompressionSink
|
||||||
{
|
{
|
||||||
Sink & nextSink;
|
Sink & nextSink;
|
||||||
|
|
|
@ -19,7 +19,6 @@ struct CompressionSink : BufferedSink, FinishSink
|
||||||
std::string decompress(const std::string & method, std::string_view in);
|
std::string decompress(const std::string & method, std::string_view in);
|
||||||
|
|
||||||
std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Sink & nextSink);
|
std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Sink & nextSink);
|
||||||
std::unique_ptr<Source> makeDecompressionSource(const std::string & method, Source & inner);
|
|
||||||
|
|
||||||
std::string compress(const std::string & method, std::string_view in, const bool parallel = false, int level = -1);
|
std::string compress(const std::string & method, std::string_view in, const bool parallel = false, int level = -1);
|
||||||
|
|
||||||
|
|
|
@ -289,17 +289,12 @@ std::string readFile(const Path & path)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
box_ptr<Source> readFileSource(const Path & path)
|
void readFile(const Path & path, Sink & sink)
|
||||||
{
|
{
|
||||||
AutoCloseFD fd{open(path.c_str(), O_RDONLY | O_CLOEXEC)};
|
AutoCloseFD fd{open(path.c_str(), O_RDONLY | O_CLOEXEC)};
|
||||||
if (!fd)
|
if (!fd)
|
||||||
throw SysError("opening file '%s'", path);
|
throw SysError("opening file '%s'", path);
|
||||||
|
drainFD(fd.get(), sink);
|
||||||
struct FileSource : FdSource {
|
|
||||||
AutoCloseFD fd;
|
|
||||||
explicit FileSource(AutoCloseFD fd) : FdSource(fd.get()), fd(std::move(fd)) {}
|
|
||||||
};
|
|
||||||
return make_box_ptr<FileSource>(std::move(fd));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -5,7 +5,6 @@
|
||||||
* Utiltities for working with the file sytem and file paths.
|
* Utiltities for working with the file sytem and file paths.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#include "box_ptr.hh"
|
|
||||||
#include "types.hh"
|
#include "types.hh"
|
||||||
#include "file-descriptor.hh"
|
#include "file-descriptor.hh"
|
||||||
|
|
||||||
|
@ -143,7 +142,7 @@ unsigned char getFileType(const Path & path);
|
||||||
* Read the contents of a file into a string.
|
* Read the contents of a file into a string.
|
||||||
*/
|
*/
|
||||||
std::string readFile(const Path & path);
|
std::string readFile(const Path & path);
|
||||||
box_ptr<Source> readFileSource(const Path & path);
|
void readFile(const Path & path, Sink & sink);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Write a string to a file.
|
* Write a string to a file.
|
||||||
|
|
|
@ -324,7 +324,7 @@ Hash hashString(HashType ht, std::string_view s)
|
||||||
Hash hashFile(HashType ht, const Path & path)
|
Hash hashFile(HashType ht, const Path & path)
|
||||||
{
|
{
|
||||||
HashSink sink(ht);
|
HashSink sink(ht);
|
||||||
readFileSource(path)->drainInto(sink);
|
readFile(path, sink);
|
||||||
return sink.finish().first;
|
return sink.finish().first;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -37,7 +37,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand
|
||||||
Hash hash = narHash;
|
Hash hash = narHash;
|
||||||
if (ingestionMethod == FileIngestionMethod::Flat) {
|
if (ingestionMethod == FileIngestionMethod::Flat) {
|
||||||
HashSink hsink(htSHA256);
|
HashSink hsink(htSHA256);
|
||||||
readFileSource(path)->drainInto(hsink);
|
readFile(path, hsink);
|
||||||
hash = hsink.finish().first;
|
hash = hsink.finish().first;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
#include "installable-flake.hh"
|
#include "installable-flake.hh"
|
||||||
#include "command.hh"
|
#include "command-installable-value.hh"
|
||||||
#include "common-args.hh"
|
#include "common-args.hh"
|
||||||
#include "shared.hh"
|
#include "shared.hh"
|
||||||
#include "store-api.hh"
|
#include "store-api.hh"
|
||||||
|
@ -9,7 +9,7 @@
|
||||||
|
|
||||||
using namespace nix;
|
using namespace nix;
|
||||||
|
|
||||||
struct CmdBundle : InstallableCommand
|
struct CmdBundle : InstallableValueCommand
|
||||||
{
|
{
|
||||||
std::string bundler = "github:NixOS/bundlers";
|
std::string bundler = "github:NixOS/bundlers";
|
||||||
std::optional<Path> outLink;
|
std::optional<Path> outLink;
|
||||||
|
@ -71,13 +71,11 @@ struct CmdBundle : InstallableCommand
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
void run(ref<Store> store, ref<Installable> installable) override
|
void run(ref<Store> store, ref<InstallableValue> installable) override
|
||||||
{
|
{
|
||||||
auto evalState = getEvalState();
|
auto evalState = getEvalState();
|
||||||
|
|
||||||
auto const installableValue = InstallableValue::require(installable);
|
auto val = installable->toValue(*evalState).first;
|
||||||
|
|
||||||
auto val = installableValue->toValue(*evalState).first;
|
|
||||||
|
|
||||||
auto [bundlerFlakeRef, bundlerName, extendedOutputsSpec] = parseFlakeRefWithFragmentAndExtendedOutputsSpec(bundler, absPath("."));
|
auto [bundlerFlakeRef, bundlerName, extendedOutputsSpec] = parseFlakeRefWithFragmentAndExtendedOutputsSpec(bundler, absPath("."));
|
||||||
const flake::LockFlags lockFlags{ .writeLockFile = false };
|
const flake::LockFlags lockFlags{ .writeLockFile = false };
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
#include "eval.hh"
|
#include "eval.hh"
|
||||||
#include "installable-flake.hh"
|
#include "installable-flake.hh"
|
||||||
#include "command.hh"
|
#include "command-installable-value.hh"
|
||||||
#include "common-args.hh"
|
#include "common-args.hh"
|
||||||
#include "shared.hh"
|
#include "shared.hh"
|
||||||
#include "store-api.hh"
|
#include "store-api.hh"
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
#include "command.hh"
|
#include "command-installable-value.hh"
|
||||||
#include "shared.hh"
|
#include "shared.hh"
|
||||||
#include "eval.hh"
|
#include "eval.hh"
|
||||||
#include "attr-path.hh"
|
#include "attr-path.hh"
|
||||||
|
@ -10,7 +10,7 @@
|
||||||
|
|
||||||
using namespace nix;
|
using namespace nix;
|
||||||
|
|
||||||
struct CmdEdit : InstallableCommand
|
struct CmdEdit : InstallableValueCommand
|
||||||
{
|
{
|
||||||
std::string description() override
|
std::string description() override
|
||||||
{
|
{
|
||||||
|
@ -26,19 +26,17 @@ struct CmdEdit : InstallableCommand
|
||||||
|
|
||||||
Category category() override { return catSecondary; }
|
Category category() override { return catSecondary; }
|
||||||
|
|
||||||
void run(ref<Store> store, ref<Installable> installable) override
|
void run(ref<Store> store, ref<InstallableValue> installable) override
|
||||||
{
|
{
|
||||||
auto state = getEvalState();
|
auto state = getEvalState();
|
||||||
|
|
||||||
auto const installableValue = InstallableValue::require(installable);
|
|
||||||
|
|
||||||
const auto [file, line] = [&] {
|
const auto [file, line] = [&] {
|
||||||
auto [v, pos] = installableValue->toValue(*state);
|
auto [v, pos] = installable->toValue(*state);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
return findPackageFilename(*state, *v, installable->what());
|
return findPackageFilename(*state, *v, installable->what());
|
||||||
} catch (NoPositionInfo &) {
|
} catch (NoPositionInfo &) {
|
||||||
throw Error("cannot find position information for '%s", installableValue->what());
|
throw Error("cannot find position information for '%s", installable->what());
|
||||||
}
|
}
|
||||||
}();
|
}();
|
||||||
|
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
#include "command.hh"
|
#include "command-installable-value.hh"
|
||||||
#include "common-args.hh"
|
#include "common-args.hh"
|
||||||
#include "print-options.hh"
|
#include "print-options.hh"
|
||||||
#include "shared.hh"
|
#include "shared.hh"
|
||||||
|
@ -12,13 +12,13 @@
|
||||||
|
|
||||||
using namespace nix;
|
using namespace nix;
|
||||||
|
|
||||||
struct CmdEval : MixJSON, InstallableCommand, MixReadOnlyOption
|
struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption
|
||||||
{
|
{
|
||||||
bool raw = false;
|
bool raw = false;
|
||||||
std::optional<std::string> apply;
|
std::optional<std::string> apply;
|
||||||
std::optional<Path> writeTo;
|
std::optional<Path> writeTo;
|
||||||
|
|
||||||
CmdEval() : InstallableCommand()
|
CmdEval() : InstallableValueCommand()
|
||||||
{
|
{
|
||||||
addFlag({
|
addFlag({
|
||||||
.longName = "raw",
|
.longName = "raw",
|
||||||
|
@ -55,16 +55,14 @@ struct CmdEval : MixJSON, InstallableCommand, MixReadOnlyOption
|
||||||
|
|
||||||
Category category() override { return catSecondary; }
|
Category category() override { return catSecondary; }
|
||||||
|
|
||||||
void run(ref<Store> store, ref<Installable> installable) override
|
void run(ref<Store> store, ref<InstallableValue> installable) override
|
||||||
{
|
{
|
||||||
if (raw && json)
|
if (raw && json)
|
||||||
throw UsageError("--raw and --json are mutually exclusive");
|
throw UsageError("--raw and --json are mutually exclusive");
|
||||||
|
|
||||||
auto const installableValue = InstallableValue::require(installable);
|
|
||||||
|
|
||||||
auto state = getEvalState();
|
auto state = getEvalState();
|
||||||
|
|
||||||
auto [v, pos] = installableValue->toValue(*state);
|
auto [v, pos] = installable->toValue(*state);
|
||||||
NixStringContext context;
|
NixStringContext context;
|
||||||
|
|
||||||
if (apply) {
|
if (apply) {
|
||||||
|
|
|
@ -85,7 +85,7 @@ struct CmdHashBase : Command
|
||||||
|
|
||||||
switch (mode) {
|
switch (mode) {
|
||||||
case FileIngestionMethod::Flat:
|
case FileIngestionMethod::Flat:
|
||||||
readFileSource(path)->drainInto(*hashSink);
|
readFile(path, *hashSink);
|
||||||
break;
|
break;
|
||||||
case FileIngestionMethod::Recursive:
|
case FileIngestionMethod::Recursive:
|
||||||
dumpPath(path, *hashSink);
|
dumpPath(path, *hashSink);
|
||||||
|
|
|
@ -98,7 +98,7 @@ std::tuple<StorePath, Hash> prefetchFile(
|
||||||
FdSink sink(fd.get());
|
FdSink sink(fd.get());
|
||||||
|
|
||||||
FileTransferRequest req(url);
|
FileTransferRequest req(url);
|
||||||
getFileTransfer()->download(std::move(req))->drainInto(sink);
|
getFileTransfer()->download(std::move(req), sink);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Optionally unpack the file. */
|
/* Optionally unpack the file. */
|
||||||
|
|
|
@ -1,7 +1,6 @@
|
||||||
#include "run.hh"
|
#include "run.hh"
|
||||||
#include "command.hh"
|
#include "command-installable-value.hh"
|
||||||
#include "common-args.hh"
|
#include "common-args.hh"
|
||||||
#include "installables.hh"
|
|
||||||
#include "shared.hh"
|
#include "shared.hh"
|
||||||
#include "store-api.hh"
|
#include "store-api.hh"
|
||||||
#include "derivations.hh"
|
#include "derivations.hh"
|
||||||
|
@ -146,7 +145,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment
|
||||||
|
|
||||||
static auto rCmdShell = registerCommand<CmdShell>("shell");
|
static auto rCmdShell = registerCommand<CmdShell>("shell");
|
||||||
|
|
||||||
struct CmdRun : InstallableCommand
|
struct CmdRun : InstallableValueCommand
|
||||||
{
|
{
|
||||||
using InstallableCommand::run;
|
using InstallableCommand::run;
|
||||||
|
|
||||||
|
@ -192,14 +191,12 @@ struct CmdRun : InstallableCommand
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
void run(ref<Store> store, ref<Installable> installable) override
|
void run(ref<Store> store, ref<InstallableValue> installable) override
|
||||||
{
|
{
|
||||||
auto state = getEvalState();
|
auto state = getEvalState();
|
||||||
|
|
||||||
auto installableValue = InstallableValue::require(installable);
|
|
||||||
|
|
||||||
lockFlags.applyNixConfig = true;
|
lockFlags.applyNixConfig = true;
|
||||||
auto app = installableValue->toApp(*state).resolve(getEvalStore(), store);
|
auto app = installable->toApp(*state).resolve(getEvalStore(), store);
|
||||||
|
|
||||||
Strings allArgs{app.program};
|
Strings allArgs{app.program};
|
||||||
for (auto & i : args) allArgs.push_back(i);
|
for (auto & i : args) allArgs.push_back(i);
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
#include "command.hh"
|
#include "command-installable-value.hh"
|
||||||
#include "globals.hh"
|
#include "globals.hh"
|
||||||
#include "eval.hh"
|
#include "eval.hh"
|
||||||
#include "eval-inline.hh"
|
#include "eval-inline.hh"
|
||||||
|
@ -23,7 +23,7 @@ std::string wrap(std::string prefix, std::string s)
|
||||||
return concatStrings(prefix, s, ANSI_NORMAL);
|
return concatStrings(prefix, s, ANSI_NORMAL);
|
||||||
}
|
}
|
||||||
|
|
||||||
struct CmdSearch : InstallableCommand, MixJSON
|
struct CmdSearch : InstallableValueCommand, MixJSON
|
||||||
{
|
{
|
||||||
std::vector<std::string> res;
|
std::vector<std::string> res;
|
||||||
std::vector<std::string> excludeRes;
|
std::vector<std::string> excludeRes;
|
||||||
|
@ -62,10 +62,8 @@ struct CmdSearch : InstallableCommand, MixJSON
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
void run(ref<Store> store, ref<Installable> installable) override
|
void run(ref<Store> store, ref<InstallableValue> installable) override
|
||||||
{
|
{
|
||||||
auto const installableValue = InstallableValue::require(installable);
|
|
||||||
|
|
||||||
settings.readOnlyMode = true;
|
settings.readOnlyMode = true;
|
||||||
evalSettings.enableImportFromDerivation.setDefault(false);
|
evalSettings.enableImportFromDerivation.setDefault(false);
|
||||||
|
|
||||||
|
@ -194,7 +192,7 @@ struct CmdSearch : InstallableCommand, MixJSON
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
for (auto & cursor : installableValue->getCursors(*state))
|
for (auto & cursor : installable->getCursors(*state))
|
||||||
visit(*cursor, cursor->getAttrPath(), true);
|
visit(*cursor, cursor->getAttrPath(), true);
|
||||||
|
|
||||||
if (json)
|
if (json)
|
||||||
|
|
|
@ -1,43 +0,0 @@
|
||||||
#include <gtest/gtest.h>
|
|
||||||
|
|
||||||
#include "eval.hh"
|
|
||||||
#include "progress-bar.hh"
|
|
||||||
#include "logging.hh"
|
|
||||||
#include "shared.hh"
|
|
||||||
|
|
||||||
constexpr std::string_view TEST_URL = "https://github.com/NixOS/nixpkgs/archive/master.tar.gz";
|
|
||||||
// Arbitrary number. We picked the size of a Nixpkgs tarball that we downloaded.
|
|
||||||
constexpr uint64_t TEST_EXPECTED = 43'370'307;
|
|
||||||
// Arbitrary number. We picked the progress made on a Nixpkgs tarball download we interrupted.
|
|
||||||
constexpr uint64_t TEST_DONE = 1'787'251;
|
|
||||||
|
|
||||||
constexpr std::string_view EXPECTED = ANSI_GREEN "1.7" ANSI_NORMAL "/41.4 MiB DL";
|
|
||||||
// Mostly here for informational purposes, but also if we change the way the escape codes
|
|
||||||
// are defined this test might break in some annoying to debug way.
|
|
||||||
constexpr std::string_view EXPECTED_RAW = "\x1b[32;1m1.7\x1b[0m/41.4 MiB DL";
|
|
||||||
static_assert(EXPECTED == EXPECTED_RAW, "Hey, hey, the ANSI escape code definitions prolly changed");
|
|
||||||
|
|
||||||
namespace nix
|
|
||||||
{
|
|
||||||
TEST(ProgressBar, basicStatusRender) {
|
|
||||||
initNix();
|
|
||||||
initGC();
|
|
||||||
|
|
||||||
startProgressBar();
|
|
||||||
ASSERT_NE(dynamic_cast<ProgressBar *>(logger), nullptr);
|
|
||||||
ProgressBar & progressBar = dynamic_cast<ProgressBar &>(*logger);
|
|
||||||
|
|
||||||
Activity act(
|
|
||||||
progressBar,
|
|
||||||
lvlDebug,
|
|
||||||
actFileTransfer,
|
|
||||||
fmt("downloading '%s'", TEST_URL),
|
|
||||||
{ "https://github.com/NixOS/nixpkgs/archive/master.tar.gz" }
|
|
||||||
);
|
|
||||||
act.progress(TEST_DONE, TEST_EXPECTED);
|
|
||||||
auto state = progressBar.state_.lock();
|
|
||||||
std::string const renderedStatus = progressBar.getStatus(*state);
|
|
||||||
|
|
||||||
ASSERT_EQ(renderedStatus, EXPECTED);
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -7,7 +7,6 @@
|
||||||
#include <gtest/gtest.h>
|
#include <gtest/gtest.h>
|
||||||
#include <netinet/in.h>
|
#include <netinet/in.h>
|
||||||
#include <stdexcept>
|
#include <stdexcept>
|
||||||
#include <string>
|
|
||||||
#include <string_view>
|
#include <string_view>
|
||||||
#include <sys/poll.h>
|
#include <sys/poll.h>
|
||||||
#include <sys/socket.h>
|
#include <sys/socket.h>
|
||||||
|
@ -137,7 +136,7 @@ TEST(FileTransfer, exceptionAbortsDownload)
|
||||||
|
|
||||||
LambdaSink broken([](auto block) { throw Done(); });
|
LambdaSink broken([](auto block) { throw Done(); });
|
||||||
|
|
||||||
ASSERT_THROW(ft->download(FileTransferRequest("file:///dev/zero"))->drainInto(broken), Done);
|
ASSERT_THROW(ft->download(FileTransferRequest("file:///dev/zero"), broken), Done);
|
||||||
|
|
||||||
// makeFileTransfer returns a ref<>, which cannot be cleared. since we also
|
// makeFileTransfer returns a ref<>, which cannot be cleared. since we also
|
||||||
// can't default-construct it we'll have to overwrite it instead, but we'll
|
// can't default-construct it we'll have to overwrite it instead, but we'll
|
||||||
|
@ -160,21 +159,16 @@ TEST(FileTransfer, NOT_ON_DARWIN(reportsSetupErrors))
|
||||||
FileTransferError);
|
FileTransferError);
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST(FileTransfer, NOT_ON_DARWIN(defersFailures))
|
TEST(FileTransfer, NOT_ON_DARWIN(reportsTransferError))
|
||||||
{
|
{
|
||||||
auto [port, srv] = serveHTTP("200 ok", "content-length: 100000000\r\n", [] {
|
auto [port, srv] = serveHTTP("200 ok", "content-length: 100\r\n", [] {
|
||||||
std::this_thread::sleep_for(10ms);
|
std::this_thread::sleep_for(10ms);
|
||||||
// just a bunch of data to fill the curl wrapper buffer, otherwise the
|
return "";
|
||||||
// initial wait for header data will also wait for the the response to
|
|
||||||
// complete (the source is only woken when curl returns data, and curl
|
|
||||||
// might only do so once its internal buffer has already been filled.)
|
|
||||||
return std::string(1024 * 1024, ' ');
|
|
||||||
});
|
});
|
||||||
auto ft = makeFileTransfer();
|
auto ft = makeFileTransfer();
|
||||||
FileTransferRequest req(fmt("http://[::1]:%d/index", port));
|
FileTransferRequest req(fmt("http://[::1]:%d/index", port));
|
||||||
req.baseRetryTimeMs = 0;
|
req.baseRetryTimeMs = 0;
|
||||||
auto src = ft->download(std::move(req));
|
ASSERT_THROW(ft->transfer(req), FileTransferError);
|
||||||
ASSERT_THROW(src->drain(), FileTransferError);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST(FileTransfer, NOT_ON_DARWIN(handlesContentEncoding))
|
TEST(FileTransfer, NOT_ON_DARWIN(handlesContentEncoding))
|
||||||
|
@ -186,7 +180,7 @@ TEST(FileTransfer, NOT_ON_DARWIN(handlesContentEncoding))
|
||||||
auto ft = makeFileTransfer();
|
auto ft = makeFileTransfer();
|
||||||
|
|
||||||
StringSink sink;
|
StringSink sink;
|
||||||
ft->download(FileTransferRequest(fmt("http://[::1]:%d/index", port)))->drainInto(sink);
|
ft->download(FileTransferRequest(fmt("http://[::1]:%d/index", port)), sink);
|
||||||
EXPECT_EQ(sink.s, original);
|
EXPECT_EQ(sink.s, original);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -66,16 +66,6 @@ namespace nix {
|
||||||
ASSERT_THROW(decompress(method, str), CompressionError);
|
ASSERT_THROW(decompress(method, str), CompressionError);
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST(decompress, veryLongBrotli) {
|
|
||||||
auto method = "br";
|
|
||||||
auto str = std::string(65536, 'a');
|
|
||||||
auto o = decompress(method, compress(method, str));
|
|
||||||
|
|
||||||
// This is just to not print 64k of "a" for most failures
|
|
||||||
ASSERT_EQ(o.length(), str.length());
|
|
||||||
ASSERT_EQ(o, str);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ----------------------------------------------------------------------------
|
/* ----------------------------------------------------------------------------
|
||||||
* compression sinks
|
* compression sinks
|
||||||
* --------------------------------------------------------------------------*/
|
* --------------------------------------------------------------------------*/
|
||||||
|
@ -91,17 +81,16 @@ namespace nix {
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST(makeCompressionSink, compressAndDecompress) {
|
TEST(makeCompressionSink, compressAndDecompress) {
|
||||||
auto inputString = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
|
|
||||||
|
|
||||||
StringSink strSink;
|
StringSink strSink;
|
||||||
auto sink = makeCompressionSink("bzip2", strSink);
|
auto inputString = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
|
||||||
|
auto decompressionSink = makeDecompressionSink("bzip2", strSink);
|
||||||
|
auto sink = makeCompressionSink("bzip2", *decompressionSink);
|
||||||
|
|
||||||
(*sink)(inputString);
|
(*sink)(inputString);
|
||||||
sink->finish();
|
sink->finish();
|
||||||
|
decompressionSink->finish();
|
||||||
|
|
||||||
StringSource strSource{strSink.s};
|
ASSERT_STREQ(strSink.s.c_str(), inputString);
|
||||||
auto decompressionSource = makeDecompressionSource("bzip2", strSource);
|
|
||||||
|
|
||||||
ASSERT_STREQ(decompressionSource->drain().c_str(), inputString);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -249,25 +249,3 @@ test(
|
||||||
suite : 'check',
|
suite : 'check',
|
||||||
protocol : 'gtest',
|
protocol : 'gtest',
|
||||||
)
|
)
|
||||||
|
|
||||||
libmain_tester = executable(
|
|
||||||
'liblixmain-tests',
|
|
||||||
files('libmain/progress-bar.cc'),
|
|
||||||
dependencies : [
|
|
||||||
liblixmain,
|
|
||||||
liblixexpr,
|
|
||||||
liblixutil,
|
|
||||||
liblixstore,
|
|
||||||
gtest,
|
|
||||||
boost,
|
|
||||||
],
|
|
||||||
cpp_pch : cpp_pch,
|
|
||||||
)
|
|
||||||
|
|
||||||
test(
|
|
||||||
'libmain-unit-tests',
|
|
||||||
libmain_tester,
|
|
||||||
args : tests_args,
|
|
||||||
suite : 'check',
|
|
||||||
protocol : 'gtest',
|
|
||||||
)
|
|
||||||
|
|
Loading…
Reference in a new issue