nix optimise-store: Show how much space has been freed
This commit is contained in:
parent
23b8b7e096
commit
b4ed97e3a3
|
@ -3257,7 +3257,7 @@ void DerivationGoal::flushLine()
|
||||||
logTail.push_back(currentLogLine);
|
logTail.push_back(currentLogLine);
|
||||||
if (logTail.size() > settings.logLines) logTail.pop_front();
|
if (logTail.size() > settings.logLines) logTail.pop_front();
|
||||||
}
|
}
|
||||||
act->progress(currentLogLine);
|
act->result(resBuildLogLine, currentLogLine);
|
||||||
currentLogLine = "";
|
currentLogLine = "";
|
||||||
currentLogLinePos = 0;
|
currentLogLinePos = 0;
|
||||||
}
|
}
|
||||||
|
|
|
@ -251,7 +251,7 @@ private:
|
||||||
|
|
||||||
InodeHash loadInodeHash();
|
InodeHash loadInodeHash();
|
||||||
Strings readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash);
|
Strings readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash);
|
||||||
void optimisePath_(OptimiseStats & stats, const Path & path, InodeHash & inodeHash);
|
void optimisePath_(Activity * act, OptimiseStats & stats, const Path & path, InodeHash & inodeHash);
|
||||||
|
|
||||||
// Internal versions that are not wrapped in retry_sqlite.
|
// Internal versions that are not wrapped in retry_sqlite.
|
||||||
bool isValidPath_(State & state, const Path & path);
|
bool isValidPath_(State & state, const Path & path);
|
||||||
|
|
|
@ -89,7 +89,8 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHash & inodeHash)
|
void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
|
||||||
|
const Path & path, InodeHash & inodeHash)
|
||||||
{
|
{
|
||||||
checkInterrupt();
|
checkInterrupt();
|
||||||
|
|
||||||
|
@ -114,7 +115,7 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa
|
||||||
if (S_ISDIR(st.st_mode)) {
|
if (S_ISDIR(st.st_mode)) {
|
||||||
Strings names = readDirectoryIgnoringInodes(path, inodeHash);
|
Strings names = readDirectoryIgnoringInodes(path, inodeHash);
|
||||||
for (auto & i : names)
|
for (auto & i : names)
|
||||||
optimisePath_(stats, path + "/" + i, inodeHash);
|
optimisePath_(act, stats, path + "/" + i, inodeHash);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -244,6 +245,9 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa
|
||||||
stats.filesLinked++;
|
stats.filesLinked++;
|
||||||
stats.bytesFreed += st.st_size;
|
stats.bytesFreed += st.st_size;
|
||||||
stats.blocksFreed += st.st_blocks;
|
stats.blocksFreed += st.st_blocks;
|
||||||
|
|
||||||
|
if (act)
|
||||||
|
act->result(resFileLinked, st.st_size, st.st_blocks);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -263,7 +267,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
|
||||||
if (!isValidPath(i)) continue; /* path was GC'ed, probably */
|
if (!isValidPath(i)) continue; /* path was GC'ed, probably */
|
||||||
{
|
{
|
||||||
Activity act(*logger, actUnknown, fmt("optimising path '%s'", i));
|
Activity act(*logger, actUnknown, fmt("optimising path '%s'", i));
|
||||||
optimisePath_(stats, realStoreDir + "/" + baseNameOf(i), inodeHash);
|
optimisePath_(&act, stats, realStoreDir + "/" + baseNameOf(i), inodeHash);
|
||||||
}
|
}
|
||||||
done++;
|
done++;
|
||||||
act.progress(done, paths.size());
|
act.progress(done, paths.size());
|
||||||
|
@ -281,7 +285,7 @@ void LocalStore::optimiseStore()
|
||||||
|
|
||||||
optimiseStore(stats);
|
optimiseStore(stats);
|
||||||
|
|
||||||
printError(
|
printInfo(
|
||||||
format("%1% freed by hard-linking %2% files")
|
format("%1% freed by hard-linking %2% files")
|
||||||
% showBytes(stats.bytesFreed)
|
% showBytes(stats.bytesFreed)
|
||||||
% stats.filesLinked);
|
% stats.filesLinked);
|
||||||
|
@ -292,7 +296,7 @@ void LocalStore::optimisePath(const Path & path)
|
||||||
OptimiseStats stats;
|
OptimiseStats stats;
|
||||||
InodeHash inodeHash;
|
InodeHash inodeHash;
|
||||||
|
|
||||||
if (settings.autoOptimiseStore) optimisePath_(stats, path, inodeHash);
|
if (settings.autoOptimiseStore) optimisePath_(nullptr, stats, path, inodeHash);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -24,6 +24,11 @@ typedef enum {
|
||||||
actOptimiseStore = 106,
|
actOptimiseStore = 106,
|
||||||
} ActivityType;
|
} ActivityType;
|
||||||
|
|
||||||
|
typedef enum {
|
||||||
|
resFileLinked = 100,
|
||||||
|
resBuildLogLine = 101,
|
||||||
|
} ResultType;
|
||||||
|
|
||||||
typedef uint64_t ActivityId;
|
typedef uint64_t ActivityId;
|
||||||
|
|
||||||
class Logger
|
class Logger
|
||||||
|
@ -49,9 +54,20 @@ public:
|
||||||
|
|
||||||
virtual void progress(ActivityId act, uint64_t done = 0, uint64_t expected = 0, uint64_t running = 0, uint64_t failed = 0) { };
|
virtual void progress(ActivityId act, uint64_t done = 0, uint64_t expected = 0, uint64_t running = 0, uint64_t failed = 0) { };
|
||||||
|
|
||||||
virtual void progress(ActivityId act, const std::string & s) { };
|
|
||||||
|
|
||||||
virtual void setExpected(ActivityId act, ActivityType type, uint64_t expected) { };
|
virtual void setExpected(ActivityId act, ActivityType type, uint64_t expected) { };
|
||||||
|
|
||||||
|
struct Field
|
||||||
|
{
|
||||||
|
// FIXME: use std::variant.
|
||||||
|
enum { tInt, tString } type;
|
||||||
|
uint64_t i = 0;
|
||||||
|
std::string s;
|
||||||
|
Field(const std::string & s) : type(tString), s(s) { }
|
||||||
|
Field(const char * s) : type(tString), s(s) { }
|
||||||
|
Field(const uint64_t & i) : type(tInt), i(i) { }
|
||||||
|
};
|
||||||
|
|
||||||
|
virtual void result(ActivityId act, ResultType type, const std::vector<Field> & fields) { };
|
||||||
};
|
};
|
||||||
|
|
||||||
struct Activity
|
struct Activity
|
||||||
|
@ -68,12 +84,17 @@ struct Activity
|
||||||
void progress(uint64_t done = 0, uint64_t expected = 0, uint64_t running = 0, uint64_t failed = 0) const
|
void progress(uint64_t done = 0, uint64_t expected = 0, uint64_t running = 0, uint64_t failed = 0) const
|
||||||
{ logger.progress(id, done, expected, running, failed); }
|
{ logger.progress(id, done, expected, running, failed); }
|
||||||
|
|
||||||
void progress(const std::string & s) const
|
|
||||||
{ logger.progress(id, s); }
|
|
||||||
|
|
||||||
void setExpected(ActivityType type2, uint64_t expected) const
|
void setExpected(ActivityType type2, uint64_t expected) const
|
||||||
{ logger.setExpected(id, type2, expected); }
|
{ logger.setExpected(id, type2, expected); }
|
||||||
|
|
||||||
|
template<typename... Args>
|
||||||
|
void result(ResultType type, const Args & ... args)
|
||||||
|
{
|
||||||
|
std::vector<Logger::Field> fields;
|
||||||
|
nop{(fields.emplace_back(Logger::Field(args)), 1)...};
|
||||||
|
logger.result(id, type, fields);
|
||||||
|
}
|
||||||
|
|
||||||
friend class Logger;
|
friend class Logger;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -12,6 +12,20 @@
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
static std::string getS(const std::vector<Logger::Field> & fields, size_t n)
|
||||||
|
{
|
||||||
|
assert(n < fields.size());
|
||||||
|
assert(fields[n].type == Logger::Field::tString);
|
||||||
|
return fields[n].s;
|
||||||
|
}
|
||||||
|
|
||||||
|
static uint64_t getI(const std::vector<Logger::Field> & fields, size_t n)
|
||||||
|
{
|
||||||
|
assert(n < fields.size());
|
||||||
|
assert(fields[n].type == Logger::Field::tInt);
|
||||||
|
return fields[n].i;
|
||||||
|
}
|
||||||
|
|
||||||
class ProgressBar : public Logger
|
class ProgressBar : public Logger
|
||||||
{
|
{
|
||||||
private:
|
private:
|
||||||
|
@ -41,6 +55,8 @@ private:
|
||||||
std::map<ActivityId, std::list<ActInfo>::iterator> its;
|
std::map<ActivityId, std::list<ActInfo>::iterator> its;
|
||||||
|
|
||||||
std::map<ActivityType, ActivitiesByType> activitiesByType;
|
std::map<ActivityType, ActivitiesByType> activitiesByType;
|
||||||
|
|
||||||
|
uint64_t filesLinked = 0, bytesLinked = 0;
|
||||||
};
|
};
|
||||||
|
|
||||||
Sync<State> state_;
|
Sync<State> state_;
|
||||||
|
@ -80,14 +96,33 @@ public:
|
||||||
void startActivity(ActivityId act, ActivityType type, const std::string & s) override
|
void startActivity(ActivityId act, ActivityType type, const std::string & s) override
|
||||||
{
|
{
|
||||||
auto state(state_.lock());
|
auto state(state_.lock());
|
||||||
createActivity(*state, act, s, type);
|
|
||||||
|
state->activities.emplace_back(ActInfo{s, "", type});
|
||||||
|
auto i = std::prev(state->activities.end());
|
||||||
|
state->its.emplace(act, i);
|
||||||
|
state->activitiesByType[type].its.emplace(act, i);
|
||||||
|
|
||||||
update(*state);
|
update(*state);
|
||||||
}
|
}
|
||||||
|
|
||||||
void stopActivity(ActivityId act) override
|
void stopActivity(ActivityId act) override
|
||||||
{
|
{
|
||||||
auto state(state_.lock());
|
auto state(state_.lock());
|
||||||
deleteActivity(*state, act);
|
|
||||||
|
auto i = state->its.find(act);
|
||||||
|
if (i != state->its.end()) {
|
||||||
|
auto & actByType = state->activitiesByType[i->second->type];
|
||||||
|
actByType.done += i->second->done;
|
||||||
|
actByType.failed += i->second->failed;
|
||||||
|
|
||||||
|
for (auto & j : i->second->expectedByType)
|
||||||
|
state->activitiesByType[j.first].expected -= j.second;
|
||||||
|
|
||||||
|
actByType.its.erase(act);
|
||||||
|
state->activities.erase(i->second);
|
||||||
|
state->its.erase(i);
|
||||||
|
}
|
||||||
|
|
||||||
update(*state);
|
update(*state);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -106,16 +141,6 @@ public:
|
||||||
update(*state);
|
update(*state);
|
||||||
}
|
}
|
||||||
|
|
||||||
void progress(ActivityId act, const std::string & s) override
|
|
||||||
{
|
|
||||||
auto state(state_.lock());
|
|
||||||
auto s2 = trim(s);
|
|
||||||
if (!s2.empty()) {
|
|
||||||
updateActivity(*state, act, s2);
|
|
||||||
update(*state);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void setExpected(ActivityId act, ActivityType type, uint64_t expected) override
|
void setExpected(ActivityId act, ActivityType type, uint64_t expected) override
|
||||||
{
|
{
|
||||||
auto state(state_.lock());
|
auto state(state_.lock());
|
||||||
|
@ -131,40 +156,29 @@ public:
|
||||||
update(*state);
|
update(*state);
|
||||||
}
|
}
|
||||||
|
|
||||||
void createActivity(State & state, ActivityId activity, const std::string & s, ActivityType type = actUnknown)
|
void result(ActivityId act, ResultType type, const std::vector<Field> & fields) override
|
||||||
{
|
{
|
||||||
state.activities.emplace_back(ActInfo{s, "", type});
|
auto state(state_.lock());
|
||||||
auto i = std::prev(state.activities.end());
|
|
||||||
state.its.emplace(activity, i);
|
|
||||||
state.activitiesByType[type].its.emplace(activity, i);
|
|
||||||
}
|
|
||||||
|
|
||||||
void deleteActivity(State & state, ActivityId activity)
|
if (type == resFileLinked) {
|
||||||
{
|
state->filesLinked++;
|
||||||
auto i = state.its.find(activity);
|
state->bytesLinked += getI(fields, 0);
|
||||||
if (i != state.its.end()) {
|
update(*state);
|
||||||
auto & act = state.activitiesByType[i->second->type];
|
|
||||||
act.done += i->second->done;
|
|
||||||
act.failed += i->second->failed;
|
|
||||||
|
|
||||||
for (auto & j : i->second->expectedByType)
|
|
||||||
state.activitiesByType[j.first].expected -= j.second;
|
|
||||||
|
|
||||||
act.its.erase(activity);
|
|
||||||
state.activities.erase(i->second);
|
|
||||||
state.its.erase(i);
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
void updateActivity(State & state, ActivityId activity, const std::string & s2)
|
else if (type == resBuildLogLine) {
|
||||||
{
|
auto s2 = trim(getS(fields, 0));
|
||||||
auto i = state.its.find(activity);
|
if (!s2.empty()) {
|
||||||
assert(i != state.its.end());
|
auto i = state->its.find(act);
|
||||||
ActInfo info = *i->second;
|
assert(i != state->its.end());
|
||||||
state.activities.erase(i->second);
|
ActInfo info = *i->second;
|
||||||
info.s2 = s2;
|
state->activities.erase(i->second);
|
||||||
state.activities.emplace_back(info);
|
info.s2 = s2;
|
||||||
i->second = std::prev(state.activities.end());
|
state->activities.emplace_back(info);
|
||||||
|
i->second = std::prev(state->activities.end());
|
||||||
|
update(*state);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void update()
|
void update()
|
||||||
|
@ -262,7 +276,14 @@ public:
|
||||||
|
|
||||||
showActivity(actDownload, "%s MiB DL", "%.1f", MiB);
|
showActivity(actDownload, "%s MiB DL", "%.1f", MiB);
|
||||||
|
|
||||||
showActivity(actOptimiseStore, "%s paths optimised");
|
{
|
||||||
|
auto s = renderActivity(actOptimiseStore, "%s paths optimised");
|
||||||
|
if (s != "") {
|
||||||
|
s += fmt(", %.1f MiB / %d inodes freed", state.bytesLinked / MiB, state.filesLinked);
|
||||||
|
if (!res.empty()) res += ", ";
|
||||||
|
res += s;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue