nix optimise-store: Show how much space has been freed

This commit is contained in:
Eelco Dolstra 2017-08-16 17:32:18 +02:00
parent 23b8b7e096
commit b4ed97e3a3
No known key found for this signature in database
GPG key ID: 8170B4726D7198DE
5 changed files with 101 additions and 55 deletions

View file

@ -3257,7 +3257,7 @@ void DerivationGoal::flushLine()
logTail.push_back(currentLogLine); logTail.push_back(currentLogLine);
if (logTail.size() > settings.logLines) logTail.pop_front(); if (logTail.size() > settings.logLines) logTail.pop_front();
} }
act->progress(currentLogLine); act->result(resBuildLogLine, currentLogLine);
currentLogLine = ""; currentLogLine = "";
currentLogLinePos = 0; currentLogLinePos = 0;
} }

View file

@ -251,7 +251,7 @@ private:
InodeHash loadInodeHash(); InodeHash loadInodeHash();
Strings readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash); Strings readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash);
void optimisePath_(OptimiseStats & stats, const Path & path, InodeHash & inodeHash); void optimisePath_(Activity * act, OptimiseStats & stats, const Path & path, InodeHash & inodeHash);
// Internal versions that are not wrapped in retry_sqlite. // Internal versions that are not wrapped in retry_sqlite.
bool isValidPath_(State & state, const Path & path); bool isValidPath_(State & state, const Path & path);

View file

@ -89,7 +89,8 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa
} }
void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHash & inodeHash) void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
const Path & path, InodeHash & inodeHash)
{ {
checkInterrupt(); checkInterrupt();
@ -114,7 +115,7 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa
if (S_ISDIR(st.st_mode)) { if (S_ISDIR(st.st_mode)) {
Strings names = readDirectoryIgnoringInodes(path, inodeHash); Strings names = readDirectoryIgnoringInodes(path, inodeHash);
for (auto & i : names) for (auto & i : names)
optimisePath_(stats, path + "/" + i, inodeHash); optimisePath_(act, stats, path + "/" + i, inodeHash);
return; return;
} }
@ -244,6 +245,9 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa
stats.filesLinked++; stats.filesLinked++;
stats.bytesFreed += st.st_size; stats.bytesFreed += st.st_size;
stats.blocksFreed += st.st_blocks; stats.blocksFreed += st.st_blocks;
if (act)
act->result(resFileLinked, st.st_size, st.st_blocks);
} }
@ -263,7 +267,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
if (!isValidPath(i)) continue; /* path was GC'ed, probably */ if (!isValidPath(i)) continue; /* path was GC'ed, probably */
{ {
Activity act(*logger, actUnknown, fmt("optimising path '%s'", i)); Activity act(*logger, actUnknown, fmt("optimising path '%s'", i));
optimisePath_(stats, realStoreDir + "/" + baseNameOf(i), inodeHash); optimisePath_(&act, stats, realStoreDir + "/" + baseNameOf(i), inodeHash);
} }
done++; done++;
act.progress(done, paths.size()); act.progress(done, paths.size());
@ -281,7 +285,7 @@ void LocalStore::optimiseStore()
optimiseStore(stats); optimiseStore(stats);
printError( printInfo(
format("%1% freed by hard-linking %2% files") format("%1% freed by hard-linking %2% files")
% showBytes(stats.bytesFreed) % showBytes(stats.bytesFreed)
% stats.filesLinked); % stats.filesLinked);
@ -292,7 +296,7 @@ void LocalStore::optimisePath(const Path & path)
OptimiseStats stats; OptimiseStats stats;
InodeHash inodeHash; InodeHash inodeHash;
if (settings.autoOptimiseStore) optimisePath_(stats, path, inodeHash); if (settings.autoOptimiseStore) optimisePath_(nullptr, stats, path, inodeHash);
} }

View file

@ -24,6 +24,11 @@ typedef enum {
actOptimiseStore = 106, actOptimiseStore = 106,
} ActivityType; } ActivityType;
typedef enum {
resFileLinked = 100,
resBuildLogLine = 101,
} ResultType;
typedef uint64_t ActivityId; typedef uint64_t ActivityId;
class Logger class Logger
@ -49,9 +54,20 @@ public:
virtual void progress(ActivityId act, uint64_t done = 0, uint64_t expected = 0, uint64_t running = 0, uint64_t failed = 0) { }; virtual void progress(ActivityId act, uint64_t done = 0, uint64_t expected = 0, uint64_t running = 0, uint64_t failed = 0) { };
virtual void progress(ActivityId act, const std::string & s) { };
virtual void setExpected(ActivityId act, ActivityType type, uint64_t expected) { }; virtual void setExpected(ActivityId act, ActivityType type, uint64_t expected) { };
struct Field
{
// FIXME: use std::variant.
enum { tInt, tString } type;
uint64_t i = 0;
std::string s;
Field(const std::string & s) : type(tString), s(s) { }
Field(const char * s) : type(tString), s(s) { }
Field(const uint64_t & i) : type(tInt), i(i) { }
};
virtual void result(ActivityId act, ResultType type, const std::vector<Field> & fields) { };
}; };
struct Activity struct Activity
@ -68,12 +84,17 @@ struct Activity
void progress(uint64_t done = 0, uint64_t expected = 0, uint64_t running = 0, uint64_t failed = 0) const void progress(uint64_t done = 0, uint64_t expected = 0, uint64_t running = 0, uint64_t failed = 0) const
{ logger.progress(id, done, expected, running, failed); } { logger.progress(id, done, expected, running, failed); }
void progress(const std::string & s) const
{ logger.progress(id, s); }
void setExpected(ActivityType type2, uint64_t expected) const void setExpected(ActivityType type2, uint64_t expected) const
{ logger.setExpected(id, type2, expected); } { logger.setExpected(id, type2, expected); }
template<typename... Args>
void result(ResultType type, const Args & ... args)
{
std::vector<Logger::Field> fields;
nop{(fields.emplace_back(Logger::Field(args)), 1)...};
logger.result(id, type, fields);
}
friend class Logger; friend class Logger;
}; };

View file

@ -12,6 +12,20 @@
namespace nix { namespace nix {
static std::string getS(const std::vector<Logger::Field> & fields, size_t n)
{
assert(n < fields.size());
assert(fields[n].type == Logger::Field::tString);
return fields[n].s;
}
static uint64_t getI(const std::vector<Logger::Field> & fields, size_t n)
{
assert(n < fields.size());
assert(fields[n].type == Logger::Field::tInt);
return fields[n].i;
}
class ProgressBar : public Logger class ProgressBar : public Logger
{ {
private: private:
@ -41,6 +55,8 @@ private:
std::map<ActivityId, std::list<ActInfo>::iterator> its; std::map<ActivityId, std::list<ActInfo>::iterator> its;
std::map<ActivityType, ActivitiesByType> activitiesByType; std::map<ActivityType, ActivitiesByType> activitiesByType;
uint64_t filesLinked = 0, bytesLinked = 0;
}; };
Sync<State> state_; Sync<State> state_;
@ -80,14 +96,33 @@ public:
void startActivity(ActivityId act, ActivityType type, const std::string & s) override void startActivity(ActivityId act, ActivityType type, const std::string & s) override
{ {
auto state(state_.lock()); auto state(state_.lock());
createActivity(*state, act, s, type);
state->activities.emplace_back(ActInfo{s, "", type});
auto i = std::prev(state->activities.end());
state->its.emplace(act, i);
state->activitiesByType[type].its.emplace(act, i);
update(*state); update(*state);
} }
void stopActivity(ActivityId act) override void stopActivity(ActivityId act) override
{ {
auto state(state_.lock()); auto state(state_.lock());
deleteActivity(*state, act);
auto i = state->its.find(act);
if (i != state->its.end()) {
auto & actByType = state->activitiesByType[i->second->type];
actByType.done += i->second->done;
actByType.failed += i->second->failed;
for (auto & j : i->second->expectedByType)
state->activitiesByType[j.first].expected -= j.second;
actByType.its.erase(act);
state->activities.erase(i->second);
state->its.erase(i);
}
update(*state); update(*state);
} }
@ -106,16 +141,6 @@ public:
update(*state); update(*state);
} }
void progress(ActivityId act, const std::string & s) override
{
auto state(state_.lock());
auto s2 = trim(s);
if (!s2.empty()) {
updateActivity(*state, act, s2);
update(*state);
}
}
void setExpected(ActivityId act, ActivityType type, uint64_t expected) override void setExpected(ActivityId act, ActivityType type, uint64_t expected) override
{ {
auto state(state_.lock()); auto state(state_.lock());
@ -131,40 +156,29 @@ public:
update(*state); update(*state);
} }
void createActivity(State & state, ActivityId activity, const std::string & s, ActivityType type = actUnknown) void result(ActivityId act, ResultType type, const std::vector<Field> & fields) override
{ {
state.activities.emplace_back(ActInfo{s, "", type}); auto state(state_.lock());
auto i = std::prev(state.activities.end());
state.its.emplace(activity, i); if (type == resFileLinked) {
state.activitiesByType[type].its.emplace(activity, i); state->filesLinked++;
state->bytesLinked += getI(fields, 0);
update(*state);
} }
void deleteActivity(State & state, ActivityId activity) else if (type == resBuildLogLine) {
{ auto s2 = trim(getS(fields, 0));
auto i = state.its.find(activity); if (!s2.empty()) {
if (i != state.its.end()) { auto i = state->its.find(act);
auto & act = state.activitiesByType[i->second->type]; assert(i != state->its.end());
act.done += i->second->done;
act.failed += i->second->failed;
for (auto & j : i->second->expectedByType)
state.activitiesByType[j.first].expected -= j.second;
act.its.erase(activity);
state.activities.erase(i->second);
state.its.erase(i);
}
}
void updateActivity(State & state, ActivityId activity, const std::string & s2)
{
auto i = state.its.find(activity);
assert(i != state.its.end());
ActInfo info = *i->second; ActInfo info = *i->second;
state.activities.erase(i->second); state->activities.erase(i->second);
info.s2 = s2; info.s2 = s2;
state.activities.emplace_back(info); state->activities.emplace_back(info);
i->second = std::prev(state.activities.end()); i->second = std::prev(state->activities.end());
update(*state);
}
}
} }
void update() void update()
@ -262,7 +276,14 @@ public:
showActivity(actDownload, "%s MiB DL", "%.1f", MiB); showActivity(actDownload, "%s MiB DL", "%.1f", MiB);
showActivity(actOptimiseStore, "%s paths optimised"); {
auto s = renderActivity(actOptimiseStore, "%s paths optimised");
if (s != "") {
s += fmt(", %.1f MiB / %d inodes freed", state.bytesLinked / MiB, state.filesLinked);
if (!res.empty()) res += ", ";
res += s;
}
}
return res; return res;
} }