puck/lix
0
0
Fork 0
forked from lix-project/lix
Code Pull requests Activity

nix flake show: Speed up eval cache bigly

Browse source 
In the fully cached case for the 'nixpkgs' flake, it went from 101s to
4.6s. Populating the cache went from 132s to 17.4s (which could
probably be improved further by combining INSERTs).
This commit is contained in:
Eelco Dolstra 2020-04-17 13:57:02 +02:00
parent 9ea4f93f88
commit aa34c0ef51
1 changed files with 47 additions and 69 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

112
src/nix/flake.cc
View file

@ -669,21 +669,12 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun
} }
}; };
// FIXME: inefficient representation of attrs / fingerprints // FIXME: inefficient representation of attrs
static const char * schema = R"sql( static const char * schema = R"sql(
create table if not exists Fingerprints (
fingerprint blob primary key not null,
timestamp integer not null
);
create table if not exists Attributes ( create table if not exists Attributes (
fingerprint blob not null, attrPath text primary key,
attrPath text not null, type integer not null,
type integer, value text not null
value text,
primary key (fingerprint, attrPath),
foreign key (fingerprint) references Fingerprints(fingerprint) on delete cascade
); );
)sql"; )sql";
@ -697,72 +688,52 @@ struct AttrDb
struct State struct State
{ {
SQLite db; SQLite db;
SQLiteStmt insertFingerprint;
SQLiteStmt insertAttribute; SQLiteStmt insertAttribute;
SQLiteStmt queryAttribute; SQLiteStmt queryAttribute;
std::set<Fingerprint> fingerprints;
}; };
std::unique_ptr<Sync<State>> _state; std::unique_ptr<Sync<State>> _state;
AttrDb() AttrDb(const Fingerprint & fingerprint)
: _state(std::make_unique<Sync<State>>()) : _state(std::make_unique<Sync<State>>())
{ {
auto state(_state->lock()); auto state(_state->lock());
Path dbPath = getCacheDir() + "/nix/eval-cache-v2.sqlite"; Path cacheDir = getCacheDir() + "/nix/eval-cache-v2";
createDirs(dirOf(dbPath)); createDirs(cacheDir);
Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
state->db = SQLite(dbPath); state->db = SQLite(dbPath);
state->db.isCache(); state->db.isCache();
state->db.exec(schema); state->db.exec(schema);
state->insertFingerprint.create(state->db,
"insert or ignore into Fingerprints(fingerprint, timestamp) values (?, ?)");
state->insertAttribute.create(state->db, state->insertAttribute.create(state->db,
"insert or replace into Attributes(fingerprint, attrPath, type, value) values (?, ?, ?, ?)"); "insert or replace into Attributes(attrPath, type, value) values (?, ?, ?)");
state->queryAttribute.create(state->db, state->queryAttribute.create(state->db,
"select type, value from Attributes where fingerprint = ? and attrPath = ?"); "select type, value from Attributes where attrPath = ?");
}
void addFingerprint(State & state, const Fingerprint & fingerprint)
{
if (state.fingerprints.insert(fingerprint).second)
// FIXME: update timestamp
state.insertFingerprint.use()
(fingerprint.hash, fingerprint.hashSize)
(time(0)).exec();
} }
void setAttr( void setAttr(
const Fingerprint & fingerprint,
const std::vector<Symbol> & attrPath, const std::vector<Symbol> & attrPath,
const std::vector<Symbol> & attrs) const std::vector<Symbol> & attrs)
{ {
auto state(_state->lock()); auto state(_state->lock());
addFingerprint(*state, fingerprint);
state->insertAttribute.use() state->insertAttribute.use()
(fingerprint.hash, fingerprint.hashSize)
(concatStringsSep(".", attrPath)) (concatStringsSep(".", attrPath))
(AttrType::Attrs) (AttrType::Attrs)
(concatStringsSep("\n", attrs)).exec(); (concatStringsSep("\n", attrs)).exec();
} }
void setAttr( void setAttr(
const Fingerprint & fingerprint,
const std::vector<Symbol> & attrPath, const std::vector<Symbol> & attrPath,
std::string_view s) std::string_view s)
{ {
auto state(_state->lock()); auto state(_state->lock());
addFingerprint(*state, fingerprint);
state->insertAttribute.use() state->insertAttribute.use()
(fingerprint.hash, fingerprint.hashSize)
(concatStringsSep(".", attrPath)) (concatStringsSep(".", attrPath))
(AttrType::String) (AttrType::String)
(s).exec(); (s).exec();
@ -771,16 +742,12 @@ struct AttrDb
typedef std::variant<std::vector<Symbol>, std::string> AttrValue; typedef std::variant<std::vector<Symbol>, std::string> AttrValue;
std::optional<AttrValue> getAttr( std::optional<AttrValue> getAttr(
const Fingerprint & fingerprint,
const std::vector<Symbol> & attrPath, const std::vector<Symbol> & attrPath,
SymbolTable & symbols) SymbolTable & symbols)
{ {
auto state(_state->lock()); auto state(_state->lock());
addFingerprint(*state, fingerprint);
auto queryAttribute(state->queryAttribute.use() auto queryAttribute(state->queryAttribute.use()
(fingerprint.hash, fingerprint.hashSize)
(concatStringsSep(".", attrPath))); (concatStringsSep(".", attrPath)));
if (!queryAttribute.next()) return {}; if (!queryAttribute.next()) return {};
@ -804,15 +771,13 @@ struct AttrRoot : std::enable_shared_from_this<AttrRoot>
{ {
std::shared_ptr<AttrDb> db; std::shared_ptr<AttrDb> db;
EvalState & state; EvalState & state;
Fingerprint fingerprint;
typedef std::function<Value *()> RootLoader; typedef std::function<Value *()> RootLoader;
RootLoader rootLoader; RootLoader rootLoader;
RootValue value; RootValue value;
AttrRoot(std::shared_ptr<AttrDb> db, EvalState & state, const Fingerprint & fingerprint, RootLoader rootLoader) AttrRoot(std::shared_ptr<AttrDb> db, EvalState & state, RootLoader rootLoader)
: db(db) : db(db)
, state(state) , state(state)
, fingerprint(fingerprint)
, rootLoader(rootLoader) , rootLoader(rootLoader)
{ {
} }
@ -838,12 +803,14 @@ struct AttrCursor : std::enable_shared_from_this<AttrCursor>
typedef std::optional<std::pair<std::shared_ptr<AttrCursor>, Symbol>> Parent; typedef std::optional<std::pair<std::shared_ptr<AttrCursor>, Symbol>> Parent;
Parent parent; Parent parent;
RootValue _value; RootValue _value;
std::optional<AttrDb::AttrValue> cachedValue;
AttrCursor( AttrCursor(
ref<AttrRoot> root, ref<AttrRoot> root,
Parent parent, Parent parent,
Value * value = nullptr) Value * value = nullptr,
: root(root), parent(parent) std::optional<AttrDb::AttrValue> && cachedValue = {})
: root(root), parent(parent), cachedValue(std::move(cachedValue))
{ {
if (value) if (value)
_value = allocRootValue(value); _value = allocRootValue(value);
@ -895,9 +862,11 @@ struct AttrCursor : std::enable_shared_from_this<AttrCursor>
std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name) std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name)
{ {
if (root->db) { if (root->db) {
auto attr = root->db->getAttr(root->fingerprint, getAttrPath(), root->state.symbols); if (!cachedValue)
if (attr) { cachedValue = root->db->getAttr(getAttrPath(), root->state.symbols);
if (auto attrs = std::get_if<std::vector<Symbol>>(&*attr)) {
if (cachedValue) {
if (auto attrs = std::get_if<std::vector<Symbol>>(&*cachedValue)) {
for (auto & attr : *attrs) for (auto & attr : *attrs)
if (attr == name) if (attr == name)
return std::make_shared<AttrCursor>(root, std::make_pair(shared_from_this(), name)); return std::make_shared<AttrCursor>(root, std::make_pair(shared_from_this(), name));
@ -905,10 +874,9 @@ struct AttrCursor : std::enable_shared_from_this<AttrCursor>
return nullptr; return nullptr;
} }
attr = root->db->getAttr(root->fingerprint, getAttrPath(name), root->state.symbols); auto attr = root->db->getAttr(getAttrPath(name), root->state.symbols);
if (attr) if (attr)
// FIXME: store *attr return std::make_shared<AttrCursor>(root, std::make_pair(shared_from_this(), name), nullptr, std::move(attr));
return std::make_shared<AttrCursor>(root, std::make_pair(shared_from_this(), name));
} }
//printError("GET ATTR %s", getAttrPathStr(name)); //printError("GET ATTR %s", getAttrPathStr(name));
@ -947,28 +915,36 @@ struct AttrCursor : std::enable_shared_from_this<AttrCursor>
std::string getString() std::string getString()
{ {
if (root->db) { if (root->db) {
auto attr = root->db->getAttr(root->fingerprint, getAttrPath(), root->state.symbols); if (!cachedValue)
if (auto s = std::get_if<std::string>(&*attr)) { cachedValue = root->db->getAttr(getAttrPath(), root->state.symbols);
if (cachedValue) {
if (auto s = std::get_if<std::string>(&*cachedValue)) {
//printError("GOT STRING %s", getAttrPathStr()); //printError("GOT STRING %s", getAttrPathStr());
return *s; return *s;
} else
throw Error("unexpected type mismatch in evaluation cache");
} }
} }
//printError("GET STRING %s", getAttrPathStr()); //printError("GET STRING %s", getAttrPathStr());
auto s = root->state.forceString(getValue()); auto s = root->state.forceString(getValue());
if (root->db) if (root->db) {
root->db->setAttr(root->fingerprint, getAttrPath(), s); root->db->setAttr(getAttrPath(), s);
cachedValue = s;
}
return s; return s;
} }
std::vector<Symbol> getAttrs() std::vector<Symbol> getAttrs()
{ {
if (root->db) { if (root->db) {
auto attr = root->db->getAttr(root->fingerprint, getAttrPath(), root->state.symbols); if (!cachedValue)
if (attr) { cachedValue = root->db->getAttr(getAttrPath(), root->state.symbols);
if (auto attrs = std::get_if<std::vector<Symbol>>(&*attr)) { if (cachedValue) {
if (auto attrs = std::get_if<std::vector<Symbol>>(&*cachedValue)) {
//printError("GOT ATTRS %s", getAttrPathStr()); //printError("GOT ATTRS %s", getAttrPathStr());
return std::move(*attrs); return *attrs;
} else } else
throw Error("unexpected type mismatch in evaluation cache"); throw Error("unexpected type mismatch in evaluation cache");
} }
@ -982,8 +958,11 @@ struct AttrCursor : std::enable_shared_from_this<AttrCursor>
std::sort(attrs.begin(), attrs.end(), [](const Symbol & a, const Symbol & b) { std::sort(attrs.begin(), attrs.end(), [](const Symbol & a, const Symbol & b) {
return (const string &) a < (const string &) b; return (const string &) a < (const string &) b;
}); });
if (root->db) if (root->db) {
root->db->setAttr(root->fingerprint, getAttrPath(), attrs); root->db->setAttr(getAttrPath(), attrs);
cachedValue = attrs;
}
return attrs; return attrs;
} }
@ -1132,10 +1111,9 @@ struct CmdFlakeShow : FlakeCommand
} }
}; };
auto db = std::make_shared<AttrDb>(); auto db = std::make_shared<AttrDb>(flake.getFingerprint());
auto root = std::make_shared<AttrRoot>(db, *state, auto root = std::make_shared<AttrRoot>(db, *state,
flake.getFingerprint(),
[&]() [&]()
{ {
auto vFlake = state->allocValue(); auto vFlake = state->allocValue();