From 50f13b06fb1b2f50a97323c000d1094d090f08ea Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 29 Jun 2020 19:08:37 +0200 Subject: [PATCH] EvalCache: Store string contexts --- src/libexpr/eval-cache.cc | 75 ++++++++++++++++++++++++++++++++------- src/libexpr/eval-cache.hh | 14 +++++++- src/libexpr/eval.cc | 23 ++++++++++++ src/libexpr/eval.hh | 2 +- src/libexpr/primops.cc | 12 ------- src/libexpr/value.hh | 2 ++ 6 files changed, 101 insertions(+), 27 deletions(-) diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 3a18b71d4..65a3fceeb 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -12,6 +12,7 @@ create table if not exists Attributes ( name text, type integer not null, value text, + context text, primary key (parent, name) ); )sql"; @@ -22,6 +23,7 @@ struct AttrDb { SQLite db; SQLiteStmt insertAttribute; + SQLiteStmt insertAttributeWithContext; SQLiteStmt queryAttribute; SQLiteStmt queryAttributes; std::unique_ptr txn; @@ -34,7 +36,7 @@ struct AttrDb { auto state(_state->lock()); - Path cacheDir = getCacheDir() + "/nix/eval-cache-v1"; + Path cacheDir = getCacheDir() + "/nix/eval-cache-v2"; createDirs(cacheDir); Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite"; @@ -46,8 +48,11 @@ struct AttrDb state->insertAttribute.create(state->db, "insert or replace into Attributes(parent, name, type, value) values (?, ?, ?, ?)"); + state->insertAttributeWithContext.create(state->db, + "insert or replace into Attributes(parent, name, type, value, context) values (?, ?, ?, ?, ?)"); + state->queryAttribute.create(state->db, - "select rowid, type, value from Attributes where parent = ? and name = ?"); + "select rowid, type, value, context from Attributes where parent = ? and name = ?"); state->queryAttributes.create(state->db, "select name from Attributes where parent = ?"); @@ -93,15 +98,30 @@ struct AttrDb AttrId setString( AttrKey key, - std::string_view s) + std::string_view s, + const char * * context = nullptr) { auto state(_state->lock()); - state->insertAttribute.use() - (key.first) - (key.second) - (AttrType::String) - (s).exec(); + if (context) { + std::string ctx; + for (const char * * p = context; *p; ++p) { + if (p != context) ctx.push_back(' '); + ctx.append(*p); + } + state->insertAttributeWithContext.use() + (key.first) + (key.second) + (AttrType::String) + (s) + (ctx).exec(); + } else { + state->insertAttribute.use() + (key.first) + (key.second) + (AttrType::String) + (s).exec(); + } return state->db.getLastInsertedRowId(); } @@ -196,8 +216,13 @@ struct AttrDb attrs.push_back(symbols.create(queryAttributes.getStr(0))); return {{rowId, attrs}}; } - case AttrType::String: - return {{rowId, queryAttribute.getStr(2)}}; + case AttrType::String: { + std::vector> context; + if (!queryAttribute.isNull(3)) + for (auto & s : tokenizeString>(queryAttribute.getStr(3), ";")) + context.push_back(decodeContext(s)); + return {{rowId, string_t{queryAttribute.getStr(2), context}}}; + } case AttrType::Bool: return {{rowId, queryAttribute.getInt(2) != 0}}; case AttrType::Missing: @@ -320,7 +345,7 @@ Value & AttrCursor::forceValue() if (root->db && (!cachedValue || std::get_if(&cachedValue->second))) { if (v.type == tString) - cachedValue = {root->db->setString(getKey(), v.string.s), v.string.s}; + cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context), v.string.s}; else if (v.type == tPath) cachedValue = {root->db->setString(getKey(), v.path), v.path}; else if (v.type == tBool) @@ -427,9 +452,9 @@ std::string AttrCursor::getString() if (!cachedValue) cachedValue = root->db->getAttr(getKey(), root->state.symbols); if (cachedValue && !std::get_if(&cachedValue->second)) { - if (auto s = std::get_if(&cachedValue->second)) { + if (auto s = std::get_if(&cachedValue->second)) { debug("using cached string attribute '%s'", getAttrPathStr()); - return *s; + return s->first; } else throw TypeError("'%s' is not a string", getAttrPathStr()); } @@ -443,6 +468,30 @@ std::string AttrCursor::getString() return v.type == tString ? v.string.s : v.path; } +string_t AttrCursor::getStringWithContext() +{ + if (root->db) { + if (!cachedValue) + cachedValue = root->db->getAttr(getKey(), root->state.symbols); + if (cachedValue && !std::get_if(&cachedValue->second)) { + if (auto s = std::get_if(&cachedValue->second)) { + debug("using cached string attribute '%s'", getAttrPathStr()); + return *s; + } else + throw TypeError("'%s' is not a string", getAttrPathStr()); + } + } + + auto & v = forceValue(); + + if (v.type == tString) + return {v.string.s, v.getContext()}; + else if (v.type == tPath) + return {v.path, {}}; + else + throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type)); +} + bool AttrCursor::getBool() { if (root->db) { diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh index 30261fd3a..674bb03c1 100644 --- a/src/libexpr/eval-cache.hh +++ b/src/libexpr/eval-cache.hh @@ -50,7 +50,17 @@ struct misc_t {}; struct failed_t {}; typedef uint64_t AttrId; typedef std::pair AttrKey; -typedef std::variant, std::string, placeholder_t, missing_t, misc_t, failed_t, bool> AttrValue; +typedef std::pair>> string_t; + +typedef std::variant< + std::vector, + string_t, + placeholder_t, + missing_t, + misc_t, + failed_t, + bool + > AttrValue; class AttrCursor : public std::enable_shared_from_this { @@ -94,6 +104,8 @@ public: std::string getString(); + string_t getStringWithContext(); + bool getBool(); std::vector getAttrs(); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 5db380f88..ba600ad0c 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1607,6 +1607,18 @@ string EvalState::forceString(Value & v, const Pos & pos) } +/* Decode a context string ‘!!’ into a pair . */ +std::pair decodeContext(std::string_view s) +{ + if (s.at(0) == '!') { + size_t index = s.find("!", 1); + return {std::string(s.substr(index + 1)), std::string(s.substr(1, index - 1))}; + } else + return {s.at(0) == '/' ? std::string(s) : std::string(s.substr(1)), ""}; +} + + void copyContext(const Value & v, PathSet & context) { if (v.string.context) @@ -1615,6 +1627,17 @@ void copyContext(const Value & v, PathSet & context) } +std::vector> Value::getContext() +{ + std::vector> res; + assert(type == tString); + if (string.context) + for (const char * * p = string.context; *p; ++p) + res.push_back(decodeContext(*p)); + return res; +} + + string EvalState::forceString(Value & v, PathSet & context, const Pos & pos) { string s = forceString(v, pos); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index be34f3f32..87bb42c11 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -333,7 +333,7 @@ string showType(const Value & v); /* Decode a context string ‘!!’ into a pair . */ -std::pair decodeContext(const string & s); +std::pair decodeContext(std::string_view s); /* If `path' refers to a directory, then append "/default.nix". */ Path resolveExprPath(Path path); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 3830d8107..96c20b227 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -30,18 +30,6 @@ namespace nix { *************************************************************/ -/* Decode a context string ‘!!’ into a pair . */ -std::pair decodeContext(const string & s) -{ - if (s.at(0) == '!') { - size_t index = s.find("!", 1); - return std::pair(string(s, index + 1), string(s, 1, index - 1)); - } else - return std::pair(s.at(0) == '/' ? s : string(s, 1), ""); -} - - InvalidPathError::InvalidPathError(const Path & path) : EvalError("path '%s' is not valid", path), path(path) {} diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 1a0738241..fe11bb2ed 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -171,6 +171,8 @@ struct Value computation. In particular, function applications are non-trivial. */ bool isTrivial() const; + + std::vector> getContext(); };