From 539a9c1c5f0e4c9ab2262e7cf48460e02b8b0e12 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 20 Apr 2020 13:13:52 +0200 Subject: [PATCH] Get rid of the old eval cache --- src/libexpr/attr-path.cc | 13 +- src/libexpr/attr-path.hh | 2 + src/libexpr/flake/eval-cache.cc | 522 ++++++++++++++++++++++++++------ src/libexpr/flake/eval-cache.hh | 107 +++++-- src/nix/flake.cc | 502 +----------------------------- src/nix/installables.cc | 92 +++--- src/nix/installables.hh | 22 +- 7 files changed, 609 insertions(+), 651 deletions(-) diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 76d101b98..0764fc05c 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -6,11 +6,11 @@ namespace nix { -static Strings parseAttrPath(const string & s) +static Strings parseAttrPath(std::string_view s) { Strings res; string cur; - string::const_iterator i = s.begin(); + auto i = s.begin(); while (i != s.end()) { if (*i == '.') { res.push_back(cur); @@ -32,6 +32,15 @@ static Strings parseAttrPath(const string & s) } +std::vector parseAttrPath(EvalState & state, std::string_view s) +{ + std::vector res; + for (auto & a : parseAttrPath(s)) + res.push_back(state.symbols.create(a)); + return res; +} + + std::pair findAlongAttrPath(EvalState & state, const string & attrPath, Bindings & autoArgs, Value & vIn) { diff --git a/src/libexpr/attr-path.hh b/src/libexpr/attr-path.hh index fce160da7..d9d74ab2d 100644 --- a/src/libexpr/attr-path.hh +++ b/src/libexpr/attr-path.hh @@ -16,4 +16,6 @@ std::pair findAlongAttrPath(EvalState & state, const string & attr /* Heuristic to find the filename and lineno or a nix value. */ Pos findDerivationFilename(EvalState & state, Value & v, std::string what); +std::vector parseAttrPath(EvalState & state, std::string_view s); + } diff --git a/src/libexpr/flake/eval-cache.cc b/src/libexpr/flake/eval-cache.cc index 8d01ef0fc..e17dbde84 100644 --- a/src/libexpr/flake/eval-cache.cc +++ b/src/libexpr/flake/eval-cache.cc @@ -1,116 +1,460 @@ #include "eval-cache.hh" #include "sqlite.hh" #include "eval.hh" +#include "eval-inline.hh" -#include - -namespace nix::flake { +namespace nix::eval_cache { +// FIXME: inefficient representation of attrs static const char * schema = R"sql( - -create table if not exists Fingerprints ( - fingerprint blob primary key not null, - timestamp integer not null -); - create table if not exists Attributes ( - fingerprint blob not null, - attrPath text not null, - type integer, + parent integer not null, + name text, + type integer not null, value text, - primary key (fingerprint, attrPath), - foreign key (fingerprint) references Fingerprints(fingerprint) on delete cascade + primary key (parent, name) ); )sql"; -struct EvalCache::State +struct AttrDb { - SQLite db; - SQLiteStmt insertFingerprint; - SQLiteStmt insertAttribute; - SQLiteStmt queryAttribute; - std::set fingerprints; + struct State + { + SQLite db; + SQLiteStmt insertAttribute; + SQLiteStmt queryAttribute; + SQLiteStmt queryAttributes; + std::unique_ptr txn; + }; + + std::unique_ptr> _state; + + AttrDb(const Hash & fingerprint) + : _state(std::make_unique>()) + { + auto state(_state->lock()); + + Path cacheDir = getCacheDir() + "/nix/eval-cache-v1"; + createDirs(cacheDir); + + Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite"; + + state->db = SQLite(dbPath); + state->db.isCache(); + state->db.exec(schema); + + state->insertAttribute.create(state->db, + "insert or replace into Attributes(parent, name, type, value) values (?, ?, ?, ?)"); + + state->queryAttribute.create(state->db, + "select rowid, type, value from Attributes where parent = ? and name = ?"); + + state->queryAttributes.create(state->db, + "select name from Attributes where parent = ?"); + + state->txn = std::make_unique(state->db); + } + + ~AttrDb() + { + try { + auto state(_state->lock()); + state->txn->commit(); + state->txn.reset(); + } catch (...) { + ignoreException(); + } + } + + AttrId setAttrs( + AttrKey key, + const std::vector & attrs) + { + auto state(_state->lock()); + + state->insertAttribute.use() + (key.first) + (key.second) + (AttrType::FullAttrs) + (0, false).exec(); + + AttrId rowId = state->db.getLastInsertedRowId(); + assert(rowId); + + for (auto & attr : attrs) + state->insertAttribute.use() + (rowId) + (attr) + (AttrType::Placeholder) + (0, false).exec(); + + return rowId; + } + + AttrId setString( + AttrKey key, + std::string_view s) + { + auto state(_state->lock()); + + state->insertAttribute.use() + (key.first) + (key.second) + (AttrType::String) + (s).exec(); + + return state->db.getLastInsertedRowId(); + } + + AttrId setPlaceholder(AttrKey key) + { + auto state(_state->lock()); + + state->insertAttribute.use() + (key.first) + (key.second) + (AttrType::Placeholder) + (0, false).exec(); + + return state->db.getLastInsertedRowId(); + } + + AttrId setMissing(AttrKey key) + { + auto state(_state->lock()); + + state->insertAttribute.use() + (key.first) + (key.second) + (AttrType::Missing) + (0, false).exec(); + + return state->db.getLastInsertedRowId(); + } + + AttrId setMisc(AttrKey key) + { + auto state(_state->lock()); + + state->insertAttribute.use() + (key.first) + (key.second) + (AttrType::Misc) + (0, false).exec(); + + return state->db.getLastInsertedRowId(); + } + + AttrId setFailed(AttrKey key) + { + auto state(_state->lock()); + + state->insertAttribute.use() + (key.first) + (key.second) + (AttrType::Failed) + (0, false).exec(); + + return state->db.getLastInsertedRowId(); + } + + std::optional> getAttr( + AttrKey key, + SymbolTable & symbols) + { + auto state(_state->lock()); + + auto queryAttribute(state->queryAttribute.use()(key.first)(key.second)); + if (!queryAttribute.next()) return {}; + + auto rowId = (AttrType) queryAttribute.getInt(0); + auto type = (AttrType) queryAttribute.getInt(1); + + if (type == AttrType::Placeholder) + return {{rowId, placeholder_t()}}; + else if (type == AttrType::FullAttrs) { + // FIXME: expensive, should separate this out. + std::vector attrs; + auto queryAttributes(state->queryAttributes.use()(rowId)); + while (queryAttributes.next()) + attrs.push_back(symbols.create(queryAttributes.getStr(0))); + return {{rowId, attrs}}; + } else if (type == AttrType::String) { + return {{rowId, queryAttribute.getStr(2)}}; + } else if (type == AttrType::Missing) { + return {{rowId, missing_t()}}; + } else if (type == AttrType::Misc) { + return {{rowId, misc_t()}}; + } else if (type == AttrType::Failed) { + return {{rowId, failed_t()}}; + } else + throw Error("unexpected type in evaluation cache"); + } }; -EvalCache::EvalCache() - : _state(std::make_unique>()) +EvalCache::EvalCache( + bool useCache, + const Hash & fingerprint, + EvalState & state, + RootLoader rootLoader) + : db(useCache ? std::make_shared(fingerprint) : nullptr) + , state(state) + , rootLoader(rootLoader) { - auto state(_state->lock()); - - Path dbPath = getCacheDir() + "/nix/eval-cache-v1.sqlite"; - createDirs(dirOf(dbPath)); - - state->db = SQLite(dbPath); - state->db.isCache(); - state->db.exec(schema); - - state->insertFingerprint.create(state->db, - "insert or ignore into Fingerprints(fingerprint, timestamp) values (?, ?)"); - - state->insertAttribute.create(state->db, - "insert or replace into Attributes(fingerprint, attrPath, type, value) values (?, ?, ?, ?)"); - - state->queryAttribute.create(state->db, - "select type, value from Attributes where fingerprint = ? and attrPath = ?"); } -enum ValueType { - Derivation = 1, -}; - -void EvalCache::addDerivation( - const Fingerprint & fingerprint, - const std::string & attrPath, - const Derivation & drv) +Value * EvalCache::getRootValue() { - if (!evalSettings.pureEval) return; - - auto state(_state->lock()); - - if (state->fingerprints.insert(fingerprint).second) - // FIXME: update timestamp - state->insertFingerprint.use() - (fingerprint.hash, fingerprint.hashSize) - (time(0)).exec(); - - state->insertAttribute.use() - (fingerprint.hash, fingerprint.hashSize) - (attrPath) - (ValueType::Derivation) - (std::string(drv.drvPath.to_string()) + " " + std::string(drv.outPath.to_string()) + " " + drv.outputName).exec(); + if (!value) { + debug("getting root value"); + value = allocRootValue(rootLoader()); + } + return *value; } -std::optional EvalCache::getDerivation( - const Fingerprint & fingerprint, - const std::string & attrPath) +std::shared_ptr EvalCache::getRoot() { - if (!evalSettings.pureEval) return {}; - - auto state(_state->lock()); - - auto queryAttribute(state->queryAttribute.use() - (fingerprint.hash, fingerprint.hashSize) - (attrPath)); - if (!queryAttribute.next()) return {}; - - // FIXME: handle negative results - - auto type = (ValueType) queryAttribute.getInt(0); - auto s = queryAttribute.getStr(1); - - if (type != ValueType::Derivation) return {}; - - auto ss = tokenizeString>(s, " "); - - debug("evaluation cache hit for '%s'", attrPath); - - return Derivation { StorePath::fromBaseName(ss[0]), StorePath::fromBaseName(ss[1]), ss[2] }; + return std::make_shared(ref(shared_from_this()), std::nullopt); } -EvalCache & EvalCache::singleton() +AttrCursor::AttrCursor( + ref root, + Parent parent, + Value * value, + std::optional> && cachedValue) + : root(root), parent(parent), cachedValue(std::move(cachedValue)) { - static std::unique_ptr evalCache(new EvalCache()); - return *evalCache; + if (value) + _value = allocRootValue(value); +} + +AttrKey AttrCursor::getKey() +{ + if (!parent) + return {0, root->state.sEpsilon}; + if (!parent->first->cachedValue) { + parent->first->cachedValue = root->db->getAttr( + parent->first->getKey(), root->state.symbols); + assert(parent->first->cachedValue); + } + return {parent->first->cachedValue->first, parent->second}; +} + +Value & AttrCursor::getValue() +{ + if (!_value) { + if (parent) { + auto & vParent = parent->first->getValue(); + root->state.forceAttrs(vParent); + auto attr = vParent.attrs->get(parent->second); + if (!attr) + throw Error("attribute '%s' is unexpectedly missing", getAttrPathStr()); + _value = allocRootValue(attr->value); + } else + _value = allocRootValue(root->getRootValue()); + } + return **_value; +} + +std::vector AttrCursor::getAttrPath() const +{ + if (parent) { + auto attrPath = parent->first->getAttrPath(); + attrPath.push_back(parent->second); + return attrPath; + } else + return {}; +} + +std::vector AttrCursor::getAttrPath(Symbol name) const +{ + auto attrPath = getAttrPath(); + attrPath.push_back(name); + return attrPath; +} + +std::string AttrCursor::getAttrPathStr() const +{ + return concatStringsSep(".", getAttrPath()); +} + +std::string AttrCursor::getAttrPathStr(Symbol name) const +{ + return concatStringsSep(".", getAttrPath(name)); +} + +Value & AttrCursor::forceValue() +{ + debug("evaluating uncached attribute %s", getAttrPathStr()); + + auto & v = getValue(); + + try { + root->state.forceValue(v); + } catch (EvalError &) { + debug("setting '%s' to failed", getAttrPathStr()); + if (root->db) + cachedValue = {root->db->setFailed(getKey()), failed_t()}; + throw; + } + + if (root->db && (!cachedValue || std::get_if(&cachedValue->second))) { + if (v.type == tString) + cachedValue = {root->db->setString(getKey(), v.string.s), v.string.s}; + else if (v.type == tAttrs) + ; // FIXME: do something? + else + cachedValue = {root->db->setMisc(getKey()), misc_t()}; + } + + return v; +} + +std::shared_ptr AttrCursor::maybeGetAttr(Symbol name) +{ + if (root->db) { + if (!cachedValue) + cachedValue = root->db->getAttr(getKey(), root->state.symbols); + + if (cachedValue) { + if (auto attrs = std::get_if>(&cachedValue->second)) { + for (auto & attr : *attrs) + if (attr == name) + return std::make_shared(root, std::make_pair(shared_from_this(), name)); + return nullptr; + } else if (std::get_if(&cachedValue->second)) { + auto attr = root->db->getAttr({cachedValue->first, name}, root->state.symbols); + if (attr) { + if (std::get_if(&attr->second)) + return nullptr; + else if (std::get_if(&attr->second)) + throw EvalError("cached failure of attribute '%s'", getAttrPathStr(name)); + else + return std::make_shared(root, + std::make_pair(shared_from_this(), name), nullptr, std::move(attr)); + } + // Incomplete attrset, so need to fall thru and + // evaluate to see whether 'name' exists + } else + return nullptr; + //throw TypeError("'%s' is not an attribute set", getAttrPathStr()); + } + } + + auto & v = forceValue(); + + if (v.type != tAttrs) + return nullptr; + //throw TypeError("'%s' is not an attribute set", getAttrPathStr()); + + auto attr = v.attrs->get(name); + + if (!attr) { + if (root->db) { + if (!cachedValue) + cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()}; + root->db->setMissing({cachedValue->first, name}); + } + return nullptr; + } + + std::optional> cachedValue2; + if (root->db) { + if (!cachedValue) + cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()}; + cachedValue2 = {root->db->setPlaceholder({cachedValue->first, name}), placeholder_t()}; + } + + return std::make_shared( + root, std::make_pair(shared_from_this(), name), attr->value, std::move(cachedValue2)); +} + +std::shared_ptr AttrCursor::maybeGetAttr(std::string_view name) +{ + return maybeGetAttr(root->state.symbols.create(name)); +} + +std::shared_ptr AttrCursor::getAttr(Symbol name) +{ + auto p = maybeGetAttr(name); + if (!p) + throw Error("attribute '%s' does not exist", getAttrPathStr(name)); + return p; +} + +std::shared_ptr AttrCursor::getAttr(std::string_view name) +{ + return getAttr(root->state.symbols.create(name)); +} + +std::shared_ptr AttrCursor::findAlongAttrPath(const std::vector & attrPath) +{ + auto res = shared_from_this(); + for (auto & attr : attrPath) { + res = res->maybeGetAttr(attr); + if (!res) return {}; + } + return res; +} + +std::string AttrCursor::getString() +{ + if (root->db) { + if (!cachedValue) + cachedValue = root->db->getAttr(getKey(), root->state.symbols); + if (cachedValue && !std::get_if(&cachedValue->second)) { + if (auto s = std::get_if(&cachedValue->second)) { + debug("using cached string attribute '%s'", getAttrPathStr()); + return *s; + } else + throw TypeError("'%s' is not a string", getAttrPathStr()); + } + } + + auto & v = forceValue(); + + if (v.type != tString) + throw TypeError("'%s' is not a string", getAttrPathStr()); + + return v.string.s; +} + +std::vector AttrCursor::getAttrs() +{ + if (root->db) { + if (!cachedValue) + cachedValue = root->db->getAttr(getKey(), root->state.symbols); + if (cachedValue && !std::get_if(&cachedValue->second)) { + if (auto attrs = std::get_if>(&cachedValue->second)) { + debug("using cached attrset attribute '%s'", getAttrPathStr()); + return *attrs; + } else + throw TypeError("'%s' is not an attribute set", getAttrPathStr()); + } + } + + auto & v = forceValue(); + + if (v.type != tAttrs) + throw TypeError("'%s' is not an attribute set", getAttrPathStr()); + + std::vector attrs; + for (auto & attr : *getValue().attrs) + attrs.push_back(attr.name); + std::sort(attrs.begin(), attrs.end(), [](const Symbol & a, const Symbol & b) { + return (const string &) a < (const string &) b; + }); + + if (root->db) + cachedValue = {root->db->setAttrs(getKey(), attrs), attrs}; + + return attrs; +} + +bool AttrCursor::isDerivation() +{ + auto aType = maybeGetAttr("type"); + return aType && aType->getString() == "derivation"; } } diff --git a/src/libexpr/flake/eval-cache.hh b/src/libexpr/flake/eval-cache.hh index f81d48ba5..a2c21b6ea 100644 --- a/src/libexpr/flake/eval-cache.hh +++ b/src/libexpr/flake/eval-cache.hh @@ -1,40 +1,103 @@ #pragma once #include "sync.hh" -#include "flake.hh" -#include "path.hh" +#include "hash.hh" +#include "eval.hh" -namespace nix { struct SQLite; struct SQLiteStmt; } +#include -namespace nix::flake { +namespace nix::eval_cache { -class EvalCache +class AttrDb; +class AttrCursor; + +class EvalCache : public std::enable_shared_from_this { - struct State; + friend class AttrCursor; - std::unique_ptr> _state; + std::shared_ptr db; + EvalState & state; + typedef std::function RootLoader; + RootLoader rootLoader; + RootValue value; - EvalCache(); + Value * getRootValue(); public: - struct Derivation - { - StorePath drvPath; - StorePath outPath; - std::string outputName; - }; + EvalCache( + bool useCache, + const Hash & fingerprint, + EvalState & state, + RootLoader rootLoader); - void addDerivation( - const Fingerprint & fingerprint, - const std::string & attrPath, - const Derivation & drv); + std::shared_ptr getRoot(); +}; - std::optional getDerivation( - const Fingerprint & fingerprint, - const std::string & attrPath); +enum AttrType { + Placeholder = 0, + FullAttrs = 1, + String = 2, + Missing = 3, + Misc = 4, + Failed = 5, +}; - static EvalCache & singleton(); +struct placeholder_t {}; +struct missing_t {}; +struct misc_t {}; +struct failed_t {}; +typedef uint64_t AttrId; +typedef std::pair AttrKey; +typedef std::variant, std::string, placeholder_t, missing_t, misc_t, failed_t> AttrValue; + +class AttrCursor : public std::enable_shared_from_this +{ + friend class EvalCache; + + ref root; + typedef std::optional, Symbol>> Parent; + Parent parent; + RootValue _value; + std::optional> cachedValue; + + AttrKey getKey(); + + Value & getValue(); + +public: + + AttrCursor( + ref root, + Parent parent, + Value * value = nullptr, + std::optional> && cachedValue = {}); + + std::vector getAttrPath() const; + + std::vector getAttrPath(Symbol name) const; + + std::string getAttrPathStr() const; + + std::string getAttrPathStr(Symbol name) const; + + std::shared_ptr maybeGetAttr(Symbol name); + + std::shared_ptr maybeGetAttr(std::string_view name); + + std::shared_ptr getAttr(Symbol name); + + std::shared_ptr getAttr(std::string_view name); + + std::shared_ptr findAlongAttrPath(const std::vector & attrPath); + + std::string getString(); + + std::vector getAttrs(); + + bool isDerivation(); + + Value & forceValue(); }; } diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 9e46cc55a..2f2dd65c7 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -11,7 +11,7 @@ #include "fetchers.hh" #include "registry.hh" #include "json.hh" -#include "sqlite.hh" +#include "flake/eval-cache.hh" #include #include @@ -669,479 +669,6 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun } }; -// FIXME: inefficient representation of attrs -static const char * schema = R"sql( -create table if not exists Attributes ( - parent integer not null, - name text, - type integer not null, - value text, - primary key (parent, name) -); -)sql"; - -enum AttrType { - Placeholder = 0, - FullAttrs = 1, - String = 2, - Missing = 3, - Misc = 4, - Failed = 5, -}; - -struct AttrDb -{ - struct State - { - SQLite db; - SQLiteStmt insertAttribute; - SQLiteStmt queryAttribute; - SQLiteStmt queryAttributes; - std::unique_ptr txn; - }; - - struct placeholder_t {}; - struct missing_t {}; - struct misc_t {}; - struct failed_t {}; - typedef uint64_t AttrId; - typedef std::pair AttrKey; - typedef std::variant, std::string, placeholder_t, missing_t, misc_t, failed_t> AttrValue; - - std::unique_ptr> _state; - - AttrDb(const Fingerprint & fingerprint) - : _state(std::make_unique>()) - { - auto state(_state->lock()); - - Path cacheDir = getCacheDir() + "/nix/eval-cache-v2"; - createDirs(cacheDir); - - Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite"; - - state->db = SQLite(dbPath); - state->db.isCache(); - state->db.exec(schema); - - state->insertAttribute.create(state->db, - "insert or replace into Attributes(parent, name, type, value) values (?, ?, ?, ?)"); - - state->queryAttribute.create(state->db, - "select rowid, type, value from Attributes where parent = ? and name = ?"); - - state->queryAttributes.create(state->db, - "select name from Attributes where parent = ?"); - - state->txn = std::make_unique(state->db); - } - - ~AttrDb() - { - try { - auto state(_state->lock()); - state->txn->commit(); - state->txn.reset(); - } catch (...) { - ignoreException(); - } - } - - AttrId setAttrs( - AttrKey key, - const std::vector & attrs) - { - auto state(_state->lock()); - - state->insertAttribute.use() - (key.first) - (key.second) - (AttrType::FullAttrs) - (0, false).exec(); - - AttrId rowId = state->db.getLastInsertedRowId(); - assert(rowId); - - for (auto & attr : attrs) - state->insertAttribute.use() - (rowId) - (attr) - (AttrType::Placeholder) - (0, false).exec(); - - return rowId; - } - - AttrId setString( - AttrKey key, - std::string_view s) - { - auto state(_state->lock()); - - state->insertAttribute.use() - (key.first) - (key.second) - (AttrType::String) - (s).exec(); - - return state->db.getLastInsertedRowId(); - } - - AttrId setPlaceholder(AttrKey key) - { - auto state(_state->lock()); - - state->insertAttribute.use() - (key.first) - (key.second) - (AttrType::Placeholder) - (0, false).exec(); - - return state->db.getLastInsertedRowId(); - } - - AttrId setMissing(AttrKey key) - { - auto state(_state->lock()); - - state->insertAttribute.use() - (key.first) - (key.second) - (AttrType::Missing) - (0, false).exec(); - - return state->db.getLastInsertedRowId(); - } - - AttrId setMisc(AttrKey key) - { - auto state(_state->lock()); - - state->insertAttribute.use() - (key.first) - (key.second) - (AttrType::Misc) - (0, false).exec(); - - return state->db.getLastInsertedRowId(); - } - - AttrId setFailed(AttrKey key) - { - auto state(_state->lock()); - - state->insertAttribute.use() - (key.first) - (key.second) - (AttrType::Failed) - (0, false).exec(); - - return state->db.getLastInsertedRowId(); - } - - std::optional> getAttr( - AttrKey key, - SymbolTable & symbols) - { - auto state(_state->lock()); - - auto queryAttribute(state->queryAttribute.use()(key.first)(key.second)); - if (!queryAttribute.next()) return {}; - - auto rowId = (AttrType) queryAttribute.getInt(0); - auto type = (AttrType) queryAttribute.getInt(1); - - if (type == AttrType::Placeholder) - return {{rowId, placeholder_t()}}; - else if (type == AttrType::FullAttrs) { - // FIXME: expensive, should separate this out. - std::vector attrs; - auto queryAttributes(state->queryAttributes.use()(rowId)); - while (queryAttributes.next()) - attrs.push_back(symbols.create(queryAttributes.getStr(0))); - return {{rowId, attrs}}; - } else if (type == AttrType::String) { - return {{rowId, queryAttribute.getStr(2)}}; - } else if (type == AttrType::Missing) { - return {{rowId, missing_t()}}; - } else if (type == AttrType::Misc) { - return {{rowId, misc_t()}}; - } else if (type == AttrType::Failed) { - return {{rowId, failed_t()}}; - } else - throw Error("unexpected type in evaluation cache"); - } -}; - -struct AttrCursor; - -struct AttrRoot : std::enable_shared_from_this -{ - std::shared_ptr db; - EvalState & state; - typedef std::function RootLoader; - RootLoader rootLoader; - RootValue value; - - AttrRoot(std::shared_ptr db, EvalState & state, RootLoader rootLoader) - : db(db) - , state(state) - , rootLoader(rootLoader) - { - } - - Value * getRootValue() - { - if (!value) { - debug("getting root value"); - value = allocRootValue(rootLoader()); - } - return *value; - } - - std::shared_ptr getRoot() - { - return std::make_shared(ref(shared_from_this()), std::nullopt); - } -}; - -struct AttrCursor : std::enable_shared_from_this -{ - ref root; - typedef std::optional, Symbol>> Parent; - Parent parent; - RootValue _value; - std::optional> cachedValue; - - AttrCursor( - ref root, - Parent parent, - Value * value = nullptr, - std::optional> && cachedValue = {}) - : root(root), parent(parent), cachedValue(std::move(cachedValue)) - { - if (value) - _value = allocRootValue(value); - } - - AttrDb::AttrKey getAttrKey() - { - if (!parent) - return {0, root->state.sEpsilon}; - if (!parent->first->cachedValue) { - parent->first->cachedValue = root->db->getAttr( - parent->first->getAttrKey(), root->state.symbols); - assert(parent->first->cachedValue); - } - return {parent->first->cachedValue->first, parent->second}; - } - - Value & getValue() - { - if (!_value) { - if (parent) { - auto & vParent = parent->first->getValue(); - root->state.forceAttrs(vParent); - auto attr = vParent.attrs->get(parent->second); - if (!attr) - throw Error("attribute '%s' is unexpectedly missing", getAttrPathStr()); - _value = allocRootValue(attr->value); - } else - _value = allocRootValue(root->getRootValue()); - } - return **_value; - } - - std::vector getAttrPath() const - { - if (parent) { - auto attrPath = parent->first->getAttrPath(); - attrPath.push_back(parent->second); - return attrPath; - } else - return {}; - } - - std::vector getAttrPath(Symbol name) const - { - auto attrPath = getAttrPath(); - attrPath.push_back(name); - return attrPath; - } - - std::string getAttrPathStr() const - { - return concatStringsSep(".", getAttrPath()); - } - - std::string getAttrPathStr(Symbol name) const - { - return concatStringsSep(".", getAttrPath(name)); - } - - Value & forceValue() - { - debug("evaluating uncached attribute %s", getAttrPathStr()); - - auto & v = getValue(); - - try { - root->state.forceValue(v); - } catch (EvalError &) { - debug("setting '%s' to failed", getAttrPathStr()); - if (root->db) - cachedValue = {root->db->setFailed(getAttrKey()), AttrDb::failed_t()}; - throw; - } - - if (root->db && (!cachedValue || std::get_if(&cachedValue->second))) { - if (v.type == tString) - cachedValue = {root->db->setString(getAttrKey(), v.string.s), v.string.s}; - else if (v.type == tAttrs) - ; // FIXME: do something? - else - cachedValue = {root->db->setMisc(getAttrKey()), AttrDb::misc_t()}; - } - - return v; - } - - std::shared_ptr maybeGetAttr(Symbol name) - { - if (root->db) { - if (!cachedValue) - cachedValue = root->db->getAttr(getAttrKey(), root->state.symbols); - - if (cachedValue) { - if (auto attrs = std::get_if>(&cachedValue->second)) { - for (auto & attr : *attrs) - if (attr == name) - return std::make_shared(root, std::make_pair(shared_from_this(), name)); - return nullptr; - } else if (std::get_if(&cachedValue->second)) { - auto attr = root->db->getAttr({cachedValue->first, name}, root->state.symbols); - if (attr) { - if (std::get_if(&attr->second)) - return nullptr; - else if (std::get_if(&attr->second)) - throw EvalError("cached failure of attribute '%s'", getAttrPathStr(name)); - else - return std::make_shared(root, - std::make_pair(shared_from_this(), name), nullptr, std::move(attr)); - } - // Incomplete attrset, so need to fall thru and - // evaluate to see whether 'name' exists - } else - return nullptr; - //throw TypeError("'%s' is not an attribute set", getAttrPathStr()); - } - } - - auto & v = forceValue(); - - if (v.type != tAttrs) - return nullptr; - //throw TypeError("'%s' is not an attribute set", getAttrPathStr()); - - auto attr = v.attrs->get(name); - - if (!attr) { - if (root->db) { - assert(cachedValue); - root->db->setMissing({cachedValue->first, name}); - } - return nullptr; - } - - std::optional> cachedValue2; - if (root->db) { - assert(cachedValue); - cachedValue2 = {root->db->setPlaceholder({cachedValue->first, name}), AttrDb::placeholder_t()}; - } - - return std::make_shared( - root, std::make_pair(shared_from_this(), name), attr->value, std::move(cachedValue2)); - } - - std::shared_ptr maybeGetAttr(std::string_view name) - { - return maybeGetAttr(root->state.symbols.create(name)); - } - - std::shared_ptr getAttr(Symbol name) - { - auto p = maybeGetAttr(name); - if (!p) - throw Error("attribute '%s' does not exist", getAttrPathStr(name)); - return p; - } - - std::shared_ptr getAttr(std::string_view name) - { - return getAttr(root->state.symbols.create(name)); - } - - std::string getString() - { - if (root->db) { - if (!cachedValue) - cachedValue = root->db->getAttr(getAttrKey(), root->state.symbols); - if (cachedValue && !std::get_if(&cachedValue->second)) { - if (auto s = std::get_if(&cachedValue->second)) { - debug("using cached string attribute '%s'", getAttrPathStr()); - return *s; - } else - throw TypeError("'%s' is not a string", getAttrPathStr()); - } - } - - auto & v = forceValue(); - - if (v.type != tString) - throw TypeError("'%s' is not a string", getAttrPathStr()); - - return v.string.s; - } - - std::vector getAttrs() - { - if (root->db) { - if (!cachedValue) - cachedValue = root->db->getAttr(getAttrKey(), root->state.symbols); - if (cachedValue && !std::get_if(&cachedValue->second)) { - if (auto attrs = std::get_if>(&cachedValue->second)) { - debug("using cached attrset attribute '%s'", getAttrPathStr()); - return *attrs; - } else - throw TypeError("'%s' is not an attribute set", getAttrPathStr()); - } - } - - auto & v = forceValue(); - - if (v.type != tAttrs) - throw TypeError("'%s' is not an attribute set", getAttrPathStr()); - - std::vector attrs; - for (auto & attr : *getValue().attrs) - attrs.push_back(attr.name); - std::sort(attrs.begin(), attrs.end(), [](const Symbol & a, const Symbol & b) { - return (const string &) a < (const string &) b; - }); - - if (root->db) - cachedValue = {root->db->setAttrs(getAttrKey(), attrs), attrs}; - - return attrs; - } - - bool isDerivation() - { - auto aType = maybeGetAttr("type"); - return aType && aType->getString() == "derivation"; - } -}; - struct CmdFlakeShow : FlakeCommand { bool showLegacy = false; @@ -1170,9 +697,9 @@ struct CmdFlakeShow : FlakeCommand auto state = getEvalState(); auto flake = lockFlake(); - std::function & attrPath, const std::string & headerPrefix, const std::string & nextPrefix)> visit; + std::function & attrPath, const std::string & headerPrefix, const std::string & nextPrefix)> visit; - visit = [&](AttrCursor & visitor, const std::vector & attrPath, const std::string & headerPrefix, const std::string & nextPrefix) + visit = [&](eval_cache::AttrCursor & visitor, const std::vector & attrPath, const std::string & headerPrefix, const std::string & nextPrefix) { Activity act(*logger, lvlInfo, actUnknown, fmt("evaluating '%s'", concatStringsSep(".", attrPath))); @@ -1286,28 +813,9 @@ struct CmdFlakeShow : FlakeCommand } }; - auto db = useEvalCache ? std::make_shared(flake.getFingerprint()) : nullptr; + auto cache = openEvalCache(*state, flake, useEvalCache); - auto root = std::make_shared(db, *state, - [&]() - { - /* For testing whether the evaluation cache is - complete. */ - if (getEnv("NIX_ALLOW_EVAL").value_or("1") == "0") - throw Error("not everything is cached, but evaluation is not allowed"); - - auto vFlake = state->allocValue(); - flake::callFlake(*state, flake, *vFlake); - - state->forceAttrs(*vFlake); - - auto aOutputs = vFlake->attrs->get(state->symbols.create("outputs")); - assert(aOutputs); - - return aOutputs->value; - }); - - visit(*root->getRoot(), {}, fmt(ANSI_BOLD "%s" ANSI_NORMAL, flake.flake.lockedRef), ""); + visit(*cache->getRoot(), {}, fmt(ANSI_BOLD "%s" ANSI_NORMAL, flake.flake.lockedRef), ""); } }; diff --git a/src/nix/installables.cc b/src/nix/installables.cc index b8c75aaaf..160fde589 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -162,7 +162,7 @@ struct InstallableStorePath : Installable } }; -std::vector InstallableValue::toDerivations() +std::vector InstallableValue::toDerivations() { auto state = cmd.getEvalState(); @@ -173,7 +173,7 @@ std::vector InstallableValue::toDerivations() DrvInfos drvInfos; getDerivations(*state, *v, "", autoArgs, drvInfos, false); - std::vector res; + std::vector res; for (auto & drvInfo : drvInfos) { res.push_back({ state->store->parseStorePath(drvInfo.queryDrvPath()), @@ -283,58 +283,76 @@ Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::Locked return aOutputs->value; } -std::tuple InstallableFlake::toDerivation() +ref openEvalCache( + EvalState & state, + const flake::LockedFlake & lockedFlake, + bool useEvalCache) +{ + return ref(std::make_shared( + useEvalCache, + lockedFlake.getFingerprint(), + state, + [&]() + { + /* For testing whether the evaluation cache is + complete. */ + if (getEnv("NIX_ALLOW_EVAL").value_or("1") == "0") + throw Error("not everything is cached, but evaluation is not allowed"); + + auto vFlake = state.allocValue(); + flake::callFlake(state, lockedFlake, *vFlake); + + state.forceAttrs(*vFlake); + + auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); + assert(aOutputs); + + return aOutputs->value; + })); +} + +std::tuple InstallableFlake::toDerivation() { auto state = cmd.getEvalState(); auto lockedFlake = lockFlake(*state, flakeRef, cmd.lockFlags); - Value * vOutputs = nullptr; - - auto emptyArgs = state->allocBindings(0); - - auto & evalCache = flake::EvalCache::singleton(); - - auto fingerprint = lockedFlake.getFingerprint(); + auto cache = openEvalCache(*state, lockedFlake, true); + auto root = cache->getRoot(); for (auto & attrPath : getActualAttrPaths()) { - auto drv = evalCache.getDerivation(fingerprint, attrPath); - if (drv) { - if (state->store->isValidPath(drv->drvPath)) - return {attrPath, lockedFlake.flake.lockedRef, std::move(*drv)}; + auto attr = root->findAlongAttrPath(parseAttrPath(*state, attrPath)); + if (!attr) continue; + + if (!attr->isDerivation()) + throw Error("flake output attribute '%s' is not a derivation", attrPath); + + auto aDrvPath = attr->getAttr(state->sDrvPath); + auto drvPath = state->store->parseStorePath(aDrvPath->getString()); + if (!state->store->isValidPath(drvPath)) { + /* The eval cache contains 'drvPath', but the actual path + has been garbage-collected. So force it to be + regenerated. */ + aDrvPath->forceValue(); + assert(state->store->isValidPath(drvPath)); } - if (!vOutputs) - vOutputs = getFlakeOutputs(*state, lockedFlake); + auto drvInfo = DerivationInfo{ + std::move(drvPath), + state->store->parseStorePath(attr->getAttr(state->sOutPath)->getString()), + attr->getAttr(state->sOutputName)->getString() + }; - try { - auto * v = findAlongAttrPath(*state, attrPath, *emptyArgs, *vOutputs).first; - state->forceValue(*v); - - auto drvInfo = getDerivation(*state, *v, false); - if (!drvInfo) - throw Error("flake output attribute '%s' is not a derivation", attrPath); - - auto drv = flake::EvalCache::Derivation{ - state->store->parseStorePath(drvInfo->queryDrvPath()), - state->store->parseStorePath(drvInfo->queryOutPath()), - drvInfo->queryOutputName() - }; - - evalCache.addDerivation(fingerprint, attrPath, drv); - - return {attrPath, lockedFlake.flake.lockedRef, std::move(drv)}; - } catch (AttrPathNotFound & e) { - } + return {attrPath, lockedFlake.flake.lockedRef, std::move(drvInfo)}; } throw Error("flake '%s' does not provide attribute %s", flakeRef, concatStringsSep(", ", quoteStrings(attrPaths))); } -std::vector InstallableFlake::toDerivations() +std::vector InstallableFlake::toDerivations() { - std::vector res; + std::vector res; res.push_back(std::get<2>(toDerivation())); return res; } diff --git a/src/nix/installables.hh b/src/nix/installables.hh index 8f2d50077..b258fb336 100644 --- a/src/nix/installables.hh +++ b/src/nix/installables.hh @@ -3,7 +3,7 @@ #include "util.hh" #include "path.hh" #include "eval.hh" -#include "flake/eval-cache.hh" +#include "flake/flake.hh" #include @@ -12,6 +12,8 @@ namespace nix { struct DrvInfo; struct SourceExprCommand; +namespace eval_cache { class EvalCache; } + struct Buildable { std::optional drvPath; @@ -63,7 +65,14 @@ struct InstallableValue : Installable InstallableValue(SourceExprCommand & cmd) : cmd(cmd) { } - virtual std::vector toDerivations(); + struct DerivationInfo + { + StorePath drvPath; + StorePath outPath; + std::string outputName; + }; + + virtual std::vector toDerivations(); Buildables toBuildables() override; }; @@ -86,11 +95,16 @@ struct InstallableFlake : InstallableValue Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake); - std::tuple toDerivation(); + std::tuple toDerivation(); - std::vector toDerivations() override; + std::vector toDerivations() override; std::pair toValue(EvalState & state) override; }; +ref openEvalCache( + EvalState & state, + const flake::LockedFlake & lockedFlake, + bool useEvalCache); + }