Get rid of the old eval cache

src/libexpr/attr-path.cc

@@ -6,11 +6,11 @@
 namespace nix {
 
 
-static Strings parseAttrPath(const string & s)
+static Strings parseAttrPath(std::string_view s)
 {
     Strings res;
     string cur;
-    string::const_iterator i = s.begin();
+    auto i = s.begin();
     while (i != s.end()) {
         if (*i == '.') {
             res.push_back(cur);
@@ -32,6 +32,15 @@ static Strings parseAttrPath(const string & s)
 }
 
 
+std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s)
+{
+    std::vector<Symbol> res;
+    for (auto & a : parseAttrPath(s))
+        res.push_back(state.symbols.create(a));
+    return res;
+}
+
+
 std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath,
     Bindings & autoArgs, Value & vIn)
 {

src/libexpr/attr-path.hh

@@ -16,4 +16,6 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
 /* Heuristic to find the filename and lineno or a nix value. */
 Pos findDerivationFilename(EvalState & state, Value & v, std::string what);
 
+std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s);
+
 }

src/libexpr/flake/eval-cache.cc

@@ -1,116 +1,460 @@
 #include "eval-cache.hh"
 #include "sqlite.hh"
 #include "eval.hh"
+#include "eval-inline.hh"
 
-#include <set>
+namespace nix::eval_cache {
-
-namespace nix::flake {
 
+// FIXME: inefficient representation of attrs
 static const char * schema = R"sql(
-
-create table if not exists Fingerprints (
-    fingerprint blob primary key not null,
-    timestamp   integer not null
-);
-
 create table if not exists Attributes (
-    fingerprint blob not null,
+    parent      integer not null,
-    attrPath    text not null,
+    name        text,
-    type        integer,
+    type        integer not null,
     value       text,
-    primary key (fingerprint, attrPath),
+    primary key (parent, name)
-    foreign key (fingerprint) references Fingerprints(fingerprint) on delete cascade
 );
 )sql";
 
-struct EvalCache::State
+struct AttrDb
 {
+    struct State
+    {
         SQLite db;
-    SQLiteStmt insertFingerprint;
         SQLiteStmt insertAttribute;
         SQLiteStmt queryAttribute;
-    std::set<Fingerprint> fingerprints;
+        SQLiteStmt queryAttributes;
-};
+        std::unique_ptr<SQLiteTxn> txn;
+    };
 
-EvalCache::EvalCache()
+    std::unique_ptr<Sync<State>> _state;
+
+    AttrDb(const Hash & fingerprint)
         : _state(std::make_unique<Sync<State>>())
-{
+    {
         auto state(_state->lock());
 
-    Path dbPath = getCacheDir() + "/nix/eval-cache-v1.sqlite";
+        Path cacheDir = getCacheDir() + "/nix/eval-cache-v1";
-    createDirs(dirOf(dbPath));
+        createDirs(cacheDir);
+
+        Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
 
         state->db = SQLite(dbPath);
         state->db.isCache();
         state->db.exec(schema);
 
-    state->insertFingerprint.create(state->db,
-        "insert or ignore into Fingerprints(fingerprint, timestamp) values (?, ?)");
-
         state->insertAttribute.create(state->db,
-        "insert or replace into Attributes(fingerprint, attrPath, type, value) values (?, ?, ?, ?)");
+            "insert or replace into Attributes(parent, name, type, value) values (?, ?, ?, ?)");
 
         state->queryAttribute.create(state->db,
-        "select type, value from Attributes where fingerprint = ? and attrPath = ?");
+            "select rowid, type, value from Attributes where parent = ? and name = ?");
-}
 
-enum ValueType {
+        state->queryAttributes.create(state->db,
-    Derivation = 1,
+            "select name from Attributes where parent = ?");
-};
 
-void EvalCache::addDerivation(
+        state->txn = std::make_unique<SQLiteTxn>(state->db);
-    const Fingerprint & fingerprint,
+    }
-    const std::string & attrPath,
-    const Derivation & drv)
-{
-    if (!evalSettings.pureEval) return;
 
+    ~AttrDb()
+    {
+        try {
             auto state(_state->lock());
+            state->txn->commit();
+            state->txn.reset();
+        } catch (...) {
+            ignoreException();
+        }
+    }
 
-    if (state->fingerprints.insert(fingerprint).second)
+    AttrId setAttrs(
-        // FIXME: update timestamp
+        AttrKey key,
-        state->insertFingerprint.use()
+        const std::vector<Symbol> & attrs)
-            (fingerprint.hash, fingerprint.hashSize)
+    {
-            (time(0)).exec();
+        auto state(_state->lock());
 
         state->insertAttribute.use()
-        (fingerprint.hash, fingerprint.hashSize)
+            (key.first)
-        (attrPath)
+            (key.second)
-        (ValueType::Derivation)
+            (AttrType::FullAttrs)
-        (std::string(drv.drvPath.to_string()) + " " + std::string(drv.outPath.to_string()) + " " + drv.outputName).exec();
+            (0, false).exec();
-}
 
-std::optional<EvalCache::Derivation> EvalCache::getDerivation(
+        AttrId rowId = state->db.getLastInsertedRowId();
-    const Fingerprint & fingerprint,
+        assert(rowId);
-    const std::string & attrPath)
-{
-    if (!evalSettings.pureEval) return {};
 
+        for (auto & attr : attrs)
+            state->insertAttribute.use()
+                (rowId)
+                (attr)
+                (AttrType::Placeholder)
+                (0, false).exec();
+
+        return rowId;
+    }
+
+    AttrId setString(
+        AttrKey key,
+        std::string_view s)
+    {
         auto state(_state->lock());
 
-    auto queryAttribute(state->queryAttribute.use()
+        state->insertAttribute.use()
-        (fingerprint.hash, fingerprint.hashSize)
+            (key.first)
-        (attrPath));
+            (key.second)
+            (AttrType::String)
+            (s).exec();
+
+        return state->db.getLastInsertedRowId();
+    }
+
+    AttrId setPlaceholder(AttrKey key)
+    {
+        auto state(_state->lock());
+
+        state->insertAttribute.use()
+            (key.first)
+            (key.second)
+            (AttrType::Placeholder)
+            (0, false).exec();
+
+        return state->db.getLastInsertedRowId();
+    }
+
+    AttrId setMissing(AttrKey key)
+    {
+        auto state(_state->lock());
+
+        state->insertAttribute.use()
+            (key.first)
+            (key.second)
+            (AttrType::Missing)
+            (0, false).exec();
+
+        return state->db.getLastInsertedRowId();
+    }
+
+    AttrId setMisc(AttrKey key)
+    {
+        auto state(_state->lock());
+
+        state->insertAttribute.use()
+            (key.first)
+            (key.second)
+            (AttrType::Misc)
+            (0, false).exec();
+
+        return state->db.getLastInsertedRowId();
+    }
+
+    AttrId setFailed(AttrKey key)
+    {
+        auto state(_state->lock());
+
+        state->insertAttribute.use()
+            (key.first)
+            (key.second)
+            (AttrType::Failed)
+            (0, false).exec();
+
+        return state->db.getLastInsertedRowId();
+    }
+
+    std::optional<std::pair<AttrId, AttrValue>> getAttr(
+        AttrKey key,
+        SymbolTable & symbols)
+    {
+        auto state(_state->lock());
+
+        auto queryAttribute(state->queryAttribute.use()(key.first)(key.second));
         if (!queryAttribute.next()) return {};
 
-    // FIXME: handle negative results
+        auto rowId = (AttrType) queryAttribute.getInt(0);
+        auto type = (AttrType) queryAttribute.getInt(1);
 
-    auto type = (ValueType) queryAttribute.getInt(0);
+        if (type == AttrType::Placeholder)
-    auto s = queryAttribute.getStr(1);
+            return {{rowId, placeholder_t()}};
+        else if (type == AttrType::FullAttrs) {
+            // FIXME: expensive, should separate this out.
+            std::vector<Symbol> attrs;
+            auto queryAttributes(state->queryAttributes.use()(rowId));
+            while (queryAttributes.next())
+                attrs.push_back(symbols.create(queryAttributes.getStr(0)));
+            return {{rowId, attrs}};
+        } else if (type == AttrType::String) {
+            return {{rowId, queryAttribute.getStr(2)}};
+        } else if (type == AttrType::Missing) {
+            return {{rowId, missing_t()}};
+        } else if (type == AttrType::Misc) {
+            return {{rowId, misc_t()}};
+        } else if (type == AttrType::Failed) {
+            return {{rowId, failed_t()}};
+        } else
+            throw Error("unexpected type in evaluation cache");
+    }
+};
 
-    if (type != ValueType::Derivation) return {};
+EvalCache::EvalCache(
-
+    bool useCache,
-    auto ss = tokenizeString<std::vector<std::string>>(s, " ");
+    const Hash & fingerprint,
-
+    EvalState & state,
-    debug("evaluation cache hit for '%s'", attrPath);
+    RootLoader rootLoader)
-
+    : db(useCache ? std::make_shared<AttrDb>(fingerprint) : nullptr)
-    return Derivation { StorePath::fromBaseName(ss[0]), StorePath::fromBaseName(ss[1]), ss[2] };
+    , state(state)
-}
+    , rootLoader(rootLoader)
-
-EvalCache & EvalCache::singleton()
 {
-    static std::unique_ptr<EvalCache> evalCache(new EvalCache());
+}
-    return *evalCache;
+
+Value * EvalCache::getRootValue()
+{
+    if (!value) {
+        debug("getting root value");
+        value = allocRootValue(rootLoader());
+    }
+    return *value;
+}
+
+std::shared_ptr<AttrCursor> EvalCache::getRoot()
+{
+    return std::make_shared<AttrCursor>(ref(shared_from_this()), std::nullopt);
+}
+
+AttrCursor::AttrCursor(
+    ref<EvalCache> root,
+    Parent parent,
+    Value * value,
+    std::optional<std::pair<AttrId, AttrValue>> && cachedValue)
+    : root(root), parent(parent), cachedValue(std::move(cachedValue))
+{
+    if (value)
+        _value = allocRootValue(value);
+}
+
+AttrKey AttrCursor::getKey()
+{
+    if (!parent)
+        return {0, root->state.sEpsilon};
+    if (!parent->first->cachedValue) {
+        parent->first->cachedValue = root->db->getAttr(
+            parent->first->getKey(), root->state.symbols);
+        assert(parent->first->cachedValue);
+    }
+    return {parent->first->cachedValue->first, parent->second};
+}
+
+Value & AttrCursor::getValue()
+{
+    if (!_value) {
+        if (parent) {
+            auto & vParent = parent->first->getValue();
+            root->state.forceAttrs(vParent);
+            auto attr = vParent.attrs->get(parent->second);
+            if (!attr)
+                throw Error("attribute '%s' is unexpectedly missing", getAttrPathStr());
+            _value = allocRootValue(attr->value);
+        } else
+            _value = allocRootValue(root->getRootValue());
+    }
+    return **_value;
+}
+
+std::vector<Symbol> AttrCursor::getAttrPath() const
+{
+    if (parent) {
+        auto attrPath = parent->first->getAttrPath();
+        attrPath.push_back(parent->second);
+        return attrPath;
+    } else
+        return {};
+}
+
+std::vector<Symbol> AttrCursor::getAttrPath(Symbol name) const
+{
+    auto attrPath = getAttrPath();
+    attrPath.push_back(name);
+    return attrPath;
+}
+
+std::string AttrCursor::getAttrPathStr() const
+{
+    return concatStringsSep(".", getAttrPath());
+}
+
+std::string AttrCursor::getAttrPathStr(Symbol name) const
+{
+    return concatStringsSep(".", getAttrPath(name));
+}
+
+Value & AttrCursor::forceValue()
+{
+    debug("evaluating uncached attribute %s", getAttrPathStr());
+
+    auto & v = getValue();
+
+    try {
+        root->state.forceValue(v);
+    } catch (EvalError &) {
+        debug("setting '%s' to failed", getAttrPathStr());
+        if (root->db)
+            cachedValue = {root->db->setFailed(getKey()), failed_t()};
+        throw;
+    }
+
+    if (root->db && (!cachedValue || std::get_if<placeholder_t>(&cachedValue->second))) {
+        if (v.type == tString)
+            cachedValue = {root->db->setString(getKey(), v.string.s), v.string.s};
+        else if (v.type == tAttrs)
+            ; // FIXME: do something?
+        else
+            cachedValue = {root->db->setMisc(getKey()), misc_t()};
+    }
+
+    return v;
+}
+
+std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name)
+{
+    if (root->db) {
+        if (!cachedValue)
+            cachedValue = root->db->getAttr(getKey(), root->state.symbols);
+
+        if (cachedValue) {
+            if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) {
+                for (auto & attr : *attrs)
+                    if (attr == name)
+                        return std::make_shared<AttrCursor>(root, std::make_pair(shared_from_this(), name));
+                return nullptr;
+            } else if (std::get_if<placeholder_t>(&cachedValue->second)) {
+                auto attr = root->db->getAttr({cachedValue->first, name}, root->state.symbols);
+                if (attr) {
+                    if (std::get_if<missing_t>(&attr->second))
+                        return nullptr;
+                    else if (std::get_if<failed_t>(&attr->second))
+                        throw EvalError("cached failure of attribute '%s'", getAttrPathStr(name));
+                    else
+                        return std::make_shared<AttrCursor>(root,
+                            std::make_pair(shared_from_this(), name), nullptr, std::move(attr));
+                }
+                // Incomplete attrset, so need to fall thru and
+                // evaluate to see whether 'name' exists
+            } else
+                return nullptr;
+                //throw TypeError("'%s' is not an attribute set", getAttrPathStr());
+        }
+    }
+
+    auto & v = forceValue();
+
+    if (v.type != tAttrs)
+        return nullptr;
+        //throw TypeError("'%s' is not an attribute set", getAttrPathStr());
+
+    auto attr = v.attrs->get(name);
+
+    if (!attr) {
+        if (root->db) {
+            if (!cachedValue)
+                cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()};
+            root->db->setMissing({cachedValue->first, name});
+        }
+        return nullptr;
+    }
+
+    std::optional<std::pair<AttrId, AttrValue>> cachedValue2;
+    if (root->db) {
+        if (!cachedValue)
+            cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()};
+        cachedValue2 = {root->db->setPlaceholder({cachedValue->first, name}), placeholder_t()};
+    }
+
+    return std::make_shared<AttrCursor>(
+        root, std::make_pair(shared_from_this(), name), attr->value, std::move(cachedValue2));
+}
+
+std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(std::string_view name)
+{
+    return maybeGetAttr(root->state.symbols.create(name));
+}
+
+std::shared_ptr<AttrCursor> AttrCursor::getAttr(Symbol name)
+{
+    auto p = maybeGetAttr(name);
+    if (!p)
+        throw Error("attribute '%s' does not exist", getAttrPathStr(name));
+    return p;
+}
+
+std::shared_ptr<AttrCursor> AttrCursor::getAttr(std::string_view name)
+{
+    return getAttr(root->state.symbols.create(name));
+}
+
+std::shared_ptr<AttrCursor> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath)
+{
+    auto res = shared_from_this();
+    for (auto & attr : attrPath) {
+        res = res->maybeGetAttr(attr);
+        if (!res) return {};
+    }
+    return res;
+}
+
+std::string AttrCursor::getString()
+{
+    if (root->db) {
+        if (!cachedValue)
+            cachedValue = root->db->getAttr(getKey(), root->state.symbols);
+        if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
+            if (auto s = std::get_if<std::string>(&cachedValue->second)) {
+                debug("using cached string attribute '%s'", getAttrPathStr());
+                return *s;
+            } else
+                throw TypeError("'%s' is not a string", getAttrPathStr());
+        }
+    }
+
+    auto & v = forceValue();
+
+    if (v.type != tString)
+        throw TypeError("'%s' is not a string", getAttrPathStr());
+
+    return v.string.s;
+}
+
+std::vector<Symbol> AttrCursor::getAttrs()
+{
+    if (root->db) {
+        if (!cachedValue)
+            cachedValue = root->db->getAttr(getKey(), root->state.symbols);
+        if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
+            if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) {
+                debug("using cached attrset attribute '%s'", getAttrPathStr());
+                return *attrs;
+            } else
+                throw TypeError("'%s' is not an attribute set", getAttrPathStr());
+        }
+    }
+
+    auto & v = forceValue();
+
+    if (v.type != tAttrs)
+        throw TypeError("'%s' is not an attribute set", getAttrPathStr());
+
+    std::vector<Symbol> attrs;
+    for (auto & attr : *getValue().attrs)
+        attrs.push_back(attr.name);
+    std::sort(attrs.begin(), attrs.end(), [](const Symbol & a, const Symbol & b) {
+        return (const string &) a < (const string &) b;
+    });
+
+    if (root->db)
+        cachedValue = {root->db->setAttrs(getKey(), attrs), attrs};
+
+    return attrs;
+}
+
+bool AttrCursor::isDerivation()
+{
+    auto aType = maybeGetAttr("type");
+    return aType && aType->getString() == "derivation";
 }
 
 }

src/libexpr/flake/eval-cache.hh

@@ -1,40 +1,103 @@
 #pragma once
 
 #include "sync.hh"
-#include "flake.hh"
+#include "hash.hh"
-#include "path.hh"
+#include "eval.hh"
 
-namespace nix { struct SQLite; struct SQLiteStmt; }
+#include <variant>
 
-namespace nix::flake {
+namespace nix::eval_cache {
 
-class EvalCache
+class AttrDb;
+class AttrCursor;
+
+class EvalCache : public std::enable_shared_from_this<EvalCache>
 {
-    struct State;
+    friend class AttrCursor;
 
-    std::unique_ptr<Sync<State>> _state;
+    std::shared_ptr<AttrDb> db;
+    EvalState & state;
+    typedef std::function<Value *()> RootLoader;
+    RootLoader rootLoader;
+    RootValue value;
 
-    EvalCache();
+    Value * getRootValue();
 
 public:
 
-    struct Derivation
+    EvalCache(
-    {
+        bool useCache,
-        StorePath drvPath;
+        const Hash & fingerprint,
-        StorePath outPath;
+        EvalState & state,
-        std::string outputName;
+        RootLoader rootLoader);
-    };
 
-    void addDerivation(
+    std::shared_ptr<AttrCursor> getRoot();
-        const Fingerprint & fingerprint,
+};
-        const std::string & attrPath,
-        const Derivation & drv);
 
-    std::optional<Derivation> getDerivation(
+enum AttrType {
-        const Fingerprint & fingerprint,
+    Placeholder = 0,
-        const std::string & attrPath);
+    FullAttrs = 1,
+    String = 2,
+    Missing = 3,
+    Misc = 4,
+    Failed = 5,
+};
 
-    static EvalCache & singleton();
+struct placeholder_t {};
+struct missing_t {};
+struct misc_t {};
+struct failed_t {};
+typedef uint64_t AttrId;
+typedef std::pair<AttrId, Symbol> AttrKey;
+typedef std::variant<std::vector<Symbol>, std::string, placeholder_t, missing_t, misc_t, failed_t> AttrValue;
+
+class AttrCursor : public std::enable_shared_from_this<AttrCursor>
+{
+    friend class EvalCache;
+
+    ref<EvalCache> root;
+    typedef std::optional<std::pair<std::shared_ptr<AttrCursor>, Symbol>> Parent;
+    Parent parent;
+    RootValue _value;
+    std::optional<std::pair<AttrId, AttrValue>> cachedValue;
+
+    AttrKey getKey();
+
+    Value & getValue();
+
+public:
+
+    AttrCursor(
+        ref<EvalCache> root,
+        Parent parent,
+        Value * value = nullptr,
+        std::optional<std::pair<AttrId, AttrValue>> && cachedValue = {});
+
+    std::vector<Symbol> getAttrPath() const;
+
+    std::vector<Symbol> getAttrPath(Symbol name) const;
+
+    std::string getAttrPathStr() const;
+
+    std::string getAttrPathStr(Symbol name) const;
+
+    std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name);
+
+    std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name);
+
+    std::shared_ptr<AttrCursor> getAttr(Symbol name);
+
+    std::shared_ptr<AttrCursor> getAttr(std::string_view name);
+
+    std::shared_ptr<AttrCursor> findAlongAttrPath(const std::vector<Symbol> & attrPath);
+
+    std::string getString();
+
+    std::vector<Symbol> getAttrs();
+
+    bool isDerivation();
+
+    Value & forceValue();
 };
 
 }

src/nix/flake.cc

@@ -11,7 +11,7 @@
 #include "fetchers.hh"
 #include "registry.hh"
 #include "json.hh"
-#include "sqlite.hh"
+#include "flake/eval-cache.hh"
 
 #include <nlohmann/json.hpp>
 #include <queue>
@@ -669,479 +669,6 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun
     }
 };
 
-// FIXME: inefficient representation of attrs
-static const char * schema = R"sql(
-create table if not exists Attributes (
-    parent      integer not null,
-    name        text,
-    type        integer not null,
-    value       text,
-    primary key (parent, name)
-);
-)sql";
-
-enum AttrType {
-    Placeholder = 0,
-    FullAttrs = 1,
-    String = 2,
-    Missing = 3,
-    Misc = 4,
-    Failed = 5,
-};
-
-struct AttrDb
-{
-    struct State
-    {
-        SQLite db;
-        SQLiteStmt insertAttribute;
-        SQLiteStmt queryAttribute;
-        SQLiteStmt queryAttributes;
-        std::unique_ptr<SQLiteTxn> txn;
-    };
-
-    struct placeholder_t {};
-    struct missing_t {};
-    struct misc_t {};
-    struct failed_t {};
-    typedef uint64_t AttrId;
-    typedef std::pair<AttrId, Symbol> AttrKey;
-    typedef std::variant<std::vector<Symbol>, std::string, placeholder_t, missing_t, misc_t, failed_t> AttrValue;
-
-    std::unique_ptr<Sync<State>> _state;
-
-    AttrDb(const Fingerprint & fingerprint)
-        : _state(std::make_unique<Sync<State>>())
-    {
-        auto state(_state->lock());
-
-        Path cacheDir = getCacheDir() + "/nix/eval-cache-v2";
-        createDirs(cacheDir);
-
-        Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
-
-        state->db = SQLite(dbPath);
-        state->db.isCache();
-        state->db.exec(schema);
-
-        state->insertAttribute.create(state->db,
-            "insert or replace into Attributes(parent, name, type, value) values (?, ?, ?, ?)");
-
-        state->queryAttribute.create(state->db,
-            "select rowid, type, value from Attributes where parent = ? and name = ?");
-
-        state->queryAttributes.create(state->db,
-            "select name from Attributes where parent = ?");
-
-        state->txn = std::make_unique<SQLiteTxn>(state->db);
-    }
-
-    ~AttrDb()
-    {
-        try {
-            auto state(_state->lock());
-            state->txn->commit();
-            state->txn.reset();
-        } catch (...) {
-            ignoreException();
-        }
-    }
-
-    AttrId setAttrs(
-        AttrKey key,
-        const std::vector<Symbol> & attrs)
-    {
-        auto state(_state->lock());
-
-        state->insertAttribute.use()
-            (key.first)
-            (key.second)
-            (AttrType::FullAttrs)
-            (0, false).exec();
-
-        AttrId rowId = state->db.getLastInsertedRowId();
-        assert(rowId);
-
-        for (auto & attr : attrs)
-            state->insertAttribute.use()
-                (rowId)
-                (attr)
-                (AttrType::Placeholder)
-                (0, false).exec();
-
-        return rowId;
-    }
-
-    AttrId setString(
-        AttrKey key,
-        std::string_view s)
-    {
-        auto state(_state->lock());
-
-        state->insertAttribute.use()
-            (key.first)
-            (key.second)
-            (AttrType::String)
-            (s).exec();
-
-        return state->db.getLastInsertedRowId();
-    }
-
-    AttrId setPlaceholder(AttrKey key)
-    {
-        auto state(_state->lock());
-
-        state->insertAttribute.use()
-            (key.first)
-            (key.second)
-            (AttrType::Placeholder)
-            (0, false).exec();
-
-        return state->db.getLastInsertedRowId();
-    }
-
-    AttrId setMissing(AttrKey key)
-    {
-        auto state(_state->lock());
-
-        state->insertAttribute.use()
-            (key.first)
-            (key.second)
-            (AttrType::Missing)
-            (0, false).exec();
-
-        return state->db.getLastInsertedRowId();
-    }
-
-    AttrId setMisc(AttrKey key)
-    {
-        auto state(_state->lock());
-
-        state->insertAttribute.use()
-            (key.first)
-            (key.second)
-            (AttrType::Misc)
-            (0, false).exec();
-
-        return state->db.getLastInsertedRowId();
-    }
-
-    AttrId setFailed(AttrKey key)
-    {
-        auto state(_state->lock());
-
-        state->insertAttribute.use()
-            (key.first)
-            (key.second)
-            (AttrType::Failed)
-            (0, false).exec();
-
-        return state->db.getLastInsertedRowId();
-    }
-
-    std::optional<std::pair<AttrId, AttrValue>> getAttr(
-        AttrKey key,
-        SymbolTable & symbols)
-    {
-        auto state(_state->lock());
-
-        auto queryAttribute(state->queryAttribute.use()(key.first)(key.second));
-        if (!queryAttribute.next()) return {};
-
-        auto rowId = (AttrType) queryAttribute.getInt(0);
-        auto type = (AttrType) queryAttribute.getInt(1);
-
-        if (type == AttrType::Placeholder)
-            return {{rowId, placeholder_t()}};
-        else if (type == AttrType::FullAttrs) {
-            // FIXME: expensive, should separate this out.
-            std::vector<Symbol> attrs;
-            auto queryAttributes(state->queryAttributes.use()(rowId));
-            while (queryAttributes.next())
-                attrs.push_back(symbols.create(queryAttributes.getStr(0)));
-            return {{rowId, attrs}};
-        } else if (type == AttrType::String) {
-            return {{rowId, queryAttribute.getStr(2)}};
-        } else if (type == AttrType::Missing) {
-            return {{rowId, missing_t()}};
-        } else if (type == AttrType::Misc) {
-            return {{rowId, misc_t()}};
-        } else if (type == AttrType::Failed) {
-            return {{rowId, failed_t()}};
-        } else
-            throw Error("unexpected type in evaluation cache");
-    }
-};
-
-struct AttrCursor;
-
-struct AttrRoot : std::enable_shared_from_this<AttrRoot>
-{
-    std::shared_ptr<AttrDb> db;
-    EvalState & state;
-    typedef std::function<Value *()> RootLoader;
-    RootLoader rootLoader;
-    RootValue value;
-
-    AttrRoot(std::shared_ptr<AttrDb> db, EvalState & state, RootLoader rootLoader)
-        : db(db)
-        , state(state)
-        , rootLoader(rootLoader)
-    {
-    }
-
-    Value * getRootValue()
-    {
-        if (!value) {
-            debug("getting root value");
-            value = allocRootValue(rootLoader());
-        }
-        return *value;
-    }
-
-    std::shared_ptr<AttrCursor> getRoot()
-    {
-        return std::make_shared<AttrCursor>(ref(shared_from_this()), std::nullopt);
-    }
-};
-
-struct AttrCursor : std::enable_shared_from_this<AttrCursor>
-{
-    ref<AttrRoot> root;
-    typedef std::optional<std::pair<std::shared_ptr<AttrCursor>, Symbol>> Parent;
-    Parent parent;
-    RootValue _value;
-    std::optional<std::pair<AttrDb::AttrId, AttrDb::AttrValue>> cachedValue;
-
-    AttrCursor(
-        ref<AttrRoot> root,
-        Parent parent,
-        Value * value = nullptr,
-        std::optional<std::pair<AttrDb::AttrId, AttrDb::AttrValue>> && cachedValue = {})
-        : root(root), parent(parent), cachedValue(std::move(cachedValue))
-    {
-        if (value)
-            _value = allocRootValue(value);
-    }
-
-    AttrDb::AttrKey getAttrKey()
-    {
-        if (!parent)
-            return {0, root->state.sEpsilon};
-        if (!parent->first->cachedValue) {
-            parent->first->cachedValue = root->db->getAttr(
-                parent->first->getAttrKey(), root->state.symbols);
-            assert(parent->first->cachedValue);
-        }
-        return {parent->first->cachedValue->first, parent->second};
-    }
-
-    Value & getValue()
-    {
-        if (!_value) {
-            if (parent) {
-                auto & vParent = parent->first->getValue();
-                root->state.forceAttrs(vParent);
-                auto attr = vParent.attrs->get(parent->second);
-                if (!attr)
-                    throw Error("attribute '%s' is unexpectedly missing", getAttrPathStr());
-                _value = allocRootValue(attr->value);
-            } else
-                _value = allocRootValue(root->getRootValue());
-        }
-        return **_value;
-    }
-
-    std::vector<Symbol> getAttrPath() const
-    {
-        if (parent) {
-            auto attrPath = parent->first->getAttrPath();
-            attrPath.push_back(parent->second);
-            return attrPath;
-        } else
-            return {};
-    }
-
-    std::vector<Symbol> getAttrPath(Symbol name) const
-    {
-        auto attrPath = getAttrPath();
-        attrPath.push_back(name);
-        return attrPath;
-    }
-
-    std::string getAttrPathStr() const
-    {
-        return concatStringsSep(".", getAttrPath());
-    }
-
-    std::string getAttrPathStr(Symbol name) const
-    {
-        return concatStringsSep(".", getAttrPath(name));
-    }
-
-    Value & forceValue()
-    {
-        debug("evaluating uncached attribute %s", getAttrPathStr());
-
-        auto & v = getValue();
-
-        try {
-            root->state.forceValue(v);
-        } catch (EvalError &) {
-            debug("setting '%s' to failed", getAttrPathStr());
-            if (root->db)
-                cachedValue = {root->db->setFailed(getAttrKey()), AttrDb::failed_t()};
-            throw;
-        }
-
-        if (root->db && (!cachedValue || std::get_if<AttrDb::placeholder_t>(&cachedValue->second))) {
-            if (v.type == tString)
-                cachedValue = {root->db->setString(getAttrKey(), v.string.s), v.string.s};
-            else if (v.type == tAttrs)
-                ; // FIXME: do something?
-            else
-                cachedValue = {root->db->setMisc(getAttrKey()), AttrDb::misc_t()};
-        }
-
-        return v;
-    }
-
-    std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name)
-    {
-        if (root->db) {
-            if (!cachedValue)
-                cachedValue = root->db->getAttr(getAttrKey(), root->state.symbols);
-
-            if (cachedValue) {
-                if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) {
-                    for (auto & attr : *attrs)
-                        if (attr == name)
-                            return std::make_shared<AttrCursor>(root, std::make_pair(shared_from_this(), name));
-                    return nullptr;
-                } else if (std::get_if<AttrDb::placeholder_t>(&cachedValue->second)) {
-                    auto attr = root->db->getAttr({cachedValue->first, name}, root->state.symbols);
-                    if (attr) {
-                        if (std::get_if<AttrDb::missing_t>(&attr->second))
-                            return nullptr;
-                        else if (std::get_if<AttrDb::failed_t>(&attr->second))
-                            throw EvalError("cached failure of attribute '%s'", getAttrPathStr(name));
-                        else
-                            return std::make_shared<AttrCursor>(root,
-                                std::make_pair(shared_from_this(), name), nullptr, std::move(attr));
-                    }
-                    // Incomplete attrset, so need to fall thru and
-                    // evaluate to see whether 'name' exists
-                } else
-                    return nullptr;
-                    //throw TypeError("'%s' is not an attribute set", getAttrPathStr());
-            }
-        }
-
-        auto & v = forceValue();
-
-        if (v.type != tAttrs)
-            return nullptr;
-            //throw TypeError("'%s' is not an attribute set", getAttrPathStr());
-
-        auto attr = v.attrs->get(name);
-
-        if (!attr) {
-            if (root->db) {
-                assert(cachedValue);
-                root->db->setMissing({cachedValue->first, name});
-            }
-            return nullptr;
-        }
-
-        std::optional<std::pair<AttrDb::AttrId, AttrDb::AttrValue>> cachedValue2;
-        if (root->db) {
-            assert(cachedValue);
-            cachedValue2 = {root->db->setPlaceholder({cachedValue->first, name}), AttrDb::placeholder_t()};
-        }
-
-        return std::make_shared<AttrCursor>(
-            root, std::make_pair(shared_from_this(), name), attr->value, std::move(cachedValue2));
-    }
-
-    std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name)
-    {
-        return maybeGetAttr(root->state.symbols.create(name));
-    }
-
-    std::shared_ptr<AttrCursor> getAttr(Symbol name)
-    {
-        auto p = maybeGetAttr(name);
-        if (!p)
-            throw Error("attribute '%s' does not exist", getAttrPathStr(name));
-        return p;
-    }
-
-    std::shared_ptr<AttrCursor> getAttr(std::string_view name)
-    {
-        return getAttr(root->state.symbols.create(name));
-    }
-
-    std::string getString()
-    {
-        if (root->db) {
-            if (!cachedValue)
-                cachedValue = root->db->getAttr(getAttrKey(), root->state.symbols);
-            if (cachedValue && !std::get_if<AttrDb::placeholder_t>(&cachedValue->second)) {
-                if (auto s = std::get_if<std::string>(&cachedValue->second)) {
-                    debug("using cached string attribute '%s'", getAttrPathStr());
-                    return *s;
-                } else
-                    throw TypeError("'%s' is not a string", getAttrPathStr());
-            }
-        }
-
-        auto & v = forceValue();
-
-        if (v.type != tString)
-            throw TypeError("'%s' is not a string", getAttrPathStr());
-
-        return v.string.s;
-    }
-
-    std::vector<Symbol> getAttrs()
-    {
-        if (root->db) {
-            if (!cachedValue)
-                cachedValue = root->db->getAttr(getAttrKey(), root->state.symbols);
-            if (cachedValue && !std::get_if<AttrDb::placeholder_t>(&cachedValue->second)) {
-                if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) {
-                    debug("using cached attrset attribute '%s'", getAttrPathStr());
-                    return *attrs;
-                } else
-                    throw TypeError("'%s' is not an attribute set", getAttrPathStr());
-            }
-        }
-
-        auto & v = forceValue();
-
-        if (v.type != tAttrs)
-            throw TypeError("'%s' is not an attribute set", getAttrPathStr());
-
-        std::vector<Symbol> attrs;
-        for (auto & attr : *getValue().attrs)
-            attrs.push_back(attr.name);
-        std::sort(attrs.begin(), attrs.end(), [](const Symbol & a, const Symbol & b) {
-            return (const string &) a < (const string &) b;
-        });
-
-        if (root->db)
-            cachedValue = {root->db->setAttrs(getAttrKey(), attrs), attrs};
-
-        return attrs;
-    }
-
-    bool isDerivation()
-    {
-        auto aType = maybeGetAttr("type");
-        return aType && aType->getString() == "derivation";
-    }
-};
-
 struct CmdFlakeShow : FlakeCommand
 {
     bool showLegacy = false;
@@ -1170,9 +697,9 @@ struct CmdFlakeShow : FlakeCommand
         auto state = getEvalState();
         auto flake = lockFlake();
 
-        std::function<void(AttrCursor & visitor, const std::vector<Symbol> & attrPath, const std::string & headerPrefix, const std::string & nextPrefix)> visit;
+        std::function<void(eval_cache::AttrCursor & visitor, const std::vector<Symbol> & attrPath, const std::string & headerPrefix, const std::string & nextPrefix)> visit;
 
-        visit = [&](AttrCursor & visitor, const std::vector<Symbol> & attrPath, const std::string & headerPrefix, const std::string & nextPrefix)
+        visit = [&](eval_cache::AttrCursor & visitor, const std::vector<Symbol> & attrPath, const std::string & headerPrefix, const std::string & nextPrefix)
         {
             Activity act(*logger, lvlInfo, actUnknown,
                 fmt("evaluating '%s'", concatStringsSep(".", attrPath)));
@@ -1286,28 +813,9 @@ struct CmdFlakeShow : FlakeCommand
             }
         };
 
-        auto db = useEvalCache ? std::make_shared<AttrDb>(flake.getFingerprint()) : nullptr;
+        auto cache = openEvalCache(*state, flake, useEvalCache);
 
-        auto root = std::make_shared<AttrRoot>(db, *state,
+        visit(*cache->getRoot(), {}, fmt(ANSI_BOLD "%s" ANSI_NORMAL, flake.flake.lockedRef), "");
-            [&]()
-            {
-                /* For testing whether the evaluation cache is
-                   complete. */
-                if (getEnv("NIX_ALLOW_EVAL").value_or("1") == "0")
-                    throw Error("not everything is cached, but evaluation is not allowed");
-
-                auto vFlake = state->allocValue();
-                flake::callFlake(*state, flake, *vFlake);
-
-                state->forceAttrs(*vFlake);
-
-                auto aOutputs = vFlake->attrs->get(state->symbols.create("outputs"));
-                assert(aOutputs);
-
-                return aOutputs->value;
-            });
-
-        visit(*root->getRoot(), {}, fmt(ANSI_BOLD "%s" ANSI_NORMAL, flake.flake.lockedRef), "");
     }
 };
 

src/nix/installables.cc

@@ -162,7 +162,7 @@ struct InstallableStorePath : Installable
     }
 };
 
-std::vector<flake::EvalCache::Derivation> InstallableValue::toDerivations()
+std::vector<InstallableValue::DerivationInfo> InstallableValue::toDerivations()
 {
     auto state = cmd.getEvalState();
 
@@ -173,7 +173,7 @@ std::vector<flake::EvalCache::Derivation> InstallableValue::toDerivations()
     DrvInfos drvInfos;
     getDerivations(*state, *v, "", autoArgs, drvInfos, false);
 
-    std::vector<flake::EvalCache::Derivation> res;
+    std::vector<DerivationInfo> res;
     for (auto & drvInfo : drvInfos) {
         res.push_back({
             state->store->parseStorePath(drvInfo.queryDrvPath()),
@@ -283,58 +283,76 @@ Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::Locked
     return aOutputs->value;
 }
 
-std::tuple<std::string, FlakeRef, flake::EvalCache::Derivation> InstallableFlake::toDerivation()
+ref<eval_cache::EvalCache> openEvalCache(
+    EvalState & state,
+    const flake::LockedFlake & lockedFlake,
+    bool useEvalCache)
+{
+    return ref(std::make_shared<nix::eval_cache::EvalCache>(
+        useEvalCache,
+        lockedFlake.getFingerprint(),
+        state,
+        [&]()
+        {
+            /* For testing whether the evaluation cache is
+               complete. */
+            if (getEnv("NIX_ALLOW_EVAL").value_or("1") == "0")
+                throw Error("not everything is cached, but evaluation is not allowed");
+
+            auto vFlake = state.allocValue();
+            flake::callFlake(state, lockedFlake, *vFlake);
+
+            state.forceAttrs(*vFlake);
+
+            auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs"));
+            assert(aOutputs);
+
+            return aOutputs->value;
+        }));
+}
+
+std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableFlake::toDerivation()
 {
     auto state = cmd.getEvalState();
 
     auto lockedFlake = lockFlake(*state, flakeRef, cmd.lockFlags);
 
-    Value * vOutputs = nullptr;
+    auto cache = openEvalCache(*state, lockedFlake, true);
-
+    auto root = cache->getRoot();
-    auto emptyArgs = state->allocBindings(0);
-
-    auto & evalCache = flake::EvalCache::singleton();
-
-    auto fingerprint = lockedFlake.getFingerprint();
 
     for (auto & attrPath : getActualAttrPaths()) {
-        auto drv = evalCache.getDerivation(fingerprint, attrPath);
+        auto attr = root->findAlongAttrPath(parseAttrPath(*state, attrPath));
-        if (drv) {
+        if (!attr) continue;
-            if (state->store->isValidPath(drv->drvPath))
-                return {attrPath, lockedFlake.flake.lockedRef, std::move(*drv)};
-        }
 
-        if (!vOutputs)
+        if (!attr->isDerivation())
-            vOutputs = getFlakeOutputs(*state, lockedFlake);
-
-        try {
-            auto * v = findAlongAttrPath(*state, attrPath, *emptyArgs, *vOutputs).first;
-            state->forceValue(*v);
-
-            auto drvInfo = getDerivation(*state, *v, false);
-            if (!drvInfo)
             throw Error("flake output attribute '%s' is not a derivation", attrPath);
 
-            auto drv = flake::EvalCache::Derivation{
+        auto aDrvPath = attr->getAttr(state->sDrvPath);
-                state->store->parseStorePath(drvInfo->queryDrvPath()),
+        auto drvPath = state->store->parseStorePath(aDrvPath->getString());
-                state->store->parseStorePath(drvInfo->queryOutPath()),
+        if (!state->store->isValidPath(drvPath)) {
-                drvInfo->queryOutputName()
+            /* The eval cache contains 'drvPath', but the actual path
+               has been garbage-collected. So force it to be
+               regenerated. */
+            aDrvPath->forceValue();
+            assert(state->store->isValidPath(drvPath));
+        }
+
+        auto drvInfo = DerivationInfo{
+            std::move(drvPath),
+            state->store->parseStorePath(attr->getAttr(state->sOutPath)->getString()),
+            attr->getAttr(state->sOutputName)->getString()
         };
 
-            evalCache.addDerivation(fingerprint, attrPath, drv);
+        return {attrPath, lockedFlake.flake.lockedRef, std::move(drvInfo)};
-
-            return {attrPath, lockedFlake.flake.lockedRef, std::move(drv)};
-        } catch (AttrPathNotFound & e) {
-        }
     }
 
     throw Error("flake '%s' does not provide attribute %s",
         flakeRef, concatStringsSep(", ", quoteStrings(attrPaths)));
 }
 
-std::vector<flake::EvalCache::Derivation> InstallableFlake::toDerivations()
+std::vector<InstallableValue::DerivationInfo> InstallableFlake::toDerivations()
 {
-    std::vector<flake::EvalCache::Derivation> res;
+    std::vector<DerivationInfo> res;
     res.push_back(std::get<2>(toDerivation()));
     return res;
 }

src/nix/installables.hh

@@ -3,7 +3,7 @@
 #include "util.hh"
 #include "path.hh"
 #include "eval.hh"
-#include "flake/eval-cache.hh"
+#include "flake/flake.hh"
 
 #include <optional>
 
@@ -12,6 +12,8 @@ namespace nix {
 struct DrvInfo;
 struct SourceExprCommand;
 
+namespace eval_cache { class EvalCache; }
+
 struct Buildable
 {
     std::optional<StorePath> drvPath;
@@ -63,7 +65,14 @@ struct InstallableValue : Installable
 
     InstallableValue(SourceExprCommand & cmd) : cmd(cmd) { }
 
-    virtual std::vector<flake::EvalCache::Derivation> toDerivations();
+    struct DerivationInfo
+    {
+        StorePath drvPath;
+        StorePath outPath;
+        std::string outputName;
+    };
+
+    virtual std::vector<DerivationInfo> toDerivations();
 
     Buildables toBuildables() override;
 };
@@ -86,11 +95,16 @@ struct InstallableFlake : InstallableValue
 
     Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake);
 
-    std::tuple<std::string, FlakeRef, flake::EvalCache::Derivation> toDerivation();
+    std::tuple<std::string, FlakeRef, DerivationInfo> toDerivation();
 
-    std::vector<flake::EvalCache::Derivation> toDerivations() override;
+    std::vector<DerivationInfo> toDerivations() override;
 
     std::pair<Value *, Pos> toValue(EvalState & state) override;
 };
 
+ref<eval_cache::EvalCache> openEvalCache(
+    EvalState & state,
+    const flake::LockedFlake & lockedFlake,
+    bool useEvalCache);
+
 }