Get rid of the old eval cache

This commit is contained in:
Eelco Dolstra 2020-04-20 13:13:52 +02:00
parent 0725ab2fd7
commit 539a9c1c5f
7 changed files with 609 additions and 651 deletions

View file

@ -6,11 +6,11 @@
namespace nix { namespace nix {
static Strings parseAttrPath(const string & s) static Strings parseAttrPath(std::string_view s)
{ {
Strings res; Strings res;
string cur; string cur;
string::const_iterator i = s.begin(); auto i = s.begin();
while (i != s.end()) { while (i != s.end()) {
if (*i == '.') { if (*i == '.') {
res.push_back(cur); res.push_back(cur);
@ -32,6 +32,15 @@ static Strings parseAttrPath(const string & s)
} }
std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s)
{
std::vector<Symbol> res;
for (auto & a : parseAttrPath(s))
res.push_back(state.symbols.create(a));
return res;
}
std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath, std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath,
Bindings & autoArgs, Value & vIn) Bindings & autoArgs, Value & vIn)
{ {

View file

@ -16,4 +16,6 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
/* Heuristic to find the filename and lineno or a nix value. */ /* Heuristic to find the filename and lineno or a nix value. */
Pos findDerivationFilename(EvalState & state, Value & v, std::string what); Pos findDerivationFilename(EvalState & state, Value & v, std::string what);
std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s);
} }

View file

@ -1,116 +1,460 @@
#include "eval-cache.hh" #include "eval-cache.hh"
#include "sqlite.hh" #include "sqlite.hh"
#include "eval.hh" #include "eval.hh"
#include "eval-inline.hh"
#include <set> namespace nix::eval_cache {
namespace nix::flake {
// FIXME: inefficient representation of attrs
static const char * schema = R"sql( static const char * schema = R"sql(
create table if not exists Fingerprints (
fingerprint blob primary key not null,
timestamp integer not null
);
create table if not exists Attributes ( create table if not exists Attributes (
fingerprint blob not null, parent integer not null,
attrPath text not null, name text,
type integer, type integer not null,
value text, value text,
primary key (fingerprint, attrPath), primary key (parent, name)
foreign key (fingerprint) references Fingerprints(fingerprint) on delete cascade
); );
)sql"; )sql";
struct EvalCache::State struct AttrDb
{
struct State
{ {
SQLite db; SQLite db;
SQLiteStmt insertFingerprint;
SQLiteStmt insertAttribute; SQLiteStmt insertAttribute;
SQLiteStmt queryAttribute; SQLiteStmt queryAttribute;
std::set<Fingerprint> fingerprints; SQLiteStmt queryAttributes;
std::unique_ptr<SQLiteTxn> txn;
}; };
EvalCache::EvalCache() std::unique_ptr<Sync<State>> _state;
AttrDb(const Hash & fingerprint)
: _state(std::make_unique<Sync<State>>()) : _state(std::make_unique<Sync<State>>())
{ {
auto state(_state->lock()); auto state(_state->lock());
Path dbPath = getCacheDir() + "/nix/eval-cache-v1.sqlite"; Path cacheDir = getCacheDir() + "/nix/eval-cache-v1";
createDirs(dirOf(dbPath)); createDirs(cacheDir);
Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
state->db = SQLite(dbPath); state->db = SQLite(dbPath);
state->db.isCache(); state->db.isCache();
state->db.exec(schema); state->db.exec(schema);
state->insertFingerprint.create(state->db,
"insert or ignore into Fingerprints(fingerprint, timestamp) values (?, ?)");
state->insertAttribute.create(state->db, state->insertAttribute.create(state->db,
"insert or replace into Attributes(fingerprint, attrPath, type, value) values (?, ?, ?, ?)"); "insert or replace into Attributes(parent, name, type, value) values (?, ?, ?, ?)");
state->queryAttribute.create(state->db, state->queryAttribute.create(state->db,
"select type, value from Attributes where fingerprint = ? and attrPath = ?"); "select rowid, type, value from Attributes where parent = ? and name = ?");
state->queryAttributes.create(state->db,
"select name from Attributes where parent = ?");
state->txn = std::make_unique<SQLiteTxn>(state->db);
} }
enum ValueType { ~AttrDb()
Derivation = 1,
};
void EvalCache::addDerivation(
const Fingerprint & fingerprint,
const std::string & attrPath,
const Derivation & drv)
{ {
if (!evalSettings.pureEval) return; try {
auto state(_state->lock()); auto state(_state->lock());
state->txn->commit();
state->txn.reset();
} catch (...) {
ignoreException();
}
}
if (state->fingerprints.insert(fingerprint).second) AttrId setAttrs(
// FIXME: update timestamp AttrKey key,
state->insertFingerprint.use() const std::vector<Symbol> & attrs)
(fingerprint.hash, fingerprint.hashSize) {
(time(0)).exec(); auto state(_state->lock());
state->insertAttribute.use() state->insertAttribute.use()
(fingerprint.hash, fingerprint.hashSize) (key.first)
(attrPath) (key.second)
(ValueType::Derivation) (AttrType::FullAttrs)
(std::string(drv.drvPath.to_string()) + " " + std::string(drv.outPath.to_string()) + " " + drv.outputName).exec(); (0, false).exec();
AttrId rowId = state->db.getLastInsertedRowId();
assert(rowId);
for (auto & attr : attrs)
state->insertAttribute.use()
(rowId)
(attr)
(AttrType::Placeholder)
(0, false).exec();
return rowId;
} }
std::optional<EvalCache::Derivation> EvalCache::getDerivation( AttrId setString(
const Fingerprint & fingerprint, AttrKey key,
const std::string & attrPath) std::string_view s)
{ {
if (!evalSettings.pureEval) return {};
auto state(_state->lock()); auto state(_state->lock());
auto queryAttribute(state->queryAttribute.use() state->insertAttribute.use()
(fingerprint.hash, fingerprint.hashSize) (key.first)
(attrPath)); (key.second)
(AttrType::String)
(s).exec();
return state->db.getLastInsertedRowId();
}
AttrId setPlaceholder(AttrKey key)
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::Placeholder)
(0, false).exec();
return state->db.getLastInsertedRowId();
}
AttrId setMissing(AttrKey key)
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::Missing)
(0, false).exec();
return state->db.getLastInsertedRowId();
}
AttrId setMisc(AttrKey key)
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::Misc)
(0, false).exec();
return state->db.getLastInsertedRowId();
}
AttrId setFailed(AttrKey key)
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::Failed)
(0, false).exec();
return state->db.getLastInsertedRowId();
}
std::optional<std::pair<AttrId, AttrValue>> getAttr(
AttrKey key,
SymbolTable & symbols)
{
auto state(_state->lock());
auto queryAttribute(state->queryAttribute.use()(key.first)(key.second));
if (!queryAttribute.next()) return {}; if (!queryAttribute.next()) return {};
// FIXME: handle negative results auto rowId = (AttrType) queryAttribute.getInt(0);
auto type = (AttrType) queryAttribute.getInt(1);
auto type = (ValueType) queryAttribute.getInt(0); if (type == AttrType::Placeholder)
auto s = queryAttribute.getStr(1); return {{rowId, placeholder_t()}};
else if (type == AttrType::FullAttrs) {
if (type != ValueType::Derivation) return {}; // FIXME: expensive, should separate this out.
std::vector<Symbol> attrs;
auto ss = tokenizeString<std::vector<std::string>>(s, " "); auto queryAttributes(state->queryAttributes.use()(rowId));
while (queryAttributes.next())
debug("evaluation cache hit for '%s'", attrPath); attrs.push_back(symbols.create(queryAttributes.getStr(0)));
return {{rowId, attrs}};
return Derivation { StorePath::fromBaseName(ss[0]), StorePath::fromBaseName(ss[1]), ss[2] }; } else if (type == AttrType::String) {
return {{rowId, queryAttribute.getStr(2)}};
} else if (type == AttrType::Missing) {
return {{rowId, missing_t()}};
} else if (type == AttrType::Misc) {
return {{rowId, misc_t()}};
} else if (type == AttrType::Failed) {
return {{rowId, failed_t()}};
} else
throw Error("unexpected type in evaluation cache");
} }
};
EvalCache & EvalCache::singleton() EvalCache::EvalCache(
bool useCache,
const Hash & fingerprint,
EvalState & state,
RootLoader rootLoader)
: db(useCache ? std::make_shared<AttrDb>(fingerprint) : nullptr)
, state(state)
, rootLoader(rootLoader)
{ {
static std::unique_ptr<EvalCache> evalCache(new EvalCache()); }
return *evalCache;
Value * EvalCache::getRootValue()
{
if (!value) {
debug("getting root value");
value = allocRootValue(rootLoader());
}
return *value;
}
std::shared_ptr<AttrCursor> EvalCache::getRoot()
{
return std::make_shared<AttrCursor>(ref(shared_from_this()), std::nullopt);
}
AttrCursor::AttrCursor(
ref<EvalCache> root,
Parent parent,
Value * value,
std::optional<std::pair<AttrId, AttrValue>> && cachedValue)
: root(root), parent(parent), cachedValue(std::move(cachedValue))
{
if (value)
_value = allocRootValue(value);
}
AttrKey AttrCursor::getKey()
{
if (!parent)
return {0, root->state.sEpsilon};
if (!parent->first->cachedValue) {
parent->first->cachedValue = root->db->getAttr(
parent->first->getKey(), root->state.symbols);
assert(parent->first->cachedValue);
}
return {parent->first->cachedValue->first, parent->second};
}
Value & AttrCursor::getValue()
{
if (!_value) {
if (parent) {
auto & vParent = parent->first->getValue();
root->state.forceAttrs(vParent);
auto attr = vParent.attrs->get(parent->second);
if (!attr)
throw Error("attribute '%s' is unexpectedly missing", getAttrPathStr());
_value = allocRootValue(attr->value);
} else
_value = allocRootValue(root->getRootValue());
}
return **_value;
}
std::vector<Symbol> AttrCursor::getAttrPath() const
{
if (parent) {
auto attrPath = parent->first->getAttrPath();
attrPath.push_back(parent->second);
return attrPath;
} else
return {};
}
std::vector<Symbol> AttrCursor::getAttrPath(Symbol name) const
{
auto attrPath = getAttrPath();
attrPath.push_back(name);
return attrPath;
}
std::string AttrCursor::getAttrPathStr() const
{
return concatStringsSep(".", getAttrPath());
}
std::string AttrCursor::getAttrPathStr(Symbol name) const
{
return concatStringsSep(".", getAttrPath(name));
}
Value & AttrCursor::forceValue()
{
debug("evaluating uncached attribute %s", getAttrPathStr());
auto & v = getValue();
try {
root->state.forceValue(v);
} catch (EvalError &) {
debug("setting '%s' to failed", getAttrPathStr());
if (root->db)
cachedValue = {root->db->setFailed(getKey()), failed_t()};
throw;
}
if (root->db && (!cachedValue || std::get_if<placeholder_t>(&cachedValue->second))) {
if (v.type == tString)
cachedValue = {root->db->setString(getKey(), v.string.s), v.string.s};
else if (v.type == tAttrs)
; // FIXME: do something?
else
cachedValue = {root->db->setMisc(getKey()), misc_t()};
}
return v;
}
std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name)
{
if (root->db) {
if (!cachedValue)
cachedValue = root->db->getAttr(getKey(), root->state.symbols);
if (cachedValue) {
if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) {
for (auto & attr : *attrs)
if (attr == name)
return std::make_shared<AttrCursor>(root, std::make_pair(shared_from_this(), name));
return nullptr;
} else if (std::get_if<placeholder_t>(&cachedValue->second)) {
auto attr = root->db->getAttr({cachedValue->first, name}, root->state.symbols);
if (attr) {
if (std::get_if<missing_t>(&attr->second))
return nullptr;
else if (std::get_if<failed_t>(&attr->second))
throw EvalError("cached failure of attribute '%s'", getAttrPathStr(name));
else
return std::make_shared<AttrCursor>(root,
std::make_pair(shared_from_this(), name), nullptr, std::move(attr));
}
// Incomplete attrset, so need to fall thru and
// evaluate to see whether 'name' exists
} else
return nullptr;
//throw TypeError("'%s' is not an attribute set", getAttrPathStr());
}
}
auto & v = forceValue();
if (v.type != tAttrs)
return nullptr;
//throw TypeError("'%s' is not an attribute set", getAttrPathStr());
auto attr = v.attrs->get(name);
if (!attr) {
if (root->db) {
if (!cachedValue)
cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()};
root->db->setMissing({cachedValue->first, name});
}
return nullptr;
}
std::optional<std::pair<AttrId, AttrValue>> cachedValue2;
if (root->db) {
if (!cachedValue)
cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()};
cachedValue2 = {root->db->setPlaceholder({cachedValue->first, name}), placeholder_t()};
}
return std::make_shared<AttrCursor>(
root, std::make_pair(shared_from_this(), name), attr->value, std::move(cachedValue2));
}
std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(std::string_view name)
{
return maybeGetAttr(root->state.symbols.create(name));
}
std::shared_ptr<AttrCursor> AttrCursor::getAttr(Symbol name)
{
auto p = maybeGetAttr(name);
if (!p)
throw Error("attribute '%s' does not exist", getAttrPathStr(name));
return p;
}
std::shared_ptr<AttrCursor> AttrCursor::getAttr(std::string_view name)
{
return getAttr(root->state.symbols.create(name));
}
std::shared_ptr<AttrCursor> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath)
{
auto res = shared_from_this();
for (auto & attr : attrPath) {
res = res->maybeGetAttr(attr);
if (!res) return {};
}
return res;
}
std::string AttrCursor::getString()
{
if (root->db) {
if (!cachedValue)
cachedValue = root->db->getAttr(getKey(), root->state.symbols);
if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
if (auto s = std::get_if<std::string>(&cachedValue->second)) {
debug("using cached string attribute '%s'", getAttrPathStr());
return *s;
} else
throw TypeError("'%s' is not a string", getAttrPathStr());
}
}
auto & v = forceValue();
if (v.type != tString)
throw TypeError("'%s' is not a string", getAttrPathStr());
return v.string.s;
}
std::vector<Symbol> AttrCursor::getAttrs()
{
if (root->db) {
if (!cachedValue)
cachedValue = root->db->getAttr(getKey(), root->state.symbols);
if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) {
debug("using cached attrset attribute '%s'", getAttrPathStr());
return *attrs;
} else
throw TypeError("'%s' is not an attribute set", getAttrPathStr());
}
}
auto & v = forceValue();
if (v.type != tAttrs)
throw TypeError("'%s' is not an attribute set", getAttrPathStr());
std::vector<Symbol> attrs;
for (auto & attr : *getValue().attrs)
attrs.push_back(attr.name);
std::sort(attrs.begin(), attrs.end(), [](const Symbol & a, const Symbol & b) {
return (const string &) a < (const string &) b;
});
if (root->db)
cachedValue = {root->db->setAttrs(getKey(), attrs), attrs};
return attrs;
}
bool AttrCursor::isDerivation()
{
auto aType = maybeGetAttr("type");
return aType && aType->getString() == "derivation";
} }
} }

View file

@ -1,40 +1,103 @@
#pragma once #pragma once
#include "sync.hh" #include "sync.hh"
#include "flake.hh" #include "hash.hh"
#include "path.hh" #include "eval.hh"
namespace nix { struct SQLite; struct SQLiteStmt; } #include <variant>
namespace nix::flake { namespace nix::eval_cache {
class EvalCache class AttrDb;
class AttrCursor;
class EvalCache : public std::enable_shared_from_this<EvalCache>
{ {
struct State; friend class AttrCursor;
std::unique_ptr<Sync<State>> _state; std::shared_ptr<AttrDb> db;
EvalState & state;
typedef std::function<Value *()> RootLoader;
RootLoader rootLoader;
RootValue value;
EvalCache(); Value * getRootValue();
public: public:
struct Derivation EvalCache(
{ bool useCache,
StorePath drvPath; const Hash & fingerprint,
StorePath outPath; EvalState & state,
std::string outputName; RootLoader rootLoader);
std::shared_ptr<AttrCursor> getRoot();
}; };
void addDerivation( enum AttrType {
const Fingerprint & fingerprint, Placeholder = 0,
const std::string & attrPath, FullAttrs = 1,
const Derivation & drv); String = 2,
Missing = 3,
Misc = 4,
Failed = 5,
};
std::optional<Derivation> getDerivation( struct placeholder_t {};
const Fingerprint & fingerprint, struct missing_t {};
const std::string & attrPath); struct misc_t {};
struct failed_t {};
typedef uint64_t AttrId;
typedef std::pair<AttrId, Symbol> AttrKey;
typedef std::variant<std::vector<Symbol>, std::string, placeholder_t, missing_t, misc_t, failed_t> AttrValue;
static EvalCache & singleton(); class AttrCursor : public std::enable_shared_from_this<AttrCursor>
{
friend class EvalCache;
ref<EvalCache> root;
typedef std::optional<std::pair<std::shared_ptr<AttrCursor>, Symbol>> Parent;
Parent parent;
RootValue _value;
std::optional<std::pair<AttrId, AttrValue>> cachedValue;
AttrKey getKey();
Value & getValue();
public:
AttrCursor(
ref<EvalCache> root,
Parent parent,
Value * value = nullptr,
std::optional<std::pair<AttrId, AttrValue>> && cachedValue = {});
std::vector<Symbol> getAttrPath() const;
std::vector<Symbol> getAttrPath(Symbol name) const;
std::string getAttrPathStr() const;
std::string getAttrPathStr(Symbol name) const;
std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name);
std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name);
std::shared_ptr<AttrCursor> getAttr(Symbol name);
std::shared_ptr<AttrCursor> getAttr(std::string_view name);
std::shared_ptr<AttrCursor> findAlongAttrPath(const std::vector<Symbol> & attrPath);
std::string getString();
std::vector<Symbol> getAttrs();
bool isDerivation();
Value & forceValue();
}; };
} }

View file

@ -11,7 +11,7 @@
#include "fetchers.hh" #include "fetchers.hh"
#include "registry.hh" #include "registry.hh"
#include "json.hh" #include "json.hh"
#include "sqlite.hh" #include "flake/eval-cache.hh"
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
#include <queue> #include <queue>
@ -669,479 +669,6 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun
} }
}; };
// FIXME: inefficient representation of attrs
static const char * schema = R"sql(
create table if not exists Attributes (
parent integer not null,
name text,
type integer not null,
value text,
primary key (parent, name)
);
)sql";
enum AttrType {
Placeholder = 0,
FullAttrs = 1,
String = 2,
Missing = 3,
Misc = 4,
Failed = 5,
};
struct AttrDb
{
struct State
{
SQLite db;
SQLiteStmt insertAttribute;
SQLiteStmt queryAttribute;
SQLiteStmt queryAttributes;
std::unique_ptr<SQLiteTxn> txn;
};
struct placeholder_t {};
struct missing_t {};
struct misc_t {};
struct failed_t {};
typedef uint64_t AttrId;
typedef std::pair<AttrId, Symbol> AttrKey;
typedef std::variant<std::vector<Symbol>, std::string, placeholder_t, missing_t, misc_t, failed_t> AttrValue;
std::unique_ptr<Sync<State>> _state;
AttrDb(const Fingerprint & fingerprint)
: _state(std::make_unique<Sync<State>>())
{
auto state(_state->lock());
Path cacheDir = getCacheDir() + "/nix/eval-cache-v2";
createDirs(cacheDir);
Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
state->db = SQLite(dbPath);
state->db.isCache();
state->db.exec(schema);
state->insertAttribute.create(state->db,
"insert or replace into Attributes(parent, name, type, value) values (?, ?, ?, ?)");
state->queryAttribute.create(state->db,
"select rowid, type, value from Attributes where parent = ? and name = ?");
state->queryAttributes.create(state->db,
"select name from Attributes where parent = ?");
state->txn = std::make_unique<SQLiteTxn>(state->db);
}
~AttrDb()
{
try {
auto state(_state->lock());
state->txn->commit();
state->txn.reset();
} catch (...) {
ignoreException();
}
}
AttrId setAttrs(
AttrKey key,
const std::vector<Symbol> & attrs)
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::FullAttrs)
(0, false).exec();
AttrId rowId = state->db.getLastInsertedRowId();
assert(rowId);
for (auto & attr : attrs)
state->insertAttribute.use()
(rowId)
(attr)
(AttrType::Placeholder)
(0, false).exec();
return rowId;
}
AttrId setString(
AttrKey key,
std::string_view s)
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::String)
(s).exec();
return state->db.getLastInsertedRowId();
}
AttrId setPlaceholder(AttrKey key)
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::Placeholder)
(0, false).exec();
return state->db.getLastInsertedRowId();
}
AttrId setMissing(AttrKey key)
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::Missing)
(0, false).exec();
return state->db.getLastInsertedRowId();
}
AttrId setMisc(AttrKey key)
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::Misc)
(0, false).exec();
return state->db.getLastInsertedRowId();
}
AttrId setFailed(AttrKey key)
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::Failed)
(0, false).exec();
return state->db.getLastInsertedRowId();
}
std::optional<std::pair<AttrId, AttrValue>> getAttr(
AttrKey key,
SymbolTable & symbols)
{
auto state(_state->lock());
auto queryAttribute(state->queryAttribute.use()(key.first)(key.second));
if (!queryAttribute.next()) return {};
auto rowId = (AttrType) queryAttribute.getInt(0);
auto type = (AttrType) queryAttribute.getInt(1);
if (type == AttrType::Placeholder)
return {{rowId, placeholder_t()}};
else if (type == AttrType::FullAttrs) {
// FIXME: expensive, should separate this out.
std::vector<Symbol> attrs;
auto queryAttributes(state->queryAttributes.use()(rowId));
while (queryAttributes.next())
attrs.push_back(symbols.create(queryAttributes.getStr(0)));
return {{rowId, attrs}};
} else if (type == AttrType::String) {
return {{rowId, queryAttribute.getStr(2)}};
} else if (type == AttrType::Missing) {
return {{rowId, missing_t()}};
} else if (type == AttrType::Misc) {
return {{rowId, misc_t()}};
} else if (type == AttrType::Failed) {
return {{rowId, failed_t()}};
} else
throw Error("unexpected type in evaluation cache");
}
};
struct AttrCursor;
struct AttrRoot : std::enable_shared_from_this<AttrRoot>
{
std::shared_ptr<AttrDb> db;
EvalState & state;
typedef std::function<Value *()> RootLoader;
RootLoader rootLoader;
RootValue value;
AttrRoot(std::shared_ptr<AttrDb> db, EvalState & state, RootLoader rootLoader)
: db(db)
, state(state)
, rootLoader(rootLoader)
{
}
Value * getRootValue()
{
if (!value) {
debug("getting root value");
value = allocRootValue(rootLoader());
}
return *value;
}
std::shared_ptr<AttrCursor> getRoot()
{
return std::make_shared<AttrCursor>(ref(shared_from_this()), std::nullopt);
}
};
struct AttrCursor : std::enable_shared_from_this<AttrCursor>
{
ref<AttrRoot> root;
typedef std::optional<std::pair<std::shared_ptr<AttrCursor>, Symbol>> Parent;
Parent parent;
RootValue _value;
std::optional<std::pair<AttrDb::AttrId, AttrDb::AttrValue>> cachedValue;
AttrCursor(
ref<AttrRoot> root,
Parent parent,
Value * value = nullptr,
std::optional<std::pair<AttrDb::AttrId, AttrDb::AttrValue>> && cachedValue = {})
: root(root), parent(parent), cachedValue(std::move(cachedValue))
{
if (value)
_value = allocRootValue(value);
}
AttrDb::AttrKey getAttrKey()
{
if (!parent)
return {0, root->state.sEpsilon};
if (!parent->first->cachedValue) {
parent->first->cachedValue = root->db->getAttr(
parent->first->getAttrKey(), root->state.symbols);
assert(parent->first->cachedValue);
}
return {parent->first->cachedValue->first, parent->second};
}
Value & getValue()
{
if (!_value) {
if (parent) {
auto & vParent = parent->first->getValue();
root->state.forceAttrs(vParent);
auto attr = vParent.attrs->get(parent->second);
if (!attr)
throw Error("attribute '%s' is unexpectedly missing", getAttrPathStr());
_value = allocRootValue(attr->value);
} else
_value = allocRootValue(root->getRootValue());
}
return **_value;
}
std::vector<Symbol> getAttrPath() const
{
if (parent) {
auto attrPath = parent->first->getAttrPath();
attrPath.push_back(parent->second);
return attrPath;
} else
return {};
}
std::vector<Symbol> getAttrPath(Symbol name) const
{
auto attrPath = getAttrPath();
attrPath.push_back(name);
return attrPath;
}
std::string getAttrPathStr() const
{
return concatStringsSep(".", getAttrPath());
}
std::string getAttrPathStr(Symbol name) const
{
return concatStringsSep(".", getAttrPath(name));
}
Value & forceValue()
{
debug("evaluating uncached attribute %s", getAttrPathStr());
auto & v = getValue();
try {
root->state.forceValue(v);
} catch (EvalError &) {
debug("setting '%s' to failed", getAttrPathStr());
if (root->db)
cachedValue = {root->db->setFailed(getAttrKey()), AttrDb::failed_t()};
throw;
}
if (root->db && (!cachedValue || std::get_if<AttrDb::placeholder_t>(&cachedValue->second))) {
if (v.type == tString)
cachedValue = {root->db->setString(getAttrKey(), v.string.s), v.string.s};
else if (v.type == tAttrs)
; // FIXME: do something?
else
cachedValue = {root->db->setMisc(getAttrKey()), AttrDb::misc_t()};
}
return v;
}
std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name)
{
if (root->db) {
if (!cachedValue)
cachedValue = root->db->getAttr(getAttrKey(), root->state.symbols);
if (cachedValue) {
if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) {
for (auto & attr : *attrs)
if (attr == name)
return std::make_shared<AttrCursor>(root, std::make_pair(shared_from_this(), name));
return nullptr;
} else if (std::get_if<AttrDb::placeholder_t>(&cachedValue->second)) {
auto attr = root->db->getAttr({cachedValue->first, name}, root->state.symbols);
if (attr) {
if (std::get_if<AttrDb::missing_t>(&attr->second))
return nullptr;
else if (std::get_if<AttrDb::failed_t>(&attr->second))
throw EvalError("cached failure of attribute '%s'", getAttrPathStr(name));
else
return std::make_shared<AttrCursor>(root,
std::make_pair(shared_from_this(), name), nullptr, std::move(attr));
}
// Incomplete attrset, so need to fall thru and
// evaluate to see whether 'name' exists
} else
return nullptr;
//throw TypeError("'%s' is not an attribute set", getAttrPathStr());
}
}
auto & v = forceValue();
if (v.type != tAttrs)
return nullptr;
//throw TypeError("'%s' is not an attribute set", getAttrPathStr());
auto attr = v.attrs->get(name);
if (!attr) {
if (root->db) {
assert(cachedValue);
root->db->setMissing({cachedValue->first, name});
}
return nullptr;
}
std::optional<std::pair<AttrDb::AttrId, AttrDb::AttrValue>> cachedValue2;
if (root->db) {
assert(cachedValue);
cachedValue2 = {root->db->setPlaceholder({cachedValue->first, name}), AttrDb::placeholder_t()};
}
return std::make_shared<AttrCursor>(
root, std::make_pair(shared_from_this(), name), attr->value, std::move(cachedValue2));
}
std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name)
{
return maybeGetAttr(root->state.symbols.create(name));
}
std::shared_ptr<AttrCursor> getAttr(Symbol name)
{
auto p = maybeGetAttr(name);
if (!p)
throw Error("attribute '%s' does not exist", getAttrPathStr(name));
return p;
}
std::shared_ptr<AttrCursor> getAttr(std::string_view name)
{
return getAttr(root->state.symbols.create(name));
}
std::string getString()
{
if (root->db) {
if (!cachedValue)
cachedValue = root->db->getAttr(getAttrKey(), root->state.symbols);
if (cachedValue && !std::get_if<AttrDb::placeholder_t>(&cachedValue->second)) {
if (auto s = std::get_if<std::string>(&cachedValue->second)) {
debug("using cached string attribute '%s'", getAttrPathStr());
return *s;
} else
throw TypeError("'%s' is not a string", getAttrPathStr());
}
}
auto & v = forceValue();
if (v.type != tString)
throw TypeError("'%s' is not a string", getAttrPathStr());
return v.string.s;
}
std::vector<Symbol> getAttrs()
{
if (root->db) {
if (!cachedValue)
cachedValue = root->db->getAttr(getAttrKey(), root->state.symbols);
if (cachedValue && !std::get_if<AttrDb::placeholder_t>(&cachedValue->second)) {
if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) {
debug("using cached attrset attribute '%s'", getAttrPathStr());
return *attrs;
} else
throw TypeError("'%s' is not an attribute set", getAttrPathStr());
}
}
auto & v = forceValue();
if (v.type != tAttrs)
throw TypeError("'%s' is not an attribute set", getAttrPathStr());
std::vector<Symbol> attrs;
for (auto & attr : *getValue().attrs)
attrs.push_back(attr.name);
std::sort(attrs.begin(), attrs.end(), [](const Symbol & a, const Symbol & b) {
return (const string &) a < (const string &) b;
});
if (root->db)
cachedValue = {root->db->setAttrs(getAttrKey(), attrs), attrs};
return attrs;
}
bool isDerivation()
{
auto aType = maybeGetAttr("type");
return aType && aType->getString() == "derivation";
}
};
struct CmdFlakeShow : FlakeCommand struct CmdFlakeShow : FlakeCommand
{ {
bool showLegacy = false; bool showLegacy = false;
@ -1170,9 +697,9 @@ struct CmdFlakeShow : FlakeCommand
auto state = getEvalState(); auto state = getEvalState();
auto flake = lockFlake(); auto flake = lockFlake();
std::function<void(AttrCursor & visitor, const std::vector<Symbol> & attrPath, const std::string & headerPrefix, const std::string & nextPrefix)> visit; std::function<void(eval_cache::AttrCursor & visitor, const std::vector<Symbol> & attrPath, const std::string & headerPrefix, const std::string & nextPrefix)> visit;
visit = [&](AttrCursor & visitor, const std::vector<Symbol> & attrPath, const std::string & headerPrefix, const std::string & nextPrefix) visit = [&](eval_cache::AttrCursor & visitor, const std::vector<Symbol> & attrPath, const std::string & headerPrefix, const std::string & nextPrefix)
{ {
Activity act(*logger, lvlInfo, actUnknown, Activity act(*logger, lvlInfo, actUnknown,
fmt("evaluating '%s'", concatStringsSep(".", attrPath))); fmt("evaluating '%s'", concatStringsSep(".", attrPath)));
@ -1286,28 +813,9 @@ struct CmdFlakeShow : FlakeCommand
} }
}; };
auto db = useEvalCache ? std::make_shared<AttrDb>(flake.getFingerprint()) : nullptr; auto cache = openEvalCache(*state, flake, useEvalCache);
auto root = std::make_shared<AttrRoot>(db, *state, visit(*cache->getRoot(), {}, fmt(ANSI_BOLD "%s" ANSI_NORMAL, flake.flake.lockedRef), "");
[&]()
{
/* For testing whether the evaluation cache is
complete. */
if (getEnv("NIX_ALLOW_EVAL").value_or("1") == "0")
throw Error("not everything is cached, but evaluation is not allowed");
auto vFlake = state->allocValue();
flake::callFlake(*state, flake, *vFlake);
state->forceAttrs(*vFlake);
auto aOutputs = vFlake->attrs->get(state->symbols.create("outputs"));
assert(aOutputs);
return aOutputs->value;
});
visit(*root->getRoot(), {}, fmt(ANSI_BOLD "%s" ANSI_NORMAL, flake.flake.lockedRef), "");
} }
}; };

View file

@ -162,7 +162,7 @@ struct InstallableStorePath : Installable
} }
}; };
std::vector<flake::EvalCache::Derivation> InstallableValue::toDerivations() std::vector<InstallableValue::DerivationInfo> InstallableValue::toDerivations()
{ {
auto state = cmd.getEvalState(); auto state = cmd.getEvalState();
@ -173,7 +173,7 @@ std::vector<flake::EvalCache::Derivation> InstallableValue::toDerivations()
DrvInfos drvInfos; DrvInfos drvInfos;
getDerivations(*state, *v, "", autoArgs, drvInfos, false); getDerivations(*state, *v, "", autoArgs, drvInfos, false);
std::vector<flake::EvalCache::Derivation> res; std::vector<DerivationInfo> res;
for (auto & drvInfo : drvInfos) { for (auto & drvInfo : drvInfos) {
res.push_back({ res.push_back({
state->store->parseStorePath(drvInfo.queryDrvPath()), state->store->parseStorePath(drvInfo.queryDrvPath()),
@ -283,58 +283,76 @@ Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::Locked
return aOutputs->value; return aOutputs->value;
} }
std::tuple<std::string, FlakeRef, flake::EvalCache::Derivation> InstallableFlake::toDerivation() ref<eval_cache::EvalCache> openEvalCache(
EvalState & state,
const flake::LockedFlake & lockedFlake,
bool useEvalCache)
{
return ref(std::make_shared<nix::eval_cache::EvalCache>(
useEvalCache,
lockedFlake.getFingerprint(),
state,
[&]()
{
/* For testing whether the evaluation cache is
complete. */
if (getEnv("NIX_ALLOW_EVAL").value_or("1") == "0")
throw Error("not everything is cached, but evaluation is not allowed");
auto vFlake = state.allocValue();
flake::callFlake(state, lockedFlake, *vFlake);
state.forceAttrs(*vFlake);
auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs"));
assert(aOutputs);
return aOutputs->value;
}));
}
std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableFlake::toDerivation()
{ {
auto state = cmd.getEvalState(); auto state = cmd.getEvalState();
auto lockedFlake = lockFlake(*state, flakeRef, cmd.lockFlags); auto lockedFlake = lockFlake(*state, flakeRef, cmd.lockFlags);
Value * vOutputs = nullptr; auto cache = openEvalCache(*state, lockedFlake, true);
auto root = cache->getRoot();
auto emptyArgs = state->allocBindings(0);
auto & evalCache = flake::EvalCache::singleton();
auto fingerprint = lockedFlake.getFingerprint();
for (auto & attrPath : getActualAttrPaths()) { for (auto & attrPath : getActualAttrPaths()) {
auto drv = evalCache.getDerivation(fingerprint, attrPath); auto attr = root->findAlongAttrPath(parseAttrPath(*state, attrPath));
if (drv) { if (!attr) continue;
if (state->store->isValidPath(drv->drvPath))
return {attrPath, lockedFlake.flake.lockedRef, std::move(*drv)};
}
if (!vOutputs) if (!attr->isDerivation())
vOutputs = getFlakeOutputs(*state, lockedFlake);
try {
auto * v = findAlongAttrPath(*state, attrPath, *emptyArgs, *vOutputs).first;
state->forceValue(*v);
auto drvInfo = getDerivation(*state, *v, false);
if (!drvInfo)
throw Error("flake output attribute '%s' is not a derivation", attrPath); throw Error("flake output attribute '%s' is not a derivation", attrPath);
auto drv = flake::EvalCache::Derivation{ auto aDrvPath = attr->getAttr(state->sDrvPath);
state->store->parseStorePath(drvInfo->queryDrvPath()), auto drvPath = state->store->parseStorePath(aDrvPath->getString());
state->store->parseStorePath(drvInfo->queryOutPath()), if (!state->store->isValidPath(drvPath)) {
drvInfo->queryOutputName() /* The eval cache contains 'drvPath', but the actual path
has been garbage-collected. So force it to be
regenerated. */
aDrvPath->forceValue();
assert(state->store->isValidPath(drvPath));
}
auto drvInfo = DerivationInfo{
std::move(drvPath),
state->store->parseStorePath(attr->getAttr(state->sOutPath)->getString()),
attr->getAttr(state->sOutputName)->getString()
}; };
evalCache.addDerivation(fingerprint, attrPath, drv); return {attrPath, lockedFlake.flake.lockedRef, std::move(drvInfo)};
return {attrPath, lockedFlake.flake.lockedRef, std::move(drv)};
} catch (AttrPathNotFound & e) {
}
} }
throw Error("flake '%s' does not provide attribute %s", throw Error("flake '%s' does not provide attribute %s",
flakeRef, concatStringsSep(", ", quoteStrings(attrPaths))); flakeRef, concatStringsSep(", ", quoteStrings(attrPaths)));
} }
std::vector<flake::EvalCache::Derivation> InstallableFlake::toDerivations() std::vector<InstallableValue::DerivationInfo> InstallableFlake::toDerivations()
{ {
std::vector<flake::EvalCache::Derivation> res; std::vector<DerivationInfo> res;
res.push_back(std::get<2>(toDerivation())); res.push_back(std::get<2>(toDerivation()));
return res; return res;
} }

View file

@ -3,7 +3,7 @@
#include "util.hh" #include "util.hh"
#include "path.hh" #include "path.hh"
#include "eval.hh" #include "eval.hh"
#include "flake/eval-cache.hh" #include "flake/flake.hh"
#include <optional> #include <optional>
@ -12,6 +12,8 @@ namespace nix {
struct DrvInfo; struct DrvInfo;
struct SourceExprCommand; struct SourceExprCommand;
namespace eval_cache { class EvalCache; }
struct Buildable struct Buildable
{ {
std::optional<StorePath> drvPath; std::optional<StorePath> drvPath;
@ -63,7 +65,14 @@ struct InstallableValue : Installable
InstallableValue(SourceExprCommand & cmd) : cmd(cmd) { } InstallableValue(SourceExprCommand & cmd) : cmd(cmd) { }
virtual std::vector<flake::EvalCache::Derivation> toDerivations(); struct DerivationInfo
{
StorePath drvPath;
StorePath outPath;
std::string outputName;
};
virtual std::vector<DerivationInfo> toDerivations();
Buildables toBuildables() override; Buildables toBuildables() override;
}; };
@ -86,11 +95,16 @@ struct InstallableFlake : InstallableValue
Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake); Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake);
std::tuple<std::string, FlakeRef, flake::EvalCache::Derivation> toDerivation(); std::tuple<std::string, FlakeRef, DerivationInfo> toDerivation();
std::vector<flake::EvalCache::Derivation> toDerivations() override; std::vector<DerivationInfo> toDerivations() override;
std::pair<Value *, Pos> toValue(EvalState & state) override; std::pair<Value *, Pos> toValue(EvalState & state) override;
}; };
ref<eval_cache::EvalCache> openEvalCache(
EvalState & state,
const flake::LockedFlake & lockedFlake,
bool useEvalCache);
} }