Get rid of the old eval cache

This commit is contained in:
Eelco Dolstra 2020-04-20 13:13:52 +02:00
parent 0725ab2fd7
commit 539a9c1c5f
7 changed files with 609 additions and 651 deletions

View file

@ -6,11 +6,11 @@
namespace nix {
static Strings parseAttrPath(const string & s)
static Strings parseAttrPath(std::string_view s)
{
Strings res;
string cur;
string::const_iterator i = s.begin();
auto i = s.begin();
while (i != s.end()) {
if (*i == '.') {
res.push_back(cur);
@ -32,6 +32,15 @@ static Strings parseAttrPath(const string & s)
}
std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s)
{
std::vector<Symbol> res;
for (auto & a : parseAttrPath(s))
res.push_back(state.symbols.create(a));
return res;
}
std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath,
Bindings & autoArgs, Value & vIn)
{

View file

@ -16,4 +16,6 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
/* Heuristic to find the filename and lineno or a nix value. */
Pos findDerivationFilename(EvalState & state, Value & v, std::string what);
std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s);
}

View file

@ -1,116 +1,460 @@
#include "eval-cache.hh"
#include "sqlite.hh"
#include "eval.hh"
#include "eval-inline.hh"
#include <set>
namespace nix::flake {
namespace nix::eval_cache {
// FIXME: inefficient representation of attrs
static const char * schema = R"sql(
create table if not exists Fingerprints (
fingerprint blob primary key not null,
timestamp integer not null
);
create table if not exists Attributes (
fingerprint blob not null,
attrPath text not null,
type integer,
parent integer not null,
name text,
type integer not null,
value text,
primary key (fingerprint, attrPath),
foreign key (fingerprint) references Fingerprints(fingerprint) on delete cascade
primary key (parent, name)
);
)sql";
struct EvalCache::State
struct AttrDb
{
SQLite db;
SQLiteStmt insertFingerprint;
SQLiteStmt insertAttribute;
SQLiteStmt queryAttribute;
std::set<Fingerprint> fingerprints;
struct State
{
SQLite db;
SQLiteStmt insertAttribute;
SQLiteStmt queryAttribute;
SQLiteStmt queryAttributes;
std::unique_ptr<SQLiteTxn> txn;
};
std::unique_ptr<Sync<State>> _state;
AttrDb(const Hash & fingerprint)
: _state(std::make_unique<Sync<State>>())
{
auto state(_state->lock());
Path cacheDir = getCacheDir() + "/nix/eval-cache-v1";
createDirs(cacheDir);
Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
state->db = SQLite(dbPath);
state->db.isCache();
state->db.exec(schema);
state->insertAttribute.create(state->db,
"insert or replace into Attributes(parent, name, type, value) values (?, ?, ?, ?)");
state->queryAttribute.create(state->db,
"select rowid, type, value from Attributes where parent = ? and name = ?");
state->queryAttributes.create(state->db,
"select name from Attributes where parent = ?");
state->txn = std::make_unique<SQLiteTxn>(state->db);
}
~AttrDb()
{
try {
auto state(_state->lock());
state->txn->commit();
state->txn.reset();
} catch (...) {
ignoreException();
}
}
AttrId setAttrs(
AttrKey key,
const std::vector<Symbol> & attrs)
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::FullAttrs)
(0, false).exec();
AttrId rowId = state->db.getLastInsertedRowId();
assert(rowId);
for (auto & attr : attrs)
state->insertAttribute.use()
(rowId)
(attr)
(AttrType::Placeholder)
(0, false).exec();
return rowId;
}
AttrId setString(
AttrKey key,
std::string_view s)
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::String)
(s).exec();
return state->db.getLastInsertedRowId();
}
AttrId setPlaceholder(AttrKey key)
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::Placeholder)
(0, false).exec();
return state->db.getLastInsertedRowId();
}
AttrId setMissing(AttrKey key)
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::Missing)
(0, false).exec();
return state->db.getLastInsertedRowId();
}
AttrId setMisc(AttrKey key)
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::Misc)
(0, false).exec();
return state->db.getLastInsertedRowId();
}
AttrId setFailed(AttrKey key)
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::Failed)
(0, false).exec();
return state->db.getLastInsertedRowId();
}
std::optional<std::pair<AttrId, AttrValue>> getAttr(
AttrKey key,
SymbolTable & symbols)
{
auto state(_state->lock());
auto queryAttribute(state->queryAttribute.use()(key.first)(key.second));
if (!queryAttribute.next()) return {};
auto rowId = (AttrType) queryAttribute.getInt(0);
auto type = (AttrType) queryAttribute.getInt(1);
if (type == AttrType::Placeholder)
return {{rowId, placeholder_t()}};
else if (type == AttrType::FullAttrs) {
// FIXME: expensive, should separate this out.
std::vector<Symbol> attrs;
auto queryAttributes(state->queryAttributes.use()(rowId));
while (queryAttributes.next())
attrs.push_back(symbols.create(queryAttributes.getStr(0)));
return {{rowId, attrs}};
} else if (type == AttrType::String) {
return {{rowId, queryAttribute.getStr(2)}};
} else if (type == AttrType::Missing) {
return {{rowId, missing_t()}};
} else if (type == AttrType::Misc) {
return {{rowId, misc_t()}};
} else if (type == AttrType::Failed) {
return {{rowId, failed_t()}};
} else
throw Error("unexpected type in evaluation cache");
}
};
EvalCache::EvalCache()
: _state(std::make_unique<Sync<State>>())
EvalCache::EvalCache(
bool useCache,
const Hash & fingerprint,
EvalState & state,
RootLoader rootLoader)
: db(useCache ? std::make_shared<AttrDb>(fingerprint) : nullptr)
, state(state)
, rootLoader(rootLoader)
{
auto state(_state->lock());
Path dbPath = getCacheDir() + "/nix/eval-cache-v1.sqlite";
createDirs(dirOf(dbPath));
state->db = SQLite(dbPath);
state->db.isCache();
state->db.exec(schema);
state->insertFingerprint.create(state->db,
"insert or ignore into Fingerprints(fingerprint, timestamp) values (?, ?)");
state->insertAttribute.create(state->db,
"insert or replace into Attributes(fingerprint, attrPath, type, value) values (?, ?, ?, ?)");
state->queryAttribute.create(state->db,
"select type, value from Attributes where fingerprint = ? and attrPath = ?");
}
enum ValueType {
Derivation = 1,
};
void EvalCache::addDerivation(
const Fingerprint & fingerprint,
const std::string & attrPath,
const Derivation & drv)
Value * EvalCache::getRootValue()
{
if (!evalSettings.pureEval) return;
auto state(_state->lock());
if (state->fingerprints.insert(fingerprint).second)
// FIXME: update timestamp
state->insertFingerprint.use()
(fingerprint.hash, fingerprint.hashSize)
(time(0)).exec();
state->insertAttribute.use()
(fingerprint.hash, fingerprint.hashSize)
(attrPath)
(ValueType::Derivation)
(std::string(drv.drvPath.to_string()) + " " + std::string(drv.outPath.to_string()) + " " + drv.outputName).exec();
if (!value) {
debug("getting root value");
value = allocRootValue(rootLoader());
}
return *value;
}
std::optional<EvalCache::Derivation> EvalCache::getDerivation(
const Fingerprint & fingerprint,
const std::string & attrPath)
std::shared_ptr<AttrCursor> EvalCache::getRoot()
{
if (!evalSettings.pureEval) return {};
auto state(_state->lock());
auto queryAttribute(state->queryAttribute.use()
(fingerprint.hash, fingerprint.hashSize)
(attrPath));
if (!queryAttribute.next()) return {};
// FIXME: handle negative results
auto type = (ValueType) queryAttribute.getInt(0);
auto s = queryAttribute.getStr(1);
if (type != ValueType::Derivation) return {};
auto ss = tokenizeString<std::vector<std::string>>(s, " ");
debug("evaluation cache hit for '%s'", attrPath);
return Derivation { StorePath::fromBaseName(ss[0]), StorePath::fromBaseName(ss[1]), ss[2] };
return std::make_shared<AttrCursor>(ref(shared_from_this()), std::nullopt);
}
EvalCache & EvalCache::singleton()
AttrCursor::AttrCursor(
ref<EvalCache> root,
Parent parent,
Value * value,
std::optional<std::pair<AttrId, AttrValue>> && cachedValue)
: root(root), parent(parent), cachedValue(std::move(cachedValue))
{
static std::unique_ptr<EvalCache> evalCache(new EvalCache());
return *evalCache;
if (value)
_value = allocRootValue(value);
}
AttrKey AttrCursor::getKey()
{
if (!parent)
return {0, root->state.sEpsilon};
if (!parent->first->cachedValue) {
parent->first->cachedValue = root->db->getAttr(
parent->first->getKey(), root->state.symbols);
assert(parent->first->cachedValue);
}
return {parent->first->cachedValue->first, parent->second};
}
Value & AttrCursor::getValue()
{
if (!_value) {
if (parent) {
auto & vParent = parent->first->getValue();
root->state.forceAttrs(vParent);
auto attr = vParent.attrs->get(parent->second);
if (!attr)
throw Error("attribute '%s' is unexpectedly missing", getAttrPathStr());
_value = allocRootValue(attr->value);
} else
_value = allocRootValue(root->getRootValue());
}
return **_value;
}
std::vector<Symbol> AttrCursor::getAttrPath() const
{
if (parent) {
auto attrPath = parent->first->getAttrPath();
attrPath.push_back(parent->second);
return attrPath;
} else
return {};
}
std::vector<Symbol> AttrCursor::getAttrPath(Symbol name) const
{
auto attrPath = getAttrPath();
attrPath.push_back(name);
return attrPath;
}
std::string AttrCursor::getAttrPathStr() const
{
return concatStringsSep(".", getAttrPath());
}
std::string AttrCursor::getAttrPathStr(Symbol name) const
{
return concatStringsSep(".", getAttrPath(name));
}
Value & AttrCursor::forceValue()
{
debug("evaluating uncached attribute %s", getAttrPathStr());
auto & v = getValue();
try {
root->state.forceValue(v);
} catch (EvalError &) {
debug("setting '%s' to failed", getAttrPathStr());
if (root->db)
cachedValue = {root->db->setFailed(getKey()), failed_t()};
throw;
}
if (root->db && (!cachedValue || std::get_if<placeholder_t>(&cachedValue->second))) {
if (v.type == tString)
cachedValue = {root->db->setString(getKey(), v.string.s), v.string.s};
else if (v.type == tAttrs)
; // FIXME: do something?
else
cachedValue = {root->db->setMisc(getKey()), misc_t()};
}
return v;
}
std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name)
{
if (root->db) {
if (!cachedValue)
cachedValue = root->db->getAttr(getKey(), root->state.symbols);
if (cachedValue) {
if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) {
for (auto & attr : *attrs)
if (attr == name)
return std::make_shared<AttrCursor>(root, std::make_pair(shared_from_this(), name));
return nullptr;
} else if (std::get_if<placeholder_t>(&cachedValue->second)) {
auto attr = root->db->getAttr({cachedValue->first, name}, root->state.symbols);
if (attr) {
if (std::get_if<missing_t>(&attr->second))
return nullptr;
else if (std::get_if<failed_t>(&attr->second))
throw EvalError("cached failure of attribute '%s'", getAttrPathStr(name));
else
return std::make_shared<AttrCursor>(root,
std::make_pair(shared_from_this(), name), nullptr, std::move(attr));
}
// Incomplete attrset, so need to fall thru and
// evaluate to see whether 'name' exists
} else
return nullptr;
//throw TypeError("'%s' is not an attribute set", getAttrPathStr());
}
}
auto & v = forceValue();
if (v.type != tAttrs)
return nullptr;
//throw TypeError("'%s' is not an attribute set", getAttrPathStr());
auto attr = v.attrs->get(name);
if (!attr) {
if (root->db) {
if (!cachedValue)
cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()};
root->db->setMissing({cachedValue->first, name});
}
return nullptr;
}
std::optional<std::pair<AttrId, AttrValue>> cachedValue2;
if (root->db) {
if (!cachedValue)
cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()};
cachedValue2 = {root->db->setPlaceholder({cachedValue->first, name}), placeholder_t()};
}
return std::make_shared<AttrCursor>(
root, std::make_pair(shared_from_this(), name), attr->value, std::move(cachedValue2));
}
std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(std::string_view name)
{
return maybeGetAttr(root->state.symbols.create(name));
}
std::shared_ptr<AttrCursor> AttrCursor::getAttr(Symbol name)
{
auto p = maybeGetAttr(name);
if (!p)
throw Error("attribute '%s' does not exist", getAttrPathStr(name));
return p;
}
std::shared_ptr<AttrCursor> AttrCursor::getAttr(std::string_view name)
{
return getAttr(root->state.symbols.create(name));
}
std::shared_ptr<AttrCursor> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath)
{
auto res = shared_from_this();
for (auto & attr : attrPath) {
res = res->maybeGetAttr(attr);
if (!res) return {};
}
return res;
}
std::string AttrCursor::getString()
{
if (root->db) {
if (!cachedValue)
cachedValue = root->db->getAttr(getKey(), root->state.symbols);
if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
if (auto s = std::get_if<std::string>(&cachedValue->second)) {
debug("using cached string attribute '%s'", getAttrPathStr());
return *s;
} else
throw TypeError("'%s' is not a string", getAttrPathStr());
}
}
auto & v = forceValue();
if (v.type != tString)
throw TypeError("'%s' is not a string", getAttrPathStr());
return v.string.s;
}
std::vector<Symbol> AttrCursor::getAttrs()
{
if (root->db) {
if (!cachedValue)
cachedValue = root->db->getAttr(getKey(), root->state.symbols);
if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) {
debug("using cached attrset attribute '%s'", getAttrPathStr());
return *attrs;
} else
throw TypeError("'%s' is not an attribute set", getAttrPathStr());
}
}
auto & v = forceValue();
if (v.type != tAttrs)
throw TypeError("'%s' is not an attribute set", getAttrPathStr());
std::vector<Symbol> attrs;
for (auto & attr : *getValue().attrs)
attrs.push_back(attr.name);
std::sort(attrs.begin(), attrs.end(), [](const Symbol & a, const Symbol & b) {
return (const string &) a < (const string &) b;
});
if (root->db)
cachedValue = {root->db->setAttrs(getKey(), attrs), attrs};
return attrs;
}
bool AttrCursor::isDerivation()
{
auto aType = maybeGetAttr("type");
return aType && aType->getString() == "derivation";
}
}

View file

@ -1,40 +1,103 @@
#pragma once
#include "sync.hh"
#include "flake.hh"
#include "path.hh"
#include "hash.hh"
#include "eval.hh"
namespace nix { struct SQLite; struct SQLiteStmt; }
#include <variant>
namespace nix::flake {
namespace nix::eval_cache {
class EvalCache
class AttrDb;
class AttrCursor;
class EvalCache : public std::enable_shared_from_this<EvalCache>
{
struct State;
friend class AttrCursor;
std::unique_ptr<Sync<State>> _state;
std::shared_ptr<AttrDb> db;
EvalState & state;
typedef std::function<Value *()> RootLoader;
RootLoader rootLoader;
RootValue value;
EvalCache();
Value * getRootValue();
public:
struct Derivation
{
StorePath drvPath;
StorePath outPath;
std::string outputName;
};
EvalCache(
bool useCache,
const Hash & fingerprint,
EvalState & state,
RootLoader rootLoader);
void addDerivation(
const Fingerprint & fingerprint,
const std::string & attrPath,
const Derivation & drv);
std::shared_ptr<AttrCursor> getRoot();
};
std::optional<Derivation> getDerivation(
const Fingerprint & fingerprint,
const std::string & attrPath);
enum AttrType {
Placeholder = 0,
FullAttrs = 1,
String = 2,
Missing = 3,
Misc = 4,
Failed = 5,
};
static EvalCache & singleton();
struct placeholder_t {};
struct missing_t {};
struct misc_t {};
struct failed_t {};
typedef uint64_t AttrId;
typedef std::pair<AttrId, Symbol> AttrKey;
typedef std::variant<std::vector<Symbol>, std::string, placeholder_t, missing_t, misc_t, failed_t> AttrValue;
class AttrCursor : public std::enable_shared_from_this<AttrCursor>
{
friend class EvalCache;
ref<EvalCache> root;
typedef std::optional<std::pair<std::shared_ptr<AttrCursor>, Symbol>> Parent;
Parent parent;
RootValue _value;
std::optional<std::pair<AttrId, AttrValue>> cachedValue;
AttrKey getKey();
Value & getValue();
public:
AttrCursor(
ref<EvalCache> root,
Parent parent,
Value * value = nullptr,
std::optional<std::pair<AttrId, AttrValue>> && cachedValue = {});
std::vector<Symbol> getAttrPath() const;
std::vector<Symbol> getAttrPath(Symbol name) const;
std::string getAttrPathStr() const;
std::string getAttrPathStr(Symbol name) const;
std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name);
std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name);
std::shared_ptr<AttrCursor> getAttr(Symbol name);
std::shared_ptr<AttrCursor> getAttr(std::string_view name);
std::shared_ptr<AttrCursor> findAlongAttrPath(const std::vector<Symbol> & attrPath);
std::string getString();
std::vector<Symbol> getAttrs();
bool isDerivation();
Value & forceValue();
};
}

View file

@ -11,7 +11,7 @@
#include "fetchers.hh"
#include "registry.hh"
#include "json.hh"
#include "sqlite.hh"
#include "flake/eval-cache.hh"
#include <nlohmann/json.hpp>
#include <queue>
@ -669,479 +669,6 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun
}
};
// FIXME: inefficient representation of attrs
static const char * schema = R"sql(
create table if not exists Attributes (
parent integer not null,
name text,
type integer not null,
value text,
primary key (parent, name)
);
)sql";
enum AttrType {
Placeholder = 0,
FullAttrs = 1,
String = 2,
Missing = 3,
Misc = 4,
Failed = 5,
};
struct AttrDb
{
struct State
{
SQLite db;
SQLiteStmt insertAttribute;
SQLiteStmt queryAttribute;
SQLiteStmt queryAttributes;
std::unique_ptr<SQLiteTxn> txn;
};
struct placeholder_t {};
struct missing_t {};
struct misc_t {};
struct failed_t {};
typedef uint64_t AttrId;
typedef std::pair<AttrId, Symbol> AttrKey;
typedef std::variant<std::vector<Symbol>, std::string, placeholder_t, missing_t, misc_t, failed_t> AttrValue;
std::unique_ptr<Sync<State>> _state;
AttrDb(const Fingerprint & fingerprint)
: _state(std::make_unique<Sync<State>>())
{
auto state(_state->lock());
Path cacheDir = getCacheDir() + "/nix/eval-cache-v2";
createDirs(cacheDir);
Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
state->db = SQLite(dbPath);
state->db.isCache();
state->db.exec(schema);
state->insertAttribute.create(state->db,
"insert or replace into Attributes(parent, name, type, value) values (?, ?, ?, ?)");
state->queryAttribute.create(state->db,
"select rowid, type, value from Attributes where parent = ? and name = ?");
state->queryAttributes.create(state->db,
"select name from Attributes where parent = ?");
state->txn = std::make_unique<SQLiteTxn>(state->db);
}
~AttrDb()
{
try {
auto state(_state->lock());
state->txn->commit();
state->txn.reset();
} catch (...) {
ignoreException();
}
}
AttrId setAttrs(
AttrKey key,
const std::vector<Symbol> & attrs)
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::FullAttrs)
(0, false).exec();
AttrId rowId = state->db.getLastInsertedRowId();
assert(rowId);
for (auto & attr : attrs)
state->insertAttribute.use()
(rowId)
(attr)
(AttrType::Placeholder)
(0, false).exec();
return rowId;
}
AttrId setString(
AttrKey key,
std::string_view s)
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::String)
(s).exec();
return state->db.getLastInsertedRowId();
}
AttrId setPlaceholder(AttrKey key)
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::Placeholder)
(0, false).exec();
return state->db.getLastInsertedRowId();
}
AttrId setMissing(AttrKey key)
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::Missing)
(0, false).exec();
return state->db.getLastInsertedRowId();
}
AttrId setMisc(AttrKey key)
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::Misc)
(0, false).exec();
return state->db.getLastInsertedRowId();
}
AttrId setFailed(AttrKey key)
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::Failed)
(0, false).exec();
return state->db.getLastInsertedRowId();
}
std::optional<std::pair<AttrId, AttrValue>> getAttr(
AttrKey key,
SymbolTable & symbols)
{
auto state(_state->lock());
auto queryAttribute(state->queryAttribute.use()(key.first)(key.second));
if (!queryAttribute.next()) return {};
auto rowId = (AttrType) queryAttribute.getInt(0);
auto type = (AttrType) queryAttribute.getInt(1);
if (type == AttrType::Placeholder)
return {{rowId, placeholder_t()}};
else if (type == AttrType::FullAttrs) {
// FIXME: expensive, should separate this out.
std::vector<Symbol> attrs;
auto queryAttributes(state->queryAttributes.use()(rowId));
while (queryAttributes.next())
attrs.push_back(symbols.create(queryAttributes.getStr(0)));
return {{rowId, attrs}};
} else if (type == AttrType::String) {
return {{rowId, queryAttribute.getStr(2)}};
} else if (type == AttrType::Missing) {
return {{rowId, missing_t()}};
} else if (type == AttrType::Misc) {
return {{rowId, misc_t()}};
} else if (type == AttrType::Failed) {
return {{rowId, failed_t()}};
} else
throw Error("unexpected type in evaluation cache");
}
};
struct AttrCursor;
struct AttrRoot : std::enable_shared_from_this<AttrRoot>
{
std::shared_ptr<AttrDb> db;
EvalState & state;
typedef std::function<Value *()> RootLoader;
RootLoader rootLoader;
RootValue value;
AttrRoot(std::shared_ptr<AttrDb> db, EvalState & state, RootLoader rootLoader)
: db(db)
, state(state)
, rootLoader(rootLoader)
{
}
Value * getRootValue()
{
if (!value) {
debug("getting root value");
value = allocRootValue(rootLoader());
}
return *value;
}
std::shared_ptr<AttrCursor> getRoot()
{
return std::make_shared<AttrCursor>(ref(shared_from_this()), std::nullopt);
}
};
struct AttrCursor : std::enable_shared_from_this<AttrCursor>
{
ref<AttrRoot> root;
typedef std::optional<std::pair<std::shared_ptr<AttrCursor>, Symbol>> Parent;
Parent parent;
RootValue _value;
std::optional<std::pair<AttrDb::AttrId, AttrDb::AttrValue>> cachedValue;
AttrCursor(
ref<AttrRoot> root,
Parent parent,
Value * value = nullptr,
std::optional<std::pair<AttrDb::AttrId, AttrDb::AttrValue>> && cachedValue = {})
: root(root), parent(parent), cachedValue(std::move(cachedValue))
{
if (value)
_value = allocRootValue(value);
}
AttrDb::AttrKey getAttrKey()
{
if (!parent)
return {0, root->state.sEpsilon};
if (!parent->first->cachedValue) {
parent->first->cachedValue = root->db->getAttr(
parent->first->getAttrKey(), root->state.symbols);
assert(parent->first->cachedValue);
}
return {parent->first->cachedValue->first, parent->second};
}
Value & getValue()
{
if (!_value) {
if (parent) {
auto & vParent = parent->first->getValue();
root->state.forceAttrs(vParent);
auto attr = vParent.attrs->get(parent->second);
if (!attr)
throw Error("attribute '%s' is unexpectedly missing", getAttrPathStr());
_value = allocRootValue(attr->value);
} else
_value = allocRootValue(root->getRootValue());
}
return **_value;
}
std::vector<Symbol> getAttrPath() const
{
if (parent) {
auto attrPath = parent->first->getAttrPath();
attrPath.push_back(parent->second);
return attrPath;
} else
return {};
}
std::vector<Symbol> getAttrPath(Symbol name) const
{
auto attrPath = getAttrPath();
attrPath.push_back(name);
return attrPath;
}
std::string getAttrPathStr() const
{
return concatStringsSep(".", getAttrPath());
}
std::string getAttrPathStr(Symbol name) const
{
return concatStringsSep(".", getAttrPath(name));
}
Value & forceValue()
{
debug("evaluating uncached attribute %s", getAttrPathStr());
auto & v = getValue();
try {
root->state.forceValue(v);
} catch (EvalError &) {
debug("setting '%s' to failed", getAttrPathStr());
if (root->db)
cachedValue = {root->db->setFailed(getAttrKey()), AttrDb::failed_t()};
throw;
}
if (root->db && (!cachedValue || std::get_if<AttrDb::placeholder_t>(&cachedValue->second))) {
if (v.type == tString)
cachedValue = {root->db->setString(getAttrKey(), v.string.s), v.string.s};
else if (v.type == tAttrs)
; // FIXME: do something?
else
cachedValue = {root->db->setMisc(getAttrKey()), AttrDb::misc_t()};
}
return v;
}
std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name)
{
if (root->db) {
if (!cachedValue)
cachedValue = root->db->getAttr(getAttrKey(), root->state.symbols);
if (cachedValue) {
if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) {
for (auto & attr : *attrs)
if (attr == name)
return std::make_shared<AttrCursor>(root, std::make_pair(shared_from_this(), name));
return nullptr;
} else if (std::get_if<AttrDb::placeholder_t>(&cachedValue->second)) {
auto attr = root->db->getAttr({cachedValue->first, name}, root->state.symbols);
if (attr) {
if (std::get_if<AttrDb::missing_t>(&attr->second))
return nullptr;
else if (std::get_if<AttrDb::failed_t>(&attr->second))
throw EvalError("cached failure of attribute '%s'", getAttrPathStr(name));
else
return std::make_shared<AttrCursor>(root,
std::make_pair(shared_from_this(), name), nullptr, std::move(attr));
}
// Incomplete attrset, so need to fall thru and
// evaluate to see whether 'name' exists
} else
return nullptr;
//throw TypeError("'%s' is not an attribute set", getAttrPathStr());
}
}
auto & v = forceValue();
if (v.type != tAttrs)
return nullptr;
//throw TypeError("'%s' is not an attribute set", getAttrPathStr());
auto attr = v.attrs->get(name);
if (!attr) {
if (root->db) {
assert(cachedValue);
root->db->setMissing({cachedValue->first, name});
}
return nullptr;
}
std::optional<std::pair<AttrDb::AttrId, AttrDb::AttrValue>> cachedValue2;
if (root->db) {
assert(cachedValue);
cachedValue2 = {root->db->setPlaceholder({cachedValue->first, name}), AttrDb::placeholder_t()};
}
return std::make_shared<AttrCursor>(
root, std::make_pair(shared_from_this(), name), attr->value, std::move(cachedValue2));
}
std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name)
{
return maybeGetAttr(root->state.symbols.create(name));
}
std::shared_ptr<AttrCursor> getAttr(Symbol name)
{
auto p = maybeGetAttr(name);
if (!p)
throw Error("attribute '%s' does not exist", getAttrPathStr(name));
return p;
}
std::shared_ptr<AttrCursor> getAttr(std::string_view name)
{
return getAttr(root->state.symbols.create(name));
}
std::string getString()
{
if (root->db) {
if (!cachedValue)
cachedValue = root->db->getAttr(getAttrKey(), root->state.symbols);
if (cachedValue && !std::get_if<AttrDb::placeholder_t>(&cachedValue->second)) {
if (auto s = std::get_if<std::string>(&cachedValue->second)) {
debug("using cached string attribute '%s'", getAttrPathStr());
return *s;
} else
throw TypeError("'%s' is not a string", getAttrPathStr());
}
}
auto & v = forceValue();
if (v.type != tString)
throw TypeError("'%s' is not a string", getAttrPathStr());
return v.string.s;
}
std::vector<Symbol> getAttrs()
{
if (root->db) {
if (!cachedValue)
cachedValue = root->db->getAttr(getAttrKey(), root->state.symbols);
if (cachedValue && !std::get_if<AttrDb::placeholder_t>(&cachedValue->second)) {
if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) {
debug("using cached attrset attribute '%s'", getAttrPathStr());
return *attrs;
} else
throw TypeError("'%s' is not an attribute set", getAttrPathStr());
}
}
auto & v = forceValue();
if (v.type != tAttrs)
throw TypeError("'%s' is not an attribute set", getAttrPathStr());
std::vector<Symbol> attrs;
for (auto & attr : *getValue().attrs)
attrs.push_back(attr.name);
std::sort(attrs.begin(), attrs.end(), [](const Symbol & a, const Symbol & b) {
return (const string &) a < (const string &) b;
});
if (root->db)
cachedValue = {root->db->setAttrs(getAttrKey(), attrs), attrs};
return attrs;
}
bool isDerivation()
{
auto aType = maybeGetAttr("type");
return aType && aType->getString() == "derivation";
}
};
struct CmdFlakeShow : FlakeCommand
{
bool showLegacy = false;
@ -1170,9 +697,9 @@ struct CmdFlakeShow : FlakeCommand
auto state = getEvalState();
auto flake = lockFlake();
std::function<void(AttrCursor & visitor, const std::vector<Symbol> & attrPath, const std::string & headerPrefix, const std::string & nextPrefix)> visit;
std::function<void(eval_cache::AttrCursor & visitor, const std::vector<Symbol> & attrPath, const std::string & headerPrefix, const std::string & nextPrefix)> visit;
visit = [&](AttrCursor & visitor, const std::vector<Symbol> & attrPath, const std::string & headerPrefix, const std::string & nextPrefix)
visit = [&](eval_cache::AttrCursor & visitor, const std::vector<Symbol> & attrPath, const std::string & headerPrefix, const std::string & nextPrefix)
{
Activity act(*logger, lvlInfo, actUnknown,
fmt("evaluating '%s'", concatStringsSep(".", attrPath)));
@ -1286,28 +813,9 @@ struct CmdFlakeShow : FlakeCommand
}
};
auto db = useEvalCache ? std::make_shared<AttrDb>(flake.getFingerprint()) : nullptr;
auto cache = openEvalCache(*state, flake, useEvalCache);
auto root = std::make_shared<AttrRoot>(db, *state,
[&]()
{
/* For testing whether the evaluation cache is
complete. */
if (getEnv("NIX_ALLOW_EVAL").value_or("1") == "0")
throw Error("not everything is cached, but evaluation is not allowed");
auto vFlake = state->allocValue();
flake::callFlake(*state, flake, *vFlake);
state->forceAttrs(*vFlake);
auto aOutputs = vFlake->attrs->get(state->symbols.create("outputs"));
assert(aOutputs);
return aOutputs->value;
});
visit(*root->getRoot(), {}, fmt(ANSI_BOLD "%s" ANSI_NORMAL, flake.flake.lockedRef), "");
visit(*cache->getRoot(), {}, fmt(ANSI_BOLD "%s" ANSI_NORMAL, flake.flake.lockedRef), "");
}
};

View file

@ -162,7 +162,7 @@ struct InstallableStorePath : Installable
}
};
std::vector<flake::EvalCache::Derivation> InstallableValue::toDerivations()
std::vector<InstallableValue::DerivationInfo> InstallableValue::toDerivations()
{
auto state = cmd.getEvalState();
@ -173,7 +173,7 @@ std::vector<flake::EvalCache::Derivation> InstallableValue::toDerivations()
DrvInfos drvInfos;
getDerivations(*state, *v, "", autoArgs, drvInfos, false);
std::vector<flake::EvalCache::Derivation> res;
std::vector<DerivationInfo> res;
for (auto & drvInfo : drvInfos) {
res.push_back({
state->store->parseStorePath(drvInfo.queryDrvPath()),
@ -283,58 +283,76 @@ Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::Locked
return aOutputs->value;
}
std::tuple<std::string, FlakeRef, flake::EvalCache::Derivation> InstallableFlake::toDerivation()
ref<eval_cache::EvalCache> openEvalCache(
EvalState & state,
const flake::LockedFlake & lockedFlake,
bool useEvalCache)
{
return ref(std::make_shared<nix::eval_cache::EvalCache>(
useEvalCache,
lockedFlake.getFingerprint(),
state,
[&]()
{
/* For testing whether the evaluation cache is
complete. */
if (getEnv("NIX_ALLOW_EVAL").value_or("1") == "0")
throw Error("not everything is cached, but evaluation is not allowed");
auto vFlake = state.allocValue();
flake::callFlake(state, lockedFlake, *vFlake);
state.forceAttrs(*vFlake);
auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs"));
assert(aOutputs);
return aOutputs->value;
}));
}
std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableFlake::toDerivation()
{
auto state = cmd.getEvalState();
auto lockedFlake = lockFlake(*state, flakeRef, cmd.lockFlags);
Value * vOutputs = nullptr;
auto emptyArgs = state->allocBindings(0);
auto & evalCache = flake::EvalCache::singleton();
auto fingerprint = lockedFlake.getFingerprint();
auto cache = openEvalCache(*state, lockedFlake, true);
auto root = cache->getRoot();
for (auto & attrPath : getActualAttrPaths()) {
auto drv = evalCache.getDerivation(fingerprint, attrPath);
if (drv) {
if (state->store->isValidPath(drv->drvPath))
return {attrPath, lockedFlake.flake.lockedRef, std::move(*drv)};
auto attr = root->findAlongAttrPath(parseAttrPath(*state, attrPath));
if (!attr) continue;
if (!attr->isDerivation())
throw Error("flake output attribute '%s' is not a derivation", attrPath);
auto aDrvPath = attr->getAttr(state->sDrvPath);
auto drvPath = state->store->parseStorePath(aDrvPath->getString());
if (!state->store->isValidPath(drvPath)) {
/* The eval cache contains 'drvPath', but the actual path
has been garbage-collected. So force it to be
regenerated. */
aDrvPath->forceValue();
assert(state->store->isValidPath(drvPath));
}
if (!vOutputs)
vOutputs = getFlakeOutputs(*state, lockedFlake);
auto drvInfo = DerivationInfo{
std::move(drvPath),
state->store->parseStorePath(attr->getAttr(state->sOutPath)->getString()),
attr->getAttr(state->sOutputName)->getString()
};
try {
auto * v = findAlongAttrPath(*state, attrPath, *emptyArgs, *vOutputs).first;
state->forceValue(*v);
auto drvInfo = getDerivation(*state, *v, false);
if (!drvInfo)
throw Error("flake output attribute '%s' is not a derivation", attrPath);
auto drv = flake::EvalCache::Derivation{
state->store->parseStorePath(drvInfo->queryDrvPath()),
state->store->parseStorePath(drvInfo->queryOutPath()),
drvInfo->queryOutputName()
};
evalCache.addDerivation(fingerprint, attrPath, drv);
return {attrPath, lockedFlake.flake.lockedRef, std::move(drv)};
} catch (AttrPathNotFound & e) {
}
return {attrPath, lockedFlake.flake.lockedRef, std::move(drvInfo)};
}
throw Error("flake '%s' does not provide attribute %s",
flakeRef, concatStringsSep(", ", quoteStrings(attrPaths)));
}
std::vector<flake::EvalCache::Derivation> InstallableFlake::toDerivations()
std::vector<InstallableValue::DerivationInfo> InstallableFlake::toDerivations()
{
std::vector<flake::EvalCache::Derivation> res;
std::vector<DerivationInfo> res;
res.push_back(std::get<2>(toDerivation()));
return res;
}

View file

@ -3,7 +3,7 @@
#include "util.hh"
#include "path.hh"
#include "eval.hh"
#include "flake/eval-cache.hh"
#include "flake/flake.hh"
#include <optional>
@ -12,6 +12,8 @@ namespace nix {
struct DrvInfo;
struct SourceExprCommand;
namespace eval_cache { class EvalCache; }
struct Buildable
{
std::optional<StorePath> drvPath;
@ -63,7 +65,14 @@ struct InstallableValue : Installable
InstallableValue(SourceExprCommand & cmd) : cmd(cmd) { }
virtual std::vector<flake::EvalCache::Derivation> toDerivations();
struct DerivationInfo
{
StorePath drvPath;
StorePath outPath;
std::string outputName;
};
virtual std::vector<DerivationInfo> toDerivations();
Buildables toBuildables() override;
};
@ -86,11 +95,16 @@ struct InstallableFlake : InstallableValue
Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake);
std::tuple<std::string, FlakeRef, flake::EvalCache::Derivation> toDerivation();
std::tuple<std::string, FlakeRef, DerivationInfo> toDerivation();
std::vector<flake::EvalCache::Derivation> toDerivations() override;
std::vector<DerivationInfo> toDerivations() override;
std::pair<Value *, Pos> toValue(EvalState & state) override;
};
ref<eval_cache::EvalCache> openEvalCache(
EvalState & state,
const flake::LockedFlake & lockedFlake,
bool useEvalCache);
}