Merge pull request #2930 from NixOS/eval-cache

Flake evaluation cache
This commit is contained in:
Eelco Dolstra 2019-06-11 12:12:59 +02:00 committed by GitHub
commit c4d740115e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 298 additions and 49 deletions

View file

@ -0,0 +1,116 @@
#include "eval-cache.hh"
#include "sqlite.hh"
#include "eval.hh"
#include <set>
namespace nix::flake {
static const char * schema = R"sql(
create table if not exists Fingerprints (
fingerprint blob primary key not null,
timestamp integer not null
);
create table if not exists Attributes (
fingerprint blob not null,
attrPath text not null,
type integer,
value text,
primary key (fingerprint, attrPath),
foreign key (fingerprint) references Fingerprints(fingerprint) on delete cascade
);
)sql";
struct EvalCache::State
{
SQLite db;
SQLiteStmt insertFingerprint;
SQLiteStmt insertAttribute;
SQLiteStmt queryAttribute;
std::set<Fingerprint> fingerprints;
};
EvalCache::EvalCache()
: _state(std::make_unique<Sync<State>>())
{
auto state(_state->lock());
Path dbPath = getCacheDir() + "/nix/eval-cache-v1.sqlite";
createDirs(dirOf(dbPath));
state->db = SQLite(dbPath);
state->db.isCache();
state->db.exec(schema);
state->insertFingerprint.create(state->db,
"insert or ignore into Fingerprints(fingerprint, timestamp) values (?, ?)");
state->insertAttribute.create(state->db,
"insert or replace into Attributes(fingerprint, attrPath, type, value) values (?, ?, ?, ?)");
state->queryAttribute.create(state->db,
"select type, value from Attributes where fingerprint = ? and attrPath = ?");
}
enum ValueType {
Derivation = 1,
};
void EvalCache::addDerivation(
const Fingerprint & fingerprint,
const std::string & attrPath,
const Derivation & drv)
{
if (!evalSettings.pureEval) return;
auto state(_state->lock());
if (state->fingerprints.insert(fingerprint).second)
// FIXME: update timestamp
state->insertFingerprint.use()
(fingerprint.hash, fingerprint.hashSize)
(time(0)).exec();
state->insertAttribute.use()
(fingerprint.hash, fingerprint.hashSize)
(attrPath)
(ValueType::Derivation)
(drv.drvPath + " " + drv.outPath + " " + drv.outputName).exec();
}
std::optional<EvalCache::Derivation> EvalCache::getDerivation(
const Fingerprint & fingerprint,
const std::string & attrPath)
{
if (!evalSettings.pureEval) return {};
auto state(_state->lock());
auto queryAttribute(state->queryAttribute.use()
(fingerprint.hash, fingerprint.hashSize)
(attrPath));
if (!queryAttribute.next()) return {};
// FIXME: handle negative results
auto type = (ValueType) queryAttribute.getInt(0);
auto s = queryAttribute.getStr(1);
if (type != ValueType::Derivation) return {};
auto ss = tokenizeString<std::vector<std::string>>(s, " ");
debug("evaluation cache hit for '%s'", attrPath);
return Derivation { ss[0], ss[1], ss[2] };
}
EvalCache & EvalCache::singleton()
{
static std::unique_ptr<EvalCache> evalCache(new EvalCache());
return *evalCache;
}
}

View file

@ -0,0 +1,39 @@
#pragma once
#include "sync.hh"
#include "flake.hh"
namespace nix { struct SQLite; struct SQLiteStmt; }
namespace nix::flake {
class EvalCache
{
struct State;
std::unique_ptr<Sync<State>> _state;
EvalCache();
public:
struct Derivation
{
Path drvPath;
Path outPath;
std::string outputName;
};
void addDerivation(
const Fingerprint & fingerprint,
const std::string & attrPath,
const Derivation & drv);
std::optional<Derivation> getDerivation(
const Fingerprint & fingerprint,
const std::string & attrPath);
static EvalCache & singleton();
};
}

View file

@ -601,4 +601,13 @@ const Registries EvalState::getFlakeRegistries()
return registries; return registries;
} }
Fingerprint ResolvedFlake::getFingerprint() const
{
// FIXME: as an optimization, if the flake contains a lockfile and
// we haven't changed it, then it's sufficient to use
// flake.sourceInfo.storePath for the fingerprint.
return hashString(htSHA256,
fmt("%s;%s", flake.sourceInfo.storePath, lockFile));
}
} }

View file

@ -83,12 +83,18 @@ struct NonFlake
Flake getFlake(EvalState &, const FlakeRef &, bool impureIsAllowed); Flake getFlake(EvalState &, const FlakeRef &, bool impureIsAllowed);
/* Fingerprint of a locked flake; used as a cache key. */
typedef Hash Fingerprint;
struct ResolvedFlake struct ResolvedFlake
{ {
Flake flake; Flake flake;
LockFile lockFile; LockFile lockFile;
ResolvedFlake(Flake && flake, LockFile && lockFile) ResolvedFlake(Flake && flake, LockFile && lockFile)
: flake(flake), lockFile(lockFile) {} : flake(flake), lockFile(lockFile) {}
Fingerprint getFingerprint() const;
}; };
ResolvedFlake resolveFlake(EvalState &, const FlakeRef &, HandleLockFile); ResolvedFlake resolveFlake(EvalState &, const FlakeRef &, HandleLockFile);

View file

@ -294,9 +294,7 @@ void LocalStore::openDB(State & state, bool create)
/* Open the Nix database. */ /* Open the Nix database. */
string dbPath = dbDir + "/db.sqlite"; string dbPath = dbDir + "/db.sqlite";
auto & db(state.db); auto & db(state.db);
if (sqlite3_open_v2(dbPath.c_str(), &db.db, state.db = SQLite(dbPath, create);
SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK)
throw Error(format("cannot open Nix database '%1%'") % dbPath);
#ifdef __CYGWIN__ #ifdef __CYGWIN__
/* The cygwin version of sqlite3 has a patch which calls /* The cygwin version of sqlite3 has a patch which calls
@ -308,11 +306,6 @@ void LocalStore::openDB(State & state, bool create)
SetDllDirectoryW(L""); SetDllDirectoryW(L"");
#endif #endif
if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK)
throwSQLiteError(db, "setting timeout");
db.exec("pragma foreign_keys = 1");
/* !!! check whether sqlite has been built with foreign key /* !!! check whether sqlite has been built with foreign key
support */ support */

View file

@ -78,12 +78,7 @@ public:
state->db = SQLite(dbPath); state->db = SQLite(dbPath);
if (sqlite3_busy_timeout(state->db, 60 * 60 * 1000) != SQLITE_OK) state->db.isCache();
throwSQLiteError(state->db, "setting timeout");
// We can always reproduce the cache.
state->db.exec("pragma synchronous = off");
state->db.exec("pragma main.journal_mode = truncate");
state->db.exec(schema); state->db.exec(schema);

View file

@ -25,11 +25,16 @@ namespace nix {
throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(exterr), path); throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(exterr), path);
} }
SQLite::SQLite(const Path & path) SQLite::SQLite(const Path & path, bool create)
{ {
if (sqlite3_open_v2(path.c_str(), &db, if (sqlite3_open_v2(path.c_str(), &db,
SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK) SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK)
throw Error(format("cannot open SQLite database '%s'") % path); throw Error(format("cannot open SQLite database '%s'") % path);
if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK)
throwSQLiteError(db, "setting timeout");
exec("pragma foreign_keys = 1");
} }
SQLite::~SQLite() SQLite::~SQLite()
@ -42,6 +47,12 @@ SQLite::~SQLite()
} }
} }
void SQLite::isCache()
{
exec("pragma synchronous = off");
exec("pragma main.journal_mode = truncate");
}
void SQLite::exec(const std::string & stmt) void SQLite::exec(const std::string & stmt)
{ {
retrySQLite<void>([&]() { retrySQLite<void>([&]() {
@ -94,6 +105,16 @@ SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool
return *this; return *this;
} }
SQLiteStmt::Use & SQLiteStmt::Use::operator () (const unsigned char * data, size_t len, bool notNull)
{
if (notNull) {
if (sqlite3_bind_blob(stmt, curArg++, data, len, SQLITE_TRANSIENT) != SQLITE_OK)
throwSQLiteError(stmt.db, "binding argument");
} else
bind();
return *this;
}
SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull) SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull)
{ {
if (notNull) { if (notNull) {

View file

@ -15,13 +15,16 @@ struct SQLite
{ {
sqlite3 * db = 0; sqlite3 * db = 0;
SQLite() { } SQLite() { }
SQLite(const Path & path); SQLite(const Path & path, bool create = true);
SQLite(const SQLite & from) = delete; SQLite(const SQLite & from) = delete;
SQLite& operator = (const SQLite & from) = delete; SQLite& operator = (const SQLite & from) = delete;
SQLite& operator = (SQLite && from) { db = from.db; from.db = 0; return *this; } SQLite& operator = (SQLite && from) { db = from.db; from.db = 0; return *this; }
~SQLite(); ~SQLite();
operator sqlite3 * () { return db; } operator sqlite3 * () { return db; }
/* Disable synchronous mode, set truncate journal mode. */
void isCache();
void exec(const std::string & stmt); void exec(const std::string & stmt);
}; };
@ -52,6 +55,7 @@ struct SQLiteStmt
/* Bind the next parameter. */ /* Bind the next parameter. */
Use & operator () (const std::string & value, bool notNull = true); Use & operator () (const std::string & value, bool notNull = true);
Use & operator () (const unsigned char * data, size_t len, bool notNull = true);
Use & operator () (int64_t value, bool notNull = true); Use & operator () (int64_t value, bool notNull = true);
Use & bind(); // null Use & bind(); // null

View file

@ -8,6 +8,7 @@
#include "store-api.hh" #include "store-api.hh"
#include "shared.hh" #include "shared.hh"
#include "flake/flake.hh" #include "flake/flake.hh"
#include "flake/eval-cache.hh"
#include <regex> #include <regex>
#include <queue> #include <queue>
@ -110,7 +111,7 @@ struct InstallableValue : Installable
InstallableValue(SourceExprCommand & cmd) : cmd(cmd) { } InstallableValue(SourceExprCommand & cmd) : cmd(cmd) { }
Buildables toBuildables() override virtual std::vector<flake::EvalCache::Derivation> toDerivations()
{ {
auto state = cmd.getEvalState(); auto state = cmd.getEvalState();
@ -118,22 +119,36 @@ struct InstallableValue : Installable
Bindings & autoArgs = *cmd.getAutoArgs(*state); Bindings & autoArgs = *cmd.getAutoArgs(*state);
DrvInfos drvs; DrvInfos drvInfos;
getDerivations(*state, *v, "", autoArgs, drvs, false); getDerivations(*state, *v, "", autoArgs, drvInfos, false);
std::vector<flake::EvalCache::Derivation> res;
for (auto & drvInfo : drvInfos) {
res.push_back({
drvInfo.queryDrvPath(),
drvInfo.queryOutPath(),
drvInfo.queryOutputName()
});
}
return res;
}
Buildables toBuildables() override
{
Buildables res; Buildables res;
PathSet drvPaths; PathSet drvPaths;
for (auto & drv : drvs) { for (auto & drv : toDerivations()) {
Buildable b{drv.queryDrvPath()}; Buildable b{drv.drvPath};
drvPaths.insert(b.drvPath); drvPaths.insert(b.drvPath);
auto outputName = drv.queryOutputName(); auto outputName = drv.outputName;
if (outputName == "") if (outputName == "")
throw Error("derivation '%s' lacks an 'outputName' attribute", b.drvPath); throw Error("derivation '%s' lacks an 'outputName' attribute", b.drvPath);
b.outputs.emplace(outputName, drv.queryOutPath()); b.outputs.emplace(outputName, drv.outPath);
res.push_back(std::move(b)); res.push_back(std::move(b));
} }
@ -254,12 +269,30 @@ struct InstallableFlake : InstallableValue
std::string what() override { return flakeRef.to_string() + ":" + *attrPaths.begin(); } std::string what() override { return flakeRef.to_string() + ":" + *attrPaths.begin(); }
Value * toValue(EvalState & state) override std::vector<std::string> getActualAttrPaths()
{
std::vector<std::string> res;
if (searchPackages) {
// As a convenience, look for the attribute in
// 'outputs.packages'.
res.push_back("packages." + *attrPaths.begin());
// As a temporary hack until Nixpkgs is properly converted
// to provide a clean 'packages' set, look in 'legacyPackages'.
res.push_back("legacyPackages." + *attrPaths.begin());
}
for (auto & s : attrPaths)
res.push_back(s);
return res;
}
Value * getFlakeOutputs(EvalState & state, const flake::ResolvedFlake & resFlake)
{ {
auto vFlake = state.allocValue(); auto vFlake = state.allocValue();
auto resFlake = resolveFlake(state, flakeRef, cmd.getLockFileMode());
callFlake(state, resFlake, *vFlake); callFlake(state, resFlake, *vFlake);
makeFlakeClosureGCRoot(*state.store, flakeRef, resFlake); makeFlakeClosureGCRoot(*state.store, flakeRef, resFlake);
@ -268,34 +301,67 @@ struct InstallableFlake : InstallableValue
state.forceValue(*vOutputs); state.forceValue(*vOutputs);
auto emptyArgs = state.allocBindings(0); return vOutputs;
}
if (searchPackages) { std::vector<flake::EvalCache::Derivation> toDerivations() override
// As a convenience, look for the attribute in {
// 'outputs.packages'. auto state = cmd.getEvalState();
if (auto aPackages = *vOutputs->attrs->get(state.symbols.create("packages"))) {
try { auto resFlake = resolveFlake(*state, flakeRef, cmd.getLockFileMode());
auto * v = findAlongAttrPath(state, *attrPaths.begin(), *emptyArgs, *aPackages->value);
state.forceValue(*v); Value * vOutputs = nullptr;
return v;
} catch (AttrPathNotFound & e) { auto emptyArgs = state->allocBindings(0);
}
auto & evalCache = flake::EvalCache::singleton();
auto fingerprint = resFlake.getFingerprint();
for (auto & attrPath : getActualAttrPaths()) {
auto drv = evalCache.getDerivation(fingerprint, attrPath);
if (drv) {
if (state->store->isValidPath(drv->drvPath))
return {*drv};
} }
// As a temporary hack until Nixpkgs is properly converted if (!vOutputs)
// to provide a clean 'packages' set, look in 'legacyPackages'. vOutputs = getFlakeOutputs(*state, resFlake);
if (auto aPackages = *vOutputs->attrs->get(state.symbols.create("legacyPackages"))) {
try { try {
auto * v = findAlongAttrPath(state, *attrPaths.begin(), *emptyArgs, *aPackages->value); auto * v = findAlongAttrPath(*state, attrPath, *emptyArgs, *vOutputs);
state.forceValue(*v); state->forceValue(*v);
return v;
} catch (AttrPathNotFound & e) { auto drvInfo = getDerivation(*state, *v, false);
} if (!drvInfo)
throw Error("flake output attribute '%s' is not a derivation", attrPath);
auto drv = flake::EvalCache::Derivation{
drvInfo->queryDrvPath(),
drvInfo->queryOutPath(),
drvInfo->queryOutputName()
};
evalCache.addDerivation(fingerprint, attrPath, drv);
return {drv};
} catch (AttrPathNotFound & e) {
} }
} }
// Otherwise, look for it in 'outputs'. throw Error("flake '%s' does not provide attribute %s",
for (auto & attrPath : attrPaths) { flakeRef, concatStringsSep(", ", quoteStrings(attrPaths)));
}
Value * toValue(EvalState & state) override
{
auto resFlake = resolveFlake(state, flakeRef, cmd.getLockFileMode());
auto vOutputs = getFlakeOutputs(state, resFlake);
auto emptyArgs = state.allocBindings(0);
for (auto & attrPath : getActualAttrPaths()) {
try { try {
auto * v = findAlongAttrPath(state, attrPath, *emptyArgs, *vOutputs); auto * v = findAlongAttrPath(state, attrPath, *emptyArgs, *vOutputs);
state.forceValue(*v); state.forceValue(*v);