Add flake evaluation cache

This exploits the hermetic nature of flake evaluation to speed up
repeated evaluations of a flake output attribute.

For example (doing 'nix build' on an already present package):

  $ time nix build nixpkgs:firefox

  real    0m1.497s
  user    0m1.160s
  sys     0m0.139s

  $ time nix build nixpkgs:firefox

  real    0m0.052s
  user    0m0.038s
  sys     0m0.007s

The cache is ~/.cache/nix/eval-cache-v1.sqlite, which has entries like

  INSERT INTO Attributes VALUES(
    X'92a907d4efe933af2a46959b082cdff176aa5bfeb47a98fabd234809a67ab195',
    'packages.firefox',
    1,
    '/nix/store/pbalzf8x19hckr8cwdv62rd6g0lqgc38-firefox-67.0.drv /nix/store/g6q0gx0v6xvdnizp8lrcw7c4gdkzana0-firefox-67.0 out');

where the hash 92a9... is a fingerprint over the flake store path and
the contents of the lockfile. Because flakes are evaluated in pure
mode, this uniquely identifies the evaluation result.
This commit is contained in:
Eelco Dolstra 2019-06-07 22:25:48 +02:00
parent 671f16aee0
commit 6644b6099b
No known key found for this signature in database
GPG key ID: 8170B4726D7198DE
9 changed files with 293 additions and 49 deletions

View file

@ -0,0 +1,111 @@
#include "eval-cache.hh"
#include "sqlite.hh"
#include <set>
namespace nix::flake {
static const char * schema = R"sql(
create table if not exists Fingerprints (
fingerprint blob primary key not null,
timestamp integer not null
);
create table if not exists Attributes (
fingerprint blob not null,
attrPath text not null,
type integer,
value text,
primary key (fingerprint, attrPath),
foreign key (fingerprint) references Fingerprints(fingerprint) on delete cascade
);
)sql";
struct EvalCache::State
{
SQLite db;
SQLiteStmt insertFingerprint;
SQLiteStmt insertAttribute;
SQLiteStmt queryAttribute;
std::set<Fingerprint> fingerprints;
};
EvalCache::EvalCache()
: _state(std::make_unique<Sync<State>>())
{
auto state(_state->lock());
Path dbPath = getCacheDir() + "/nix/eval-cache-v1.sqlite";
createDirs(dirOf(dbPath));
state->db = SQLite(dbPath);
state->db.isCache();
state->db.exec(schema);
state->insertFingerprint.create(state->db,
"insert or ignore into Fingerprints(fingerprint, timestamp) values (?, ?)");
state->insertAttribute.create(state->db,
"insert or replace into Attributes(fingerprint, attrPath, type, value) values (?, ?, ?, ?)");
state->queryAttribute.create(state->db,
"select type, value from Attributes where fingerprint = ? and attrPath = ?");
}
enum ValueType {
Derivation = 1,
};
void EvalCache::addDerivation(
const Fingerprint & fingerprint,
const std::string & attrPath,
const Derivation & drv)
{
auto state(_state->lock());
if (state->fingerprints.insert(fingerprint).second)
// FIXME: update timestamp
state->insertFingerprint.use()
(fingerprint.hash, fingerprint.hashSize)
(time(0)).exec();
state->insertAttribute.use()
(fingerprint.hash, fingerprint.hashSize)
(attrPath)
(ValueType::Derivation)
(drv.drvPath + " " + drv.outPath + " " + drv.outputName).exec();
}
std::optional<EvalCache::Derivation> EvalCache::getDerivation(
const Fingerprint & fingerprint,
const std::string & attrPath)
{
auto state(_state->lock());
auto queryAttribute(state->queryAttribute.use()
(fingerprint.hash, fingerprint.hashSize)
(attrPath));
if (!queryAttribute.next()) return {};
// FIXME: handle negative results
auto type = (ValueType) queryAttribute.getInt(0);
auto s = queryAttribute.getStr(1);
if (type != ValueType::Derivation) return {};
auto ss = tokenizeString<std::vector<std::string>>(s, " ");
debug("evaluation cache hit for '%s'", attrPath);
return Derivation { ss[0], ss[1], ss[2] };
}
EvalCache & EvalCache::singleton()
{
static std::unique_ptr<EvalCache> evalCache(new EvalCache());
return *evalCache;
}
}

View file

@ -0,0 +1,39 @@
#pragma once
#include "sync.hh"
#include "flake.hh"
namespace nix { struct SQLite; struct SQLiteStmt; }
namespace nix::flake {
class EvalCache
{
struct State;
std::unique_ptr<Sync<State>> _state;
EvalCache();
public:
struct Derivation
{
Path drvPath;
Path outPath;
std::string outputName;
};
void addDerivation(
const Fingerprint & fingerprint,
const std::string & attrPath,
const Derivation & drv);
std::optional<Derivation> getDerivation(
const Fingerprint & fingerprint,
const std::string & attrPath);
static EvalCache & singleton();
};
}

View file

@ -601,4 +601,13 @@ const Registries EvalState::getFlakeRegistries()
return registries;
}
Fingerprint ResolvedFlake::getFingerprint() const
{
// FIXME: as an optimization, if the flake contains a lockfile and
// we haven't changed it, then it's sufficient to use
// flake.sourceInfo.storePath for the fingerprint.
return hashString(htSHA256,
fmt("%s;%s", flake.sourceInfo.storePath, lockFile));
}
}

View file

@ -83,12 +83,18 @@ struct NonFlake
Flake getFlake(EvalState &, const FlakeRef &, bool impureIsAllowed);
/* Fingerprint of a locked flake; used as a cache key. */
typedef Hash Fingerprint;
struct ResolvedFlake
{
Flake flake;
LockFile lockFile;
ResolvedFlake(Flake && flake, LockFile && lockFile)
: flake(flake), lockFile(lockFile) {}
Fingerprint getFingerprint() const;
};
ResolvedFlake resolveFlake(EvalState &, const FlakeRef &, HandleLockFile);

View file

@ -294,9 +294,7 @@ void LocalStore::openDB(State & state, bool create)
/* Open the Nix database. */
string dbPath = dbDir + "/db.sqlite";
auto & db(state.db);
if (sqlite3_open_v2(dbPath.c_str(), &db.db,
SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK)
throw Error(format("cannot open Nix database '%1%'") % dbPath);
state.db = SQLite(dbPath, create);
#ifdef __CYGWIN__
/* The cygwin version of sqlite3 has a patch which calls
@ -308,11 +306,6 @@ void LocalStore::openDB(State & state, bool create)
SetDllDirectoryW(L"");
#endif
if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK)
throwSQLiteError(db, "setting timeout");
db.exec("pragma foreign_keys = 1");
/* !!! check whether sqlite has been built with foreign key
support */

View file

@ -78,12 +78,7 @@ public:
state->db = SQLite(dbPath);
if (sqlite3_busy_timeout(state->db, 60 * 60 * 1000) != SQLITE_OK)
throwSQLiteError(state->db, "setting timeout");
// We can always reproduce the cache.
state->db.exec("pragma synchronous = off");
state->db.exec("pragma main.journal_mode = truncate");
state->db.isCache();
state->db.exec(schema);

View file

@ -25,11 +25,16 @@ namespace nix {
throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(exterr), path);
}
SQLite::SQLite(const Path & path)
SQLite::SQLite(const Path & path, bool create)
{
if (sqlite3_open_v2(path.c_str(), &db,
SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK)
SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK)
throw Error(format("cannot open SQLite database '%s'") % path);
if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK)
throwSQLiteError(db, "setting timeout");
exec("pragma foreign_keys = 1");
}
SQLite::~SQLite()
@ -42,6 +47,12 @@ SQLite::~SQLite()
}
}
void SQLite::isCache()
{
exec("pragma synchronous = off");
exec("pragma main.journal_mode = truncate");
}
void SQLite::exec(const std::string & stmt)
{
retrySQLite<void>([&]() {
@ -94,6 +105,16 @@ SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool
return *this;
}
SQLiteStmt::Use & SQLiteStmt::Use::operator () (const unsigned char * data, size_t len, bool notNull)
{
if (notNull) {
if (sqlite3_bind_blob(stmt, curArg++, data, len, SQLITE_TRANSIENT) != SQLITE_OK)
throwSQLiteError(stmt.db, "binding argument");
} else
bind();
return *this;
}
SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull)
{
if (notNull) {

View file

@ -15,13 +15,16 @@ struct SQLite
{
sqlite3 * db = 0;
SQLite() { }
SQLite(const Path & path);
SQLite(const Path & path, bool create = true);
SQLite(const SQLite & from) = delete;
SQLite& operator = (const SQLite & from) = delete;
SQLite& operator = (SQLite && from) { db = from.db; from.db = 0; return *this; }
~SQLite();
operator sqlite3 * () { return db; }
/* Disable synchronous mode, set truncate journal mode. */
void isCache();
void exec(const std::string & stmt);
};
@ -52,6 +55,7 @@ struct SQLiteStmt
/* Bind the next parameter. */
Use & operator () (const std::string & value, bool notNull = true);
Use & operator () (const unsigned char * data, size_t len, bool notNull = true);
Use & operator () (int64_t value, bool notNull = true);
Use & bind(); // null

View file

@ -8,6 +8,7 @@
#include "store-api.hh"
#include "shared.hh"
#include "flake/flake.hh"
#include "flake/eval-cache.hh"
#include <regex>
#include <queue>
@ -110,7 +111,7 @@ struct InstallableValue : Installable
InstallableValue(SourceExprCommand & cmd) : cmd(cmd) { }
Buildables toBuildables() override
virtual std::vector<flake::EvalCache::Derivation> toDerivations()
{
auto state = cmd.getEvalState();
@ -118,22 +119,36 @@ struct InstallableValue : Installable
Bindings & autoArgs = *cmd.getAutoArgs(*state);
DrvInfos drvs;
getDerivations(*state, *v, "", autoArgs, drvs, false);
DrvInfos drvInfos;
getDerivations(*state, *v, "", autoArgs, drvInfos, false);
std::vector<flake::EvalCache::Derivation> res;
for (auto & drvInfo : drvInfos) {
res.push_back({
drvInfo.queryDrvPath(),
drvInfo.queryOutPath(),
drvInfo.queryOutputName()
});
}
return res;
}
Buildables toBuildables() override
{
Buildables res;
PathSet drvPaths;
for (auto & drv : drvs) {
Buildable b{drv.queryDrvPath()};
for (auto & drv : toDerivations()) {
Buildable b{drv.drvPath};
drvPaths.insert(b.drvPath);
auto outputName = drv.queryOutputName();
auto outputName = drv.outputName;
if (outputName == "")
throw Error("derivation '%s' lacks an 'outputName' attribute", b.drvPath);
b.outputs.emplace(outputName, drv.queryOutPath());
b.outputs.emplace(outputName, drv.outPath);
res.push_back(std::move(b));
}
@ -254,12 +269,30 @@ struct InstallableFlake : InstallableValue
std::string what() override { return flakeRef.to_string() + ":" + *attrPaths.begin(); }
Value * toValue(EvalState & state) override
std::vector<std::string> getActualAttrPaths()
{
std::vector<std::string> res;
if (searchPackages) {
// As a convenience, look for the attribute in
// 'outputs.packages'.
res.push_back("packages." + *attrPaths.begin());
// As a temporary hack until Nixpkgs is properly converted
// to provide a clean 'packages' set, look in 'legacyPackages'.
res.push_back("legacyPackages." + *attrPaths.begin());
}
for (auto & s : attrPaths)
res.push_back(s);
return res;
}
Value * getFlakeOutputs(EvalState & state, const flake::ResolvedFlake & resFlake)
{
auto vFlake = state.allocValue();
auto resFlake = resolveFlake(state, flakeRef, cmd.getLockFileMode());
callFlake(state, resFlake, *vFlake);
makeFlakeClosureGCRoot(*state.store, flakeRef, resFlake);
@ -268,34 +301,67 @@ struct InstallableFlake : InstallableValue
state.forceValue(*vOutputs);
auto emptyArgs = state.allocBindings(0);
return vOutputs;
}
if (searchPackages) {
// As a convenience, look for the attribute in
// 'outputs.packages'.
if (auto aPackages = *vOutputs->attrs->get(state.symbols.create("packages"))) {
try {
auto * v = findAlongAttrPath(state, *attrPaths.begin(), *emptyArgs, *aPackages->value);
state.forceValue(*v);
return v;
} catch (AttrPathNotFound & e) {
}
std::vector<flake::EvalCache::Derivation> toDerivations() override
{
auto state = cmd.getEvalState();
auto resFlake = resolveFlake(*state, flakeRef, cmd.getLockFileMode());
Value * vOutputs = nullptr;
auto emptyArgs = state->allocBindings(0);
auto & evalCache = flake::EvalCache::singleton();
auto fingerprint = resFlake.getFingerprint();
for (auto & attrPath : getActualAttrPaths()) {
auto drv = evalCache.getDerivation(fingerprint, attrPath);
if (drv) {
if (state->store->isValidPath(drv->drvPath))
return {*drv};
}
// As a temporary hack until Nixpkgs is properly converted
// to provide a clean 'packages' set, look in 'legacyPackages'.
if (auto aPackages = *vOutputs->attrs->get(state.symbols.create("legacyPackages"))) {
try {
auto * v = findAlongAttrPath(state, *attrPaths.begin(), *emptyArgs, *aPackages->value);
state.forceValue(*v);
return v;
} catch (AttrPathNotFound & e) {
}
if (!vOutputs)
vOutputs = getFlakeOutputs(*state, resFlake);
try {
auto * v = findAlongAttrPath(*state, attrPath, *emptyArgs, *vOutputs);
state->forceValue(*v);
auto drvInfo = getDerivation(*state, *v, false);
if (!drvInfo)
throw Error("flake output attribute '%s' is not a derivation", attrPath);
auto drv = flake::EvalCache::Derivation{
drvInfo->queryDrvPath(),
drvInfo->queryOutPath(),
drvInfo->queryOutputName()
};
evalCache.addDerivation(fingerprint, attrPath, drv);
return {drv};
} catch (AttrPathNotFound & e) {
}
}
// Otherwise, look for it in 'outputs'.
for (auto & attrPath : attrPaths) {
throw Error("flake '%s' does not provide attribute %s",
flakeRef, concatStringsSep(", ", quoteStrings(attrPaths)));
}
Value * toValue(EvalState & state) override
{
auto resFlake = resolveFlake(state, flakeRef, cmd.getLockFileMode());
auto vOutputs = getFlakeOutputs(state, resFlake);
auto emptyArgs = state.allocBindings(0);
for (auto & attrPath : getActualAttrPaths()) {
try {
auto * v = findAlongAttrPath(state, attrPath, *emptyArgs, *vOutputs);
state.forceValue(*v);