From 6644b6099be2d3393206bf1c9c091c888c0a0f57 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 7 Jun 2019 22:25:48 +0200 Subject: [PATCH 1/2] Add flake evaluation cache This exploits the hermetic nature of flake evaluation to speed up repeated evaluations of a flake output attribute. For example (doing 'nix build' on an already present package): $ time nix build nixpkgs:firefox real 0m1.497s user 0m1.160s sys 0m0.139s $ time nix build nixpkgs:firefox real 0m0.052s user 0m0.038s sys 0m0.007s The cache is ~/.cache/nix/eval-cache-v1.sqlite, which has entries like INSERT INTO Attributes VALUES( X'92a907d4efe933af2a46959b082cdff176aa5bfeb47a98fabd234809a67ab195', 'packages.firefox', 1, '/nix/store/pbalzf8x19hckr8cwdv62rd6g0lqgc38-firefox-67.0.drv /nix/store/g6q0gx0v6xvdnizp8lrcw7c4gdkzana0-firefox-67.0 out'); where the hash 92a9... is a fingerprint over the flake store path and the contents of the lockfile. Because flakes are evaluated in pure mode, this uniquely identifies the evaluation result. --- src/libexpr/flake/eval-cache.cc | 111 ++++++++++++++++++++++++ src/libexpr/flake/eval-cache.hh | 39 +++++++++ src/libexpr/flake/flake.cc | 9 ++ src/libexpr/flake/flake.hh | 6 ++ src/libstore/local-store.cc | 9 +- src/libstore/nar-info-disk-cache.cc | 7 +- src/libstore/sqlite.cc | 25 +++++- src/libstore/sqlite.hh | 6 +- src/nix/installables.cc | 130 +++++++++++++++++++++------- 9 files changed, 293 insertions(+), 49 deletions(-) create mode 100644 src/libexpr/flake/eval-cache.cc create mode 100644 src/libexpr/flake/eval-cache.hh diff --git a/src/libexpr/flake/eval-cache.cc b/src/libexpr/flake/eval-cache.cc new file mode 100644 index 000000000..fece1a2b5 --- /dev/null +++ b/src/libexpr/flake/eval-cache.cc @@ -0,0 +1,111 @@ +#include "eval-cache.hh" +#include "sqlite.hh" + +#include + +namespace nix::flake { + +static const char * schema = R"sql( + +create table if not exists Fingerprints ( + fingerprint blob primary key not null, + timestamp integer not null +); + +create table if not exists Attributes ( + fingerprint blob not null, + attrPath text not null, + type integer, + value text, + primary key (fingerprint, attrPath), + foreign key (fingerprint) references Fingerprints(fingerprint) on delete cascade +); +)sql"; + +struct EvalCache::State +{ + SQLite db; + SQLiteStmt insertFingerprint; + SQLiteStmt insertAttribute; + SQLiteStmt queryAttribute; + std::set fingerprints; +}; + +EvalCache::EvalCache() + : _state(std::make_unique>()) +{ + auto state(_state->lock()); + + Path dbPath = getCacheDir() + "/nix/eval-cache-v1.sqlite"; + createDirs(dirOf(dbPath)); + + state->db = SQLite(dbPath); + state->db.isCache(); + state->db.exec(schema); + + state->insertFingerprint.create(state->db, + "insert or ignore into Fingerprints(fingerprint, timestamp) values (?, ?)"); + + state->insertAttribute.create(state->db, + "insert or replace into Attributes(fingerprint, attrPath, type, value) values (?, ?, ?, ?)"); + + state->queryAttribute.create(state->db, + "select type, value from Attributes where fingerprint = ? and attrPath = ?"); +} + +enum ValueType { + Derivation = 1, +}; + +void EvalCache::addDerivation( + const Fingerprint & fingerprint, + const std::string & attrPath, + const Derivation & drv) +{ + auto state(_state->lock()); + + if (state->fingerprints.insert(fingerprint).second) + // FIXME: update timestamp + state->insertFingerprint.use() + (fingerprint.hash, fingerprint.hashSize) + (time(0)).exec(); + + state->insertAttribute.use() + (fingerprint.hash, fingerprint.hashSize) + (attrPath) + (ValueType::Derivation) + (drv.drvPath + " " + drv.outPath + " " + drv.outputName).exec(); +} + +std::optional EvalCache::getDerivation( + const Fingerprint & fingerprint, + const std::string & attrPath) +{ + auto state(_state->lock()); + + auto queryAttribute(state->queryAttribute.use() + (fingerprint.hash, fingerprint.hashSize) + (attrPath)); + if (!queryAttribute.next()) return {}; + + // FIXME: handle negative results + + auto type = (ValueType) queryAttribute.getInt(0); + auto s = queryAttribute.getStr(1); + + if (type != ValueType::Derivation) return {}; + + auto ss = tokenizeString>(s, " "); + + debug("evaluation cache hit for '%s'", attrPath); + + return Derivation { ss[0], ss[1], ss[2] }; +} + +EvalCache & EvalCache::singleton() +{ + static std::unique_ptr evalCache(new EvalCache()); + return *evalCache; +} + +} diff --git a/src/libexpr/flake/eval-cache.hh b/src/libexpr/flake/eval-cache.hh new file mode 100644 index 000000000..03aea142e --- /dev/null +++ b/src/libexpr/flake/eval-cache.hh @@ -0,0 +1,39 @@ +#pragma once + +#include "sync.hh" +#include "flake.hh" + +namespace nix { struct SQLite; struct SQLiteStmt; } + +namespace nix::flake { + +class EvalCache +{ + struct State; + + std::unique_ptr> _state; + + EvalCache(); + +public: + + struct Derivation + { + Path drvPath; + Path outPath; + std::string outputName; + }; + + void addDerivation( + const Fingerprint & fingerprint, + const std::string & attrPath, + const Derivation & drv); + + std::optional getDerivation( + const Fingerprint & fingerprint, + const std::string & attrPath); + + static EvalCache & singleton(); +}; + +} diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index bb0543541..0018a0d07 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -601,4 +601,13 @@ const Registries EvalState::getFlakeRegistries() return registries; } +Fingerprint ResolvedFlake::getFingerprint() const +{ + // FIXME: as an optimization, if the flake contains a lockfile and + // we haven't changed it, then it's sufficient to use + // flake.sourceInfo.storePath for the fingerprint. + return hashString(htSHA256, + fmt("%s;%s", flake.sourceInfo.storePath, lockFile)); +} + } diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index b8d0da252..81b6541f0 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -83,12 +83,18 @@ struct NonFlake Flake getFlake(EvalState &, const FlakeRef &, bool impureIsAllowed); +/* Fingerprint of a locked flake; used as a cache key. */ +typedef Hash Fingerprint; + struct ResolvedFlake { Flake flake; LockFile lockFile; + ResolvedFlake(Flake && flake, LockFile && lockFile) : flake(flake), lockFile(lockFile) {} + + Fingerprint getFingerprint() const; }; ResolvedFlake resolveFlake(EvalState &, const FlakeRef &, HandleLockFile); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 485fdd691..f39c73b23 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -294,9 +294,7 @@ void LocalStore::openDB(State & state, bool create) /* Open the Nix database. */ string dbPath = dbDir + "/db.sqlite"; auto & db(state.db); - if (sqlite3_open_v2(dbPath.c_str(), &db.db, - SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) - throw Error(format("cannot open Nix database '%1%'") % dbPath); + state.db = SQLite(dbPath, create); #ifdef __CYGWIN__ /* The cygwin version of sqlite3 has a patch which calls @@ -308,11 +306,6 @@ void LocalStore::openDB(State & state, bool create) SetDllDirectoryW(L""); #endif - if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) - throwSQLiteError(db, "setting timeout"); - - db.exec("pragma foreign_keys = 1"); - /* !!! check whether sqlite has been built with foreign key support */ diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 32ad7f2b2..3f6dbbcf5 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -78,12 +78,7 @@ public: state->db = SQLite(dbPath); - if (sqlite3_busy_timeout(state->db, 60 * 60 * 1000) != SQLITE_OK) - throwSQLiteError(state->db, "setting timeout"); - - // We can always reproduce the cache. - state->db.exec("pragma synchronous = off"); - state->db.exec("pragma main.journal_mode = truncate"); + state->db.isCache(); state->db.exec(schema); diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index a061d64f3..eb1daafc5 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -25,11 +25,16 @@ namespace nix { throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(exterr), path); } -SQLite::SQLite(const Path & path) +SQLite::SQLite(const Path & path, bool create) { if (sqlite3_open_v2(path.c_str(), &db, - SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK) + SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) throw Error(format("cannot open SQLite database '%s'") % path); + + if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) + throwSQLiteError(db, "setting timeout"); + + exec("pragma foreign_keys = 1"); } SQLite::~SQLite() @@ -42,6 +47,12 @@ SQLite::~SQLite() } } +void SQLite::isCache() +{ + exec("pragma synchronous = off"); + exec("pragma main.journal_mode = truncate"); +} + void SQLite::exec(const std::string & stmt) { retrySQLite([&]() { @@ -94,6 +105,16 @@ SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool return *this; } +SQLiteStmt::Use & SQLiteStmt::Use::operator () (const unsigned char * data, size_t len, bool notNull) +{ + if (notNull) { + if (sqlite3_bind_blob(stmt, curArg++, data, len, SQLITE_TRANSIENT) != SQLITE_OK) + throwSQLiteError(stmt.db, "binding argument"); + } else + bind(); + return *this; +} + SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull) { if (notNull) { diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index 115679b84..78e53fa32 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -15,13 +15,16 @@ struct SQLite { sqlite3 * db = 0; SQLite() { } - SQLite(const Path & path); + SQLite(const Path & path, bool create = true); SQLite(const SQLite & from) = delete; SQLite& operator = (const SQLite & from) = delete; SQLite& operator = (SQLite && from) { db = from.db; from.db = 0; return *this; } ~SQLite(); operator sqlite3 * () { return db; } + /* Disable synchronous mode, set truncate journal mode. */ + void isCache(); + void exec(const std::string & stmt); }; @@ -52,6 +55,7 @@ struct SQLiteStmt /* Bind the next parameter. */ Use & operator () (const std::string & value, bool notNull = true); + Use & operator () (const unsigned char * data, size_t len, bool notNull = true); Use & operator () (int64_t value, bool notNull = true); Use & bind(); // null diff --git a/src/nix/installables.cc b/src/nix/installables.cc index a85295a09..86e601bc4 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -8,6 +8,7 @@ #include "store-api.hh" #include "shared.hh" #include "flake/flake.hh" +#include "flake/eval-cache.hh" #include #include @@ -110,7 +111,7 @@ struct InstallableValue : Installable InstallableValue(SourceExprCommand & cmd) : cmd(cmd) { } - Buildables toBuildables() override + virtual std::vector toDerivations() { auto state = cmd.getEvalState(); @@ -118,22 +119,36 @@ struct InstallableValue : Installable Bindings & autoArgs = *cmd.getAutoArgs(*state); - DrvInfos drvs; - getDerivations(*state, *v, "", autoArgs, drvs, false); + DrvInfos drvInfos; + getDerivations(*state, *v, "", autoArgs, drvInfos, false); + std::vector res; + for (auto & drvInfo : drvInfos) { + res.push_back({ + drvInfo.queryDrvPath(), + drvInfo.queryOutPath(), + drvInfo.queryOutputName() + }); + } + + return res; + } + + Buildables toBuildables() override + { Buildables res; PathSet drvPaths; - for (auto & drv : drvs) { - Buildable b{drv.queryDrvPath()}; + for (auto & drv : toDerivations()) { + Buildable b{drv.drvPath}; drvPaths.insert(b.drvPath); - auto outputName = drv.queryOutputName(); + auto outputName = drv.outputName; if (outputName == "") throw Error("derivation '%s' lacks an 'outputName' attribute", b.drvPath); - b.outputs.emplace(outputName, drv.queryOutPath()); + b.outputs.emplace(outputName, drv.outPath); res.push_back(std::move(b)); } @@ -254,12 +269,30 @@ struct InstallableFlake : InstallableValue std::string what() override { return flakeRef.to_string() + ":" + *attrPaths.begin(); } - Value * toValue(EvalState & state) override + std::vector getActualAttrPaths() + { + std::vector res; + + if (searchPackages) { + // As a convenience, look for the attribute in + // 'outputs.packages'. + res.push_back("packages." + *attrPaths.begin()); + + // As a temporary hack until Nixpkgs is properly converted + // to provide a clean 'packages' set, look in 'legacyPackages'. + res.push_back("legacyPackages." + *attrPaths.begin()); + } + + for (auto & s : attrPaths) + res.push_back(s); + + return res; + } + + Value * getFlakeOutputs(EvalState & state, const flake::ResolvedFlake & resFlake) { auto vFlake = state.allocValue(); - auto resFlake = resolveFlake(state, flakeRef, cmd.getLockFileMode()); - callFlake(state, resFlake, *vFlake); makeFlakeClosureGCRoot(*state.store, flakeRef, resFlake); @@ -268,34 +301,67 @@ struct InstallableFlake : InstallableValue state.forceValue(*vOutputs); - auto emptyArgs = state.allocBindings(0); + return vOutputs; + } - if (searchPackages) { - // As a convenience, look for the attribute in - // 'outputs.packages'. - if (auto aPackages = *vOutputs->attrs->get(state.symbols.create("packages"))) { - try { - auto * v = findAlongAttrPath(state, *attrPaths.begin(), *emptyArgs, *aPackages->value); - state.forceValue(*v); - return v; - } catch (AttrPathNotFound & e) { - } + std::vector toDerivations() override + { + auto state = cmd.getEvalState(); + + auto resFlake = resolveFlake(*state, flakeRef, cmd.getLockFileMode()); + + Value * vOutputs = nullptr; + + auto emptyArgs = state->allocBindings(0); + + auto & evalCache = flake::EvalCache::singleton(); + + auto fingerprint = resFlake.getFingerprint(); + + for (auto & attrPath : getActualAttrPaths()) { + auto drv = evalCache.getDerivation(fingerprint, attrPath); + if (drv) { + if (state->store->isValidPath(drv->drvPath)) + return {*drv}; } - // As a temporary hack until Nixpkgs is properly converted - // to provide a clean 'packages' set, look in 'legacyPackages'. - if (auto aPackages = *vOutputs->attrs->get(state.symbols.create("legacyPackages"))) { - try { - auto * v = findAlongAttrPath(state, *attrPaths.begin(), *emptyArgs, *aPackages->value); - state.forceValue(*v); - return v; - } catch (AttrPathNotFound & e) { - } + if (!vOutputs) + vOutputs = getFlakeOutputs(*state, resFlake); + + try { + auto * v = findAlongAttrPath(*state, attrPath, *emptyArgs, *vOutputs); + state->forceValue(*v); + + auto drvInfo = getDerivation(*state, *v, false); + if (!drvInfo) + throw Error("flake output attribute '%s' is not a derivation", attrPath); + + auto drv = flake::EvalCache::Derivation{ + drvInfo->queryDrvPath(), + drvInfo->queryOutPath(), + drvInfo->queryOutputName() + }; + + evalCache.addDerivation(fingerprint, attrPath, drv); + + return {drv}; + } catch (AttrPathNotFound & e) { } } - // Otherwise, look for it in 'outputs'. - for (auto & attrPath : attrPaths) { + throw Error("flake '%s' does not provide attribute %s", + flakeRef, concatStringsSep(", ", quoteStrings(attrPaths))); + } + + Value * toValue(EvalState & state) override + { + auto resFlake = resolveFlake(state, flakeRef, cmd.getLockFileMode()); + + auto vOutputs = getFlakeOutputs(state, resFlake); + + auto emptyArgs = state.allocBindings(0); + + for (auto & attrPath : getActualAttrPaths()) { try { auto * v = findAlongAttrPath(state, attrPath, *emptyArgs, *vOutputs); state.forceValue(*v); From c47d2dac6c7b404714e4c3429f26791790a483f5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 7 Jun 2019 22:38:39 +0200 Subject: [PATCH 2/2] Disable EvalCache in impure mode --- src/libexpr/flake/eval-cache.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libexpr/flake/eval-cache.cc b/src/libexpr/flake/eval-cache.cc index fece1a2b5..b32d502f7 100644 --- a/src/libexpr/flake/eval-cache.cc +++ b/src/libexpr/flake/eval-cache.cc @@ -1,5 +1,6 @@ #include "eval-cache.hh" #include "sqlite.hh" +#include "eval.hh" #include @@ -62,6 +63,8 @@ void EvalCache::addDerivation( const std::string & attrPath, const Derivation & drv) { + if (!evalSettings.pureEval) return; + auto state(_state->lock()); if (state->fingerprints.insert(fingerprint).second) @@ -81,6 +84,8 @@ std::optional EvalCache::getDerivation( const Fingerprint & fingerprint, const std::string & attrPath) { + if (!evalSettings.pureEval) return {}; + auto state(_state->lock()); auto queryAttribute(state->queryAttribute.use()