Make LocalStore thread-safe

Necessary for multi-threaded commands like "nix verify-paths".
This commit is contained in:
Eelco Dolstra 2016-04-08 18:07:13 +02:00
parent 05fbc606fc
commit f398949b40
5 changed files with 246 additions and 217 deletions

View file

@ -239,6 +239,9 @@ private:
/* Last time the goals in `waitingForAWhile' where woken up. */
time_t lastWokenUp;
/* Cache for pathContentsGood(). */
std::map<Path, bool> pathContentsGoodCache;
public:
/* Set if at least one derivation had a BuildError (i.e. permanent
@ -304,6 +307,12 @@ public:
void waitForInput();
unsigned int exitStatus();
/* Check whether the given valid path exists and has the right
contents. */
bool pathContentsGood(const Path & path);
void markContentsGood(const Path & path);
};
@ -1159,7 +1168,7 @@ void DerivationGoal::repairClosure()
/* Check each path (slow!). */
PathSet broken;
for (auto & i : outputClosure) {
if (worker.store.pathContentsGood(i)) continue;
if (worker.pathContentsGood(i)) continue;
printMsg(lvlError, format("found corrupted or missing path %1% in the output closure of %2%") % i % drvPath);
Path drvPath2 = outputsToDrv[i];
if (drvPath2 == "")
@ -2799,7 +2808,7 @@ void DerivationGoal::registerOutputs()
if (curRound == nrRounds) {
worker.store.optimisePath(path); // FIXME: combine with scanForReferences()
worker.store.markContentsGood(path);
worker.markContentsGood(path);
}
ValidPathInfo info;
@ -2977,7 +2986,7 @@ PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash)
if (!wantOutput(i.first, wantedOutputs)) continue;
bool good =
worker.store.isValidPath(i.second.path) &&
(!checkHash || worker.store.pathContentsGood(i.second.path));
(!checkHash || worker.pathContentsGood(i.second.path));
if (good == returnValid) result.insert(i.second.path);
}
return result;
@ -3385,7 +3394,7 @@ void SubstitutionGoal::finished()
outputLock->setDeletion(true);
outputLock.reset();
worker.store.markContentsGood(storePath);
worker.markContentsGood(storePath);
printMsg(lvlChatty,
format("substitution of path %1% succeeded") % storePath);
@ -3785,6 +3794,32 @@ unsigned int Worker::exitStatus()
}
bool Worker::pathContentsGood(const Path & path)
{
std::map<Path, bool>::iterator i = pathContentsGoodCache.find(path);
if (i != pathContentsGoodCache.end()) return i->second;
printMsg(lvlInfo, format("checking path %1%...") % path);
ValidPathInfo info = store.queryPathInfo(path);
bool res;
if (!pathExists(path))
res = false;
else {
HashResult current = hashPath(info.narHash.type, path);
Hash nullHash(htSHA256);
res = info.narHash == nullHash || info.narHash == current.first;
}
pathContentsGoodCache[path] = res;
if (!res) printMsg(lvlError, format("path %1% is corrupted or missing!") % path);
return res;
}
void Worker::markContentsGood(const Path & path)
{
pathContentsGoodCache[path] = true;
}
//////////////////////////////////////////////////////////////////////

View file

@ -147,35 +147,36 @@ Path Store::addPermRoot(const Path & _storePath,
void LocalStore::addTempRoot(const Path & path)
{
auto state(_state.lock());
/* Create the temporary roots file for this process. */
if (fdTempRoots == -1) {
if (state->fdTempRoots == -1) {
while (1) {
Path dir = (format("%1%/%2%") % settings.nixStateDir % tempRootsDir).str();
createDirs(dir);
fnTempRoots = (format("%1%/%2%")
% dir % getpid()).str();
state->fnTempRoots = (format("%1%/%2%") % dir % getpid()).str();
AutoCloseFD fdGCLock = openGCLock(ltRead);
if (pathExists(fnTempRoots))
if (pathExists(state->fnTempRoots))
/* It *must* be stale, since there can be no two
processes with the same pid. */
unlink(fnTempRoots.c_str());
unlink(state->fnTempRoots.c_str());
fdTempRoots = openLockFile(fnTempRoots, true);
state->fdTempRoots = openLockFile(state->fnTempRoots, true);
fdGCLock.close();
debug(format("acquiring read lock on %1%") % fnTempRoots);
lockFile(fdTempRoots, ltRead, true);
debug(format("acquiring read lock on %1%") % state->fnTempRoots);
lockFile(state->fdTempRoots, ltRead, true);
/* Check whether the garbage collector didn't get in our
way. */
struct stat st;
if (fstat(fdTempRoots, &st) == -1)
throw SysError(format("statting %1%") % fnTempRoots);
if (fstat(state->fdTempRoots, &st) == -1)
throw SysError(format("statting %1%") % state->fnTempRoots);
if (st.st_size == 0) break;
/* The garbage collector deleted this file before we could
@ -187,15 +188,15 @@ void LocalStore::addTempRoot(const Path & path)
/* Upgrade the lock to a write lock. This will cause us to block
if the garbage collector is holding our lock. */
debug(format("acquiring write lock on %1%") % fnTempRoots);
lockFile(fdTempRoots, ltWrite, true);
debug(format("acquiring write lock on %1%") % state->fnTempRoots);
lockFile(state->fdTempRoots, ltWrite, true);
string s = path + '\0';
writeFull(fdTempRoots, s);
writeFull(state->fdTempRoots, s);
/* Downgrade to a read lock. */
debug(format("downgrading to read lock on %1%") % fnTempRoots);
lockFile(fdTempRoots, ltRead, true);
debug(format("downgrading to read lock on %1%") % state->fnTempRoots);
lockFile(state->fdTempRoots, ltRead, true);
}

View file

@ -55,20 +55,21 @@ void checkStoreNotSymlink()
LocalStore::LocalStore()
: reservedPath(settings.nixDBPath + "/reserved")
, didSetSubstituterEnv(false)
: linksDir(settings.nixStore + "/.links")
, reservedPath(settings.nixDBPath + "/reserved")
, schemaPath(settings.nixDBPath + "/schema")
{
schemaPath = settings.nixDBPath + "/schema";
auto state(_state.lock());
if (settings.readOnlyMode) {
openDB(false);
openDB(*state, false);
return;
}
/* Create missing state directories if they don't already exist. */
createDirs(settings.nixStore);
makeStoreWritable();
createDirs(linksDir = settings.nixStore + "/.links");
createDirs(linksDir);
Path profilesDir = settings.nixStateDir + "/profiles";
createDirs(profilesDir);
createDirs(settings.nixStateDir + "/temproots");
@ -140,7 +141,7 @@ LocalStore::LocalStore()
} catch (SysError & e) {
if (e.errNo != EACCES) throw;
settings.readOnlyMode = true;
openDB(false);
openDB(*state, false);
return;
}
@ -158,7 +159,7 @@ LocalStore::LocalStore()
else if (curSchema == 0) { /* new store */
curSchema = nixSchemaVersion;
openDB(true);
openDB(*state, true);
writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str());
}
@ -186,14 +187,14 @@ LocalStore::LocalStore()
if (curSchema < 7) { upgradeStore7(); }
openDB(false);
openDB(*state, false);
if (curSchema < 8) {
SQLiteTxn txn(db);
if (sqlite3_exec(db, "alter table ValidPaths add column ultimate integer", 0, 0, 0) != SQLITE_OK)
throwSQLiteError(db, "upgrading database schema");
if (sqlite3_exec(db, "alter table ValidPaths add column sigs text", 0, 0, 0) != SQLITE_OK)
throwSQLiteError(db, "upgrading database schema");
SQLiteTxn txn(state->db);
if (sqlite3_exec(state->db, "alter table ValidPaths add column ultimate integer", 0, 0, 0) != SQLITE_OK)
throwSQLiteError(state->db, "upgrading database schema");
if (sqlite3_exec(state->db, "alter table ValidPaths add column sigs text", 0, 0, 0) != SQLITE_OK)
throwSQLiteError(state->db, "upgrading database schema");
txn.commit();
}
@ -202,14 +203,16 @@ LocalStore::LocalStore()
lockFile(globalLock, ltRead, true);
}
else openDB(false);
else openDB(*state, false);
}
LocalStore::~LocalStore()
{
auto state(_state.lock());
try {
for (auto & i : runningSubstituters) {
for (auto & i : state->runningSubstituters) {
if (i.second.disabled) continue;
i.second.to.close();
i.second.from.close();
@ -222,9 +225,9 @@ LocalStore::~LocalStore()
}
try {
if (fdTempRoots != -1) {
fdTempRoots.close();
unlink(fnTempRoots.c_str());
if (state->fdTempRoots != -1) {
state->fdTempRoots.close();
unlink(state->fnTempRoots.c_str());
}
} catch (...) {
ignoreException();
@ -250,13 +253,14 @@ bool LocalStore::haveWriteAccess()
}
void LocalStore::openDB(bool create)
void LocalStore::openDB(State & state, bool create)
{
if (!haveWriteAccess())
throw SysError(format("Nix database directory %1% is not writable") % settings.nixDBPath);
/* Open the Nix database. */
string dbPath = settings.nixDBPath + "/db.sqlite";
auto & db(state.db);
if (sqlite3_open_v2(dbPath.c_str(), &db.db,
SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK)
throw Error(format("cannot open Nix database %1%") % dbPath);
@ -309,41 +313,41 @@ void LocalStore::openDB(bool create)
}
/* Prepare SQL statements. */
stmtRegisterValidPath.create(db,
state.stmtRegisterValidPath.create(db,
"insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs) values (?, ?, ?, ?, ?, ?, ?);");
stmtUpdatePathInfo.create(db,
state.stmtUpdatePathInfo.create(db,
"update ValidPaths set narSize = ?, hash = ?, ultimate = ?, sigs = ? where path = ?;");
stmtAddReference.create(db,
state.stmtAddReference.create(db,
"insert or replace into Refs (referrer, reference) values (?, ?);");
stmtQueryPathInfo.create(db,
state.stmtQueryPathInfo.create(db,
"select id, hash, registrationTime, deriver, narSize, ultimate, sigs from ValidPaths where path = ?;");
stmtQueryReferences.create(db,
state.stmtQueryReferences.create(db,
"select path from Refs join ValidPaths on reference = id where referrer = ?;");
stmtQueryReferrers.create(db,
state.stmtQueryReferrers.create(db,
"select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);");
stmtInvalidatePath.create(db,
state.stmtInvalidatePath.create(db,
"delete from ValidPaths where path = ?;");
stmtRegisterFailedPath.create(db,
state.stmtRegisterFailedPath.create(db,
"insert or ignore into FailedPaths (path, time) values (?, ?);");
stmtHasPathFailed.create(db,
state.stmtHasPathFailed.create(db,
"select time from FailedPaths where path = ?;");
stmtQueryFailedPaths.create(db,
state.stmtQueryFailedPaths.create(db,
"select path from FailedPaths;");
// If the path is a derivation, then clear its outputs.
stmtClearFailedPath.create(db,
state.stmtClearFailedPath.create(db,
"delete from FailedPaths where ?1 = '*' or path = ?1 "
"or path in (select d.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where v.path = ?1);");
stmtAddDerivationOutput.create(db,
state.stmtAddDerivationOutput.create(db,
"insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);");
stmtQueryValidDerivers.create(db,
state.stmtQueryValidDerivers.create(db,
"select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;");
stmtQueryDerivationOutputs.create(db,
state.stmtQueryDerivationOutputs.create(db,
"select id, path from DerivationOutputs where drv = ?;");
// Use "path >= ?" with limit 1 rather than "path like '?%'" to
// ensure efficient lookup.
stmtQueryPathFromHashPart.create(db,
state.stmtQueryPathFromHashPart.create(db,
"select path from ValidPaths where path >= ? limit 1;");
stmtQueryValidPaths.create(db, "select path from ValidPaths");
state.stmtQueryValidPaths.create(db, "select path from ValidPaths");
}
@ -538,9 +542,10 @@ void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation &
}
uint64_t LocalStore::addValidPath(const ValidPathInfo & info, bool checkOutputs)
uint64_t LocalStore::addValidPath(State & state,
const ValidPathInfo & info, bool checkOutputs)
{
stmtRegisterValidPath.use()
state.stmtRegisterValidPath.use()
(info.path)
("sha256:" + printHash(info.narHash))
(info.registrationTime == 0 ? time(0) : info.registrationTime)
@ -549,7 +554,7 @@ uint64_t LocalStore::addValidPath(const ValidPathInfo & info, bool checkOutputs)
(info.ultimate ? 1 : 0, info.ultimate)
(concatStringsSep(" ", info.sigs), !info.sigs.empty())
.exec();
uint64_t id = sqlite3_last_insert_rowid(db);
uint64_t id = sqlite3_last_insert_rowid(state.db);
/* If this is a derivation, then store the derivation outputs in
the database. This is useful for the garbage collector: it can
@ -566,7 +571,7 @@ uint64_t LocalStore::addValidPath(const ValidPathInfo & info, bool checkOutputs)
if (checkOutputs) checkDerivationOutputs(info.path, drv);
for (auto & i : drv.outputs) {
stmtAddDerivationOutput.use()
state.stmtAddDerivationOutput.use()
(id)
(i.first)
(i.second.path)
@ -578,16 +583,11 @@ uint64_t LocalStore::addValidPath(const ValidPathInfo & info, bool checkOutputs)
}
void LocalStore::addReference(uint64_t referrer, uint64_t reference)
{
stmtAddReference.use()(referrer)(reference).exec();
}
void LocalStore::registerFailedPath(const Path & path)
{
retrySQLite<void>([&]() {
stmtRegisterFailedPath.use()(path)(time(0)).step();
auto state(_state.lock());
state->stmtRegisterFailedPath.use()(path)(time(0)).step();
});
}
@ -595,7 +595,8 @@ void LocalStore::registerFailedPath(const Path & path)
bool LocalStore::hasPathFailed(const Path & path)
{
return retrySQLite<bool>([&]() {
return stmtHasPathFailed.use()(path).next();
auto state(_state.lock());
return state->stmtHasPathFailed.use()(path).next();
});
}
@ -603,7 +604,9 @@ bool LocalStore::hasPathFailed(const Path & path)
PathSet LocalStore::queryFailedPaths()
{
return retrySQLite<PathSet>([&]() {
auto useQueryFailedPaths(stmtQueryFailedPaths.use());
auto state(_state.lock());
auto useQueryFailedPaths(state->stmtQueryFailedPaths.use());
PathSet res;
while (useQueryFailedPaths.next())
@ -617,10 +620,12 @@ PathSet LocalStore::queryFailedPaths()
void LocalStore::clearFailedPaths(const PathSet & paths)
{
retrySQLite<void>([&]() {
SQLiteTxn txn(db);
auto state(_state.lock());
SQLiteTxn txn(state->db);
for (auto & path : paths)
stmtClearFailedPath.use()(path).exec();
state->stmtClearFailedPath.use()(path).exec();
txn.commit();
});
@ -649,9 +654,10 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path)
assertStorePath(path);
return retrySQLite<ValidPathInfo>([&]() {
auto state(_state.lock());
/* Get the path info. */
auto useQueryPathInfo(stmtQueryPathInfo.use()(path));
auto useQueryPathInfo(state->stmtQueryPathInfo.use()(path));
if (!useQueryPathInfo.next())
throw Error(format("path %1% is not valid") % path);
@ -662,19 +668,19 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path)
info.registrationTime = useQueryPathInfo.getInt(2);
auto s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 3);
auto s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 3);
if (s) info.deriver = s;
/* Note that narSize = NULL yields 0. */
info.narSize = useQueryPathInfo.getInt(4);
info.ultimate = sqlite3_column_int(stmtQueryPathInfo, 5) == 1;
info.ultimate = useQueryPathInfo.getInt(5) == 1;
s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 6);
s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 6);
if (s) info.sigs = tokenizeString<StringSet>(s, " ");
/* Get the references. */
auto useQueryReferences(stmtQueryReferences.use()(info.id));
auto useQueryReferences(state->stmtQueryReferences.use()(info.id));
while (useQueryReferences.next())
info.references.insert(useQueryReferences.getStr(0));
@ -685,9 +691,9 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path)
/* Update path info in the database. */
void LocalStore::updatePathInfo(const ValidPathInfo & info)
void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
{
stmtUpdatePathInfo.use()
state.stmtUpdatePathInfo.use()
(info.narSize, info.narSize != 0)
("sha256:" + printHash(info.narHash))
(info.ultimate ? 1 : 0, info.ultimate)
@ -697,44 +703,44 @@ void LocalStore::updatePathInfo(const ValidPathInfo & info)
}
uint64_t LocalStore::queryValidPathId(const Path & path)
uint64_t LocalStore::queryValidPathId(State & state, const Path & path)
{
auto use(stmtQueryPathInfo.use()(path));
auto use(state.stmtQueryPathInfo.use()(path));
if (!use.next())
throw Error(format("path %1% is not valid") % path);
return use.getInt(0);
}
bool LocalStore::isValidPath_(const Path & path)
bool LocalStore::isValidPath(State & state, const Path & path)
{
return stmtQueryPathInfo.use()(path).next();
return state.stmtQueryPathInfo.use()(path).next();
}
bool LocalStore::isValidPath(const Path & path)
{
return retrySQLite<bool>([&]() {
return isValidPath_(path);
auto state(_state.lock());
return isValidPath(*state, path);
});
}
PathSet LocalStore::queryValidPaths(const PathSet & paths)
{
return retrySQLite<PathSet>([&]() {
PathSet res;
for (auto & i : paths)
if (isValidPath_(i)) res.insert(i);
if (isValidPath(i)) res.insert(i);
return res;
});
}
PathSet LocalStore::queryAllValidPaths()
{
return retrySQLite<PathSet>([&]() {
auto use(stmtQueryValidPaths.use());
auto state(_state.lock());
auto use(state->stmtQueryValidPaths.use());
PathSet res;
while (use.next()) res.insert(use.getStr(0));
return res;
@ -742,9 +748,9 @@ PathSet LocalStore::queryAllValidPaths()
}
void LocalStore::queryReferrers_(const Path & path, PathSet & referrers)
void LocalStore::queryReferrers(State & state, const Path & path, PathSet & referrers)
{
auto useQueryReferrers(stmtQueryReferrers.use()(path));
auto useQueryReferrers(state.stmtQueryReferrers.use()(path));
while (useQueryReferrers.next())
referrers.insert(useQueryReferrers.getStr(0));
@ -755,7 +761,8 @@ void LocalStore::queryReferrers(const Path & path, PathSet & referrers)
{
assertStorePath(path);
return retrySQLite<void>([&]() {
queryReferrers_(path, referrers);
auto state(_state.lock());
queryReferrers(*state, path, referrers);
});
}
@ -771,7 +778,9 @@ PathSet LocalStore::queryValidDerivers(const Path & path)
assertStorePath(path);
return retrySQLite<PathSet>([&]() {
auto useQueryValidDerivers(stmtQueryValidDerivers.use()(path));
auto state(_state.lock());
auto useQueryValidDerivers(state->stmtQueryValidDerivers.use()(path));
PathSet derivers;
while (useQueryValidDerivers.next())
@ -785,7 +794,10 @@ PathSet LocalStore::queryValidDerivers(const Path & path)
PathSet LocalStore::queryDerivationOutputs(const Path & path)
{
return retrySQLite<PathSet>([&]() {
auto useQueryDerivationOutputs(stmtQueryDerivationOutputs.use()(queryValidPathId(path)));
auto state(_state.lock());
auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use()
(queryValidPathId(*state, path)));
PathSet outputs;
while (useQueryDerivationOutputs.next())
@ -799,7 +811,10 @@ PathSet LocalStore::queryDerivationOutputs(const Path & path)
StringSet LocalStore::queryDerivationOutputNames(const Path & path)
{
return retrySQLite<StringSet>([&]() {
auto useQueryDerivationOutputs(stmtQueryDerivationOutputs.use()(queryValidPathId(path)));
auto state(_state.lock());
auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use()
(queryValidPathId(*state, path)));
StringSet outputNames;
while (useQueryDerivationOutputs.next())
@ -817,11 +832,13 @@ Path LocalStore::queryPathFromHashPart(const string & hashPart)
Path prefix = settings.nixStore + "/" + hashPart;
return retrySQLite<Path>([&]() {
auto useQueryPathFromHashPart(stmtQueryPathFromHashPart.use()(prefix));
auto state(_state.lock());
auto useQueryPathFromHashPart(state->stmtQueryPathFromHashPart.use()(prefix));
if (!useQueryPathFromHashPart.next()) return "";
const char * s = (const char *) sqlite3_column_text(stmtQueryPathFromHashPart, 0);
const char * s = (const char *) sqlite3_column_text(state->stmtQueryPathFromHashPart, 0);
return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : "";
});
}
@ -829,13 +846,13 @@ Path LocalStore::queryPathFromHashPart(const string & hashPart)
void LocalStore::setSubstituterEnv()
{
if (didSetSubstituterEnv) return;
static std::atomic_flag done;
if (done.test_and_set()) return;
/* Pass configuration options (including those overridden with
--option) to substituters. */
setenv("_NIX_OPTIONS", settings.pack().c_str(), 1);
didSetSubstituterEnv = true;
}
@ -957,10 +974,12 @@ template<class T> T LocalStore::getIntLineFromSubstituter(RunningSubstituter & r
PathSet LocalStore::querySubstitutablePaths(const PathSet & paths)
{
auto state(_state.lock());
PathSet res;
for (auto & i : settings.substituters) {
if (res.size() == paths.size()) break;
RunningSubstituter & run(runningSubstituters[i]);
RunningSubstituter & run(state->runningSubstituters[i]);
startSubstituter(i, run);
if (run.disabled) continue;
string s = "have ";
@ -977,6 +996,7 @@ PathSet LocalStore::querySubstitutablePaths(const PathSet & paths)
res.insert(path);
}
}
return res;
}
@ -984,7 +1004,9 @@ PathSet LocalStore::querySubstitutablePaths(const PathSet & paths)
void LocalStore::querySubstitutablePathInfos(const Path & substituter,
PathSet & paths, SubstitutablePathInfos & infos)
{
RunningSubstituter & run(runningSubstituters[substituter]);
auto state(_state.lock());
RunningSubstituter & run(state->runningSubstituters[substituter]);
startSubstituter(substituter, run);
if (run.disabled) return;
@ -1048,22 +1070,24 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
if (settings.syncBeforeRegistering) sync();
return retrySQLite<void>([&]() {
SQLiteTxn txn(db);
auto state(_state.lock());
SQLiteTxn txn(state->db);
PathSet paths;
for (auto & i : infos) {
assert(i.narHash.type == htSHA256);
if (isValidPath_(i.path))
updatePathInfo(i);
if (isValidPath(*state, i.path))
updatePathInfo(*state, i);
else
addValidPath(i, false);
addValidPath(*state, i, false);
paths.insert(i.path);
}
for (auto & i : infos) {
auto referrer = queryValidPathId(i.path);
auto referrer = queryValidPathId(*state, i.path);
for (auto & j : i.references)
addReference(referrer, queryValidPathId(j));
state->stmtAddReference.use()(referrer)(queryValidPathId(*state, j)).exec();
}
/* Check that the derivation outputs are correct. We can't do
@ -1090,13 +1114,11 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
/* Invalidate a path. The caller is responsible for checking that
there are no referrers. */
void LocalStore::invalidatePath(const Path & path)
void LocalStore::invalidatePath(State & state, const Path & path)
{
debug(format("invalidating path %1%") % path);
drvHashes.erase(path);
stmtInvalidatePath.use()(path).exec();
state.stmtInvalidatePath.use()(path).exec();
/* Note that the foreign key constraints on the Refs table take
care of deleting the references entries for `path'. */
@ -1465,15 +1487,17 @@ void LocalStore::invalidatePathChecked(const Path & path)
assertStorePath(path);
retrySQLite<void>([&]() {
SQLiteTxn txn(db);
auto state(_state.lock());
if (isValidPath_(path)) {
PathSet referrers; queryReferrers_(path, referrers);
SQLiteTxn txn(state->db);
if (isValidPath(*state, path)) {
PathSet referrers; queryReferrers(*state, path, referrers);
referrers.erase(path); /* ignore self-references */
if (!referrers.empty())
throw PathInUse(format("cannot delete path %1% because it is in use by %2%")
% path % showPaths(referrers));
invalidatePath(path);
invalidatePath(*state, path);
}
txn.commit();
@ -1542,7 +1566,10 @@ bool LocalStore::verifyStore(bool checkContents, bool repair)
update = true;
}
if (update) updatePathInfo(info);
if (update) {
auto state(_state.lock());
updatePathInfo(*state, info);
}
}
@ -1572,7 +1599,8 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store,
if (!isStorePath(path)) {
printMsg(lvlError, format("path %1% is not in the Nix store") % path);
invalidatePath(path);
auto state(_state.lock());
invalidatePath(*state, path);
return;
}
@ -1590,7 +1618,8 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store,
if (canInvalidate) {
printMsg(lvlError, format("path %1% disappeared, removing from database...") % path);
invalidatePath(path);
auto state(_state.lock());
invalidatePath(*state, path);
} else {
printMsg(lvlError, format("path %1% disappeared, but it still has valid referrers!") % path);
if (repair)
@ -1610,32 +1639,6 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store,
}
bool LocalStore::pathContentsGood(const Path & path)
{
std::map<Path, bool>::iterator i = pathContentsGoodCache.find(path);
if (i != pathContentsGoodCache.end()) return i->second;
printMsg(lvlInfo, format("checking path %1%...") % path);
ValidPathInfo info = queryPathInfo(path);
bool res;
if (!pathExists(path))
res = false;
else {
HashResult current = hashPath(info.narHash.type, path);
Hash nullHash(htSHA256);
res = info.narHash == nullHash || info.narHash == current.first;
}
pathContentsGoodCache[path] = res;
if (!res) printMsg(lvlError, format("path %1% is corrupted or missing!") % path);
return res;
}
void LocalStore::markContentsGood(const Path & path)
{
pathContentsGoodCache[path] = true;
}
#if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL)
static void makeMutable(const Path & path)
@ -1690,21 +1693,25 @@ void LocalStore::upgradeStore7()
void LocalStore::vacuumDB()
{
if (sqlite3_exec(db, "vacuum;", 0, 0, 0) != SQLITE_OK)
throwSQLiteError(db, "vacuuming SQLite database");
auto state(_state.lock());
if (sqlite3_exec(state->db, "vacuum;", 0, 0, 0) != SQLITE_OK)
throwSQLiteError(state->db, "vacuuming SQLite database");
}
void LocalStore::addSignatures(const Path & storePath, const StringSet & sigs)
{
retrySQLite<void>([&]() {
SQLiteTxn txn(db);
auto state(_state.lock());
SQLiteTxn txn(state->db);
auto info = queryPathInfo(storePath);
info.sigs.insert(sigs.begin(), sigs.end());
updatePathInfo(info);
updatePathInfo(*state, info);
txn.commit();
});

View file

@ -1,13 +1,15 @@
#pragma once
#include "sqlite.hh"
#include <string>
#include <unordered_set>
#include "pathlocks.hh"
#include "store-api.hh"
#include "sync.hh"
#include "util.hh"
#include <string>
#include <unordered_set>
namespace nix {
@ -52,12 +54,47 @@ struct RunningSubstituter
class LocalStore : public LocalFSStore
{
private:
/* Lock file used for upgrading. */
AutoCloseFD globalLock;
struct State
{
/* The SQLite database object. */
SQLite db;
/* Some precompiled SQLite statements. */
SQLiteStmt stmtRegisterValidPath;
SQLiteStmt stmtUpdatePathInfo;
SQLiteStmt stmtAddReference;
SQLiteStmt stmtQueryPathInfo;
SQLiteStmt stmtQueryReferences;
SQLiteStmt stmtQueryReferrers;
SQLiteStmt stmtInvalidatePath;
SQLiteStmt stmtRegisterFailedPath;
SQLiteStmt stmtHasPathFailed;
SQLiteStmt stmtQueryFailedPaths;
SQLiteStmt stmtClearFailedPath;
SQLiteStmt stmtAddDerivationOutput;
SQLiteStmt stmtQueryValidDerivers;
SQLiteStmt stmtQueryDerivationOutputs;
SQLiteStmt stmtQueryPathFromHashPart;
SQLiteStmt stmtQueryValidPaths;
/* The file to which we write our temporary roots. */
Path fnTempRoots;
AutoCloseFD fdTempRoots;
typedef std::map<Path, RunningSubstituter> RunningSubstituters;
RunningSubstituters runningSubstituters;
Path linksDir;
};
Path reservedPath;
Sync<State, std::recursive_mutex> _state;
const Path linksDir;
const Path reservedPath;
const Path schemaPath;
public:
@ -174,76 +211,25 @@ public:
a substituter (if available). */
void repairPath(const Path & path);
/* Check whether the given valid path exists and has the right
contents. */
bool pathContentsGood(const Path & path);
void markContentsGood(const Path & path);
void setSubstituterEnv();
void addSignatures(const Path & storePath, const StringSet & sigs) override;
private:
Path schemaPath;
/* Lock file used for upgrading. */
AutoCloseFD globalLock;
/* The SQLite database object. */
SQLite db;
/* Some precompiled SQLite statements. */
SQLiteStmt stmtRegisterValidPath;
SQLiteStmt stmtUpdatePathInfo;
SQLiteStmt stmtAddReference;
SQLiteStmt stmtQueryPathInfo;
SQLiteStmt stmtQueryReferences;
SQLiteStmt stmtQueryReferrers;
SQLiteStmt stmtInvalidatePath;
SQLiteStmt stmtRegisterFailedPath;
SQLiteStmt stmtHasPathFailed;
SQLiteStmt stmtQueryFailedPaths;
SQLiteStmt stmtClearFailedPath;
SQLiteStmt stmtAddDerivationOutput;
SQLiteStmt stmtQueryValidDerivers;
SQLiteStmt stmtQueryDerivationOutputs;
SQLiteStmt stmtQueryPathFromHashPart;
SQLiteStmt stmtQueryValidPaths;
/* Cache for pathContentsGood(). */
std::map<Path, bool> pathContentsGoodCache;
bool didSetSubstituterEnv;
/* The file to which we write our temporary roots. */
Path fnTempRoots;
AutoCloseFD fdTempRoots;
int getSchema();
public:
static bool haveWriteAccess();
private:
void openDB(bool create);
int getSchema();
void openDB(State & state, bool create);
void makeStoreWritable();
uint64_t queryValidPathId(const Path & path);
uint64_t queryValidPathId(State & state, const Path & path);
uint64_t addValidPath(const ValidPathInfo & info, bool checkOutputs = true);
uint64_t addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs = true);
void addReference(uint64_t referrer, uint64_t reference);
void appendReferrer(const Path & from, const Path & to, bool lock);
void rewriteReferrers(const Path & path, bool purge, PathSet referrers);
void invalidatePath(const Path & path);
void invalidatePath(State & state, const Path & path);
/* Delete a path from the Nix store. */
void invalidatePathChecked(const Path & path);
@ -251,7 +237,7 @@ private:
void verifyPath(const Path & path, const PathSet & store,
PathSet & done, PathSet & validPaths, bool repair, bool & errors);
void updatePathInfo(const ValidPathInfo & info);
void updatePathInfo(State & state, const ValidPathInfo & info);
void upgradeStore6();
void upgradeStore7();
@ -299,8 +285,8 @@ private:
void optimisePath_(OptimiseStats & stats, const Path & path, InodeHash & inodeHash);
// Internal versions that are not wrapped in retry_sqlite.
bool isValidPath_(const Path & path);
void queryReferrers_(const Path & path, PathSet & referrers);
bool isValidPath(State & state, const Path & path);
void queryReferrers(State & state, const Path & path, PathSet & referrers);
/* Add signatures to a ValidPathInfo using the secret keys
specified by the secret-key-files option. */

View file

@ -22,11 +22,11 @@ namespace nix {
scope.
*/
template<class T>
template<class T, class M = std::mutex>
class Sync
{
private:
std::mutex mutex;
M mutex;
T data;
public:
@ -38,7 +38,7 @@ public:
{
private:
Sync * s;
std::unique_lock<std::mutex> lk;
std::unique_lock<M> lk;
friend Sync;
Lock(Sync * s) : s(s), lk(s->mutex) { }
public: