arcuru/lix
0
0
Fork 0
forked from lix-project/lix
Code Pull requests Activity

* Retry a transaction if SQLite returns SQLITE_BUSY. This can happen

Browse source 
even with a very long busy timeout, because SQLITE_BUSY is also
  returned to resolve deadlocks.  This should get rid of random
  "database is locked" errors.  This is kind of hard to test though.
* Fix a horrible bug in deleteFromStore(): deletePathWrapped() should
  be called after committing the transaction, not before, because the
  commit might not succeed.
This commit is contained in:
Eelco Dolstra 2010-12-05 18:23:19 +00:00
parent 365f3028dd
commit de79d23f76
1 changed files with 83 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

144
src/libstore/local-store.cc
View file

@ -22,22 +22,31 @@
namespace nix { namespace nix {
class SQLiteError : public Error MakeError(SQLiteError, Error);
MakeError(SQLiteBusy, SQLiteError);
static void throwSQLiteError(sqlite3 * db, const format & f)
__attribute__ ((noreturn));
static void throwSQLiteError(sqlite3 * db, const format & f)
{ {
public: int err = sqlite3_errcode(db);
SQLiteError(sqlite3 * db, const format & f) if (err == SQLITE_BUSY) {
: Error(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)) printMsg(lvlError, "warning: SQLite database is busy");
{ throw SQLiteBusy(format("%1%: %2%") % f.str() % sqlite3_errmsg(db));
} }
}; else
throw SQLiteError(format("%1%: %2%") % f.str() % sqlite3_errmsg(db));
}
SQLite::~SQLite() SQLite::~SQLite()
{ {
try { try {
if (db && sqlite3_close(db) != SQLITE_OK) if (db && sqlite3_close(db) != SQLITE_OK)
throw SQLiteError(db, "closing database"); throwSQLiteError(db, "closing database");
} catch (...) { } catch (...) {
ignoreException(); ignoreException();
} }
@ -49,7 +58,7 @@ void SQLiteStmt::create(sqlite3 * db, const string & s)
checkInterrupt(); checkInterrupt();
assert(!stmt); assert(!stmt);
if (sqlite3_prepare_v2(db, s.c_str(), -1, &stmt, 0) != SQLITE_OK) if (sqlite3_prepare_v2(db, s.c_str(), -1, &stmt, 0) != SQLITE_OK)
throw SQLiteError(db, "creating statement"); throwSQLiteError(db, "creating statement");
this->db = db; this->db = db;
} }
@ -58,7 +67,7 @@ void SQLiteStmt::reset()
{ {
assert(stmt); assert(stmt);
if (sqlite3_reset(stmt) != SQLITE_OK) if (sqlite3_reset(stmt) != SQLITE_OK)
throw SQLiteError(db, "resetting statement"); throwSQLiteError(db, "resetting statement");
curArg = 1; curArg = 1;
} }
@ -67,7 +76,7 @@ SQLiteStmt::~SQLiteStmt()
{ {
try { try {
if (stmt && sqlite3_finalize(stmt) != SQLITE_OK) if (stmt && sqlite3_finalize(stmt) != SQLITE_OK)
throw SQLiteError(db, "finalizing statement"); throwSQLiteError(db, "finalizing statement");
} catch (...) { } catch (...) {
ignoreException(); ignoreException();
} }
@ -77,28 +86,28 @@ SQLiteStmt::~SQLiteStmt()
void SQLiteStmt::bind(const string & value) void SQLiteStmt::bind(const string & value)
{ {
if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK)
throw SQLiteError(db, "binding argument"); throwSQLiteError(db, "binding argument");
} }
void SQLiteStmt::bind(int value) void SQLiteStmt::bind(int value)
{ {
if (sqlite3_bind_int(stmt, curArg++, value) != SQLITE_OK) if (sqlite3_bind_int(stmt, curArg++, value) != SQLITE_OK)
throw SQLiteError(db, "binding argument"); throwSQLiteError(db, "binding argument");
} }
void SQLiteStmt::bind64(long long value) void SQLiteStmt::bind64(long long value)
{ {
if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK) if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK)
throw SQLiteError(db, "binding argument"); throwSQLiteError(db, "binding argument");
} }
void SQLiteStmt::bind() void SQLiteStmt::bind()
{ {
if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK) if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK)
throw SQLiteError(db, "binding argument"); throwSQLiteError(db, "binding argument");
} }
@ -132,14 +141,14 @@ struct SQLiteTxn
SQLiteTxn(sqlite3 * db) : active(false) { SQLiteTxn(sqlite3 * db) : active(false) {
this->db = db; this->db = db;
if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK)
throw SQLiteError(db, "starting transaction"); throwSQLiteError(db, "starting transaction");
active = true; active = true;
} }
void commit() void commit()
{ {
if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK)
throw SQLiteError(db, "committing transaction"); throwSQLiteError(db, "committing transaction");
active = false; active = false;
} }
@ -147,7 +156,7 @@ struct SQLiteTxn
{ {
try { try {
if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK) if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK)
throw SQLiteError(db, "aborting transaction"); throwSQLiteError(db, "aborting transaction");
} catch (...) { } catch (...) {
ignoreException(); ignoreException();
} }
@ -289,10 +298,10 @@ void LocalStore::openDB(bool create)
throw Error("cannot open SQLite database"); throw Error("cannot open SQLite database");
if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK)
throw SQLiteError(db, "setting timeout"); throwSQLiteError(db, "setting timeout");
if (sqlite3_exec(db, "pragma foreign_keys = 1;", 0, 0, 0) != SQLITE_OK) if (sqlite3_exec(db, "pragma foreign_keys = 1;", 0, 0, 0) != SQLITE_OK)
throw SQLiteError(db, "enabling foreign keys"); throwSQLiteError(db, "enabling foreign keys");
/* !!! check whether sqlite has been built with foreign key /* !!! check whether sqlite has been built with foreign key
support */ support */
@ -303,7 +312,7 @@ void LocalStore::openDB(bool create)
crashes. */ crashes. */
string syncMode = queryBoolSetting("fsync-metadata", true) ? "normal" : "off"; string syncMode = queryBoolSetting("fsync-metadata", true) ? "normal" : "off";
if (sqlite3_exec(db, ("pragma synchronous = " + syncMode + ";").c_str(), 0, 0, 0) != SQLITE_OK) if (sqlite3_exec(db, ("pragma synchronous = " + syncMode + ";").c_str(), 0, 0, 0) != SQLITE_OK)
throw SQLiteError(db, "setting synchronous mode"); throwSQLiteError(db, "setting synchronous mode");
/* Set the SQLite journal mode. WAL mode is fastest, but doesn't /* Set the SQLite journal mode. WAL mode is fastest, but doesn't
seem entirely stable at the moment (Oct. 2010). Thus, use seem entirely stable at the moment (Oct. 2010). Thus, use
@ -314,31 +323,31 @@ void LocalStore::openDB(bool create)
SQLiteStmt stmt; SQLiteStmt stmt;
stmt.create(db, "pragma main.journal_mode;"); stmt.create(db, "pragma main.journal_mode;");
if (sqlite3_step(stmt) != SQLITE_ROW) if (sqlite3_step(stmt) != SQLITE_ROW)
throw SQLiteError(db, "querying journal mode"); throwSQLiteError(db, "querying journal mode");
prevMode = string((const char *) sqlite3_column_text(stmt, 0)); prevMode = string((const char *) sqlite3_column_text(stmt, 0));
} }
if (prevMode != mode && if (prevMode != mode &&
sqlite3_exec(db, ("pragma main.journal_mode = " + mode + ";").c_str(), 0, 0, 0) != SQLITE_OK) sqlite3_exec(db, ("pragma main.journal_mode = " + mode + ";").c_str(), 0, 0, 0) != SQLITE_OK)
throw SQLiteError(db, "setting journal mode"); throwSQLiteError(db, "setting journal mode");
/* Increase the auto-checkpoint interval to 8192 pages. This /* Increase the auto-checkpoint interval to 8192 pages. This
seems enough to ensure that instantiating the NixOS system seems enough to ensure that instantiating the NixOS system
derivation is done in a single fsync(). */ derivation is done in a single fsync(). */
if (sqlite3_exec(db, "pragma wal_autocheckpoint = 8192;", 0, 0, 0) != SQLITE_OK) if (sqlite3_exec(db, "pragma wal_autocheckpoint = 8192;", 0, 0, 0) != SQLITE_OK)
throw SQLiteError(db, "setting autocheckpoint interval"); throwSQLiteError(db, "setting autocheckpoint interval");
/* Initialise the database schema, if necessary. */ /* Initialise the database schema, if necessary. */
if (create) { if (create) {
#include "schema.sql.hh" #include "schema.sql.hh"
if (sqlite3_exec(db, (const char *) schema, 0, 0, 0) != SQLITE_OK) if (sqlite3_exec(db, (const char *) schema, 0, 0, 0) != SQLITE_OK)
throw SQLiteError(db, "initialising database schema"); throwSQLiteError(db, "initialising database schema");
} }
/* Backwards compatibility with old (pre-release) databases. Can /* Backwards compatibility with old (pre-release) databases. Can
remove this eventually. */ remove this eventually. */
if (sqlite3_table_column_metadata(db, 0, "ValidPaths", "narSize", 0, 0, 0, 0, 0) != SQLITE_OK) { if (sqlite3_table_column_metadata(db, 0, "ValidPaths", "narSize", 0, 0, 0, 0, 0) != SQLITE_OK) {
if (sqlite3_exec(db, "alter table ValidPaths add column narSize integer" , 0, 0, 0) != SQLITE_OK) if (sqlite3_exec(db, "alter table ValidPaths add column narSize integer" , 0, 0, 0) != SQLITE_OK)
throw SQLiteError(db, "adding column narSize"); throwSQLiteError(db, "adding column narSize");
} }
/* Prepare SQL statements. */ /* Prepare SQL statements. */
@ -460,7 +469,7 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info)
else else
stmtRegisterValidPath.bind(); // null stmtRegisterValidPath.bind(); // null
if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE) if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE)
throw SQLiteError(db, format("registering valid path `%1%' in database") % info.path); throwSQLiteError(db, format("registering valid path `%1%' in database") % info.path);
unsigned long long id = sqlite3_last_insert_rowid(db); unsigned long long id = sqlite3_last_insert_rowid(db);
/* If this is a derivation, then store the derivation outputs in /* If this is a derivation, then store the derivation outputs in
@ -475,7 +484,7 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info)
stmtAddDerivationOutput.bind(i->first); stmtAddDerivationOutput.bind(i->first);
stmtAddDerivationOutput.bind(i->second.path); stmtAddDerivationOutput.bind(i->second.path);
if (sqlite3_step(stmtAddDerivationOutput) != SQLITE_DONE) if (sqlite3_step(stmtAddDerivationOutput) != SQLITE_DONE)
throw SQLiteError(db, format("adding derivation output for `%1%' in database") % info.path); throwSQLiteError(db, format("adding derivation output for `%1%' in database") % info.path);
} }
} }
@ -489,7 +498,7 @@ void LocalStore::addReference(unsigned long long referrer, unsigned long long re
stmtAddReference.bind(referrer); stmtAddReference.bind(referrer);
stmtAddReference.bind(reference); stmtAddReference.bind(reference);
if (sqlite3_step(stmtAddReference) != SQLITE_DONE) if (sqlite3_step(stmtAddReference) != SQLITE_DONE)
throw SQLiteError(db, "adding reference to database"); throwSQLiteError(db, "adding reference to database");
} }
@ -498,15 +507,23 @@ void LocalStore::registerValidPath(const ValidPathInfo & info)
assert(info.hash.type == htSHA256); assert(info.hash.type == htSHA256);
ValidPathInfo info2(info); ValidPathInfo info2(info);
if (info2.registrationTime == 0) info2.registrationTime = time(0); if (info2.registrationTime == 0) info2.registrationTime = time(0);
SQLiteTxn txn(db);
unsigned long long id = addValidPath(info2);
foreach (PathSet::const_iterator, i, info2.references) while (1) {
addReference(id, queryValidPathId(*i)); try {
SQLiteTxn txn(db);
txn.commit();
unsigned long long id = addValidPath(info2);
foreach (PathSet::const_iterator, i, info2.references)
addReference(id, queryValidPathId(*i));
txn.commit();
break;
} catch (SQLiteBusy & e) {
/* Retry; the `txn' destructor will roll back the current
transaction. */
}
}
} }
@ -517,7 +534,7 @@ void LocalStore::registerFailedPath(const Path & path)
stmtRegisterFailedPath.bind(path); stmtRegisterFailedPath.bind(path);
stmtRegisterFailedPath.bind(time(0)); stmtRegisterFailedPath.bind(time(0));
if (sqlite3_step(stmtRegisterFailedPath) != SQLITE_DONE) if (sqlite3_step(stmtRegisterFailedPath) != SQLITE_DONE)
throw SQLiteError(db, format("registering failed path `%1%'") % path); throwSQLiteError(db, format("registering failed path `%1%'") % path);
} }
@ -527,7 +544,7 @@ bool LocalStore::hasPathFailed(const Path & path)
stmtHasPathFailed.bind(path); stmtHasPathFailed.bind(path);
int res = sqlite3_step(stmtHasPathFailed); int res = sqlite3_step(stmtHasPathFailed);
if (res != SQLITE_DONE && res != SQLITE_ROW) if (res != SQLITE_DONE && res != SQLITE_ROW)
throw SQLiteError(db, "querying whether path failed"); throwSQLiteError(db, "querying whether path failed");
return res == SQLITE_ROW; return res == SQLITE_ROW;
} }
@ -545,7 +562,7 @@ PathSet LocalStore::queryFailedPaths()
} }
if (r != SQLITE_DONE) if (r != SQLITE_DONE)
throw SQLiteError(db, "error querying failed paths"); throwSQLiteError(db, "error querying failed paths");
return res; return res;
} }
@ -559,7 +576,7 @@ void LocalStore::clearFailedPaths(const PathSet & paths)
SQLiteStmtUse use(stmtClearFailedPath); SQLiteStmtUse use(stmtClearFailedPath);
stmtClearFailedPath.bind(*i); stmtClearFailedPath.bind(*i);
if (sqlite3_step(stmtClearFailedPath) != SQLITE_DONE) if (sqlite3_step(stmtClearFailedPath) != SQLITE_DONE)
throw SQLiteError(db, format("clearing failed path `%1%' in database") % *i); throwSQLiteError(db, format("clearing failed path `%1%' in database") % *i);
} }
txn.commit(); txn.commit();
@ -594,7 +611,7 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path)
int r = sqlite3_step(stmtQueryPathInfo); int r = sqlite3_step(stmtQueryPathInfo);
if (r == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path); if (r == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path);
if (r != SQLITE_ROW) throw SQLiteError(db, "querying path in database"); if (r != SQLITE_ROW) throwSQLiteError(db, "querying path in database");
info.id = sqlite3_column_int(stmtQueryPathInfo, 0); info.id = sqlite3_column_int(stmtQueryPathInfo, 0);
@ -622,7 +639,7 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path)
} }
if (r != SQLITE_DONE) if (r != SQLITE_DONE)
throw SQLiteError(db, format("error getting references of `%1%'") % path); throwSQLiteError(db, format("error getting references of `%1%'") % path);
return info; return info;
} }
@ -635,7 +652,7 @@ unsigned long long LocalStore::queryValidPathId(const Path & path)
int res = sqlite3_step(stmtQueryPathInfo); int res = sqlite3_step(stmtQueryPathInfo);
if (res == SQLITE_ROW) return sqlite3_column_int(stmtQueryPathInfo, 0); if (res == SQLITE_ROW) return sqlite3_column_int(stmtQueryPathInfo, 0);
if (res == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path); if (res == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path);
throw SQLiteError(db, "querying path in database"); throwSQLiteError(db, "querying path in database");
} }
@ -645,7 +662,7 @@ bool LocalStore::isValidPath(const Path & path)
stmtQueryPathInfo.bind(path); stmtQueryPathInfo.bind(path);
int res = sqlite3_step(stmtQueryPathInfo); int res = sqlite3_step(stmtQueryPathInfo);
if (res != SQLITE_DONE && res != SQLITE_ROW) if (res != SQLITE_DONE && res != SQLITE_ROW)
throw SQLiteError(db, "querying path in database"); throwSQLiteError(db, "querying path in database");
return res == SQLITE_ROW; return res == SQLITE_ROW;
} }
@ -665,7 +682,7 @@ PathSet LocalStore::queryValidPaths()
} }
if (r != SQLITE_DONE) if (r != SQLITE_DONE)
throw SQLiteError(db, "error getting valid paths"); throwSQLiteError(db, "error getting valid paths");
return res; return res;
} }
@ -695,7 +712,7 @@ void LocalStore::queryReferrers(const Path & path, PathSet & referrers)
} }
if (r != SQLITE_DONE) if (r != SQLITE_DONE)
throw SQLiteError(db, format("error getting references of `%1%'") % path); throwSQLiteError(db, format("error getting references of `%1%'") % path);
} }
@ -721,7 +738,7 @@ PathSet LocalStore::queryValidDerivers(const Path & path)
} }
if (r != SQLITE_DONE) if (r != SQLITE_DONE)
throw SQLiteError(db, format("error getting valid derivers of `%1%'") % path); throwSQLiteError(db, format("error getting valid derivers of `%1%'") % path);
return derivers; return derivers;
} }
@ -743,7 +760,7 @@ PathSet LocalStore::queryDerivationOutputs(const Path & path)
} }
if (r != SQLITE_DONE) if (r != SQLITE_DONE)
throw SQLiteError(db, format("error getting outputs of `%1%'") % path); throwSQLiteError(db, format("error getting outputs of `%1%'") % path);
return outputs; return outputs;
} }
@ -890,7 +907,7 @@ void LocalStore::invalidatePath(const Path & path)
stmtInvalidatePath.bind(path); stmtInvalidatePath.bind(path);
if (sqlite3_step(stmtInvalidatePath) != SQLITE_DONE) if (sqlite3_step(stmtInvalidatePath) != SQLITE_DONE)
throw SQLiteError(db, format("invalidating path `%1%' in database") % path); throwSQLiteError(db, format("invalidating path `%1%' in database") % path);
/* Note that the foreign key constraints on the Refs table take /* Note that the foreign key constraints on the Refs table take
care of deleting the references entries for `path'. */ care of deleting the references entries for `path'. */
@ -1248,20 +1265,25 @@ void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFr
assertStorePath(path); assertStorePath(path);
SQLiteTxn txn(db); while (1) {
try {
SQLiteTxn txn(db);
if (isValidPath(path)) { if (isValidPath(path)) {
PathSet referrers; queryReferrers(path, referrers); PathSet referrers; queryReferrers(path, referrers);
referrers.erase(path); /* ignore self-references */ referrers.erase(path); /* ignore self-references */
if (!referrers.empty()) if (!referrers.empty())
throw PathInUse(format("cannot delete path `%1%' because it is in use by `%2%'") throw PathInUse(format("cannot delete path `%1%' because it is in use by `%2%'")
% path % showPaths(referrers)); % path % showPaths(referrers));
invalidatePath(path); invalidatePath(path);
}
txn.commit();
break;
} catch (SQLiteBusy & e) { };
} }
deletePathWrapped(path, bytesFreed, blocksFreed); deletePathWrapped(path, bytesFreed, blocksFreed);
txn.commit();
} }