lix-project/lix
20
57
Fork 31
Code Issues 347 Code Review (Gerrit) Projects 3 Releases Wiki Activity

* Started using Berkeley DB environments. This is necessary for

Browse source 
transaction support (but we don't actually use transactions yet).
This commit is contained in:
Eelco Dolstra 2003-07-31 13:47:13 +00:00
parent 758bd4673a
commit 4a013962bd
12 changed files with 256 additions and 107 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/Makefile.am
View file

@ -42,6 +42,7 @@ nix.o: nix-help.txt.hh
install-data-local:
$(INSTALL) -d $(localstatedir)/nix
$(INSTALL) -d $(localstatedir)/nix/db
$(INSTALL) -d $(localstatedir)/nix/links
ln -sf $(localstatedir)/nix/links/current $(prefix)/current
$(INSTALL) -d $(localstatedir)/log/nix

170
src/db.cc
View file

@ -3,66 +3,142 @@
#include <memory>
#include <db_cxx.h>
/* Wrapper classes that ensures that the database is closed upon
object destruction. */
class Db2 : public Db
/* Wrapper class to ensure proper destruction. */
class DestroyDb
{
Db * db;
public:
Db2(DbEnv *env, u_int32_t flags) : Db(env, flags) { }
~Db2() { close(0); }
DestroyDb(Db * _db) : db(_db) { }
~DestroyDb() { db->close(0); delete db; }
};
class DbcClose
class DestroyDbc
{
Dbc * cursor;
Dbc * dbc;
public:
DbcClose(Dbc * c) : cursor(c) { }
~DbcClose() { cursor->close(); }
DestroyDbc(Dbc * _dbc) : dbc(_dbc) { }
~DestroyDbc() { dbc->close(); /* close() frees dbc */ }
};
static auto_ptr<Db2> openDB(const string & filename, const string & dbname,
bool readonly)
{
auto_ptr<Db2> db(new Db2(0, 0));
db->open(filename.c_str(), dbname.c_str(),
DB_HASH, readonly ? DB_RDONLY : DB_CREATE, 0666);
return db;
}
static void rethrow(DbException & e)
{
throw Error(e.what());
}
void createDB(const string & filename, const string & dbname)
Transaction::Transaction()
: txn(0)
{
}
Transaction::Transaction(Database & db)
{
db.requireEnv();
db.env->txn_begin(0, &txn, 0);
}
Transaction::~Transaction()
{
if (txn) {
txn->abort();
txn = 0;
}
}
void Transaction::commit()
{
if (!txn) throw Error("commit called on null transaction");
txn->commit(0);
txn = 0;
}
void Database::requireEnv()
{
if (!env) throw Error("database environment not open");
}
Db * Database::openDB(const Transaction & txn,
const string & table, bool create)
{
requireEnv();
Db * db = new Db(env, 0);
try {
// !!! fixme when switching to BDB 4.1: use txn.
db->open(table.c_str(), 0,
DB_HASH, create ? DB_CREATE : 0, 0666);
} catch (...) {
delete db;
throw;
}
return db;
}
Database::Database()
: env(0)
{
}
Database::~Database()
{
if (env) {
env->close(0);
delete env;
}
}
void Database::open(const string & path)
{
try {
openDB(filename, dbname, false);
if (env) throw Error(format("environment already open"));
env = new DbEnv(0);
debug("foo" + path);
env->open(path.c_str(),
DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN |
DB_CREATE,
0666);
} catch (DbException e) { rethrow(e); }
}
bool queryDB(const string & filename, const string & dbname,
void Database::createTable(const string & table)
{
try {
Db * db = openDB(noTxn, table, true);
DestroyDb destroyDb(db);
} catch (DbException e) { rethrow(e); }
}
bool Database::queryString(const Transaction & txn, const string & table,
const string & key, string & data)
{
try {
int err;
auto_ptr<Db2> db = openDB(filename, dbname, true);
Db * db = openDB(txn, table, false);
DestroyDb destroyDb(db);
Dbt kt((void *) key.c_str(), key.length());
Dbt dt;
err = db->get(0, &kt, &dt, 0);
int err = db->get(txn.txn, &kt, &dt, 0);
if (err) return false;
if (!dt.get_data())
@ -76,12 +152,12 @@ bool queryDB(const string & filename, const string & dbname,
}
bool queryListDB(const string & filename, const string & dbname,
bool Database::queryStrings(const Transaction & txn, const string & table,
const string & key, Strings & data)
{
string d;
if (!queryDB(filename, dbname, key, d))
if (!queryString(txn, table, key, d))
return false;
string::iterator it = d.begin();
@ -110,19 +186,21 @@ bool queryListDB(const string & filename, const string & dbname,
}
void setDB(const string & filename, const string & dbname,
void Database::setString(const Transaction & txn, const string & table,
const string & key, const string & data)
{
try {
auto_ptr<Db2> db = openDB(filename, dbname, false);
Db * db = openDB(txn, table, false);
DestroyDb destroyDb(db);
Dbt kt((void *) key.c_str(), key.length());
Dbt dt((void *) data.c_str(), data.length());
db->put(0, &kt, &dt, 0);
db->put(txn.txn, &kt, &dt, 0);
} catch (DbException e) { rethrow(e); }
}
void setListDB(const string & filename, const string & dbname,
void Database::setStrings(const Transaction & txn, const string & table,
const string & key, const Strings & data)
{
string d;
@ -141,34 +219,36 @@ void setListDB(const string & filename, const string & dbname,
d += s;
}
setDB(filename, dbname, key, d);
setString(txn, table, key, d);
}
void delDB(const string & filename, const string & dbname,
void Database::delPair(const Transaction & txn, const string & table,
const string & key)
{
try {
auto_ptr<Db2> db = openDB(filename, dbname, false);
Db * db = openDB(txn, table, false);
DestroyDb destroyDb(db);
Dbt kt((void *) key.c_str(), key.length());
db->del(0, &kt, 0);
db->del(txn.txn, &kt, 0);
} catch (DbException e) { rethrow(e); }
}
void enumDB(const string & filename, const string & dbname,
void Database::enumTable(const Transaction & txn, const string & table,
Strings & keys)
{
try {
auto_ptr<Db2> db = openDB(filename, dbname, true);
Db * db = openDB(txn, table, false);
DestroyDb destroyDb(db);
Dbc * cursor;
db->cursor(0, &cursor, 0);
DbcClose cursorCloser(cursor);
Dbc * dbc;
db->cursor(0, &dbc, 0);
DestroyDbc destroyDbc(dbc);
Dbt kt, dt;
while (cursor->get(&kt, &dt, DB_NEXT) != DB_NOTFOUND)
while (dbc->get(&kt, &dt, DB_NEXT) != DB_NOTFOUND)
keys.push_back(
string((char *) kt.get_data(), kt.get_size()));

71
src/db.hh
View file

@ -4,28 +4,73 @@
#include <string>
#include <list>
#include <db_cxx.h>
#include "util.hh"
using namespace std;
void createDB(const string & filename, const string & dbname);
bool queryDB(const string & filename, const string & dbname,
const string & key, string & data);
class Database;
bool queryListDB(const string & filename, const string & dbname,
const string & key, Strings & data);
void setDB(const string & filename, const string & dbname,
const string & key, const string & data);
class Transaction
{
friend class Database;
void setListDB(const string & filename, const string & dbname,
const string & key, const Strings & data);
private:
DbTxn * txn;
public:
Transaction();
Transaction(Database & _db);
~Transaction();
void delDB(const string & filename, const string & dbname,
const string & key);
void commit();
};
#define noTxn Transaction()
class Database
{
friend class Transaction;
private:
DbEnv * env;
void requireEnv();
Db * openDB(const Transaction & txn,
const string & table, bool create);
public:
Database();
~Database();
void open(const string & path);
void createTable(const string & table);
bool queryString(const Transaction & txn, const string & table,
const string & key, string & data);
bool queryStrings(const Transaction & txn, const string & table,
const string & key, Strings & data);
void setString(const Transaction & txn, const string & table,
const string & key, const string & data);
void setStrings(const Transaction & txn, const string & table,
const string & key, const Strings & data);
void delPair(const Transaction & txn, const string & table,
const string & key);
void enumTable(const Transaction & txn, const string & table,
Strings & keys);
};
void enumDB(const string & filename, const string & dbname,
Strings & keys);
#endif /* !__DB_H */

2
src/fix.cc
View file

@ -301,6 +301,8 @@ static Expr evalFile(EvalState & state, string relPath)
void run(Strings args)
{
openDB();
EvalState state;
Strings files;

19
src/globals.cc
View file

@ -2,6 +2,9 @@
#include "db.hh"
Database nixDB;
string dbPath2Id = "path2id";
string dbId2Paths = "id2paths";
string dbSuccessors = "successors";
@ -11,13 +14,19 @@ string dbSubstitutes = "substitutes";
string nixStore = "/UNINIT";
string nixDataDir = "/UNINIT";
string nixLogDir = "/UNINIT";
string nixDB = "/UNINIT";
string nixDBPath = "/UNINIT";
void openDB()
{
nixDB.open(nixDBPath);
}
void initDB()
{
createDB(nixDB, dbPath2Id);
createDB(nixDB, dbId2Paths);
createDB(nixDB, dbSuccessors);
createDB(nixDB, dbSubstitutes);
nixDB.createTable(dbPath2Id);
nixDB.createTable(dbId2Paths);
nixDB.createTable(dbSuccessors);
nixDB.createTable(dbSubstitutes);
}

17
src/globals.hh
View file

@ -3,10 +3,15 @@
#include <string>
#include "db.hh"
using namespace std;
/* Database names. */
extern Database nixDB;
/* Database tables. */
/* dbPath2Id :: Path -> FSId
@ -60,12 +65,14 @@ extern string nixDataDir; /* !!! fix */
/* nixLogDir is the directory where we log various operations. */
extern string nixLogDir;
/* nixDB is the file name of the Berkeley DB database where we
maintain the dbXXX mappings. */
extern string nixDB;
/* nixDBPath is the path name of our Berkeley DB environment. */
extern string nixDBPath;
/* Initialize the databases. */
/* Open the database environment. */
void openDB();
/* Create the required database tables. */
void initDB();

2
src/nix.cc
View file

@ -335,6 +335,8 @@ static void opVerify(Strings opFlags, Strings opArgs)
list. */
void run(Strings args)
{
openDB();
Strings opFlags, opArgs;
Operation op = 0;

12
src/normalise.cc
View file

@ -9,7 +9,7 @@
void registerSuccessor(const FSId & id1, const FSId & id2)
{
setDB(nixDB, dbSuccessors, id1, id2);
nixDB.setString(noTxn, dbSuccessors, id1, id2);
}
@ -31,7 +31,7 @@ FSId normaliseFState(FSId id, FSIdSet pending)
/* Try to substitute $id$ by any known successors in order to
speed up the rewrite process. */
string idSucc;
while (queryDB(nixDB, dbSuccessors, id, idSucc)) {
while (nixDB.queryString(noTxn, dbSuccessors, id, idSucc)) {
debug(format("successor %1% -> %2%") % (string) id % idSucc);
id = parseHash(idSucc);
}
@ -191,7 +191,7 @@ void realiseSlice(const FSId & id, FSIdSet pending)
{
SliceElem elem = *i;
string id;
if (!queryDB(nixDB, dbPath2Id, elem.path, id)) {
if (!nixDB.queryString(noTxn, dbPath2Id, elem.path, id)) {
if (pathExists(elem.path))
throw Error(format("path `%1%' obstructed") % elem.path);
missing = true;
@ -269,7 +269,7 @@ static void fstateRequisitesSet(const FSId & id,
string idSucc;
if (includeSuccessors &&
queryDB(nixDB, dbSuccessors, id, idSucc))
nixDB.queryString(noTxn, dbSuccessors, id, idSucc))
fstateRequisitesSet(parseHash(idSucc),
includeExprs, includeSuccessors, paths);
}
@ -293,13 +293,13 @@ FSIds findGenerators(const FSIds & _ids)
mappings, since we know that those are Nix expressions. */
Strings sucs;
enumDB(nixDB, dbSuccessors, sucs);
nixDB.enumTable(noTxn, dbSuccessors, sucs);
for (Strings::iterator i = sucs.begin();
i != sucs.end(); i++)
{
string s;
if (!queryDB(nixDB, dbSuccessors, *i, s)) continue;
if (!nixDB.queryString(noTxn, dbSuccessors, *i, s)) continue;
FSId id = parseHash(s);
FState fs;

2
src/shared.cc
View file

@ -19,7 +19,7 @@ static void initAndRun(int argc, char * * argv)
nixStore = NIX_STORE_DIR;
nixDataDir = NIX_DATA_DIR;
nixLogDir = NIX_LOG_DIR;
nixDB = (string) NIX_STATE_DIR + "/nixstate.db";
nixDBPath = (string) NIX_STATE_DIR + "/db";
/* Put the arguments in a vector. */
Strings args;

57
src/store.cc
View file

@ -96,7 +96,7 @@ void registerSubstitute(const FSId & srcId, const FSId & subId)
/* For now, accept only one substitute per id. */
Strings subs;
subs.push_back(subId);
setListDB(nixDB, dbSubstitutes, srcId, subs);
nixDB.setStrings(noTxn, dbSubstitutes, srcId, subs);
}
@ -104,10 +104,10 @@ void registerPath(const string & _path, const FSId & id)
{
string path(canonPath(_path));
setDB(nixDB, dbPath2Id, path, id);
nixDB.setString(noTxn, dbPath2Id, path, id);
Strings paths;
queryListDB(nixDB, dbId2Paths, id, paths); /* non-existence = ok */
nixDB.queryStrings(noTxn, dbId2Paths, id, paths); /* non-existence = ok */
for (Strings::iterator it = paths.begin();
it != paths.end(); it++)
@ -115,7 +115,7 @@ void registerPath(const string & _path, const FSId & id)
paths.push_back(path);
setListDB(nixDB, dbId2Paths, id, paths);
nixDB.setStrings(noTxn, dbId2Paths, id, paths);
}
@ -124,16 +124,15 @@ void unregisterPath(const string & _path)
string path(canonPath(_path));
string _id;
if (!queryDB(nixDB, dbPath2Id, path, _id))
return;
if (!nixDB.queryString(noTxn, dbPath2Id, path, _id)) return;
FSId id(parseHash(_id));
delDB(nixDB, dbPath2Id, path);
nixDB.delPair(noTxn, dbPath2Id, path);
/* begin transaction */
Strings paths, paths2;
queryListDB(nixDB, dbId2Paths, id, paths); /* non-existence = ok */
nixDB.queryStrings(noTxn, dbId2Paths, id, paths); /* non-existence = ok */
bool changed = false;
for (Strings::iterator it = paths.begin();
@ -141,7 +140,7 @@ void unregisterPath(const string & _path)
if (*it != path) paths2.push_back(*it); else changed = true;
if (changed)
setListDB(nixDB, dbId2Paths, id, paths2);
nixDB.setStrings(noTxn, dbId2Paths, id, paths2);
/* end transaction */
@ -151,7 +150,7 @@ void unregisterPath(const string & _path)
bool queryPathId(const string & path, FSId & id)
{
string s;
if (!queryDB(nixDB, dbPath2Id, absPath(path), s)) return false;
if (!nixDB.queryString(noTxn, dbPath2Id, absPath(path), s)) return false;
id = parseHash(s);
return true;
}
@ -174,7 +173,7 @@ string expandId(const FSId & id, const string & target,
if (!target.empty() && !isInPrefix(target, prefix))
abort();
queryListDB(nixDB, dbId2Paths, id, paths);
nixDB.queryStrings(noTxn, dbId2Paths, id, paths);
/* Pick one equal to `target'. */
if (!target.empty()) {
@ -212,7 +211,7 @@ string expandId(const FSId & id, const string & target,
/* Try to realise the substitutes, but only if this id is not
already being realised by a substitute. */
Strings subs;
queryListDB(nixDB, dbSubstitutes, id, subs); /* non-existence = ok */
nixDB.queryStrings(noTxn, dbSubstitutes, id, subs); /* non-existence = ok */
for (Strings::iterator it = subs.begin(); it != subs.end(); it++) {
FSId subId = parseHash(*it);
@ -264,7 +263,7 @@ void deleteFromStore(const string & path)
void verifyStore()
{
Strings paths;
enumDB(nixDB, dbPath2Id, paths);
nixDB.enumTable(noTxn, dbPath2Id, paths);
for (Strings::iterator i = paths.begin();
i != paths.end(); i++)
@ -278,10 +277,10 @@ void verifyStore()
else {
string id;
if (!queryDB(nixDB, dbPath2Id, path, id)) abort();
if (!nixDB.queryString(noTxn, dbPath2Id, path, id)) abort();
Strings idPaths;
queryListDB(nixDB, dbId2Paths, id, idPaths);
nixDB.queryStrings(noTxn, dbId2Paths, id, idPaths);
bool found = false;
for (Strings::iterator j = idPaths.begin();
@ -298,11 +297,11 @@ void verifyStore()
debug(format("reverse mapping for path `%1%' missing") % path);
}
if (erase) delDB(nixDB, dbPath2Id, path);
if (erase) nixDB.delPair(noTxn, dbPath2Id, path);
}
Strings ids;
enumDB(nixDB, dbId2Paths, ids);
nixDB.enumTable(noTxn, dbId2Paths, ids);
for (Strings::iterator i = ids.begin();
i != ids.end(); i++)
@ -310,13 +309,13 @@ void verifyStore()
FSId id = parseHash(*i);
Strings idPaths;
queryListDB(nixDB, dbId2Paths, id, idPaths);
nixDB.queryStrings(noTxn, dbId2Paths, id, idPaths);
for (Strings::iterator j = idPaths.begin();
j != idPaths.end(); )
{
string id2;
if (!queryDB(nixDB, dbPath2Id, *j, id2) ||
if (!nixDB.queryString(noTxn, dbPath2Id, *j, id2) ||
id != parseHash(id2)) {
debug(format("erasing path `%1%' from mapping for id %2%")
% *j % (string) id);
@ -324,12 +323,12 @@ void verifyStore()
} else j++;
}
setListDB(nixDB, dbId2Paths, id, idPaths);
nixDB.setStrings(noTxn, dbId2Paths, id, idPaths);
}
Strings subs;
enumDB(nixDB, dbSubstitutes, subs);
nixDB.enumTable(noTxn, dbSubstitutes, subs);
for (Strings::iterator i = subs.begin();
i != subs.end(); i++)
@ -337,7 +336,7 @@ void verifyStore()
FSId srcId = parseHash(*i);
Strings subIds;
queryListDB(nixDB, dbSubstitutes, srcId, subIds);
nixDB.queryStrings(noTxn, dbSubstitutes, srcId, subIds);
for (Strings::iterator j = subIds.begin();
j != subIds.end(); )
@ -345,7 +344,7 @@ void verifyStore()
FSId subId = parseHash(*j);
Strings subPaths;
queryListDB(nixDB, dbId2Paths, subId, subPaths);
nixDB.queryStrings(noTxn, dbId2Paths, subId, subPaths);
if (subPaths.size() == 0) {
debug(format("erasing substitute %1% for %2%")
% (string) subId % (string) srcId);
@ -353,11 +352,11 @@ void verifyStore()
} else j++;
}
setListDB(nixDB, dbSubstitutes, srcId, subIds);
nixDB.setStrings(noTxn, dbSubstitutes, srcId, subIds);
}
Strings sucs;
enumDB(nixDB, dbSuccessors, sucs);
nixDB.enumTable(noTxn, dbSuccessors, sucs);
for (Strings::iterator i = sucs.begin();
i != sucs.end(); i++)
@ -365,17 +364,17 @@ void verifyStore()
FSId id1 = parseHash(*i);
string id2;
if (!queryDB(nixDB, dbSuccessors, id1, id2)) abort();
if (!nixDB.queryString(noTxn, dbSuccessors, id1, id2)) abort();
Strings id2Paths;
queryListDB(nixDB, dbId2Paths, id2, id2Paths);
nixDB.queryStrings(noTxn, dbId2Paths, id2, id2Paths);
if (id2Paths.size() == 0) {
Strings id2Subs;
queryListDB(nixDB, dbSubstitutes, id2, id2Subs);
nixDB.queryStrings(noTxn, dbSubstitutes, id2, id2Subs);
if (id2Subs.size() == 0) {
debug(format("successor %1% for %2% missing")
% id2 % (string) id1);
delDB(nixDB, dbSuccessors, (string) id1);
nixDB.delPair(noTxn, dbSuccessors, (string) id1);
}
}
}

6
src/test.cc
View file

@ -51,6 +51,8 @@ struct MySource : RestoreSource
void runTests()
{
verbosity = (Verbosity) 100;
/* Hashing. */
string s = "0b0ffd0538622bfe20b92c4aa57254d9";
Hash h = parseHash(s);
@ -94,14 +96,16 @@ void runTests()
/* Set up the test environment. */
mkdir("scratch", 0777);
mkdir("scratch/db", 0777);
string testDir = absPath("scratch");
cout << testDir << endl;
nixStore = testDir;
nixLogDir = testDir;
nixDB = testDir + "/db";
nixDBPath = testDir + "/db";
openDB();
initDB();
/* Expression evaluation. */

4
testpkgs/slow2/slow-build.sh
View file

@ -2,13 +2,13 @@
echo "builder started..."
for i in $(seq 1 30); do
for i in $(seq 1 10); do
echo $i
sleep 1
done
mkdir $out
echo "done" > $out/bla
echo "done" >> $out/bla
echo "builder finished"