* Started using Berkeley DB environments. This is necessary for

transaction support (but we don't actually use transactions yet).

src/Makefile.am

@@ -42,6 +42,7 @@ nix.o: nix-help.txt.hh
 
 install-data-local:
 	$(INSTALL) -d $(localstatedir)/nix
+	$(INSTALL) -d $(localstatedir)/nix/db
 	$(INSTALL) -d $(localstatedir)/nix/links
 	ln -sf $(localstatedir)/nix/links/current $(prefix)/current
 	$(INSTALL) -d $(localstatedir)/log/nix

src/db.cc

@@ -3,66 +3,142 @@
 
 #include <memory>
 
-#include <db_cxx.h>
 
-
+/* Wrapper class to ensure proper destruction. */
-/* Wrapper classes that ensures that the database is closed upon
+class DestroyDb
-   object destruction. */
-class Db2 : public Db 
 {
+    Db * db;
 public:
-    Db2(DbEnv *env, u_int32_t flags) : Db(env, flags) { }
+    DestroyDb(Db * _db) : db(_db) { }
-    ~Db2() { close(0); }
+    ~DestroyDb() { db->close(0); delete db; }
 };
 
 
-class DbcClose 
+class DestroyDbc 
 {
-    Dbc * cursor;
+    Dbc * dbc;
 public:
-    DbcClose(Dbc * c) : cursor(c) { }
+    DestroyDbc(Dbc * _dbc) : dbc(_dbc) { }
-    ~DbcClose() { cursor->close(); }
+    ~DestroyDbc() { dbc->close(); /* close() frees dbc */ }
 };
 
 
-static auto_ptr<Db2> openDB(const string & filename, const string & dbname,
-    bool readonly)
-{
-    auto_ptr<Db2> db(new Db2(0, 0));
-
-    db->open(filename.c_str(), dbname.c_str(),
-        DB_HASH, readonly ? DB_RDONLY : DB_CREATE, 0666);
-
-    return db;
-}
-
-
 static void rethrow(DbException & e)
 {
     throw Error(e.what());
 }
 
 
-void createDB(const string & filename, const string & dbname)
+Transaction::Transaction()
+    : txn(0)
+{
+}
+
+
+Transaction::Transaction(Database & db)
+{
+    db.requireEnv();
+    db.env->txn_begin(0, &txn, 0);
+}
+
+
+Transaction::~Transaction()
+{
+    if (txn) {
+        txn->abort();
+        txn = 0;
+    }
+}
+
+
+void Transaction::commit()
+{
+    if (!txn) throw Error("commit called on null transaction");
+    txn->commit(0);
+    txn = 0;
+}
+
+
+void Database::requireEnv()
+{
+    if (!env) throw Error("database environment not open");
+}
+
+
+Db * Database::openDB(const Transaction & txn,
+    const string & table, bool create)
+{
+    requireEnv();
+
+    Db * db = new Db(env, 0);
+
+    try {
+        // !!! fixme when switching to BDB 4.1: use txn.
+        db->open(table.c_str(), 0, 
+            DB_HASH, create ? DB_CREATE : 0, 0666);
+    } catch (...) {
+        delete db;
+        throw;
+    }
+
+    return db;
+}
+
+
+Database::Database()
+    : env(0)
+{
+}
+
+
+Database::~Database()
+{
+    if (env) {
+        env->close(0);
+        delete env;
+    }
+}
+
+
+void Database::open(const string & path)
 {
     try {
-        openDB(filename, dbname, false);
+        
+        if (env) throw Error(format("environment already open"));
+
+        env = new DbEnv(0);
+
+        debug("foo" + path);
+        env->open(path.c_str(), 
+            DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN |
+            DB_CREATE,
+            0666);
+        
     } catch (DbException e) { rethrow(e); }
 }
 
 
-bool queryDB(const string & filename, const string & dbname,
+void Database::createTable(const string & table)
+{
+    try {
+        Db * db = openDB(noTxn, table, true);
+        DestroyDb destroyDb(db);
+    } catch (DbException e) { rethrow(e); }
+}
+
+
+bool Database::queryString(const Transaction & txn, const string & table, 
     const string & key, string & data)
 {
     try {
 
-        int err;
+        Db * db = openDB(txn, table, false);
-        auto_ptr<Db2> db = openDB(filename, dbname, true);
+        DestroyDb destroyDb(db);
 
         Dbt kt((void *) key.c_str(), key.length());
         Dbt dt;
 
-        err = db->get(0, &kt, &dt, 0);
+        int err = db->get(txn.txn, &kt, &dt, 0);
         if (err) return false;
 
         if (!dt.get_data())
@@ -76,12 +152,12 @@ bool queryDB(const string & filename, const string & dbname,
 }
 
 
-bool queryListDB(const string & filename, const string & dbname,
+bool Database::queryStrings(const Transaction & txn, const string & table, 
     const string & key, Strings & data)
 {
     string d;
 
-    if (!queryDB(filename, dbname, key, d))
+    if (!queryString(txn, table, key, d))
         return false;
 
     string::iterator it = d.begin();
@@ -110,19 +186,21 @@ bool queryListDB(const string & filename, const string & dbname,
 }
 
 
-void setDB(const string & filename, const string & dbname,
+void Database::setString(const Transaction & txn, const string & table,
     const string & key, const string & data)
 {
     try {
-        auto_ptr<Db2> db = openDB(filename, dbname, false);
+        Db * db = openDB(txn, table, false);
+        DestroyDb destroyDb(db);
+
         Dbt kt((void *) key.c_str(), key.length());
         Dbt dt((void *) data.c_str(), data.length());
-        db->put(0, &kt, &dt, 0);
+        db->put(txn.txn, &kt, &dt, 0);
     } catch (DbException e) { rethrow(e); }
 }
 
 
-void setListDB(const string & filename, const string & dbname,
+void Database::setStrings(const Transaction & txn, const string & table,
     const string & key, const Strings & data)
 {
     string d;
@@ -141,34 +219,36 @@ void setListDB(const string & filename, const string & dbname,
         d += s;
     }
 
-    setDB(filename, dbname, key, d);
+    setString(txn, table, key, d);
 }
 
 
-void delDB(const string & filename, const string & dbname,
+void Database::delPair(const Transaction & txn, const string & table,
     const string & key)
 {
     try {
-        auto_ptr<Db2> db = openDB(filename, dbname, false);
+        Db * db = openDB(txn, table, false);
+        DestroyDb destroyDb(db);
         Dbt kt((void *) key.c_str(), key.length());
-        db->del(0, &kt, 0);
+        db->del(txn.txn, &kt, 0);
     } catch (DbException e) { rethrow(e); }
 }
 
 
-void enumDB(const string & filename, const string & dbname,
+void Database::enumTable(const Transaction & txn, const string & table,
     Strings & keys)
 {
     try {
 
-        auto_ptr<Db2> db = openDB(filename, dbname, true);
+        Db * db = openDB(txn, table, false);
+        DestroyDb destroyDb(db);
 
-        Dbc * cursor;
+        Dbc * dbc;
-        db->cursor(0, &cursor, 0);
+        db->cursor(0, &dbc, 0);
-        DbcClose cursorCloser(cursor);
+        DestroyDbc destroyDbc(dbc);
 
         Dbt kt, dt;
-        while (cursor->get(&kt, &dt, DB_NEXT) != DB_NOTFOUND)
+        while (dbc->get(&kt, &dt, DB_NEXT) != DB_NOTFOUND)
             keys.push_back(
                 string((char *) kt.get_data(), kt.get_size()));
 

src/db.hh

@@ -4,28 +4,73 @@
 #include <string>
 #include <list>
 
+#include <db_cxx.h>
+
 #include "util.hh"
 
 using namespace std;
 
-void createDB(const string & filename, const string & dbname);
 
-bool queryDB(const string & filename, const string & dbname,
+class Database;
-    const string & key, string & data);
 
-bool queryListDB(const string & filename, const string & dbname,
-    const string & key, Strings & data);
 
-void setDB(const string & filename, const string & dbname,
+class Transaction
-    const string & key, const string & data);
+{
+    friend class Database;
 
-void setListDB(const string & filename, const string & dbname,
+private:
-    const string & key, const Strings & data);
+    DbTxn * txn;
     
-void delDB(const string & filename, const string & dbname,
+public:
-    const string & key);
+    Transaction();
+    Transaction(Database & _db);
+    ~Transaction();
+
+    void commit();
+};
+
+
+#define noTxn Transaction()
+
+
+class Database
+{
+    friend class Transaction;
+
+private:
+    DbEnv * env;
+
+    void requireEnv();
+
+    Db * openDB(const Transaction & txn,
+        const string & table, bool create);
+
+public:
+    Database();
+    ~Database();
+    
+    void open(const string & path);
+
+    void createTable(const string & table);
+
+    bool queryString(const Transaction & txn, const string & table, 
+        const string & key, string & data);
+
+    bool queryStrings(const Transaction & txn, const string & table, 
+        const string & key, Strings & data);
+
+    void setString(const Transaction & txn, const string & table,
+        const string & key, const string & data);
+
+    void setStrings(const Transaction & txn, const string & table,
+        const string & key, const Strings & data);
+
+    void delPair(const Transaction & txn, const string & table,
+        const string & key);
+
+    void enumTable(const Transaction & txn, const string & table,
+        Strings & keys);
+};
 
-void enumDB(const string & filename, const string & dbname,
-    Strings & keys);
 
 #endif /* !__DB_H */

src/fix.cc

@@ -301,6 +301,8 @@ static Expr evalFile(EvalState & state, string relPath)
 
 void run(Strings args)
 {
+    openDB();
+
     EvalState state;
     Strings files;
 

src/globals.cc

@@ -2,6 +2,9 @@
 #include "db.hh"
 
 
+Database nixDB;
+
+
 string dbPath2Id = "path2id";
 string dbId2Paths = "id2paths";
 string dbSuccessors = "successors";
@@ -11,13 +14,19 @@ string dbSubstitutes = "substitutes";
 string nixStore = "/UNINIT";
 string nixDataDir = "/UNINIT";
 string nixLogDir = "/UNINIT";
-string nixDB = "/UNINIT";
+string nixDBPath = "/UNINIT";
+
+
+void openDB()
+{
+    nixDB.open(nixDBPath);
+}
 
 
 void initDB()
 {
-    createDB(nixDB, dbPath2Id);
+    nixDB.createTable(dbPath2Id);
-    createDB(nixDB, dbId2Paths);
+    nixDB.createTable(dbId2Paths);
-    createDB(nixDB, dbSuccessors);
+    nixDB.createTable(dbSuccessors);
-    createDB(nixDB, dbSubstitutes);
+    nixDB.createTable(dbSubstitutes);
 }

src/globals.hh

@@ -3,10 +3,15 @@
 
 #include <string>
 
+#include "db.hh"
+
 using namespace std;
 
 
-/* Database names. */
+extern Database nixDB;
+
+
+/* Database tables. */
 
 /* dbPath2Id :: Path -> FSId
 
@@ -60,12 +65,14 @@ extern string nixDataDir; /* !!! fix */
 /* nixLogDir is the directory where we log various operations. */ 
 extern string nixLogDir;
 
-/* nixDB is the file name of the Berkeley DB database where we
+/* nixDBPath is the path name of our Berkeley DB environment. */
-   maintain the dbXXX mappings. */
+extern string nixDBPath;
-extern string nixDB;
 
 
-/* Initialize the databases. */
+/* Open the database environment. */
+void openDB();
+
+/* Create the required database tables. */
 void initDB();
 
 

src/nix.cc

@@ -335,6 +335,8 @@ static void opVerify(Strings opFlags, Strings opArgs)
    list. */
 void run(Strings args)
 {
+    openDB();
+
     Strings opFlags, opArgs;
     Operation op = 0;
 

src/normalise.cc

@@ -9,7 +9,7 @@
 
 void registerSuccessor(const FSId & id1, const FSId & id2)
 {
-    setDB(nixDB, dbSuccessors, id1, id2);
+    nixDB.setString(noTxn, dbSuccessors, id1, id2);
 }
 
 
@@ -31,7 +31,7 @@ FSId normaliseFState(FSId id, FSIdSet pending)
     /* Try to substitute $id$ by any known successors in order to
        speed up the rewrite process. */
     string idSucc;
-    while (queryDB(nixDB, dbSuccessors, id, idSucc)) {
+    while (nixDB.queryString(noTxn, dbSuccessors, id, idSucc)) {
         debug(format("successor %1% -> %2%") % (string) id % idSucc);
         id = parseHash(idSucc);
     }
@@ -191,7 +191,7 @@ void realiseSlice(const FSId & id, FSIdSet pending)
     {
         SliceElem elem = *i;
         string id;
-        if (!queryDB(nixDB, dbPath2Id, elem.path, id)) {
+        if (!nixDB.queryString(noTxn, dbPath2Id, elem.path, id)) {
             if (pathExists(elem.path))
                 throw Error(format("path `%1%' obstructed") % elem.path);
             missing = true;
@@ -269,7 +269,7 @@ static void fstateRequisitesSet(const FSId & id,
 
     string idSucc;
     if (includeSuccessors &&
-        queryDB(nixDB, dbSuccessors, id, idSucc))
+        nixDB.queryString(noTxn, dbSuccessors, id, idSucc))
         fstateRequisitesSet(parseHash(idSucc), 
             includeExprs, includeSuccessors, paths);
 }
@@ -293,13 +293,13 @@ FSIds findGenerators(const FSIds & _ids)
        mappings, since we know that those are Nix expressions. */
 
     Strings sucs;
-    enumDB(nixDB, dbSuccessors, sucs);
+    nixDB.enumTable(noTxn, dbSuccessors, sucs);
 
     for (Strings::iterator i = sucs.begin();
          i != sucs.end(); i++)
     {
         string s;
-        if (!queryDB(nixDB, dbSuccessors, *i, s)) continue;
+        if (!nixDB.queryString(noTxn, dbSuccessors, *i, s)) continue;
         FSId id = parseHash(s);
 
         FState fs;

src/shared.cc

@@ -19,7 +19,7 @@ static void initAndRun(int argc, char * * argv)
     nixStore = NIX_STORE_DIR;
     nixDataDir = NIX_DATA_DIR;
     nixLogDir = NIX_LOG_DIR;
-    nixDB = (string) NIX_STATE_DIR + "/nixstate.db";
+    nixDBPath = (string) NIX_STATE_DIR + "/db";
 
     /* Put the arguments in a vector. */
     Strings args;

src/store.cc

@@ -96,7 +96,7 @@ void registerSubstitute(const FSId & srcId, const FSId & subId)
     /* For now, accept only one substitute per id. */
     Strings subs;
     subs.push_back(subId);
-    setListDB(nixDB, dbSubstitutes, srcId, subs);
+    nixDB.setStrings(noTxn, dbSubstitutes, srcId, subs);
 }
 
 
@@ -104,10 +104,10 @@ void registerPath(const string & _path, const FSId & id)
 {
     string path(canonPath(_path));
 
-    setDB(nixDB, dbPath2Id, path, id);
+    nixDB.setString(noTxn, dbPath2Id, path, id);
 
     Strings paths;
-    queryListDB(nixDB, dbId2Paths, id, paths); /* non-existence = ok */
+    nixDB.queryStrings(noTxn, dbId2Paths, id, paths); /* non-existence = ok */
 
     for (Strings::iterator it = paths.begin();
          it != paths.end(); it++)
@@ -115,7 +115,7 @@ void registerPath(const string & _path, const FSId & id)
     
     paths.push_back(path);
     
-    setListDB(nixDB, dbId2Paths, id, paths);
+    nixDB.setStrings(noTxn, dbId2Paths, id, paths);
 }
 
 
@@ -124,16 +124,15 @@ void unregisterPath(const string & _path)
     string path(canonPath(_path));
 
     string _id;
-    if (!queryDB(nixDB, dbPath2Id, path, _id))
+    if (!nixDB.queryString(noTxn, dbPath2Id, path, _id)) return;
-        return;
     FSId id(parseHash(_id));
 
-    delDB(nixDB, dbPath2Id, path);
+    nixDB.delPair(noTxn, dbPath2Id, path);
 
     /* begin transaction */
     
     Strings paths, paths2;
-    queryListDB(nixDB, dbId2Paths, id, paths); /* non-existence = ok */
+    nixDB.queryStrings(noTxn, dbId2Paths, id, paths); /* non-existence = ok */
 
     bool changed = false;
     for (Strings::iterator it = paths.begin();
@@ -141,7 +140,7 @@ void unregisterPath(const string & _path)
         if (*it != path) paths2.push_back(*it); else changed = true;
 
     if (changed)
-        setListDB(nixDB, dbId2Paths, id, paths2);
+        nixDB.setStrings(noTxn, dbId2Paths, id, paths2);
 
     /* end transaction */
 
@@ -151,7 +150,7 @@ void unregisterPath(const string & _path)
 bool queryPathId(const string & path, FSId & id)
 {
     string s;
-    if (!queryDB(nixDB, dbPath2Id, absPath(path), s)) return false;
+    if (!nixDB.queryString(noTxn, dbPath2Id, absPath(path), s)) return false;
     id = parseHash(s);
     return true;
 }
@@ -174,7 +173,7 @@ string expandId(const FSId & id, const string & target,
     if (!target.empty() && !isInPrefix(target, prefix))
         abort();
 
-    queryListDB(nixDB, dbId2Paths, id, paths);
+    nixDB.queryStrings(noTxn, dbId2Paths, id, paths);
 
     /* Pick one equal to `target'. */
     if (!target.empty()) {
@@ -212,7 +211,7 @@ string expandId(const FSId & id, const string & target,
     /* Try to realise the substitutes, but only if this id is not
        already being realised by a substitute. */
     Strings subs;
-    queryListDB(nixDB, dbSubstitutes, id, subs); /* non-existence = ok */
+    nixDB.queryStrings(noTxn, dbSubstitutes, id, subs); /* non-existence = ok */
 
     for (Strings::iterator it = subs.begin(); it != subs.end(); it++) {
         FSId subId = parseHash(*it);
@@ -264,7 +263,7 @@ void deleteFromStore(const string & path)
 void verifyStore()
 {
     Strings paths;
-    enumDB(nixDB, dbPath2Id, paths);
+    nixDB.enumTable(noTxn, dbPath2Id, paths);
 
     for (Strings::iterator i = paths.begin();
          i != paths.end(); i++)
@@ -278,10 +277,10 @@ void verifyStore()
 
         else {
             string id;
-            if (!queryDB(nixDB, dbPath2Id, path, id)) abort();
+            if (!nixDB.queryString(noTxn, dbPath2Id, path, id)) abort();
 
             Strings idPaths;
-            queryListDB(nixDB, dbId2Paths, id, idPaths);
+            nixDB.queryStrings(noTxn, dbId2Paths, id, idPaths);
 
             bool found = false;
             for (Strings::iterator j = idPaths.begin();     
@@ -298,11 +297,11 @@ void verifyStore()
                 debug(format("reverse mapping for path `%1%' missing") % path);
         }
 
-        if (erase) delDB(nixDB, dbPath2Id, path);
+        if (erase) nixDB.delPair(noTxn, dbPath2Id, path);
     }
 
     Strings ids;
-    enumDB(nixDB, dbId2Paths, ids);
+    nixDB.enumTable(noTxn, dbId2Paths, ids);
 
     for (Strings::iterator i = ids.begin();
          i != ids.end(); i++)
@@ -310,13 +309,13 @@ void verifyStore()
         FSId id = parseHash(*i);
 
         Strings idPaths;
-        queryListDB(nixDB, dbId2Paths, id, idPaths);
+        nixDB.queryStrings(noTxn, dbId2Paths, id, idPaths);
 
         for (Strings::iterator j = idPaths.begin();     
              j != idPaths.end(); )
         {
             string id2;
-            if (!queryDB(nixDB, dbPath2Id, *j, id2) || 
+            if (!nixDB.queryString(noTxn, dbPath2Id, *j, id2) || 
                 id != parseHash(id2)) {
                 debug(format("erasing path `%1%' from mapping for id %2%") 
                     % *j % (string) id);
@@ -324,12 +323,12 @@ void verifyStore()
             } else j++;
         }
 
-        setListDB(nixDB, dbId2Paths, id, idPaths);
+        nixDB.setStrings(noTxn, dbId2Paths, id, idPaths);
     }
 
     
     Strings subs;
-    enumDB(nixDB, dbSubstitutes, subs);
+    nixDB.enumTable(noTxn, dbSubstitutes, subs);
 
     for (Strings::iterator i = subs.begin();
          i != subs.end(); i++)
@@ -337,7 +336,7 @@ void verifyStore()
         FSId srcId = parseHash(*i);
 
         Strings subIds;
-        queryListDB(nixDB, dbSubstitutes, srcId, subIds);
+        nixDB.queryStrings(noTxn, dbSubstitutes, srcId, subIds);
 
         for (Strings::iterator j = subIds.begin();     
              j != subIds.end(); )
@@ -345,7 +344,7 @@ void verifyStore()
             FSId subId = parseHash(*j);
             
             Strings subPaths;
-            queryListDB(nixDB, dbId2Paths, subId, subPaths);
+            nixDB.queryStrings(noTxn, dbId2Paths, subId, subPaths);
             if (subPaths.size() == 0) {
                 debug(format("erasing substitute %1% for %2%") 
                     % (string) subId % (string) srcId);
@@ -353,11 +352,11 @@ void verifyStore()
             } else j++;
         }
 
-        setListDB(nixDB, dbSubstitutes, srcId, subIds);
+        nixDB.setStrings(noTxn, dbSubstitutes, srcId, subIds);
     }
 
     Strings sucs;
-    enumDB(nixDB, dbSuccessors, sucs);
+    nixDB.enumTable(noTxn, dbSuccessors, sucs);
 
     for (Strings::iterator i = sucs.begin();
          i != sucs.end(); i++)
@@ -365,17 +364,17 @@ void verifyStore()
         FSId id1 = parseHash(*i);
 
         string id2;
-        if (!queryDB(nixDB, dbSuccessors, id1, id2)) abort();
+        if (!nixDB.queryString(noTxn, dbSuccessors, id1, id2)) abort();
         
         Strings id2Paths;
-        queryListDB(nixDB, dbId2Paths, id2, id2Paths);
+        nixDB.queryStrings(noTxn, dbId2Paths, id2, id2Paths);
         if (id2Paths.size() == 0) {
             Strings id2Subs;
-            queryListDB(nixDB, dbSubstitutes, id2, id2Subs);
+            nixDB.queryStrings(noTxn, dbSubstitutes, id2, id2Subs);
             if (id2Subs.size() == 0) {
                 debug(format("successor %1% for %2% missing") 
                     % id2 % (string) id1);
-                delDB(nixDB, dbSuccessors, (string) id1);
+                nixDB.delPair(noTxn, dbSuccessors, (string) id1);
             }
         }
     }

src/test.cc

@@ -51,6 +51,8 @@ struct MySource : RestoreSource
 
 void runTests()
 {
+    verbosity = (Verbosity) 100;
+
     /* Hashing. */
     string s = "0b0ffd0538622bfe20b92c4aa57254d9";
     Hash h = parseHash(s);
@@ -94,14 +96,16 @@ void runTests()
     /* Set up the test environment. */
 
     mkdir("scratch", 0777);
+    mkdir("scratch/db", 0777);
 
     string testDir = absPath("scratch");
     cout << testDir << endl;
 
     nixStore = testDir;
     nixLogDir = testDir;
-    nixDB = testDir + "/db";
+    nixDBPath = testDir + "/db";
 
+    openDB();
     initDB();
 
     /* Expression evaluation. */

testpkgs/slow2/slow-build.sh

@@ -2,13 +2,13 @@
 
 echo "builder started..."
 
-for i in $(seq 1 30); do
+for i in $(seq 1 10); do
     echo $i
     sleep 1
 done
 
 mkdir $out
 
-echo "done" > $out/bla
+echo "done" >> $out/bla
 
 echo "builder finished"