* Implement isValidPath().

src/libstore/local-store.cc

@@ -54,6 +54,14 @@ void SQLiteStmt::create(sqlite3 * db, const string & s)
 }
 
 
+void SQLiteStmt::reset()
+{
+    assert(stmt);
+    if (sqlite3_reset(stmt) != SQLITE_OK)
+        throw SQLiteError(db, "resetting statement");
+}
+
+
 SQLiteStmt::~SQLiteStmt()
 {
     try {
@@ -188,14 +196,11 @@ LocalStore::LocalStore()
 LocalStore::~LocalStore()
 {
     try {
-        flushDelayedUpdates();
-
         foreach (RunningSubstituters::iterator, i, runningSubstituters) {
             i->second.to.close();
             i->second.from.close();
             i->second.pid.wait(true);
         }
-                
     } catch (...) {
         ignoreException();
     }
@@ -231,6 +236,7 @@ void LocalStore::prepareStatements()
         "insert into ValidPaths (path, hash, registrationTime, deriver) values (?, ?, ?, ?);");
     stmtAddReference.create(db,
         "insert into Refs (referrer, reference) values (?, ?);");
+    stmtIsValidPath.create(db, "select 1 from ValidPaths where path = ?;");
 }
 
 
@@ -308,98 +314,6 @@ void canonicalisePathMetaData(const Path & path)
 }
 
 
-static Path infoFileFor(const Path & path)
-{
-    string baseName = baseNameOf(path);
-    return (format("%1%/info/%2%") % nixDBPath % baseName).str();
-}
-
-
-static Path referrersFileFor(const Path & path)
-{
-    string baseName = baseNameOf(path);
-    return (format("%1%/referrer/%2%") % nixDBPath % baseName).str();
-}
-
-
-static Path failedFileFor(const Path & path)
-{
-    string baseName = baseNameOf(path);
-    return (format("%1%/failed/%2%") % nixDBPath % baseName).str();
-}
-
-
-static Path tmpFileForAtomicUpdate(const Path & path)
-{
-    return (format("%1%/.%2%.%3%") % dirOf(path) % getpid() % baseNameOf(path)).str();
-}
-
-
-void LocalStore::appendReferrer(const Path & from, const Path & to, bool lock)
-{
-    Path referrersFile = referrersFileFor(from);
-    
-    PathLocks referrersLock;
-    if (lock) {
-        referrersLock.lockPaths(singleton<PathSet, Path>(referrersFile));
-        referrersLock.setDeletion(true);
-    }
-
-    AutoCloseFD fd = open(referrersFile.c_str(), O_WRONLY | O_APPEND | O_CREAT, 0666);
-    if (fd == -1) throw SysError(format("opening file `%1%'") % referrersFile);
-    
-    string s = " " + to;
-    writeFull(fd, (const unsigned char *) s.c_str(), s.size());
-
-    if (doFsync) fdatasync(fd);
-}
-
-
-/* Atomically update the referrers file.  If `purge' is true, the set
-   of referrers is set to `referrers'.  Otherwise, the current set of
-   referrers is purged of invalid paths. */
-void LocalStore::rewriteReferrers(const Path & path, bool purge, PathSet referrers)
-{
-    Path referrersFile = referrersFileFor(path);
-    
-    PathLocks referrersLock(singleton<PathSet, Path>(referrersFile));
-    referrersLock.setDeletion(true);
-
-    if (purge)
-        /* queryReferrers() purges invalid paths, so that's all we
-           need. */
-        queryReferrers(path, referrers);
-
-    Path tmpFile = tmpFileForAtomicUpdate(referrersFile);
-    
-    AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0666);
-    if (fd == -1) throw SysError(format("opening file `%1%'") % referrersFile);
-    
-    string s;
-    foreach (PathSet::const_iterator, i, referrers) {
-        s += " "; s += *i;
-    }
-    
-    writeFull(fd, (const unsigned char *) s.c_str(), s.size());
-
-    if (doFsync) fdatasync(fd);
-    
-    fd.close(); /* for Windows; can't rename open file */
-
-    if (rename(tmpFile.c_str(), referrersFile.c_str()) == -1)
-        throw SysError(format("cannot rename `%1%' to `%2%'") % tmpFile % referrersFile);
-}
-
-
-void LocalStore::flushDelayedUpdates()
-{
-    foreach (PathSet::iterator, i, delayedUpdates) {
-        rewriteReferrers(*i, true, PathSet());
-    }
-    delayedUpdates.clear();
-}
-
-
 void LocalStore::registerValidPath(const Path & path,
     const Hash & hash, const PathSet & references,
     const Path & deriver)
@@ -415,6 +329,7 @@ void LocalStore::registerValidPath(const Path & path,
 
 void LocalStore::registerValidPath(const ValidPathInfo & info, bool ignoreValidity)
 {
+#if 0    
     Path infoFile = infoFileFor(info.path);
 
     ValidPathInfo oldInfo;
@@ -467,22 +382,25 @@ void LocalStore::registerValidPath(const ValidPathInfo & info, bool ignoreValidi
     writeFile(tmpFile, s, doFsync);
     if (rename(tmpFile.c_str(), infoFile.c_str()) == -1)
         throw SysError(format("cannot rename `%1%' to `%2%'") % tmpFile % infoFile);
-
+#endif
-    pathInfoCache[info.path] = info;
 }
 
 
 void LocalStore::registerFailedPath(const Path & path)
 {
+#if 0
     /* Write an empty file in the .../failed directory to denote the
        failure of the builder for `path'. */
     writeFile(failedFileFor(path), "");
+#endif
 }
 
 
 bool LocalStore::hasPathFailed(const Path & path)
 {
+#if 0
     return pathExists(failedFileFor(path));
+#endif
 }
 
 
@@ -502,6 +420,7 @@ Hash parseHashField(const Path & path, const string & s)
 
 ValidPathInfo LocalStore::queryPathInfo(const Path & path, bool ignoreErrors)
 {
+#if 0
     ValidPathInfo res;
     res.path = path;
 
@@ -510,9 +429,6 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path, bool ignoreErrors)
     if (!isValidPath(path))
         throw Error(format("path `%1%' is not valid") % path);
 
-    std::map<Path, ValidPathInfo>::iterator lookup = pathInfoCache.find(path);
-    if (lookup != pathInfoCache.end()) return lookup->second;
-    
     /* Read the info file. */
     Path infoFile = infoFileFor(path);
     if (!pathExists(infoFile))
@@ -550,31 +466,20 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path, bool ignoreErrors)
         }
     }
 
-    return pathInfoCache[path] = res;
+    return res;
+#endif
 }
 
 
 bool LocalStore::isValidPath(const Path & path)
 {
-    /* Files in the info directory starting with a `.' are temporary
+    stmtIsValidPath.reset();
-       files. */
+    if (sqlite3_bind_text(stmtIsValidPath, 1, path.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK)
-    if (baseNameOf(path).at(0) == '.') return false;
+        throw SQLiteError(db, "binding argument");
-
+    int res = sqlite3_step(stmtIsValidPath);
-    /* A path is valid if its info file exists and has a non-zero
+    if (res != SQLITE_DONE && res != SQLITE_ROW)
-       size.  (The non-zero size restriction is to be robust to
+        throw SQLiteError(db, "querying path in database");
-       certain kinds of filesystem corruption, particularly with
+    return res == SQLITE_ROW;
-       ext4.) */
-    Path infoFile = infoFileFor(path);
-
-    struct stat st;
-    if (lstat(infoFile.c_str(), &st)) {
-        if (errno == ENOENT) return false;
-        throw SysError(format("getting status of `%1%'") % infoFile);
-    }
-
-    if (st.st_size == 0) return false;
-    
-    return true;
 }
 
 
@@ -598,6 +503,7 @@ void LocalStore::queryReferences(const Path & path,
 
 bool LocalStore::queryReferrersInternal(const Path & path, PathSet & referrers)
 {
+#if 0
     bool allValid = true;
     
     if (!isValidPath(path))
@@ -623,6 +529,7 @@ bool LocalStore::queryReferrersInternal(const Path & path, PathSet & referrers)
         if (isStorePath(*i) && isValidPath(*i)) referrers.insert(*i); else allValid = false;
 
     return allValid;
+#endif
 }
 
 
@@ -788,6 +695,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
    there are no referrers. */
 void LocalStore::invalidatePath(const Path & path)
 {
+#if 0    
     debug(format("invalidating path `%1%'") % path);
 
     ValidPathInfo info;
@@ -805,28 +713,7 @@ void LocalStore::invalidatePath(const Path & path)
     Path p = referrersFileFor(path);
     if (pathExists(p) && unlink(p.c_str()) == -1)
         throw SysError(format("unlinking `%1%'") % p);
-
+#endif
-    /* Clear `path' from the info cache. */
-    pathInfoCache.erase(path);
-    delayedUpdates.erase(path);
-
-    /* Cause the referrer files for each path referenced by this one
-       to be updated.  This has to happen after removing the info file
-       to preserve the invariant (see registerValidPath()).
-
-       The referrer files are updated lazily in flushDelayedUpdates()
-       to prevent quadratic performance in the garbage collector
-       (i.e., when N referrers to some path X are deleted, we have to
-       rewrite the referrers file for X N times, causing O(N^2) I/O).
-
-       What happens if we die before the referrer file can be updated?
-       That's not a problem, because stale (invalid) entries in the
-       referrer file are ignored by queryReferrers().  Thus a referrer
-       file is allowed to have stale entries; removing them is just an
-       optimisation.  verifyStore() gets rid of them eventually.
-    */
-    foreach (PathSet::iterator, i, info.references)
-        if (*i != path) delayedUpdates.insert(*i);
 }
 
 
@@ -1130,6 +1017,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source)
 void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFreed,
     unsigned long long & blocksFreed)
 {
+#if 0
     bytesFreed = 0;
 
     assertStorePath(path);
@@ -1149,11 +1037,13 @@ void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFr
     }
 
     deletePathWrapped(path, bytesFreed, blocksFreed);
+#endif
 }
 
 
 void LocalStore::verifyStore(bool checkContents)
 {
+#if 0
     /* Check whether all valid paths actually exist. */
     printMsg(lvlInfo, "checking path existence");
 
@@ -1279,6 +1169,64 @@ void LocalStore::verifyStore(bool checkContents)
 
         if (update) rewriteReferrers(from, false, referrersNew);
     }
+#endif
+}
+
+
+/* Functions for upgrading from the pre-SQLite database. */
+
+static Path infoFileFor(const Path & path)
+{
+    string baseName = baseNameOf(path);
+    return (format("%1%/info/%2%") % nixDBPath % baseName).str();
+}
+
+
+PathSet LocalStore::queryValidPathsOld()
+{
+    PathSet paths;
+    Strings entries = readDirectory(nixDBPath + "/info");
+    foreach (Strings::iterator, i, entries)
+        if (i->at(0) != '.') paths.insert(nixStore + "/" + *i);
+    return paths;
+}
+
+
+ValidPathInfo LocalStore::queryPathInfoOld(const Path & path)
+{
+    ValidPathInfo res;
+    res.path = path;
+
+    /* Read the info file. */
+    Path infoFile = infoFileFor(path);
+    if (!pathExists(infoFile))
+        throw Error(format("path `%1%' is not valid") % path);
+    string info = readFile(infoFile);
+
+    /* Parse it. */
+    Strings lines = tokenizeString(info, "\n");
+
+    foreach (Strings::iterator, i, lines) {
+        string::size_type p = i->find(':');
+        if (p == string::npos)
+            throw Error(format("corrupt line in `%1%': %2%") % infoFile % *i);
+        string name(*i, 0, p);
+        string value(*i, p + 2);
+        if (name == "References") {
+            Strings refs = tokenizeString(value, " ");
+            res.references = PathSet(refs.begin(), refs.end());
+        } else if (name == "Deriver") {
+            res.deriver = value;
+        } else if (name == "Hash") {
+            res.hash = parseHashField(path, value);
+        } else if (name == "Registered-At") {
+            int n = 0;
+            string2Int(value, n);
+            res.registrationTime = n;
+        }
+    }
+
+    return res;
 }
 
 
@@ -1294,17 +1242,16 @@ void LocalStore::upgradeStore6()
 
     initSchema();
 
-    PathSet validPaths = queryValidPaths();
+    PathSet validPaths = queryValidPathsOld();
 
     SQLiteTxn txn(db);
     
     std::map<Path, sqlite3_int64> pathToId;
     
     foreach (PathSet::iterator, i, validPaths) {
-        ValidPathInfo info = queryPathInfo(*i, true);
+        ValidPathInfo info = queryPathInfoOld(*i);
         
-        if (sqlite3_reset(stmtRegisterValidPath) != SQLITE_OK)
+        stmtRegisterValidPath.reset();
-            throw SQLiteError(db, "resetting statement");
         if (sqlite3_bind_text(stmtRegisterValidPath, 1, i->c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK)
             throw SQLiteError(db, "binding argument 1");
         string h = "sha256:" + printHash(info.hash);
@@ -1330,11 +1277,10 @@ void LocalStore::upgradeStore6()
     std::cerr << "|";
     
     foreach (PathSet::iterator, i, validPaths) {
-        ValidPathInfo info = queryPathInfo(*i, true);
+        ValidPathInfo info = queryPathInfoOld(*i);
         
         foreach (PathSet::iterator, j, info.references) {
-            if (sqlite3_reset(stmtAddReference) != SQLITE_OK)
+            stmtAddReference.reset();
-                throw SQLiteError(db, "resetting statement");
             if (sqlite3_bind_int(stmtAddReference, 1, pathToId[*i]) != SQLITE_OK)
                 throw SQLiteError(db, "binding argument 1");
             if (pathToId.find(*j) == pathToId.end())

src/libstore/local-store.hh

@@ -63,6 +63,7 @@ struct SQLiteStmt
     sqlite3_stmt * stmt;
     SQLiteStmt() { stmt = 0; }
     void create(sqlite3 * db, const string & s);
+    void reset();
     ~SQLiteStmt();
     operator sqlite3_stmt * () { return stmt; }
 };
@@ -178,13 +179,6 @@ private:
     /* Lock file used for upgrading. */
     AutoCloseFD globalLock;
 
-    /* !!! The cache can grow very big.  Maybe it should be pruned
-       every once in a while. */
-    std::map<Path, ValidPathInfo> pathInfoCache;
-
-    /* Store paths for which the referrers file must be purged. */
-    PathSet delayedUpdates;
-
     /* Whether to do an fsync() after writing Nix metadata. */
     bool doFsync;
 
@@ -194,6 +188,7 @@ private:
     /* Some precompiled SQLite statements. */
     SQLiteStmt stmtRegisterValidPath;
     SQLiteStmt stmtAddReference;
+    SQLiteStmt stmtIsValidPath;
 
     int getSchema();
 
@@ -209,13 +204,13 @@ private:
     
     void rewriteReferrers(const Path & path, bool purge, PathSet referrers);
 
-    void flushDelayedUpdates();
-    
     bool queryReferrersInternal(const Path & path, PathSet & referrers);
     
     void invalidatePath(const Path & path);
 
     void upgradeStore6();
+    PathSet queryValidPathsOld();
+    ValidPathInfo queryPathInfoOld(const Path & path);
 
     struct GCState;