Support opening local store with database on read-only filesystem (#8356)

Previously it was not possible to open a local store when its database is on a read-only filesystem. Obviously a store on a read-only filesystem cannot be modified, but it would still be useful to be able to query it.

This change adds a new read-only setting to LocalStore. When set to true, Nix will skip operations that fail when the database is on a read-only filesystem (acquiring big-lock, schema migration, etc), and the store database will be opened in immutable mode.

Co-authored-by: Ben Radford <benradf@users.noreply.github.com>
Co-authored-by: cidkidnix <cidkidnix@protonmail.com>
Co-authored-by: Dylan Green <67574902+cidkidnix@users.noreply.github.com>
Co-authored-by: John Ericson <git@JohnEricson.me>
Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>

src/libstore/gc.cc

@@ -110,6 +110,11 @@ void LocalStore::createTempRootsFile()
 
 void LocalStore::addTempRoot(const StorePath & path)
 {
+    if (readOnly) {
+      debug("Read-only store doesn't support creating lock files for temp roots, but nothing can be deleted anyways.");
+      return;
+    }
+
     createTempRootsFile();
 
     /* Open/create the global GC lock file. */

src/libstore/local-store.cc

@@ -190,7 +190,11 @@ LocalStore::LocalStore(const Params & params)
 
     /* Create missing state directories if they don't already exist. */
     createDirs(realStoreDir);
+    if (readOnly) {
+        experimentalFeatureSettings.require(Xp::ReadOnlyLocalStore);
+    } else {
         makeStoreWritable();
+    }
     createDirs(linksDir);
     Path profilesDir = stateDir + "/profiles";
     createDirs(profilesDir);
@@ -204,9 +208,11 @@ LocalStore::LocalStore(const Params & params)
 
     for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) {
         createDirs(perUserDir);
+        if (!readOnly) {
             if (chmod(perUserDir.c_str(), 0755) == -1)
                 throw SysError("could not set permissions on '%s' to 755", perUserDir);
         }
+    }
 
     /* Optionally, create directories and set permissions for a
        multi-user install. */
@@ -269,10 +275,12 @@ LocalStore::LocalStore(const Params & params)
 
     /* Acquire the big fat lock in shared mode to make sure that no
        schema upgrade is in progress. */
+    if (!readOnly) {
         Path globalLockPath = dbDir + "/big-lock";
         globalLock = openLockFile(globalLockPath.c_str(), true);
+    }
 
-    if (!lockFile(globalLock.get(), ltRead, false)) {
+    if (!readOnly && !lockFile(globalLock.get(), ltRead, false)) {
         printInfo("waiting for the big Nix store lock...");
         lockFile(globalLock.get(), ltRead, true);
     }
@@ -280,6 +288,14 @@ LocalStore::LocalStore(const Params & params)
     /* Check the current database schema and if necessary do an
        upgrade.  */
     int curSchema = getSchema();
+    if (readOnly && curSchema < nixSchemaVersion) {
+        debug("current schema version: %d", curSchema);
+        debug("supported schema version: %d", nixSchemaVersion);
+        throw Error(curSchema == 0 ?
+            "database does not exist, and cannot be created in read-only mode" :
+            "database schema needs migrating, but this cannot be done in read-only mode");
+    }
+
     if (curSchema > nixSchemaVersion)
         throw Error("current Nix store schema is version %1%, but I only support %2%",
              curSchema, nixSchemaVersion);
@@ -344,7 +360,11 @@ LocalStore::LocalStore(const Params & params)
     else openDB(*state, false);
 
     if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
+        if (!readOnly) {
             migrateCASchema(state->db, dbDir + "/ca-schema", globalLock);
+        } else {
+            throw Error("need to migrate to content-addressed schema, but this cannot be done in read-only mode");
+        }
     }
 
     /* Prepare SQL statements. */
@@ -475,13 +495,20 @@ int LocalStore::getSchema()
 
 void LocalStore::openDB(State & state, bool create)
 {
-    if (access(dbDir.c_str(), R_OK | W_OK))
+    if (create && readOnly) {
+        throw Error("cannot create database while in read-only mode");
+    }
+
+    if (access(dbDir.c_str(), R_OK | (readOnly ? 0 : W_OK)))
         throw SysError("Nix database directory '%1%' is not writable", dbDir);
 
     /* Open the Nix database. */
     std::string dbPath = dbDir + "/db.sqlite";
     auto & db(state.db);
-    state.db = SQLite(dbPath, create);
+    auto openMode = readOnly ? SQLiteOpenMode::Immutable
+                  : create ? SQLiteOpenMode::Normal
+                  : SQLiteOpenMode::NoCreate;
+    state.db = SQLite(dbPath, openMode);
 
 #ifdef __CYGWIN__
     /* The cygwin version of sqlite3 has a patch which calls

src/libstore/local-store.hh

@@ -46,6 +46,23 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig
         "require-sigs",
         "Whether store paths copied into this store should have a trusted signature."};
 
+    Setting<bool> readOnly{(StoreConfig*) this,
+        false,
+        "read-only",
+        R"(
+          Allow this store to be opened when its [database](@docroot@/glossary.md#gloss-nix-database) is on a read-only filesystem.
+
+          Normally Nix will attempt to open the store database in read-write mode, even for querying (when write access is not needed), causing it to fail if the database is on a read-only filesystem.
+
+          Enable read-only mode to disable locking and open the SQLite database with the [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set.
+
+          > **Warning**
+          > Do not use this unless the filesystem is read-only.
+          >
+          > Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process.
+          > While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it.
+        )"};
+
     const std::string name() override { return "Local Store"; }
 
     std::string doc() override;
@@ -269,6 +286,10 @@ public:
 
 private:
 
+    /**
+     * Retrieve the current version of the database schema.
+     * If the database does not exist yet, the version returned will be 0.
+     */
     int getSchema();
 
     void openDB(State & state, bool create);

src/libstore/sqlite.cc

@@ -1,6 +1,7 @@
 #include "sqlite.hh"
 #include "globals.hh"
 #include "util.hh"
+#include "url.hh"
 
 #include <sqlite3.h>
 
@@ -50,15 +51,17 @@ static void traceSQL(void * x, const char * sql)
     notice("SQL<[%1%]>", sql);
 };
 
-SQLite::SQLite(const Path & path, bool create)
+SQLite::SQLite(const Path & path, SQLiteOpenMode mode)
 {
     // useSQLiteWAL also indicates what virtual file system we need.  Using
     // `unix-dotfile` is needed on NFS file systems and on Windows' Subsystem
     // for Linux (WSL) where useSQLiteWAL should be false by default.
     const char *vfs = settings.useSQLiteWAL ? 0 : "unix-dotfile";
-    int flags = SQLITE_OPEN_READWRITE;
-    if (create) flags |= SQLITE_OPEN_CREATE;
-    int ret = sqlite3_open_v2(path.c_str(), &db, flags, vfs);
+    bool immutable = mode == SQLiteOpenMode::Immutable;
+    int flags = immutable ? SQLITE_OPEN_READONLY : SQLITE_OPEN_READWRITE;
+    if (mode == SQLiteOpenMode::Normal) flags |= SQLITE_OPEN_CREATE;
+    auto uri = "file:" + percentEncode(path) + "?immutable=" + (immutable ? "1" : "0");
+    int ret = sqlite3_open_v2(uri.c_str(), &db, SQLITE_OPEN_URI | flags, vfs);
     if (ret != SQLITE_OK) {
         const char * err = sqlite3_errstr(ret);
         throw Error("cannot open SQLite database '%s': %s", path, err);

src/libstore/sqlite.hh

@@ -11,6 +11,27 @@ struct sqlite3_stmt;
 
 namespace nix {
 
+enum class SQLiteOpenMode {
+    /**
+     * Open the database in read-write mode.
+     * If the database does not exist, it will be created.
+     */
+    Normal,
+    /**
+     * Open the database in read-write mode.
+     * Fails with an error if the database does not exist.
+     */
+    NoCreate,
+    /**
+     * Open the database in immutable mode.
+     * In addition to the database being read-only,
+     * no wal or journal files will be created by sqlite.
+     * Use this mode if the database is on a read-only filesystem.
+     * Fails with an error if the database does not exist.
+     */
+    Immutable,
+};
+
 /**
  * RAII wrapper to close a SQLite database automatically.
  */
@@ -18,7 +39,7 @@ struct SQLite
 {
     sqlite3 * db = 0;
     SQLite() { }
-    SQLite(const Path & path, bool create = true);
+    SQLite(const Path & path, SQLiteOpenMode mode = SQLiteOpenMode::Normal);
     SQLite(const SQLite & from) = delete;
     SQLite& operator = (const SQLite & from) = delete;
     SQLite& operator = (SQLite && from) { db = from.db; from.db = 0; return *this; }

src/libutil/experimental-features.cc

@@ -12,7 +12,7 @@ struct ExperimentalFeatureDetails
     std::string_view description;
 };
 
-constexpr std::array<ExperimentalFeatureDetails, 14> xpFeatureDetails = {{
+constexpr std::array<ExperimentalFeatureDetails, 15> xpFeatureDetails = {{
     {
         .tag = Xp::CaDerivations,
         .name = "ca-derivations",
@@ -221,6 +221,13 @@ constexpr std::array<ExperimentalFeatureDetails, 14> xpFeatureDetails = {{
             Allow parsing of timestamps in builtins.fromTOML.
         )",
     },
+    {
+        .tag = Xp::ReadOnlyLocalStore,
+        .name = "read-only-local-store",
+        .description = R"(
+            Allow the use of the `read-only` parameter in [local store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) URIs.
+        )",
+    },
 }};
 
 static_assert(

src/libutil/experimental-features.hh

@@ -31,6 +31,7 @@ enum struct ExperimentalFeature
     DaemonTrustOverride,
     DynamicDerivations,
     ParseTomlTimestamps,
+    ReadOnlyLocalStore,
 };
 
 /**

tests/local.mk

@@ -136,7 +136,8 @@ nix_tests = \
   impure-derivations.sh \
   path-from-hash-part.sh \
   test-libstoreconsumer.sh \
-  toString-path.sh
+  toString-path.sh \
+  read-only-store.sh
 
 ifeq ($(HAVE_LIBCPUID), 1)
 	nix_tests += compute-levels.sh

tests/read-only-store.sh

@@ -0,0 +1,42 @@
+source common.sh
+
+enableFeatures "read-only-local-store"
+
+needLocalStore "cannot open store read-only when daemon has already opened it writeable"
+
+clearStore
+
+happy () {
+    # We can do a read-only query just fine with a read-only store
+    nix --store local?read-only=true path-info $dummyPath
+    
+    # We can "write" an already-present store-path a read-only store, because no IO is actually required
+    nix-store --store local?read-only=true --add dummy
+}
+## Testing read-only mode without forcing the underlying store to actually be read-only
+
+# Make sure the command fails when the store doesn't already have a database
+expectStderr 1 nix-store --store local?read-only=true --add dummy | grepQuiet "database does not exist, and cannot be created in read-only mode"
+
+# Make sure the store actually has a current-database, with at least one store object
+dummyPath=$(nix-store --add dummy)
+
+# Try again and make sure we fail when adding a item not already in the store
+expectStderr 1 nix-store --store local?read-only=true --add eval.nix | grepQuiet "attempt to write a readonly database"
+
+# Test a few operations that should work with the read-only store in its current state
+happy
+
+## Testing read-only mode with an underlying store that is actually read-only
+
+# Ensure store is actually read-only
+chmod -R -w $TEST_ROOT/store
+chmod -R -w $TEST_ROOT/var
+
+# Make sure we fail on add operations on the read-only store
+# This is only for adding files that are not *already* in the store
+expectStderr 1 nix-store --add eval.nix | grepQuiet "error: opening lock file '$(readlink -e $TEST_ROOT)/var/nix/db/big-lock'"
+expectStderr 1 nix-store --store local?read-only=true --add eval.nix | grepQuiet "Permission denied"
+
+# Test the same operations from before should again succeed
+happy