From b2ac6fc040223a58f9b923a89798f72b48e310e5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Nov 2023 14:36:40 +0100 Subject: [PATCH 01/11] Remove FSAccessor::Type::tMissing Instead stat() now returns std::nullopt to denote that the file doesn't exist. --- src/libstore/binary-cache-store.cc | 6 +-- src/libstore/fs-accessor.hh | 19 +++++---- src/libstore/local-fs-store.cc | 8 ++-- src/libstore/nar-accessor.cc | 68 +++++++++++++++--------------- src/libstore/remote-fs-accessor.cc | 2 +- src/libstore/remote-fs-accessor.hh | 2 +- src/nix/cat.cc | 11 +++-- src/nix/ls.cc | 26 ++++++------ src/nix/run.cc | 2 +- src/nix/why-depends.cc | 7 +-- 10 files changed, 77 insertions(+), 74 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 2a91233ec..06d89c478 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -208,7 +208,7 @@ ref BinaryCacheStore::addToStoreCommon( std::string buildIdDir = "/lib/debug/.build-id"; - if (narAccessor->stat(buildIdDir).type == FSAccessor::tDirectory) { + if (auto st = narAccessor->stat(buildIdDir); st && st->type == FSAccessor::tDirectory) { ThreadPool threadPool(25); @@ -234,14 +234,14 @@ ref BinaryCacheStore::addToStoreCommon( for (auto & s1 : narAccessor->readDirectory(buildIdDir)) { auto dir = buildIdDir + "/" + s1; - if (narAccessor->stat(dir).type != FSAccessor::tDirectory + if (auto st = narAccessor->stat(dir); !st || st->type != FSAccessor::tDirectory || !std::regex_match(s1, regex1)) continue; for (auto & s2 : narAccessor->readDirectory(dir)) { auto debugPath = dir + "/" + s2; - if (narAccessor->stat(debugPath).type != FSAccessor::tRegular + if (auto st = narAccessor->stat(debugPath); !st || st->type != FSAccessor::tRegular || !std::regex_match(s2, regex2)) continue; diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh index 1df19e647..9bae0be74 100644 --- a/src/libstore/fs-accessor.hh +++ b/src/libstore/fs-accessor.hh @@ -3,6 +3,8 @@ #include "types.hh" +#include + namespace nix { /** @@ -12,28 +14,29 @@ namespace nix { class FSAccessor { public: - enum Type { tMissing, tRegular, tSymlink, tDirectory }; + enum Type { tRegular, tSymlink, tDirectory }; struct Stat { - Type type = tMissing; + Type type; /** - * regular files only + * For regular files only: the size of the file. */ uint64_t fileSize = 0; /** - * regular files only + * For regular files only: whether this is an executable. */ - bool isExecutable = false; // regular files only + bool isExecutable = false; /** - * regular files only + * For regular files only: the position of the contents of this + * file in the NAR. */ - uint64_t narOffset = 0; // regular files only + uint64_t narOffset = 0; }; virtual ~FSAccessor() { } - virtual Stat stat(const Path & path) = 0; + virtual std::optional stat(const Path & path) = 0; virtual StringSet readDirectory(const Path & path) = 0; diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index b224fc3e9..bb83a9cd4 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -27,25 +27,25 @@ struct LocalStoreAccessor : public FSAccessor return store->getRealStoreDir() + std::string(path, store->storeDir.size()); } - FSAccessor::Stat stat(const Path & path) override + std::optional stat(const Path & path) override { auto realPath = toRealPath(path); struct stat st; if (lstat(realPath.c_str(), &st)) { - if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false}; + if (errno == ENOENT || errno == ENOTDIR) return std::nullopt; throw SysError("getting status of '%1%'", path); } if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode)) throw Error("file '%1%' has unsupported type", path); - return { + return {{ S_ISREG(st.st_mode) ? Type::tRegular : S_ISLNK(st.st_mode) ? Type::tSymlink : Type::tDirectory, S_ISREG(st.st_mode) ? (uint64_t) st.st_size : 0, - S_ISREG(st.st_mode) && st.st_mode & S_IXUSR}; + S_ISREG(st.st_mode) && st.st_mode & S_IXUSR}}; } StringSet readDirectory(const Path & path) override diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index f0dfcb19b..9123bd59d 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -11,13 +11,7 @@ namespace nix { struct NarMember { - FSAccessor::Type type = FSAccessor::Type::tMissing; - - bool isExecutable = false; - - /* If this is a regular file, position of the contents of this - file in the NAR. */ - uint64_t start = 0, size = 0; + FSAccessor::Stat stat; std::string target; @@ -57,7 +51,7 @@ struct NarAccessor : public FSAccessor acc.root = std::move(member); parents.push(&acc.root); } else { - if (parents.top()->type != FSAccessor::Type::tDirectory) + if (parents.top()->stat.type != FSAccessor::Type::tDirectory) throw Error("NAR file missing parent directory of path '%s'", path); auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member)); parents.push(&result.first->second); @@ -79,14 +73,15 @@ struct NarAccessor : public FSAccessor void isExecutable() override { - parents.top()->isExecutable = true; + parents.top()->stat.isExecutable = true; } void preallocateContents(uint64_t size) override { assert(size <= std::numeric_limits::max()); - parents.top()->size = (uint64_t) size; - parents.top()->start = pos; + auto & st = parents.top()->stat; + st.fileSize = (uint64_t) size; + st.narOffset = pos; } void receiveContents(std::string_view data) override @@ -95,7 +90,9 @@ struct NarAccessor : public FSAccessor void createSymlink(const Path & path, const std::string & target) override { createMember(path, - NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target}); + NarMember{ + .stat = {.type = FSAccessor::Type::tSymlink}, + .target = target}); } size_t read(char * data, size_t len) override @@ -130,18 +127,20 @@ struct NarAccessor : public FSAccessor std::string type = v["type"]; if (type == "directory") { - member.type = FSAccessor::Type::tDirectory; + member.stat = {.type = FSAccessor::Type::tDirectory}; for (auto i = v["entries"].begin(); i != v["entries"].end(); ++i) { std::string name = i.key(); recurse(member.children[name], i.value()); } } else if (type == "regular") { - member.type = FSAccessor::Type::tRegular; - member.size = v["size"]; - member.isExecutable = v.value("executable", false); - member.start = v["narOffset"]; + member.stat = { + .type = FSAccessor::Type::tRegular, + .fileSize = v["size"], + .isExecutable = v.value("executable", false), + .narOffset = v["narOffset"] + }; } else if (type == "symlink") { - member.type = FSAccessor::Type::tSymlink; + member.stat = {.type = FSAccessor::Type::tSymlink}; member.target = v.value("target", ""); } else return; }; @@ -158,7 +157,7 @@ struct NarAccessor : public FSAccessor for (auto it = path.begin(); it != end; ) { // because it != end, the remaining component is non-empty so we need // a directory - if (current->type != FSAccessor::Type::tDirectory) return nullptr; + if (current->stat.type != FSAccessor::Type::tDirectory) return nullptr; // skip slash (canonPath above ensures that this is always a slash) assert(*it == '/'); @@ -183,19 +182,19 @@ struct NarAccessor : public FSAccessor return *result; } - Stat stat(const Path & path) override + std::optional stat(const Path & path) override { auto i = find(path); if (i == nullptr) - return {FSAccessor::Type::tMissing, 0, false}; - return {i->type, i->size, i->isExecutable, i->start}; + return std::nullopt; + return i->stat; } StringSet readDirectory(const Path & path) override { auto i = get(path); - if (i.type != FSAccessor::Type::tDirectory) + if (i.stat.type != FSAccessor::Type::tDirectory) throw Error("path '%1%' inside NAR file is not a directory", path); StringSet res; @@ -208,19 +207,19 @@ struct NarAccessor : public FSAccessor std::string readFile(const Path & path, bool requireValidPath = true) override { auto i = get(path); - if (i.type != FSAccessor::Type::tRegular) + if (i.stat.type != FSAccessor::Type::tRegular) throw Error("path '%1%' inside NAR file is not a regular file", path); - if (getNarBytes) return getNarBytes(i.start, i.size); + if (getNarBytes) return getNarBytes(i.stat.narOffset, i.stat.fileSize); assert(nar); - return std::string(*nar, i.start, i.size); + return std::string(*nar, i.stat.narOffset, i.stat.fileSize); } std::string readLink(const Path & path) override { auto i = get(path); - if (i.type != FSAccessor::Type::tSymlink) + if (i.stat.type != FSAccessor::Type::tSymlink) throw Error("path '%1%' inside NAR file is not a symlink", path); return i.target; } @@ -246,17 +245,19 @@ using nlohmann::json; json listNar(ref accessor, const Path & path, bool recurse) { auto st = accessor->stat(path); + if (!st) + throw Error("path '%s' does not exist in NAR", path); json obj = json::object(); - switch (st.type) { + switch (st->type) { case FSAccessor::Type::tRegular: obj["type"] = "regular"; - obj["size"] = st.fileSize; - if (st.isExecutable) + obj["size"] = st->fileSize; + if (st->isExecutable) obj["executable"] = true; - if (st.narOffset) - obj["narOffset"] = st.narOffset; + if (st->narOffset) + obj["narOffset"] = st->narOffset; break; case FSAccessor::Type::tDirectory: obj["type"] = "directory"; @@ -275,9 +276,6 @@ json listNar(ref accessor, const Path & path, bool recurse) obj["type"] = "symlink"; obj["target"] = accessor->readLink(path); break; - case FSAccessor::Type::tMissing: - default: - throw Error("path '%s' does not exist in NAR", path); } return obj; } diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index fcfb527f5..6c87ebeaa 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -101,7 +101,7 @@ std::pair, Path> RemoteFSAccessor::fetch(const Path & path_, boo return {addToCache(storePath.hashPart(), std::move(sink.s)), restPath}; } -FSAccessor::Stat RemoteFSAccessor::stat(const Path & path) +std::optional RemoteFSAccessor::stat(const Path & path) { auto res = fetch(path); return res.first->stat(res.second); diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh index e2673b6f6..5cf759aa0 100644 --- a/src/libstore/remote-fs-accessor.hh +++ b/src/libstore/remote-fs-accessor.hh @@ -28,7 +28,7 @@ public: RemoteFSAccessor(ref store, const /* FIXME: use std::optional */ Path & cacheDir = ""); - Stat stat(const Path & path) override; + std::optional stat(const Path & path) override; StringSet readDirectory(const Path & path) override; diff --git a/src/nix/cat.cc b/src/nix/cat.cc index 60aa66ce0..b5fe2506f 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -11,13 +11,12 @@ struct MixCat : virtual Args void cat(ref accessor) { - auto st = accessor->stat(path); - if (st.type == FSAccessor::Type::tMissing) + if (auto st = accessor->stat(path)) { + if (st->type != FSAccessor::Type::tRegular) + throw Error("path '%1%' is not a regular file", path); + writeFull(STDOUT_FILENO, accessor->readFile(path)); + } else throw Error("path '%1%' does not exist", path); - if (st.type != FSAccessor::Type::tRegular) - throw Error("path '%1%' is not a regular file", path); - - writeFull(STDOUT_FILENO, accessor->readFile(path)); } }; diff --git a/src/nix/ls.cc b/src/nix/ls.cc index c990a303c..8dc8a47b4 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -46,23 +46,25 @@ struct MixLs : virtual Args, MixJSON auto showFile = [&](const Path & curPath, const std::string & relPath) { if (verbose) { auto st = accessor->stat(curPath); + assert(st); std::string tp = - st.type == FSAccessor::Type::tRegular ? - (st.isExecutable ? "-r-xr-xr-x" : "-r--r--r--") : - st.type == FSAccessor::Type::tSymlink ? "lrwxrwxrwx" : + st->type == FSAccessor::Type::tRegular ? + (st->isExecutable ? "-r-xr-xr-x" : "-r--r--r--") : + st->type == FSAccessor::Type::tSymlink ? "lrwxrwxrwx" : "dr-xr-xr-x"; - auto line = fmt("%s %20d %s", tp, st.fileSize, relPath); - if (st.type == FSAccessor::Type::tSymlink) + auto line = fmt("%s %20d %s", tp, st->fileSize, relPath); + if (st->type == FSAccessor::Type::tSymlink) line += " -> " + accessor->readLink(curPath); logger->cout(line); - if (recursive && st.type == FSAccessor::Type::tDirectory) - doPath(st, curPath, relPath, false); + if (recursive && st->type == FSAccessor::Type::tDirectory) + doPath(*st, curPath, relPath, false); } else { logger->cout(relPath); if (recursive) { auto st = accessor->stat(curPath); - if (st.type == FSAccessor::Type::tDirectory) - doPath(st, curPath, relPath, false); + assert(st); + if (st->type == FSAccessor::Type::tDirectory) + doPath(*st, curPath, relPath, false); } } }; @@ -79,10 +81,10 @@ struct MixLs : virtual Args, MixJSON }; auto st = accessor->stat(path); - if (st.type == FSAccessor::Type::tMissing) + if (!st) throw Error("path '%1%' does not exist", path); - doPath(st, path, - st.type == FSAccessor::Type::tDirectory ? "." : std::string(baseNameOf(path)), + doPath(*st, path, + st->type == FSAccessor::Type::tDirectory ? "." : std::string(baseNameOf(path)), showDirectory); } diff --git a/src/nix/run.cc b/src/nix/run.cc index 1baf299ab..f6c229adc 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -120,7 +120,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment unixPath.push_front(store->printStorePath(path) + "/bin"); auto propPath = store->printStorePath(path) + "/nix-support/propagated-user-env-packages"; - if (accessor->stat(propPath).type == FSAccessor::tRegular) { + if (auto st = accessor->stat(propPath); st && st->type == FSAccessor::tRegular) { for (auto & p : tokenizeString(readFile(propPath))) todo.push(store->parseStorePath(p)); } diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 055cf6d0d..912ba72fb 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -214,6 +214,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions visitPath = [&](const Path & p) { auto st = accessor->stat(p); + assert(st); auto p2 = p == pathS ? "/" : std::string(p, pathS.size() + 1); @@ -221,13 +222,13 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions return hash == dependencyPathHash ? ANSI_GREEN : ANSI_BLUE; }; - if (st.type == FSAccessor::Type::tDirectory) { + if (st->type == FSAccessor::Type::tDirectory) { auto names = accessor->readDirectory(p); for (auto & name : names) visitPath(p + "/" + name); } - else if (st.type == FSAccessor::Type::tRegular) { + else if (st->type == FSAccessor::Type::tRegular) { auto contents = accessor->readFile(p); for (auto & hash : hashes) { @@ -245,7 +246,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions } } - else if (st.type == FSAccessor::Type::tSymlink) { + else if (st->type == FSAccessor::Type::tSymlink) { auto target = accessor->readLink(p); for (auto & hash : hashes) { From 8ffd1695ce31ff81b038fdc995dd8da03b180f03 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Nov 2023 14:43:20 +0100 Subject: [PATCH 02/11] Unify FSAccessor::Type and SourceAccessor::Type --- src/libstore/binary-cache-store.cc | 6 +++--- src/libstore/fs-accessor.hh | 3 ++- src/libstore/nar-accessor.cc | 30 ++++++++++++++++-------------- src/nix/run.cc | 2 +- 4 files changed, 22 insertions(+), 19 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 06d89c478..f9abd8cbd 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -208,7 +208,7 @@ ref BinaryCacheStore::addToStoreCommon( std::string buildIdDir = "/lib/debug/.build-id"; - if (auto st = narAccessor->stat(buildIdDir); st && st->type == FSAccessor::tDirectory) { + if (auto st = narAccessor->stat(buildIdDir); st && st->type == SourceAccessor::tDirectory) { ThreadPool threadPool(25); @@ -234,14 +234,14 @@ ref BinaryCacheStore::addToStoreCommon( for (auto & s1 : narAccessor->readDirectory(buildIdDir)) { auto dir = buildIdDir + "/" + s1; - if (auto st = narAccessor->stat(dir); !st || st->type != FSAccessor::tDirectory + if (auto st = narAccessor->stat(dir); !st || st->type != SourceAccessor::tDirectory || !std::regex_match(s1, regex1)) continue; for (auto & s2 : narAccessor->readDirectory(dir)) { auto debugPath = dir + "/" + s2; - if (auto st = narAccessor->stat(debugPath); !st || st->type != FSAccessor::tRegular + if (auto st = narAccessor->stat(debugPath); !st || st->type != SourceAccessor::tRegular || !std::regex_match(s2, regex2)) continue; diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh index 9bae0be74..1e951ec57 100644 --- a/src/libstore/fs-accessor.hh +++ b/src/libstore/fs-accessor.hh @@ -2,6 +2,7 @@ ///@file #include "types.hh" +#include "source-accessor.hh" #include @@ -14,7 +15,7 @@ namespace nix { class FSAccessor { public: - enum Type { tRegular, tSymlink, tDirectory }; + using Type = SourceAccessor::Type; struct Stat { diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 9123bd59d..43a78a362 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -51,7 +51,7 @@ struct NarAccessor : public FSAccessor acc.root = std::move(member); parents.push(&acc.root); } else { - if (parents.top()->stat.type != FSAccessor::Type::tDirectory) + if (parents.top()->stat.type != Type::tDirectory) throw Error("NAR file missing parent directory of path '%s'", path); auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member)); parents.push(&result.first->second); @@ -60,12 +60,12 @@ struct NarAccessor : public FSAccessor void createDirectory(const Path & path) override { - createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0}); + createMember(path, {Type::tDirectory, false, 0, 0}); } void createRegularFile(const Path & path) override { - createMember(path, {FSAccessor::Type::tRegular, false, 0, 0}); + createMember(path, {Type::tRegular, false, 0, 0}); } void closeRegularFile() override @@ -91,7 +91,7 @@ struct NarAccessor : public FSAccessor { createMember(path, NarMember{ - .stat = {.type = FSAccessor::Type::tSymlink}, + .stat = {.type = Type::tSymlink}, .target = target}); } @@ -127,20 +127,20 @@ struct NarAccessor : public FSAccessor std::string type = v["type"]; if (type == "directory") { - member.stat = {.type = FSAccessor::Type::tDirectory}; + member.stat = {.type = Type::tDirectory}; for (auto i = v["entries"].begin(); i != v["entries"].end(); ++i) { std::string name = i.key(); recurse(member.children[name], i.value()); } } else if (type == "regular") { member.stat = { - .type = FSAccessor::Type::tRegular, + .type = Type::tRegular, .fileSize = v["size"], .isExecutable = v.value("executable", false), .narOffset = v["narOffset"] }; } else if (type == "symlink") { - member.stat = {.type = FSAccessor::Type::tSymlink}; + member.stat = {.type = Type::tSymlink}; member.target = v.value("target", ""); } else return; }; @@ -157,7 +157,7 @@ struct NarAccessor : public FSAccessor for (auto it = path.begin(); it != end; ) { // because it != end, the remaining component is non-empty so we need // a directory - if (current->stat.type != FSAccessor::Type::tDirectory) return nullptr; + if (current->stat.type != Type::tDirectory) return nullptr; // skip slash (canonPath above ensures that this is always a slash) assert(*it == '/'); @@ -194,7 +194,7 @@ struct NarAccessor : public FSAccessor { auto i = get(path); - if (i.stat.type != FSAccessor::Type::tDirectory) + if (i.stat.type != Type::tDirectory) throw Error("path '%1%' inside NAR file is not a directory", path); StringSet res; @@ -207,7 +207,7 @@ struct NarAccessor : public FSAccessor std::string readFile(const Path & path, bool requireValidPath = true) override { auto i = get(path); - if (i.stat.type != FSAccessor::Type::tRegular) + if (i.stat.type != Type::tRegular) throw Error("path '%1%' inside NAR file is not a regular file", path); if (getNarBytes) return getNarBytes(i.stat.narOffset, i.stat.fileSize); @@ -219,7 +219,7 @@ struct NarAccessor : public FSAccessor std::string readLink(const Path & path) override { auto i = get(path); - if (i.stat.type != FSAccessor::Type::tSymlink) + if (i.stat.type != Type::tSymlink) throw Error("path '%1%' inside NAR file is not a symlink", path); return i.target; } @@ -251,7 +251,7 @@ json listNar(ref accessor, const Path & path, bool recurse) json obj = json::object(); switch (st->type) { - case FSAccessor::Type::tRegular: + case SourceAccessor::Type::tRegular: obj["type"] = "regular"; obj["size"] = st->fileSize; if (st->isExecutable) @@ -259,7 +259,7 @@ json listNar(ref accessor, const Path & path, bool recurse) if (st->narOffset) obj["narOffset"] = st->narOffset; break; - case FSAccessor::Type::tDirectory: + case SourceAccessor::Type::tDirectory: obj["type"] = "directory"; { obj["entries"] = json::object(); @@ -272,10 +272,12 @@ json listNar(ref accessor, const Path & path, bool recurse) } } break; - case FSAccessor::Type::tSymlink: + case SourceAccessor::Type::tSymlink: obj["type"] = "symlink"; obj["target"] = accessor->readLink(path); break; + case SourceAccessor::Type::tMisc: + assert(false); // cannot happen for NARs } return obj; } diff --git a/src/nix/run.cc b/src/nix/run.cc index f6c229adc..07806283c 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -120,7 +120,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment unixPath.push_front(store->printStorePath(path) + "/bin"); auto propPath = store->printStorePath(path) + "/nix-support/propagated-user-env-packages"; - if (auto st = accessor->stat(propPath); st && st->type == FSAccessor::tRegular) { + if (auto st = accessor->stat(propPath); st && st->type == SourceAccessor::tRegular) { for (auto & p : tokenizeString(readFile(propPath))) todo.push(store->parseStorePath(p)); } From cdb27c1519cd802f477e8fa90beabe1bddc4bac7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Nov 2023 15:26:07 +0100 Subject: [PATCH 03/11] SourceAccessor: Change the main interface from lstat() to maybeLstat() --- src/libexpr/primops.cc | 6 ++---- src/libfetchers/fs-input-accessor.cc | 4 ++-- src/libfetchers/memory-input-accessor.cc | 4 ++-- src/libutil/posix-source-accessor.cc | 8 ++++++-- src/libutil/posix-source-accessor.hh | 2 +- src/libutil/source-accessor.cc | 10 +++++----- src/libutil/source-accessor.hh | 4 ++-- .../lang/eval-fail-bad-string-interpolation-2.err.exp | 2 +- tests/functional/lang/eval-fail-nonexist-path.err.exp | 2 +- 9 files changed, 22 insertions(+), 20 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 704e7007b..e3c775d90 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1548,10 +1548,8 @@ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args, try { auto checked = state.checkSourcePath(path); - auto exists = checked.pathExists(); - if (exists && mustBeDir) { - exists = checked.lstat().type == InputAccessor::tDirectory; - } + auto st = checked.maybeLstat(); + auto exists = st && (!mustBeDir || st->type == SourceAccessor::tDirectory); v.mkBool(exists); } catch (SysError & e) { /* Don't give away info from errors while canonicalising diff --git a/src/libfetchers/fs-input-accessor.cc b/src/libfetchers/fs-input-accessor.cc index 7638d2d82..81be64482 100644 --- a/src/libfetchers/fs-input-accessor.cc +++ b/src/libfetchers/fs-input-accessor.cc @@ -36,11 +36,11 @@ struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor return isAllowed(absPath) && PosixSourceAccessor::pathExists(absPath); } - Stat lstat(const CanonPath & path) override + std::optional maybeLstat(const CanonPath & path) override { auto absPath = makeAbsPath(path); checkAllowed(absPath); - return PosixSourceAccessor::lstat(absPath); + return PosixSourceAccessor::maybeLstat(absPath); } DirEntries readDirectory(const CanonPath & path) override diff --git a/src/libfetchers/memory-input-accessor.cc b/src/libfetchers/memory-input-accessor.cc index 817d063ba..6468ece41 100644 --- a/src/libfetchers/memory-input-accessor.cc +++ b/src/libfetchers/memory-input-accessor.cc @@ -20,12 +20,12 @@ struct MemoryInputAccessorImpl : MemoryInputAccessor return i != files.end(); } - Stat lstat(const CanonPath & path) override + std::optional maybeLstat(const CanonPath & path) override { auto i = files.find(path); if (i != files.end()) return Stat { .type = tRegular, .isExecutable = false }; - throw Error("file '%s' does not exist", path); + return std::nullopt; } DirEntries readDirectory(const CanonPath & path) override diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 48b4fe626..8a8d64f3f 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -44,9 +44,13 @@ bool PosixSourceAccessor::pathExists(const CanonPath & path) return nix::pathExists(path.abs()); } -SourceAccessor::Stat PosixSourceAccessor::lstat(const CanonPath & path) +std::optional PosixSourceAccessor::maybeLstat(const CanonPath & path) { - auto st = nix::lstat(path.abs()); + struct stat st; + if (::lstat(path.c_str(), &st)) { + if (errno == ENOENT) return std::nullopt; + throw SysError("getting status of '%s'", showPath(path)); + } mtime = std::max(mtime, st.st_mtime); return Stat { .type = diff --git a/src/libutil/posix-source-accessor.hh b/src/libutil/posix-source-accessor.hh index 608f96ee2..cf087d26e 100644 --- a/src/libutil/posix-source-accessor.hh +++ b/src/libutil/posix-source-accessor.hh @@ -22,7 +22,7 @@ struct PosixSourceAccessor : SourceAccessor bool pathExists(const CanonPath & path) override; - Stat lstat(const CanonPath & path) override; + std::optional maybeLstat(const CanonPath & path) override; DirEntries readDirectory(const CanonPath & path) override; diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index d168a9667..5b0c7dd34 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -42,12 +42,12 @@ Hash SourceAccessor::hashPath( return sink.finish().first; } -std::optional SourceAccessor::maybeLstat(const CanonPath & path) +SourceAccessor::Stat SourceAccessor::lstat(const CanonPath & path) { - // FIXME: merge these into one operation. - if (!pathExists(path)) - return {}; - return lstat(path); + if (auto st = maybeLstat(path)) + return *st; + else + throw Error("path '%s' does not exist", showPath(path)); } std::string SourceAccessor::showPath(const CanonPath & path) diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index fd823aa39..80bc02b48 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -61,9 +61,9 @@ struct SourceAccessor bool isExecutable = false; // regular files only }; - virtual Stat lstat(const CanonPath & path) = 0; + Stat lstat(const CanonPath & path); - std::optional maybeLstat(const CanonPath & path); + virtual std::optional maybeLstat(const CanonPath & path) = 0; typedef std::optional DirEntry; diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp index dea119ae8..a287067cd 100644 --- a/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp @@ -1 +1 @@ -error: getting status of '/pwd/lang/fnord': No such file or directory +error: path '/pwd/lang/fnord' does not exist diff --git a/tests/functional/lang/eval-fail-nonexist-path.err.exp b/tests/functional/lang/eval-fail-nonexist-path.err.exp index dea119ae8..a287067cd 100644 --- a/tests/functional/lang/eval-fail-nonexist-path.err.exp +++ b/tests/functional/lang/eval-fail-nonexist-path.err.exp @@ -1 +1 @@ -error: getting status of '/pwd/lang/fnord': No such file or directory +error: path '/pwd/lang/fnord' does not exist From 53811238790f4bb5f9df74bb25047fe5b734a61f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Nov 2023 15:33:19 +0100 Subject: [PATCH 04/11] Unify DirEntries types --- src/libstore/binary-cache-store.cc | 4 ++-- src/libstore/fs-accessor.hh | 4 +++- src/libstore/local-fs-store.cc | 6 +++--- src/libstore/nar-accessor.cc | 8 ++++---- src/libstore/remote-fs-accessor.cc | 2 +- src/libstore/remote-fs-accessor.hh | 2 +- src/nix/ls.cc | 2 +- src/nix/why-depends.cc | 2 +- 8 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index f9abd8cbd..b61868413 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -231,14 +231,14 @@ ref BinaryCacheStore::addToStoreCommon( std::regex regex1("^[0-9a-f]{2}$"); std::regex regex2("^[0-9a-f]{38}\\.debug$"); - for (auto & s1 : narAccessor->readDirectory(buildIdDir)) { + for (auto & [s1, _type] : narAccessor->readDirectory(buildIdDir)) { auto dir = buildIdDir + "/" + s1; if (auto st = narAccessor->stat(dir); !st || st->type != SourceAccessor::tDirectory || !std::regex_match(s1, regex1)) continue; - for (auto & s2 : narAccessor->readDirectory(dir)) { + for (auto & [s2, _type] : narAccessor->readDirectory(dir)) { auto debugPath = dir + "/" + s2; if (auto st = narAccessor->stat(debugPath); !st || st->type != SourceAccessor::tRegular diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh index 1e951ec57..f04a92206 100644 --- a/src/libstore/fs-accessor.hh +++ b/src/libstore/fs-accessor.hh @@ -39,7 +39,9 @@ public: virtual std::optional stat(const Path & path) = 0; - virtual StringSet readDirectory(const Path & path) = 0; + using DirEntries = SourceAccessor::DirEntries; + + virtual DirEntries readDirectory(const Path & path) = 0; /** * Read a file inside the store. diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index bb83a9cd4..65cbb9e35 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -48,15 +48,15 @@ struct LocalStoreAccessor : public FSAccessor S_ISREG(st.st_mode) && st.st_mode & S_IXUSR}}; } - StringSet readDirectory(const Path & path) override + DirEntries readDirectory(const Path & path) override { auto realPath = toRealPath(path); auto entries = nix::readDirectory(realPath); - StringSet res; + DirEntries res; for (auto & entry : entries) - res.insert(entry.name); + res.insert_or_assign(entry.name, std::nullopt); return res; } diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 43a78a362..fe857a60e 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -190,16 +190,16 @@ struct NarAccessor : public FSAccessor return i->stat; } - StringSet readDirectory(const Path & path) override + DirEntries readDirectory(const Path & path) override { auto i = get(path); if (i.stat.type != Type::tDirectory) throw Error("path '%1%' inside NAR file is not a directory", path); - StringSet res; + DirEntries res; for (auto & child : i.children) - res.insert(child.first); + res.insert_or_assign(child.first, std::nullopt); return res; } @@ -264,7 +264,7 @@ json listNar(ref accessor, const Path & path, bool recurse) { obj["entries"] = json::object(); json &res2 = obj["entries"]; - for (auto & name : accessor->readDirectory(path)) { + for (auto & [name, type] : accessor->readDirectory(path)) { if (recurse) { res2[name] = listNar(accessor, path + "/" + name, true); } else diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 6c87ebeaa..21419700c 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -107,7 +107,7 @@ std::optional RemoteFSAccessor::stat(const Path & path) return res.first->stat(res.second); } -StringSet RemoteFSAccessor::readDirectory(const Path & path) +SourceAccessor::DirEntries RemoteFSAccessor::readDirectory(const Path & path) { auto res = fetch(path); return res.first->readDirectory(res.second); diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh index 5cf759aa0..8de3b7bcd 100644 --- a/src/libstore/remote-fs-accessor.hh +++ b/src/libstore/remote-fs-accessor.hh @@ -30,7 +30,7 @@ public: std::optional stat(const Path & path) override; - StringSet readDirectory(const Path & path) override; + DirEntries readDirectory(const Path & path) override; std::string readFile(const Path & path, bool requireValidPath = true) override; diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 8dc8a47b4..0ca08cea8 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -74,7 +74,7 @@ struct MixLs : virtual Args, MixJSON { if (st.type == FSAccessor::Type::tDirectory && !showDirectory) { auto names = accessor->readDirectory(curPath); - for (auto & name : names) + for (auto & [name, type] : names) showFile(curPath + "/" + name, relPath + "/" + name); } else showFile(curPath, relPath); diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 912ba72fb..04c1a0c1c 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -224,7 +224,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions if (st->type == FSAccessor::Type::tDirectory) { auto names = accessor->readDirectory(p); - for (auto & name : names) + for (auto & [name, type] : names) visitPath(p + "/" + name); } From 50aae0a14c5bbbde5785ead8f46b28333e6248ae Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Nov 2023 15:39:40 +0100 Subject: [PATCH 05/11] FSAccessor: Make the fileSize and narOffset fields optional The narOffset field only applies to NAR accessors. The fileSize field may be too expensive to compute for certain accessors (e.g. libgit). --- src/libstore/fs-accessor.hh | 4 ++-- src/libstore/nar-accessor.cc | 11 ++++++----- src/nix/ls.cc | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh index f04a92206..f6c002a2d 100644 --- a/src/libstore/fs-accessor.hh +++ b/src/libstore/fs-accessor.hh @@ -23,7 +23,7 @@ public: /** * For regular files only: the size of the file. */ - uint64_t fileSize = 0; + std::optional fileSize; /** * For regular files only: whether this is an executable. */ @@ -32,7 +32,7 @@ public: * For regular files only: the position of the contents of this * file in the NAR. */ - uint64_t narOffset = 0; + std::optional narOffset; }; virtual ~FSAccessor() { } diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index fe857a60e..f1be5606e 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -210,10 +210,10 @@ struct NarAccessor : public FSAccessor if (i.stat.type != Type::tRegular) throw Error("path '%1%' inside NAR file is not a regular file", path); - if (getNarBytes) return getNarBytes(i.stat.narOffset, i.stat.fileSize); + if (getNarBytes) return getNarBytes(*i.stat.narOffset, *i.stat.fileSize); assert(nar); - return std::string(*nar, i.stat.narOffset, i.stat.fileSize); + return std::string(*nar, *i.stat.narOffset, *i.stat.fileSize); } std::string readLink(const Path & path) override @@ -253,11 +253,12 @@ json listNar(ref accessor, const Path & path, bool recurse) switch (st->type) { case SourceAccessor::Type::tRegular: obj["type"] = "regular"; - obj["size"] = st->fileSize; + if (st->fileSize) + obj["size"] = *st->fileSize; if (st->isExecutable) obj["executable"] = true; - if (st->narOffset) - obj["narOffset"] = st->narOffset; + if (st->narOffset && *st->narOffset) + obj["narOffset"] = *st->narOffset; break; case SourceAccessor::Type::tDirectory: obj["type"] = "directory"; diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 0ca08cea8..da978f379 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -52,7 +52,7 @@ struct MixLs : virtual Args, MixJSON (st->isExecutable ? "-r-xr-xr-x" : "-r--r--r--") : st->type == FSAccessor::Type::tSymlink ? "lrwxrwxrwx" : "dr-xr-xr-x"; - auto line = fmt("%s %20d %s", tp, st->fileSize, relPath); + auto line = fmt("%s %20d %s", tp, st->fileSize.value_or(0), relPath); if (st->type == FSAccessor::Type::tSymlink) line += " -> " + accessor->readLink(curPath); logger->cout(line); From 581693bdea3981eb0b106c904c7a1fed7f7582ae Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Nov 2023 16:33:22 +0100 Subject: [PATCH 06/11] fmt(): Handle std::string_view --- src/libutil/fmt.hh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index 727255b45..ac72e47fb 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -44,6 +44,11 @@ inline std::string fmt(const std::string & s) return s; } +inline std::string fmt(std::string_view s) +{ + return std::string(s); +} + inline std::string fmt(const char * s) { return s; From 1a902f5fa7d4f268d0fec3e44a48ecc2445b3b6b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Nov 2023 17:09:28 +0100 Subject: [PATCH 07/11] Merge FSAccessor into SourceAccessor --- src/libstore/binary-cache-store.cc | 24 +++++------ src/libstore/binary-cache-store.hh | 2 +- src/libstore/derivations.cc | 1 - src/libstore/dummy-store.cc | 2 +- src/libstore/fs-accessor.hh | 58 --------------------------- src/libstore/legacy-ssh-store.cc | 2 +- src/libstore/local-fs-store.cc | 34 +++++++++------- src/libstore/local-fs-store.hh | 2 +- src/libstore/nar-accessor.cc | 64 ++++++++++++------------------ src/libstore/nar-accessor.hh | 11 ++--- src/libstore/remote-fs-accessor.cc | 28 ++++++------- src/libstore/remote-fs-accessor.hh | 21 +++++----- src/libstore/remote-store.cc | 2 +- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.cc | 8 ++-- src/libstore/store-api.hh | 4 +- src/libstore/uds-remote-store.hh | 4 +- src/libutil/source-accessor.cc | 5 +++ src/libutil/source-accessor.hh | 22 ++++++++-- src/nix/bundle.cc | 1 - src/nix/cat.cc | 13 +++--- src/nix/ls.cc | 54 +++++++++++-------------- src/nix/run.cc | 8 ++-- src/nix/why-depends.cc | 22 +++++----- tests/functional/nar-access.sh | 8 +++- 25 files changed, 178 insertions(+), 224 deletions(-) delete mode 100644 src/libstore/fs-accessor.hh diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index b61868413..dd9e2f3af 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -2,7 +2,7 @@ #include "binary-cache-store.hh" #include "compression.hh" #include "derivations.hh" -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "globals.hh" #include "nar-info.hh" #include "sync.hh" @@ -143,7 +143,7 @@ ref BinaryCacheStore::addToStoreCommon( write the compressed NAR to disk), into a HashSink (to get the NAR hash), and into a NarAccessor (to get the NAR listing). */ HashSink fileHashSink { htSHA256 }; - std::shared_ptr narAccessor; + std::shared_ptr narAccessor; HashSink narHashSink { htSHA256 }; { FdSink fileSink(fdTemp.get()); @@ -195,7 +195,7 @@ ref BinaryCacheStore::addToStoreCommon( if (writeNARListing) { nlohmann::json j = { {"version", 1}, - {"root", listNar(ref(narAccessor), "", true)}, + {"root", listNar(ref(narAccessor), CanonPath::root, true)}, }; upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json"); @@ -206,9 +206,9 @@ ref BinaryCacheStore::addToStoreCommon( specify the NAR file and member containing the debug info. */ if (writeDebugInfo) { - std::string buildIdDir = "/lib/debug/.build-id"; + CanonPath buildIdDir("lib/debug/.build-id"); - if (auto st = narAccessor->stat(buildIdDir); st && st->type == SourceAccessor::tDirectory) { + if (auto st = narAccessor->maybeLstat(buildIdDir); st && st->type == SourceAccessor::tDirectory) { ThreadPool threadPool(25); @@ -232,16 +232,16 @@ ref BinaryCacheStore::addToStoreCommon( std::regex regex2("^[0-9a-f]{38}\\.debug$"); for (auto & [s1, _type] : narAccessor->readDirectory(buildIdDir)) { - auto dir = buildIdDir + "/" + s1; + auto dir = buildIdDir + s1; - if (auto st = narAccessor->stat(dir); !st || st->type != SourceAccessor::tDirectory + if (narAccessor->lstat(dir).type != SourceAccessor::tDirectory || !std::regex_match(s1, regex1)) continue; for (auto & [s2, _type] : narAccessor->readDirectory(dir)) { - auto debugPath = dir + "/" + s2; + auto debugPath = dir + s2; - if (auto st = narAccessor->stat(debugPath); !st || st->type != SourceAccessor::tRegular + if ( narAccessor->lstat(debugPath).type != SourceAccessor::tRegular || !std::regex_match(s2, regex2)) continue; @@ -250,7 +250,7 @@ ref BinaryCacheStore::addToStoreCommon( std::string key = "debuginfo/" + buildId; std::string target = "../" + narInfo->url; - threadPool.enqueue(std::bind(doFile, std::string(debugPath, 1), key, target)); + threadPool.enqueue(std::bind(doFile, std::string(debugPath.rel()), key, target)); } } @@ -503,9 +503,9 @@ void BinaryCacheStore::registerDrvOutput(const Realisation& info) { upsertFile(filePath, info.toJSON().dump(), "application/json"); } -ref BinaryCacheStore::getFSAccessor() +ref BinaryCacheStore::getFSAccessor(bool requireValidPath) { - return make_ref(ref(shared_from_this()), localNarCache); + return make_ref(ref(shared_from_this()), requireValidPath, localNarCache); } void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSet & sigs) diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 218a888e3..cea2a571f 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -148,7 +148,7 @@ public: void narFromPath(const StorePath & path, Sink & sink) override; - ref getFSAccessor() override; + ref getFSAccessor(bool requireValidPath) override; void addSignatures(const StorePath & storePath, const StringSet & sigs) override; diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index a5ceb29dc..efdad18e1 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -6,7 +6,6 @@ #include "split.hh" #include "common-protocol.hh" #include "common-protocol-impl.hh" -#include "fs-accessor.hh" #include #include diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 74d6ed3b5..821cda399 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -72,7 +72,7 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store Callback> callback) noexcept override { callback(nullptr); } - virtual ref getFSAccessor() override + virtual ref getFSAccessor(bool requireValidPath) override { unsupported("getFSAccessor"); } }; diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh deleted file mode 100644 index f6c002a2d..000000000 --- a/src/libstore/fs-accessor.hh +++ /dev/null @@ -1,58 +0,0 @@ -#pragma once -///@file - -#include "types.hh" -#include "source-accessor.hh" - -#include - -namespace nix { - -/** - * An abstract class for accessing a filesystem-like structure, such - * as a (possibly remote) Nix store or the contents of a NAR file. - */ -class FSAccessor -{ -public: - using Type = SourceAccessor::Type; - - struct Stat - { - Type type; - /** - * For regular files only: the size of the file. - */ - std::optional fileSize; - /** - * For regular files only: whether this is an executable. - */ - bool isExecutable = false; - /** - * For regular files only: the position of the contents of this - * file in the NAR. - */ - std::optional narOffset; - }; - - virtual ~FSAccessor() { } - - virtual std::optional stat(const Path & path) = 0; - - using DirEntries = SourceAccessor::DirEntries; - - virtual DirEntries readDirectory(const Path & path) = 0; - - /** - * Read a file inside the store. - * - * If `requireValidPath` is set to `true` (the default), the path must be - * inside a valid store path, otherwise it just needs to be physically - * present (but not necessarily properly registered) - */ - virtual std::string readFile(const Path & path, bool requireValidPath = true) = 0; - - virtual std::string readLink(const Path & path) = 0; -}; - -} diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 38fdf118f..731457354 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -363,7 +363,7 @@ public: void ensurePath(const StorePath & path) override { unsupported("ensurePath"); } - virtual ref getFSAccessor() override + virtual ref getFSAccessor(bool requireValidPath) override { unsupported("getFSAccessor"); } /** diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 65cbb9e35..63497acbd 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -1,5 +1,5 @@ #include "archive.hh" -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "globals.hh" @@ -13,26 +13,31 @@ LocalFSStore::LocalFSStore(const Params & params) { } -struct LocalStoreAccessor : public FSAccessor +struct LocalStoreAccessor : public SourceAccessor { ref store; + bool requireValidPath; - LocalStoreAccessor(ref store) : store(store) { } + LocalStoreAccessor(ref store, bool requireValidPath) + : store(store) + , requireValidPath(requireValidPath) + { } - Path toRealPath(const Path & path, bool requireValidPath = true) + Path toRealPath(const CanonPath & path) { - auto storePath = store->toStorePath(path).first; + auto storePath = store->toStorePath(path.abs()).first; if (requireValidPath && !store->isValidPath(storePath)) throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); - return store->getRealStoreDir() + std::string(path, store->storeDir.size()); + return store->getRealStoreDir() + path.abs().substr(store->storeDir.size()); } - std::optional stat(const Path & path) override + std::optional maybeLstat(const CanonPath & path) override { auto realPath = toRealPath(path); + // FIXME: use PosixSourceAccessor. struct stat st; - if (lstat(realPath.c_str(), &st)) { + if (::lstat(realPath.c_str(), &st)) { if (errno == ENOENT || errno == ENOTDIR) return std::nullopt; throw SysError("getting status of '%1%'", path); } @@ -48,7 +53,7 @@ struct LocalStoreAccessor : public FSAccessor S_ISREG(st.st_mode) && st.st_mode & S_IXUSR}}; } - DirEntries readDirectory(const Path & path) override + DirEntries readDirectory(const CanonPath & path) override { auto realPath = toRealPath(path); @@ -61,21 +66,22 @@ struct LocalStoreAccessor : public FSAccessor return res; } - std::string readFile(const Path & path, bool requireValidPath = true) override + std::string readFile(const CanonPath & path) override { - return nix::readFile(toRealPath(path, requireValidPath)); + return nix::readFile(toRealPath(path)); } - std::string readLink(const Path & path) override + std::string readLink(const CanonPath & path) override { return nix::readLink(toRealPath(path)); } }; -ref LocalFSStore::getFSAccessor() +ref LocalFSStore::getFSAccessor(bool requireValidPath) { return make_ref(ref( - std::dynamic_pointer_cast(shared_from_this()))); + std::dynamic_pointer_cast(shared_from_this())), + requireValidPath); } void LocalFSStore::narFromPath(const StorePath & path, Sink & sink) diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index d6bda05d1..bf855b67e 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -43,7 +43,7 @@ public: LocalFSStore(const Params & params); void narFromPath(const StorePath & path, Sink & sink) override; - ref getFSAccessor() override; + ref getFSAccessor(bool requireValidPath) override; /** * Creates symlink from the `gcRoot` to the `storePath` and diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index f1be5606e..02993680f 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -11,7 +11,7 @@ namespace nix { struct NarMember { - FSAccessor::Stat stat; + SourceAccessor::Stat stat; std::string target; @@ -19,7 +19,7 @@ struct NarMember std::map children; }; -struct NarAccessor : public FSAccessor +struct NarAccessor : public SourceAccessor { std::optional nar; @@ -149,48 +149,36 @@ struct NarAccessor : public FSAccessor recurse(root, v); } - NarMember * find(const Path & path) + NarMember * find(const CanonPath & path) { - Path canon = path == "" ? "" : canonPath(path); NarMember * current = &root; - auto end = path.end(); - for (auto it = path.begin(); it != end; ) { - // because it != end, the remaining component is non-empty so we need - // a directory + + for (auto & i : path) { if (current->stat.type != Type::tDirectory) return nullptr; - - // skip slash (canonPath above ensures that this is always a slash) - assert(*it == '/'); - it += 1; - - // lookup current component - auto next = std::find(it, end, '/'); - auto child = current->children.find(std::string(it, next)); + auto child = current->children.find(std::string(i)); if (child == current->children.end()) return nullptr; current = &child->second; - - it = next; } return current; } - NarMember & get(const Path & path) { + NarMember & get(const CanonPath & path) { auto result = find(path); - if (result == nullptr) + if (!result) throw Error("NAR file does not contain path '%1%'", path); return *result; } - std::optional stat(const Path & path) override + std::optional maybeLstat(const CanonPath & path) override { auto i = find(path); - if (i == nullptr) + if (!i) return std::nullopt; return i->stat; } - DirEntries readDirectory(const Path & path) override + DirEntries readDirectory(const CanonPath & path) override { auto i = get(path); @@ -204,7 +192,7 @@ struct NarAccessor : public FSAccessor return res; } - std::string readFile(const Path & path, bool requireValidPath = true) override + std::string readFile(const CanonPath & path) override { auto i = get(path); if (i.stat.type != Type::tRegular) @@ -216,7 +204,7 @@ struct NarAccessor : public FSAccessor return std::string(*nar, *i.stat.narOffset, *i.stat.fileSize); } - std::string readLink(const Path & path) override + std::string readLink(const CanonPath & path) override { auto i = get(path); if (i.stat.type != Type::tSymlink) @@ -225,40 +213,38 @@ struct NarAccessor : public FSAccessor } }; -ref makeNarAccessor(std::string && nar) +ref makeNarAccessor(std::string && nar) { return make_ref(std::move(nar)); } -ref makeNarAccessor(Source & source) +ref makeNarAccessor(Source & source) { return make_ref(source); } -ref makeLazyNarAccessor(const std::string & listing, +ref makeLazyNarAccessor(const std::string & listing, GetNarBytes getNarBytes) { return make_ref(listing, getNarBytes); } using nlohmann::json; -json listNar(ref accessor, const Path & path, bool recurse) +json listNar(ref accessor, const CanonPath & path, bool recurse) { - auto st = accessor->stat(path); - if (!st) - throw Error("path '%s' does not exist in NAR", path); + auto st = accessor->lstat(path); json obj = json::object(); - switch (st->type) { + switch (st.type) { case SourceAccessor::Type::tRegular: obj["type"] = "regular"; - if (st->fileSize) - obj["size"] = *st->fileSize; - if (st->isExecutable) + if (st.fileSize) + obj["size"] = *st.fileSize; + if (st.isExecutable) obj["executable"] = true; - if (st->narOffset && *st->narOffset) - obj["narOffset"] = *st->narOffset; + if (st.narOffset && *st.narOffset) + obj["narOffset"] = *st.narOffset; break; case SourceAccessor::Type::tDirectory: obj["type"] = "directory"; @@ -267,7 +253,7 @@ json listNar(ref accessor, const Path & path, bool recurse) json &res2 = obj["entries"]; for (auto & [name, type] : accessor->readDirectory(path)) { if (recurse) { - res2[name] = listNar(accessor, path + "/" + name, true); + res2[name] = listNar(accessor, path + name, true); } else res2[name] = json::object(); } diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh index 5e19bd3c7..433774524 100644 --- a/src/libstore/nar-accessor.hh +++ b/src/libstore/nar-accessor.hh @@ -1,10 +1,11 @@ #pragma once ///@file +#include "source-accessor.hh" + #include #include -#include "fs-accessor.hh" namespace nix { @@ -14,9 +15,9 @@ struct Source; * Return an object that provides access to the contents of a NAR * file. */ -ref makeNarAccessor(std::string && nar); +ref makeNarAccessor(std::string && nar); -ref makeNarAccessor(Source & source); +ref makeNarAccessor(Source & source); /** * Create a NAR accessor from a NAR listing (in the format produced by @@ -26,7 +27,7 @@ ref makeNarAccessor(Source & source); */ typedef std::function GetNarBytes; -ref makeLazyNarAccessor( +ref makeLazyNarAccessor( const std::string & listing, GetNarBytes getNarBytes); @@ -34,6 +35,6 @@ ref makeLazyNarAccessor( * Write a JSON representation of the contents of a NAR (except file * contents). */ -nlohmann::json listNar(ref accessor, const Path & path, bool recurse); +nlohmann::json listNar(ref accessor, const CanonPath & path, bool recurse); } diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 21419700c..03e57a565 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -8,8 +8,9 @@ namespace nix { -RemoteFSAccessor::RemoteFSAccessor(ref store, const Path & cacheDir) +RemoteFSAccessor::RemoteFSAccessor(ref store, bool requireValidPath, const Path & cacheDir) : store(store) + , requireValidPath(requireValidPath) , cacheDir(cacheDir) { if (cacheDir != "") @@ -22,7 +23,7 @@ Path RemoteFSAccessor::makeCacheFile(std::string_view hashPart, const std::strin return fmt("%s/%s.%s", cacheDir, hashPart, ext); } -ref RemoteFSAccessor::addToCache(std::string_view hashPart, std::string && nar) +ref RemoteFSAccessor::addToCache(std::string_view hashPart, std::string && nar) { if (cacheDir != "") { try { @@ -38,7 +39,7 @@ ref RemoteFSAccessor::addToCache(std::string_view hashPart, std::str if (cacheDir != "") { try { - nlohmann::json j = listNar(narAccessor, "", true); + nlohmann::json j = listNar(narAccessor, CanonPath::root, true); writeFile(makeCacheFile(hashPart, "ls"), j.dump()); } catch (...) { ignoreException(); @@ -48,11 +49,10 @@ ref RemoteFSAccessor::addToCache(std::string_view hashPart, std::str return narAccessor; } -std::pair, Path> RemoteFSAccessor::fetch(const Path & path_, bool requireValidPath) +std::pair, CanonPath> RemoteFSAccessor::fetch(const CanonPath & path) { - auto path = canonPath(path_); - - auto [storePath, restPath] = store->toStorePath(path); + auto [storePath, restPath_] = store->toStorePath(path.abs()); + auto restPath = CanonPath(restPath_); if (requireValidPath && !store->isValidPath(storePath)) throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); @@ -63,7 +63,7 @@ std::pair, Path> RemoteFSAccessor::fetch(const Path & path_, boo std::string listing; Path cacheFile; - if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) { + if (cacheDir != "" && nix::pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) { try { listing = nix::readFile(makeCacheFile(storePath.hashPart(), "ls")); @@ -101,25 +101,25 @@ std::pair, Path> RemoteFSAccessor::fetch(const Path & path_, boo return {addToCache(storePath.hashPart(), std::move(sink.s)), restPath}; } -std::optional RemoteFSAccessor::stat(const Path & path) +std::optional RemoteFSAccessor::maybeLstat(const CanonPath & path) { auto res = fetch(path); - return res.first->stat(res.second); + return res.first->maybeLstat(res.second); } -SourceAccessor::DirEntries RemoteFSAccessor::readDirectory(const Path & path) +SourceAccessor::DirEntries RemoteFSAccessor::readDirectory(const CanonPath & path) { auto res = fetch(path); return res.first->readDirectory(res.second); } -std::string RemoteFSAccessor::readFile(const Path & path, bool requireValidPath) +std::string RemoteFSAccessor::readFile(const CanonPath & path) { - auto res = fetch(path, requireValidPath); + auto res = fetch(path); return res.first->readFile(res.second); } -std::string RemoteFSAccessor::readLink(const Path & path) +std::string RemoteFSAccessor::readLink(const CanonPath & path) { auto res = fetch(path); return res.first->readLink(res.second); diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh index 8de3b7bcd..d09762a53 100644 --- a/src/libstore/remote-fs-accessor.hh +++ b/src/libstore/remote-fs-accessor.hh @@ -1,40 +1,43 @@ #pragma once ///@file -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "ref.hh" #include "store-api.hh" namespace nix { -class RemoteFSAccessor : public FSAccessor +class RemoteFSAccessor : public SourceAccessor { ref store; - std::map> nars; + std::map> nars; + + bool requireValidPath; Path cacheDir; - std::pair, Path> fetch(const Path & path_, bool requireValidPath = true); + std::pair, CanonPath> fetch(const CanonPath & path); friend class BinaryCacheStore; Path makeCacheFile(std::string_view hashPart, const std::string & ext); - ref addToCache(std::string_view hashPart, std::string && nar); + ref addToCache(std::string_view hashPart, std::string && nar); public: RemoteFSAccessor(ref store, + bool requireValidPath = true, const /* FIXME: use std::optional */ Path & cacheDir = ""); - std::optional stat(const Path & path) override; + std::optional maybeLstat(const CanonPath & path) override; - DirEntries readDirectory(const Path & path) override; + DirEntries readDirectory(const CanonPath & path) override; - std::string readFile(const Path & path, bool requireValidPath = true) override; + std::string readFile(const CanonPath & path) override; - std::string readLink(const Path & path) override; + std::string readLink(const CanonPath & path) override; }; } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 7bdc25433..f16949f42 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -970,7 +970,7 @@ void RemoteStore::narFromPath(const StorePath & path, Sink & sink) copyNAR(conn->from, sink); } -ref RemoteStore::getFSAccessor() +ref RemoteStore::getFSAccessor(bool requireValidPath) { return make_ref(ref(shared_from_this())); } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index f0985fdc1..1cc11af86 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -185,7 +185,7 @@ protected: friend struct ConnectionHandle; - virtual ref getFSAccessor() override; + virtual ref getFSAccessor(bool requireValidPath) override; virtual void narFromPath(const StorePath & path, Sink & sink) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 0399120d1..665b5fed7 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1,5 +1,5 @@ #include "crypto.hh" -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "globals.hh" #include "derivations.hh" #include "store-api.hh" @@ -1338,12 +1338,12 @@ Derivation Store::derivationFromPath(const StorePath & drvPath) return readDerivation(drvPath); } -Derivation readDerivationCommon(Store& store, const StorePath& drvPath, bool requireValidPath) +static Derivation readDerivationCommon(Store & store, const StorePath & drvPath, bool requireValidPath) { - auto accessor = store.getFSAccessor(); + auto accessor = store.getFSAccessor(requireValidPath); try { return parseDerivation(store, - accessor->readFile(store.printStorePath(drvPath), requireValidPath), + accessor->readFile(CanonPath(store.printStorePath(drvPath))), Derivation::nameFromPath(drvPath)); } catch (FormatError & e) { throw Error("error parsing derivation '%s': %s", store.printStorePath(drvPath), e.msg()); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index e123fccc5..6aa317e3d 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -70,7 +70,7 @@ MakeError(InvalidStoreURI, Error); struct BasicDerivation; struct Derivation; -class FSAccessor; +struct SourceAccessor; class NarInfoDiskCache; class Store; @@ -703,7 +703,7 @@ public: /** * @return An object to access files in the Nix store. */ - virtual ref getFSAccessor() = 0; + virtual ref getFSAccessor(bool requireValidPath = true) = 0; /** * Repair the contents of the given path by redownloading it using diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index cdb28a001..a5ac9080a 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -35,8 +35,8 @@ public: static std::set uriSchemes() { return {"unix"}; } - ref getFSAccessor() override - { return LocalFSStore::getFSAccessor(); } + ref getFSAccessor(bool requireValidPath) override + { return LocalFSStore::getFSAccessor(requireValidPath); } void narFromPath(const StorePath & path, Sink & sink) override { LocalFSStore::narFromPath(path, sink); } diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index 5b0c7dd34..e2114e18f 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -10,6 +10,11 @@ SourceAccessor::SourceAccessor() { } +bool SourceAccessor::pathExists(const CanonPath & path) +{ + return maybeLstat(path).has_value(); +} + std::string SourceAccessor::readFile(const CanonPath & path) { StringSink sink; diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index 80bc02b48..1a4e80361 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -40,7 +40,7 @@ struct SourceAccessor Sink & sink, std::function sizeCallback = [](uint64_t size){}); - virtual bool pathExists(const CanonPath & path) = 0; + virtual bool pathExists(const CanonPath & path); enum Type { tRegular, tSymlink, tDirectory, @@ -57,8 +57,24 @@ struct SourceAccessor struct Stat { Type type = tMisc; - //uint64_t fileSize = 0; // regular files only - bool isExecutable = false; // regular files only + + /** + * For regular files only: the size of the file. Not all + * accessors return this since it may be too expensive to + * compute. + */ + std::optional fileSize; + + /** + * For regular files only: whether this is an executable. + */ + bool isExecutable = false; + + /** + * For regular files only: the position of the contents of this + * file in the NAR. Only returned by NAR accessors. + */ + std::optional narOffset; }; Stat lstat(const CanonPath & path); diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 504e35c81..54cc6a17f 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -4,7 +4,6 @@ #include "shared.hh" #include "store-api.hh" #include "local-fs-store.hh" -#include "fs-accessor.hh" #include "eval-inline.hh" using namespace nix; diff --git a/src/nix/cat.cc b/src/nix/cat.cc index b5fe2506f..6e5a736f2 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -1,6 +1,5 @@ #include "command.hh" #include "store-api.hh" -#include "fs-accessor.hh" #include "nar-accessor.hh" using namespace nix; @@ -9,14 +8,12 @@ struct MixCat : virtual Args { std::string path; - void cat(ref accessor) + void cat(ref accessor) { - if (auto st = accessor->stat(path)) { - if (st->type != FSAccessor::Type::tRegular) - throw Error("path '%1%' is not a regular file", path); - writeFull(STDOUT_FILENO, accessor->readFile(path)); - } else - throw Error("path '%1%' does not exist", path); + auto st = accessor->lstat(CanonPath(path)); + if (st.type != SourceAccessor::Type::tRegular) + throw Error("path '%1%' is not a regular file", path); + writeFull(STDOUT_FILENO, accessor->readFile(CanonPath(path))); } }; diff --git a/src/nix/ls.cc b/src/nix/ls.cc index da978f379..231456c9c 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -1,6 +1,5 @@ #include "command.hh" #include "store-api.hh" -#include "fs-accessor.hh" #include "nar-accessor.hh" #include "common-args.hh" #include @@ -39,63 +38,58 @@ struct MixLs : virtual Args, MixJSON }); } - void listText(ref accessor) + void listText(ref accessor) { - std::function doPath; + std::function doPath; - auto showFile = [&](const Path & curPath, const std::string & relPath) { + auto showFile = [&](const CanonPath & curPath, std::string_view relPath) { if (verbose) { - auto st = accessor->stat(curPath); - assert(st); + auto st = accessor->lstat(curPath); std::string tp = - st->type == FSAccessor::Type::tRegular ? - (st->isExecutable ? "-r-xr-xr-x" : "-r--r--r--") : - st->type == FSAccessor::Type::tSymlink ? "lrwxrwxrwx" : + st.type == SourceAccessor::Type::tRegular ? + (st.isExecutable ? "-r-xr-xr-x" : "-r--r--r--") : + st.type == SourceAccessor::Type::tSymlink ? "lrwxrwxrwx" : "dr-xr-xr-x"; - auto line = fmt("%s %20d %s", tp, st->fileSize.value_or(0), relPath); - if (st->type == FSAccessor::Type::tSymlink) + auto line = fmt("%s %20d %s", tp, st.fileSize.value_or(0), relPath); + if (st.type == SourceAccessor::Type::tSymlink) line += " -> " + accessor->readLink(curPath); logger->cout(line); - if (recursive && st->type == FSAccessor::Type::tDirectory) - doPath(*st, curPath, relPath, false); + if (recursive && st.type == SourceAccessor::Type::tDirectory) + doPath(st, curPath, relPath, false); } else { logger->cout(relPath); if (recursive) { - auto st = accessor->stat(curPath); - assert(st); - if (st->type == FSAccessor::Type::tDirectory) - doPath(*st, curPath, relPath, false); + auto st = accessor->lstat(curPath); + if (st.type == SourceAccessor::Type::tDirectory) + doPath(st, curPath, relPath, false); } } }; - doPath = [&](const FSAccessor::Stat & st, const Path & curPath, - const std::string & relPath, bool showDirectory) + doPath = [&](const SourceAccessor::Stat & st, const CanonPath & curPath, + std::string_view relPath, bool showDirectory) { - if (st.type == FSAccessor::Type::tDirectory && !showDirectory) { + if (st.type == SourceAccessor::Type::tDirectory && !showDirectory) { auto names = accessor->readDirectory(curPath); for (auto & [name, type] : names) - showFile(curPath + "/" + name, relPath + "/" + name); + showFile(curPath + name, relPath + "/" + name); } else showFile(curPath, relPath); }; - auto st = accessor->stat(path); - if (!st) - throw Error("path '%1%' does not exist", path); - doPath(*st, path, - st->type == FSAccessor::Type::tDirectory ? "." : std::string(baseNameOf(path)), + auto path2 = CanonPath(path); + auto st = accessor->lstat(path2); + doPath(st, path2, + st.type == SourceAccessor::Type::tDirectory ? "." : path2.baseName().value_or(""), showDirectory); } - void list(ref accessor) + void list(ref accessor) { - if (path == "/") path = ""; - if (json) { if (showDirectory) throw UsageError("'--directory' is useless with '--json'"); - logger->cout("%s", listNar(accessor, path, recursive)); + logger->cout("%s", listNar(accessor, CanonPath(path), recursive)); } else listText(accessor); } diff --git a/src/nix/run.cc b/src/nix/run.cc index 07806283c..1465e8cde 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -6,7 +6,7 @@ #include "derivations.hh" #include "local-store.hh" #include "finally.hh" -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "progress-bar.hh" #include "eval.hh" #include "build/personality.hh" @@ -119,9 +119,9 @@ struct CmdShell : InstallablesCommand, MixEnvironment if (true) unixPath.push_front(store->printStorePath(path) + "/bin"); - auto propPath = store->printStorePath(path) + "/nix-support/propagated-user-env-packages"; - if (auto st = accessor->stat(propPath); st && st->type == SourceAccessor::tRegular) { - for (auto & p : tokenizeString(readFile(propPath))) + auto propPath = CanonPath(store->printStorePath(path)) + "nix-support" + "propagated-user-env-packages"; + if (auto st = accessor->maybeLstat(propPath); st && st->type == SourceAccessor::tRegular) { + for (auto & p : tokenizeString(accessor->readFile(propPath))) todo.push(store->parseStorePath(p)); } } diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 04c1a0c1c..aecf65922 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -1,7 +1,7 @@ #include "command.hh" #include "store-api.hh" #include "progress-bar.hh" -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "shared.hh" #include @@ -175,7 +175,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions struct BailOut { }; printNode = [&](Node & node, const std::string & firstPad, const std::string & tailPad) { - auto pathS = store->printStorePath(node.path); + CanonPath pathS(store->printStorePath(node.path)); assert(node.dist != inf); if (precise) { @@ -183,7 +183,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions firstPad, node.visited ? "\e[38;5;244m" : "", firstPad != "" ? "→ " : "", - pathS); + pathS.abs()); } if (node.path == dependencyPath && !all @@ -210,25 +210,25 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions contain the reference. */ std::map hits; - std::function visitPath; + std::function visitPath; - visitPath = [&](const Path & p) { - auto st = accessor->stat(p); + visitPath = [&](const CanonPath & p) { + auto st = accessor->maybeLstat(p); assert(st); - auto p2 = p == pathS ? "/" : std::string(p, pathS.size() + 1); + auto p2 = p == pathS ? "/" : p.abs().substr(pathS.abs().size() + 1); auto getColour = [&](const std::string & hash) { return hash == dependencyPathHash ? ANSI_GREEN : ANSI_BLUE; }; - if (st->type == FSAccessor::Type::tDirectory) { + if (st->type == SourceAccessor::Type::tDirectory) { auto names = accessor->readDirectory(p); for (auto & [name, type] : names) - visitPath(p + "/" + name); + visitPath(p + name); } - else if (st->type == FSAccessor::Type::tRegular) { + else if (st->type == SourceAccessor::Type::tRegular) { auto contents = accessor->readFile(p); for (auto & hash : hashes) { @@ -246,7 +246,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions } } - else if (st->type == FSAccessor::Type::tSymlink) { + else if (st->type == SourceAccessor::Type::tSymlink) { auto target = accessor->readLink(p); for (auto & hash : hashes) { diff --git a/tests/functional/nar-access.sh b/tests/functional/nar-access.sh index d487d58d2..13d23c342 100644 --- a/tests/functional/nar-access.sh +++ b/tests/functional/nar-access.sh @@ -25,6 +25,12 @@ diff -u baz.cat-nar $storePath/foo/baz nix store cat $storePath/foo/baz > baz.cat-nar diff -u baz.cat-nar $storePath/foo/baz +# Check that 'nix store cat' fails on invalid store paths. +invalidPath="$(dirname $storePath)/99999999999999999999999999999999-foo" +mv $storePath $invalidPath +(! nix store cat $invalidPath/foo/baz) +mv $invalidPath $storePath + # Test --json. diff -u \ <(nix nar ls --json $narFile / | jq -S) \ @@ -46,7 +52,7 @@ diff -u \ <(echo '{"type":"regular","size":0}' | jq -S) # Test missing files. -expect 1 nix store ls --json -R $storePath/xyzzy 2>&1 | grep 'does not exist in NAR' +expect 1 nix store ls --json -R $storePath/xyzzy 2>&1 | grep 'does not exist' expect 1 nix store ls $storePath/xyzzy 2>&1 | grep 'does not exist' # Test failure to dump. From 2f5c1a27dc71275c1d4c96cff42beffed0d4d2f7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Nov 2023 17:22:25 +0100 Subject: [PATCH 08/11] LocalStoreAccessor: Reuse PosixSourceAccessor --- src/libstore/local-fs-store.cc | 48 ++++++++-------------------- src/libutil/posix-source-accessor.cc | 3 +- 2 files changed, 15 insertions(+), 36 deletions(-) diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 63497acbd..953f3a264 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -1,5 +1,5 @@ #include "archive.hh" -#include "source-accessor.hh" +#include "posix-source-accessor.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "globals.hh" @@ -13,7 +13,7 @@ LocalFSStore::LocalFSStore(const Params & params) { } -struct LocalStoreAccessor : public SourceAccessor +struct LocalStoreAccessor : PosixSourceAccessor { ref store; bool requireValidPath; @@ -23,57 +23,35 @@ struct LocalStoreAccessor : public SourceAccessor , requireValidPath(requireValidPath) { } - Path toRealPath(const CanonPath & path) + CanonPath toRealPath(const CanonPath & path) { - auto storePath = store->toStorePath(path.abs()).first; + auto [storePath, rest] = store->toStorePath(path.abs()); if (requireValidPath && !store->isValidPath(storePath)) throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); - return store->getRealStoreDir() + path.abs().substr(store->storeDir.size()); + return CanonPath(store->getRealStoreDir()) + storePath.to_string() + CanonPath(rest); } std::optional maybeLstat(const CanonPath & path) override { - auto realPath = toRealPath(path); - - // FIXME: use PosixSourceAccessor. - struct stat st; - if (::lstat(realPath.c_str(), &st)) { - if (errno == ENOENT || errno == ENOTDIR) return std::nullopt; - throw SysError("getting status of '%1%'", path); - } - - if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode)) - throw Error("file '%1%' has unsupported type", path); - - return {{ - S_ISREG(st.st_mode) ? Type::tRegular : - S_ISLNK(st.st_mode) ? Type::tSymlink : - Type::tDirectory, - S_ISREG(st.st_mode) ? (uint64_t) st.st_size : 0, - S_ISREG(st.st_mode) && st.st_mode & S_IXUSR}}; + return PosixSourceAccessor::maybeLstat(toRealPath(path)); } DirEntries readDirectory(const CanonPath & path) override { - auto realPath = toRealPath(path); - - auto entries = nix::readDirectory(realPath); - - DirEntries res; - for (auto & entry : entries) - res.insert_or_assign(entry.name, std::nullopt); - - return res; + return PosixSourceAccessor::readDirectory(toRealPath(path)); } - std::string readFile(const CanonPath & path) override + void readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) override { - return nix::readFile(toRealPath(path)); + return PosixSourceAccessor::readFile(toRealPath(path), sink, sizeCallback); } std::string readLink(const CanonPath & path) override { - return nix::readLink(toRealPath(path)); + return PosixSourceAccessor::readLink(toRealPath(path)); } }; diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 8a8d64f3f..d5e32d989 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -58,7 +58,8 @@ std::optional PosixSourceAccessor::maybeLstat(const CanonP S_ISDIR(st.st_mode) ? tDirectory : S_ISLNK(st.st_mode) ? tSymlink : tMisc, - .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR + .fileSize = S_ISREG(st.st_mode) ? std::optional(st.st_size) : std::nullopt, + .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR, }; } From e47984ce0b37cb8e00b66e85703c1ff72de80a73 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Nov 2023 20:19:08 +0100 Subject: [PATCH 09/11] Fix whitespace Co-authored-by: John Ericson --- src/libstore/binary-cache-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index dd9e2f3af..6a52c4c51 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -241,7 +241,7 @@ ref BinaryCacheStore::addToStoreCommon( for (auto & [s2, _type] : narAccessor->readDirectory(dir)) { auto debugPath = dir + s2; - if ( narAccessor->lstat(debugPath).type != SourceAccessor::tRegular + if (narAccessor->lstat(debugPath).type != SourceAccessor::tRegular || !std::regex_match(s2, regex2)) continue; From d7710a40be1a871859d331e9a50cc7f31797d792 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Nov 2023 20:05:23 -0400 Subject: [PATCH 10/11] flake: Temporarily get Nixpkgs ahead of Hydra MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/31ed632c692e6a36cfc18083b88ece892f863ed4' (2023-09-21) → 'github:NixOS/nixpkgs/9eb24edd6a0027fed010ccfe300a9734d029983c' (2023-11-01) --- flake.lock | 8 ++++---- flake.nix | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index 56df9c3fb..991cef1ee 100644 --- a/flake.lock +++ b/flake.lock @@ -34,16 +34,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1695283060, - "narHash": "sha256-CJz71xhCLlRkdFUSQEL0pIAAfcnWFXMzd9vXhPrnrEg=", + "lastModified": 1698876495, + "narHash": "sha256-nsQo2/mkDUFeAjuu92p0dEqhRvHHiENhkKVIV1y0/Oo=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "31ed632c692e6a36cfc18083b88ece892f863ed4", + "rev": "9eb24edd6a0027fed010ccfe300a9734d029983c", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-23.05-small", + "ref": "release-23.05", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index 398ba10a0..7cc4ed7fe 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,9 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; + # FIXME go back to nixos-23.05-small once + # https://github.com/NixOS/nixpkgs/pull/264875 is included. + inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-23.05"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; From d26c317b14bc3f0ce82d5a91acc63e62a8836dee Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 Nov 2023 13:40:54 +0100 Subject: [PATCH 11/11] Use expect Co-authored-by: John Ericson --- tests/functional/nar-access.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/nar-access.sh b/tests/functional/nar-access.sh index 13d23c342..218b521fb 100644 --- a/tests/functional/nar-access.sh +++ b/tests/functional/nar-access.sh @@ -28,7 +28,7 @@ diff -u baz.cat-nar $storePath/foo/baz # Check that 'nix store cat' fails on invalid store paths. invalidPath="$(dirname $storePath)/99999999999999999999999999999999-foo" mv $storePath $invalidPath -(! nix store cat $invalidPath/foo/baz) +expect 1 nix store cat $invalidPath/foo/baz mv $invalidPath $storePath # Test --json.