Merge remote-tracking branch 'origin/master' into flakes

nix-rust/src/c.rs

@@ -34,7 +34,8 @@ pub extern "C" fn ffi_StorePath_new(
     path: &str,
     store_dir: &str,
 ) -> Result<StorePath, error::CppException> {
-    StorePath::new(std::path::Path::new(path), store_dir).map_err(|err| err.into())
+    StorePath::new(std::path::Path::new(path), std::path::Path::new(store_dir))
+        .map_err(|err| err.into())
 }
 
 #[no_mangle]

nix-rust/src/error.rs

@@ -4,6 +4,7 @@ use std::fmt;
 pub enum Error {
     InvalidPath(crate::store::StorePath),
     BadStorePath(std::path::PathBuf),
+    NotInStore(std::path::PathBuf),
     BadNarInfo,
     BadBase32,
     StorePathNameEmpty,
@@ -46,6 +47,9 @@ impl fmt::Display for Error {
             Error::InvalidPath(_) => write!(f, "invalid path"),
             Error::BadNarInfo => write!(f, ".narinfo file is corrupt"),
             Error::BadStorePath(path) => write!(f, "path '{}' is not a store path", path.display()),
+            Error::NotInStore(path) => {
+                write!(f, "path '{}' is not in the Nix store", path.display())
+            }
             Error::BadBase32 => write!(f, "invalid base32 string"),
             Error::StorePathNameEmpty => write!(f, "store path name is empty"),
             Error::StorePathNameTooLong => {

nix-rust/src/store/path.rs

@@ -13,8 +13,10 @@ pub const STORE_PATH_HASH_BYTES: usize = 20;
 pub const STORE_PATH_HASH_CHARS: usize = 32;
 
 impl StorePath {
-    pub fn new(path: &Path, _store_dir: &str) -> Result<Self, Error> {
+    pub fn new(path: &Path, store_dir: &Path) -> Result<Self, Error> {
-        // FIXME: check store_dir
+        if path.parent() != Some(store_dir) {
+            return Err(Error::NotInStore(path.into()));
+        }
         Self::new_from_base_name(
             path.file_name()
                 .ok_or(Error::BadStorePath(path.into()))?

src/libexpr/get-drvs.cc

@@ -19,7 +19,7 @@ DrvInfo::DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs)
 DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs)
     : state(&state), attrs(nullptr), attrPath("")
 {
-    auto [drvPath, selectedOutputs] = store->parseDrvPathWithOutputs(drvPathWithOutputs);
+    auto [drvPath, selectedOutputs] = store->parsePathWithOutputs(drvPathWithOutputs);
 
     this->drvPath = store->printStorePath(drvPath);
 

src/libstore/binary-cache-store.cc

@@ -49,9 +49,9 @@ void BinaryCacheStore::init()
                     throw Error(format("binary cache '%s' is for Nix stores with prefix '%s', not '%s'")
                         % getUri() % value % storeDir);
             } else if (name == "WantMassQuery") {
-                wantMassQuery_ = value == "1";
+                wantMassQuery.setDefault(value == "1" ? "true" : "false");
             } else if (name == "Priority") {
-                string2Int(value, priority);
+                priority.setDefault(fmt("%d", std::stoi(value)));
             }
         }
     }

src/libstore/binary-cache-store.hh

@@ -52,11 +52,6 @@ public:
 
     std::shared_ptr<std::string> getFile(const std::string & path);
 
-protected:
-
-    bool wantMassQuery_ = false;
-    int priority = 50;
-
 public:
 
     virtual void init();
@@ -79,8 +74,6 @@ public:
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
     { unsupported("queryPathFromHashPart"); }
 
-    bool wantMassQuery() override { return wantMassQuery_; }
-
     void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
         RepairFlag repair, CheckSigsFlag checkSigs,
         std::shared_ptr<FSAccessor> accessor) override;
@@ -107,8 +100,6 @@ public:
 
     std::shared_ptr<std::string> getBuildLog(const StorePath & path) override;
 
-    int getPriority() override { return priority; }
-
 };
 
 MakeError(NoSuchBinaryCacheFile, Error);

src/libstore/daemon.cc

@@ -424,7 +424,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopBuildPaths: {
         std::vector<StorePathWithOutputs> drvs;
         for (auto & s : readStrings<Strings>(from))
-            drvs.push_back(store->parseDrvPathWithOutputs(s));
+            drvs.push_back(store->parsePathWithOutputs(s));
         BuildMode mode = bmNormal;
         if (GET_PROTOCOL_MINOR(clientVersion) >= 15) {
             mode = (BuildMode) readInt(from);
@@ -721,7 +721,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopQueryMissing: {
         std::vector<StorePathWithOutputs> targets;
         for (auto & s : readStrings<Strings>(from))
-            targets.push_back(store->parseDrvPathWithOutputs(s));
+            targets.push_back(store->parsePathWithOutputs(s));
         logger->startWork();
         StorePathSet willBuild, willSubstitute, unknown;
         unsigned long long downloadSize, narSize;

src/libstore/derivations.cc

@@ -375,16 +375,6 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput
 }
 
 
-StorePathWithOutputs Store::parseDrvPathWithOutputs(const std::string & s)
-{
-    size_t n = s.find("!");
-    return n == s.npos
-        ? StorePathWithOutputs{parseStorePath(s), std::set<string>()}
-        : StorePathWithOutputs{parseStorePath(std::string_view(s.data(), n)),
-            tokenizeString<std::set<string>>(string(s, n + 1), ",")};
-}
-
-
 std::string StorePathWithOutputs::to_string(const Store & store) const
 {
     return outputs.empty()

src/libstore/globals.cc

@@ -127,7 +127,7 @@ template<> void BaseSetting<SandboxMode>::set(const std::string & str)
     else throw UsageError("option '%s' has invalid value '%s'", name, str);
 }
 
-template<> std::string BaseSetting<SandboxMode>::to_string()
+template<> std::string BaseSetting<SandboxMode>::to_string() const
 {
     if (value == smEnabled) return "true";
     else if (value == smRelaxed) return "relaxed";

src/libstore/http-binary-cache-store.cc

@@ -42,13 +42,16 @@ public:
     void init() override
     {
         // FIXME: do this lazily?
-        if (!diskCache->cacheExists(cacheUri, wantMassQuery_, priority)) {
+        if (auto cacheInfo = diskCache->cacheExists(cacheUri)) {
+            wantMassQuery.setDefault(cacheInfo->wantMassQuery ? "true" : "false");
+            priority.setDefault(fmt("%d", cacheInfo->priority));
+        } else {
             try {
                 BinaryCacheStore::init();
             } catch (UploadToHTTP &) {
                 throw Error("'%s' does not appear to be a binary cache", cacheUri);
             }
-            diskCache->createCache(cacheUri, storeDir, wantMassQuery_, priority);
+            diskCache->createCache(cacheUri, storeDir, wantMassQuery, priority);
         }
     }
 

src/libstore/local-store.cc

@@ -833,7 +833,7 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths)
     for (auto & sub : getDefaultSubstituters()) {
         if (remaining.empty()) break;
         if (sub->storeDir != storeDir) continue;
-        if (!sub->wantMassQuery()) continue;
+        if (!sub->wantMassQuery) continue;
 
         auto valid = sub->queryValidPaths(remaining);
 

src/libstore/nar-info-disk-cache.cc

@@ -142,26 +142,26 @@ public:
         });
     }
 
-    bool cacheExists(const std::string & uri,
+    std::optional<CacheInfo> cacheExists(const std::string & uri) override
-        bool & wantMassQuery, int & priority) override
     {
-        return retrySQLite<bool>([&]() {
+        return retrySQLite<std::optional<CacheInfo>>([&]() -> std::optional<CacheInfo> {
             auto state(_state.lock());
 
             auto i = state->caches.find(uri);
             if (i == state->caches.end()) {
                 auto queryCache(state->queryCache.use()(uri));
-                if (!queryCache.next()) return false;
+                if (!queryCache.next())
+                    return std::nullopt;
                 state->caches.emplace(uri,
                     Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2) != 0, (int) queryCache.getInt(3)});
             }
 
             auto & cache(getCache(*state, uri));
 
-            wantMassQuery = cache.wantMassQuery;
+            return CacheInfo {
-            priority = cache.priority;
+                .wantMassQuery = cache.wantMassQuery,
-
+                .priority = cache.priority
-            return true;
+            };
         });
     }
 

src/libstore/nar-info-disk-cache.hh

@@ -15,8 +15,13 @@ public:
     virtual void createCache(const std::string & uri, const Path & storeDir,
         bool wantMassQuery, int priority) = 0;
 
-    virtual bool cacheExists(const std::string & uri,
+    struct CacheInfo
-        bool & wantMassQuery, int & priority) = 0;
+    {
+        bool wantMassQuery;
+        int priority;
+    };
+
+    virtual std::optional<CacheInfo> cacheExists(const std::string & uri) = 0;
 
     virtual std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo(
         const std::string & uri, const std::string & hashPart) = 0;

src/libstore/path.cc

@@ -96,4 +96,19 @@ StorePathSet singleton(const StorePath & path)
     return res;
 }
 
+std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s)
+{
+    size_t n = s.find("!");
+    return n == s.npos
+        ? std::make_pair(s, std::set<string>())
+        : std::make_pair(((std::string_view) s).substr(0, n),
+            tokenizeString<std::set<string>>(((std::string_view) s).substr(n + 1), ","));
+}
+
+StorePathWithOutputs Store::parsePathWithOutputs(const std::string & s)
+{
+    auto [path, outputs] = nix::parsePathWithOutputs(s);
+    return {parseStorePath(path), std::move(outputs)};
+}
+
 }

src/libstore/path.hh

@@ -7,6 +7,8 @@ namespace nix {
 /* See path.rs. */
 struct StorePath;
 
+struct Store;
+
 extern "C" {
     void ffi_StorePath_drop(void *);
     bool ffi_StorePath_less_than(const StorePath & a, const StorePath & b);
@@ -67,6 +69,28 @@ const size_t storePathHashLen = 32; // i.e. 160 bits
 /* Extension of derivations in the Nix store. */
 const std::string drvExtension = ".drv";
 
+struct StorePathWithOutputs
+{
+    StorePath path;
+    std::set<std::string> outputs;
+
+    StorePathWithOutputs(const StorePath & path, const std::set<std::string> & outputs = {})
+        : path(path.clone()), outputs(outputs)
+    { }
+
+    StorePathWithOutputs(StorePath && path, std::set<std::string> && outputs)
+        : path(std::move(path)), outputs(std::move(outputs))
+    { }
+
+    StorePathWithOutputs(const StorePathWithOutputs & other)
+        : path(other.path.clone()), outputs(other.outputs)
+    { }
+
+    std::string to_string(const Store & store) const;
+};
+
+std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s);
+
 }
 
 namespace std {

src/libstore/s3-binary-cache-store.cc

@@ -205,11 +205,12 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
 
     void init() override
     {
-        if (!diskCache->cacheExists(getUri(), wantMassQuery_, priority)) {
+        if (auto cacheInfo = diskCache->cacheExists(getUri())) {
-
+            wantMassQuery.setDefault(cacheInfo->wantMassQuery ? "true" : "false");
+            priority.setDefault(fmt("%d", cacheInfo->priority));
+        } else {
             BinaryCacheStore::init();
-
+            diskCache->createCache(getUri(), storeDir, wantMassQuery, priority);
-            diskCache->createCache(getUri(), storeDir, wantMassQuery_, priority);
         }
     }
 

src/libstore/sqlite.hh

@@ -103,8 +103,8 @@ void handleSQLiteBusy(const SQLiteBusy & e);
 
 /* Convenience function for retrying a SQLite transaction when the
    database is busy. */
-template<typename T>
+template<typename T, typename F>
-T retrySQLite(std::function<T()> fun)
+T retrySQLite(F && fun)
 {
     while (true) {
         try {

src/libstore/store-api.cc

@@ -39,9 +39,9 @@ Path Store::toStorePath(const Path & path) const
 }
 
 
-Path Store::followLinksToStore(const Path & _path) const
+Path Store::followLinksToStore(std::string_view _path) const
 {
-    Path path = absPath(_path);
+    Path path = absPath(std::string(_path));
     while (!isInStore(path)) {
         if (!isLink(path)) break;
         string target = readLink(path);
@@ -53,12 +53,19 @@ Path Store::followLinksToStore(const Path & _path) const
 }
 
 
-StorePath Store::followLinksToStorePath(const Path & path) const
+StorePath Store::followLinksToStorePath(std::string_view path) const
 {
     return parseStorePath(toStorePath(followLinksToStore(path)));
 }
 
 
+StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view path) const
+{
+    auto [path2, outputs] = nix::parsePathWithOutputs(path);
+    return StorePathWithOutputs(followLinksToStorePath(path2), std::move(outputs));
+}
+
+
 string storePathToHash(const Path & path)
 {
     auto base = baseNameOf(path);
@@ -963,7 +970,7 @@ std::list<ref<Store>> getDefaultSubstituters()
             addStore(uri);
 
         stores.sort([](ref<Store> & a, ref<Store> & b) {
-            return a->getPriority() < b->getPriority();
+            return a->priority < b->priority;
         });
 
         return stores;

src/libstore/store-api.hh

@@ -242,23 +242,6 @@ struct BuildResult
 };
 
 
-struct StorePathWithOutputs
-{
-    StorePath path;
-    std::set<std::string> outputs;
-
-    StorePathWithOutputs(const StorePath & path, const std::set<std::string> & outputs = {})
-        : path(path.clone()), outputs(outputs)
-    { }
-
-    StorePathWithOutputs(const StorePathWithOutputs & other)
-        : path(other.path.clone()), outputs(other.outputs)
-    { }
-
-    std::string to_string(const Store & store) const;
-};
-
-
 class Store : public std::enable_shared_from_this<Store>, public Config
 {
 public:
@@ -273,6 +256,10 @@ public:
 
     const Setting<bool> isTrusted{this, false, "trusted", "whether paths from this store can be used as substitutes even when they lack trusted signatures"};
 
+    Setting<int> priority{this, 0, "priority", "priority of this substituter (lower value means higher priority)"};
+
+    Setting<bool> wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"};
+
 protected:
 
     struct State
@@ -305,7 +292,7 @@ public:
     /* Split a string specifying a derivation and a set of outputs
        (/nix/store/hash-foo!out1,out2,...) into the derivation path
        and the outputs. */
-    StorePathWithOutputs parseDrvPathWithOutputs(const string & s);
+    StorePathWithOutputs parsePathWithOutputs(const string & s);
 
     /* Display a set of paths in human-readable form (i.e., between quotes
        and separated by commas). */
@@ -324,11 +311,13 @@ public:
     Path toStorePath(const Path & path) const;
 
     /* Follow symlinks until we end up with a path in the Nix store. */
-    Path followLinksToStore(const Path & path) const;
+    Path followLinksToStore(std::string_view path) const;
 
     /* Same as followLinksToStore(), but apply toStorePath() to the
        result. */
-    StorePath followLinksToStorePath(const Path & path) const;
+    StorePath followLinksToStorePath(std::string_view path) const;
+
+    StorePathWithOutputs followLinksToStorePathWithOutputs(std::string_view path) const;
 
     /* Constructs a unique store path name. */
     StorePath makeStorePath(const string & type,
@@ -438,8 +427,6 @@ public:
     virtual void querySubstitutablePathInfos(const StorePathSet & paths,
         SubstitutablePathInfos & infos) { return; };
 
-    virtual bool wantMassQuery() { return false; }
-
     /* Import a path into the store. */
     virtual void addToStore(const ValidPathInfo & info, Source & narSource,
         RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,
@@ -664,11 +651,6 @@ public:
         return 0;
     };
 
-    /* Get the priority of the store, used to order substituters. In
-       particular, binary caches can specify a priority field in their
-       "nix-cache-info" file. Lower value means higher priority. */
-    virtual int getPriority() { return 0; }
-
     virtual Path toRealPath(const Path & storePath)
     {
         return storePath;

src/libutil/config.cc

@@ -154,6 +154,11 @@ AbstractSetting::AbstractSetting(
 {
 }
 
+void AbstractSetting::setDefault(const std::string & str)
+{
+    if (!overriden) set(str);
+}
+
 void AbstractSetting::toJSON(JSONPlaceholder & out)
 {
     out.write(to_string());
@@ -185,7 +190,7 @@ template<> void BaseSetting<std::string>::set(const std::string & str)
     value = str;
 }
 
-template<> std::string BaseSetting<std::string>::to_string()
+template<> std::string BaseSetting<std::string>::to_string() const
 {
     return value;
 }
@@ -199,7 +204,7 @@ void BaseSetting<T>::set(const std::string & str)
 }
 
 template<typename T>
-std::string BaseSetting<T>::to_string()
+std::string BaseSetting<T>::to_string() const
 {
     static_assert(std::is_integral<T>::value, "Integer required.");
     return std::to_string(value);
@@ -215,7 +220,7 @@ template<> void BaseSetting<bool>::set(const std::string & str)
         throw UsageError("Boolean setting '%s' has invalid value '%s'", name, str);
 }
 
-template<> std::string BaseSetting<bool>::to_string()
+template<> std::string BaseSetting<bool>::to_string() const
 {
     return value ? "true" : "false";
 }
@@ -239,7 +244,7 @@ template<> void BaseSetting<Strings>::set(const std::string & str)
     value = tokenizeString<Strings>(str);
 }
 
-template<> std::string BaseSetting<Strings>::to_string()
+template<> std::string BaseSetting<Strings>::to_string() const
 {
     return concatStringsSep(" ", value);
 }
@@ -256,7 +261,7 @@ template<> void BaseSetting<StringSet>::set(const std::string & str)
     value = tokenizeString<StringSet>(str);
 }
 
-template<> std::string BaseSetting<StringSet>::to_string()
+template<> std::string BaseSetting<StringSet>::to_string() const
 {
     return concatStringsSep(" ", value);
 }

src/libutil/config.hh

@@ -115,6 +115,8 @@ public:
 
     bool overriden = false;
 
+    void setDefault(const std::string & str);
+
 protected:
 
     AbstractSetting(
@@ -131,13 +133,13 @@ protected:
 
     virtual void set(const std::string & value) = 0;
 
-    virtual std::string to_string() = 0;
+    virtual std::string to_string() const = 0;
 
     virtual void toJSON(JSONPlaceholder & out);
 
     virtual void convertToArg(Args & args, const std::string & category);
 
-    bool isOverriden() { return overriden; }
+    bool isOverriden() const { return overriden; }
 };
 
 /* A setting of type T. */
@@ -174,7 +176,7 @@ public:
         value = v;
     }
 
-    std::string to_string() override;
+    std::string to_string() const override;
 
     void convertToArg(Args & args, const std::string & category) override;
 

src/libutil/util.cc

@@ -1213,7 +1213,7 @@ void _interrupted()
 //////////////////////////////////////////////////////////////////////
 
 
-template<class C> C tokenizeString(const string & s, const string & separators)
+template<class C> C tokenizeString(std::string_view s, const string & separators)
 {
     C result;
     string::size_type pos = s.find_first_not_of(separators, 0);
@@ -1227,9 +1227,9 @@ template<class C> C tokenizeString(const string & s, const string & separators)
     return result;
 }
 
-template Strings tokenizeString(const string & s, const string & separators);
+template Strings tokenizeString(std::string_view s, const string & separators);
-template StringSet tokenizeString(const string & s, const string & separators);
+template StringSet tokenizeString(std::string_view s, const string & separators);
-template vector<string> tokenizeString(const string & s, const string & separators);
+template vector<string> tokenizeString(std::string_view s, const string & separators);
 
 
 string chomp(const string & s)

src/libutil/util.hh

@@ -346,7 +346,7 @@ MakeError(FormatError, Error);
 
 
 /* String tokenizer. */
-template<class C> C tokenizeString(const string & s, const string & separators = " \t\n\r");
+template<class C> C tokenizeString(std::string_view s, const string & separators = " \t\n\r");
 
 
 /* Concatenate the given strings with a separator between the

src/nix-store/nix-store.cc

@@ -128,7 +128,7 @@ static void opRealise(Strings opFlags, Strings opArgs)
 
     std::vector<StorePathWithOutputs> paths;
     for (auto & i : opArgs)
-        paths.push_back(store->parseDrvPathWithOutputs(i));
+        paths.push_back(store->followLinksToStorePathWithOutputs(i));
 
     unsigned long long downloadSize, narSize;
     StorePathSet willBuild, willSubstitute, unknown;
@@ -893,7 +893,7 @@ static void opServe(Strings opFlags, Strings opArgs)
 
                 std::vector<StorePathWithOutputs> paths;
                 for (auto & s : readStrings<Strings>(in))
-                    paths.emplace_back(store->parseDrvPathWithOutputs(s));
+                    paths.emplace_back(store->parsePathWithOutputs(s));
 
                 getBuildSettings();
 

tests/nix-build.sh

@@ -23,3 +23,6 @@ outPath2=$(nix-build $(nix-instantiate dependencies.nix) --no-out-link)
 
 outPath2=$(nix-build $(nix-instantiate dependencies.nix)!out --no-out-link)
 [[ $outPath = $outPath2 ]]
+
+outPath2=$(nix-store -r $(nix-instantiate --indirect --add-root $TEST_ROOT/indirect dependencies.nix)!out)
+[[ $outPath = $outPath2 ]]