Merge pull request #9413 from edolstra/fetchToStore-cache

Persistently cache `InputAccessor::fetchToStore()`

src/libfetchers/fetchers.cc

@@ -108,6 +108,11 @@ Input Input::fromAttrs(Attrs && attrs)
     return std::move(*res);
 }
 
+std::optional<std::string> Input::getFingerprint(ref<Store> store) const
+{
+    return scheme ? scheme->getFingerprint(store, *this) : std::nullopt;
+}
+
 ParsedURL Input::toURL() const
 {
     if (!scheme)

src/libfetchers/fetchers.hh

@@ -113,6 +113,12 @@ public:
     std::optional<Hash> getRev() const;
     std::optional<uint64_t> getRevCount() const;
     std::optional<time_t> getLastModified() const;
+
+    /**
+     * For locked inputs, return a string that uniquely specifies the
+     * content of the input (typically a commit hash or content hash).
+     */
+    std::optional<std::string> getFingerprint(ref<Store> store) const;
 };
 
 
@@ -180,6 +186,9 @@ struct InputScheme
 
     virtual bool isDirect(const Input & input) const
     { return true; }
+
+    virtual std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const
+    { return std::nullopt; }
 };
 
 void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);

src/libfetchers/git.cc

@@ -705,10 +705,22 @@ struct GitInputScheme : InputScheme
 
         auto repoInfo = getRepoInfo(input);
 
-        return
+        auto [accessor, final] =
             input.getRef() || input.getRev() || !repoInfo.isLocal
             ? getAccessorFromCommit(store, repoInfo, std::move(input))
             : getAccessorFromWorkdir(store, repoInfo, std::move(input));
+
+        accessor->fingerprint = final.getFingerprint(store);
+
+        return {accessor, std::move(final)};
+    }
+
+    std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
+    {
+        if (auto rev = input.getRev())
+            return rev->gitRev() + (getSubmodulesAttr(input) ? ";s" : "");
+        else
+            return std::nullopt;
     }
 };
 

src/libfetchers/github.cc

@@ -229,6 +229,14 @@ struct GitArchiveInputScheme : InputScheme
     {
         return Xp::Flakes;
     }
+
+    std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
+    {
+        if (auto rev = input.getRev())
+            return rev->gitRev();
+        else
+            return std::nullopt;
+    }
 };
 
 struct GitHubInputScheme : GitArchiveInputScheme

src/libfetchers/input-accessor.cc

@@ -1,5 +1,6 @@
 #include "input-accessor.hh"
 #include "store-api.hh"
+#include "cache.hh"
 
 namespace nix {
 
@@ -11,6 +12,27 @@ StorePath InputAccessor::fetchToStore(
     PathFilter * filter,
     RepairFlag repair)
 {
+    // FIXME: add an optimisation for the case where the accessor is
+    // an FSInputAccessor pointing to a store path.
+
+    std::optional<fetchers::Attrs> cacheKey;
+
+    if (!filter && fingerprint) {
+        cacheKey = fetchers::Attrs{
+            {"_what", "fetchToStore"},
+            {"store", store->storeDir},
+            {"name", std::string(name)},
+            {"fingerprint", *fingerprint},
+            {"method", (uint8_t) method},
+            {"path", path.abs()}
+        };
+        if (auto res = fetchers::getCache()->lookup(store, *cacheKey)) {
+            debug("store path cache hit for '%s'", showPath(path));
+            return res->second;
+        }
+    } else
+        debug("source path '%s' is uncacheable", showPath(path));
+
     Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", showPath(path)));
 
     auto source = sinkToSource([&](Sink & sink) {
@@ -25,6 +47,9 @@ StorePath InputAccessor::fetchToStore(
         ? store->computeStorePathFromDump(*source, name, method, htSHA256).first
         : store->addToStoreFromDump(*source, name, method, htSHA256, repair);
 
+    if (cacheKey)
+        fetchers::getCache()->add(store, *cacheKey, {}, storePath, true);
+
     return storePath;
 }
 

src/libfetchers/input-accessor.hh

@@ -18,6 +18,8 @@ class Store;
 
 struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this<InputAccessor>
 {
+    std::optional<std::string> fingerprint;
+
     /**
      * Return the maximum last-modified time of the files in this
      * tree, if available.

src/libfetchers/mercurial.cc

@@ -339,6 +339,14 @@ struct MercurialInputScheme : InputScheme
 
         return makeResult(infoAttrs, std::move(storePath));
     }
+
+    std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
+    {
+        if (auto rev = input.getRev())
+            return rev->gitRev();
+        else
+            return std::nullopt;
+    }
 };
 
 static auto rMercurialInputScheme = OnStartup([] { registerInputScheme(std::make_unique<MercurialInputScheme>()); });

src/libstore/content-address.hh

@@ -39,12 +39,12 @@ enum struct FileIngestionMethod : uint8_t {
     /**
      * Flat-file hashing. Directly ingest the contents of a single file
      */
-    Flat = false,
+    Flat = 0,
     /**
      * Recursive (or NAR) hashing. Serializes the file-system object in Nix
      * Archive format and ingest that
      */
-    Recursive = true
+    Recursive = 1
 };
 
 /**