Unified fetcher caching system

src/libexpr/common-eval-args.cc

@@ -44,7 +44,7 @@ MixEvalArgs::MixEvalArgs()
       .handler([&](std::vector<std::string> ss) {
           auto from = parseFlakeRef(ss[0], absPath("."));
           auto to = parseFlakeRef(ss[1], absPath("."));
-          fetchers::Input::Attrs extraAttrs;
+          fetchers::Attrs extraAttrs;
           if (to.subdir != "") extraAttrs["dir"] = to.subdir;
           fetchers::overrideRegistry(from.input, to.input, extraAttrs);
       });

src/libexpr/flake/flake.cc

@@ -132,10 +132,10 @@ static FlakeInput parseFlakeInput(EvalState & state,
     auto sFlake = state.symbols.create("flake");
     auto sFollows = state.symbols.create("follows");
 
-    fetchers::Input::Attrs attrs;
+    fetchers::Attrs attrs;
     std::optional<std::string> url;
 
-    for (Attr attr : *(value->attrs)) {
+    for (nix::Attr attr : *(value->attrs)) {
         try {
             if (attr.name == sUrl || attr.name == sUri) {
                 expectType(state, tString, *attr.value, *attr.pos);
@@ -188,7 +188,7 @@ static std::map<FlakeId, FlakeInput> parseFlakeInputs(
 
     expectType(state, tAttrs, *value, pos);
 
-    for (Attr & inputAttr : *(*value).attrs) {
+    for (nix::Attr & inputAttr : *(*value).attrs) {
         inputs.emplace(inputAttr.name,
             parseFlakeInput(state,
                 inputAttr.name,

src/libexpr/flake/flakeref.cc

@@ -19,7 +19,7 @@ std::string FlakeRef::to_string() const
     return input->to_string();
 }
 
-fetchers::Input::Attrs FlakeRef::toAttrs() const
+fetchers::Attrs FlakeRef::toAttrs() const
 {
     auto attrs = input->toAttrs();
     if (subdir != "")
@@ -168,7 +168,7 @@ std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment(
     }
 }
 
-FlakeRef FlakeRef::fromAttrs(const fetchers::Input::Attrs & attrs)
+FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs)
 {
     auto attrs2(attrs);
     attrs2.erase("dir");

src/libexpr/flake/flakeref.hh

@@ -29,11 +29,11 @@ struct FlakeRef
     // FIXME: change to operator <<.
     std::string to_string() const;
 
-    fetchers::Input::Attrs toAttrs() const;
+    fetchers::Attrs toAttrs() const;
 
     FlakeRef resolve(ref<Store> store) const;
 
-    static FlakeRef fromAttrs(const fetchers::Input::Attrs & attrs);
+    static FlakeRef fromAttrs(const fetchers::Attrs & attrs);
 
     std::pair<fetchers::Tree, FlakeRef> fetchTree(ref<Store> store) const;
 };

src/libexpr/primops/fetchTree.cc

@@ -52,7 +52,7 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
     if (args[0]->type == tAttrs) {
         state.forceAttrs(*args[0], pos);
 
-        fetchers::Input::Attrs attrs;
+        fetchers::Attrs attrs;
 
         for (auto & attr : *args[0]->attrs) {
             state.forceValue(*attr.value);

src/libstore/fetchers/attrs.cc

@@ -0,0 +1,71 @@
+#include "attrs.hh"
+#include "fetchers.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix::fetchers {
+
+Attrs jsonToAttrs(const nlohmann::json & json)
+{
+    Attrs attrs;
+
+    for (auto & i : json.items()) {
+        if (i.value().is_number())
+            attrs.emplace(i.key(), i.value().get<int64_t>());
+        else if (i.value().is_string())
+            attrs.emplace(i.key(), i.value().get<std::string>());
+        else
+            throw Error("unsupported input attribute type in lock file");
+    }
+
+    return attrs;
+}
+
+nlohmann::json attrsToJson(const Attrs & attrs)
+{
+    nlohmann::json json;
+    for (auto & attr : attrs) {
+        if (auto v = std::get_if<int64_t>(&attr.second)) {
+            json[attr.first] = *v;
+        } else if (auto v = std::get_if<std::string>(&attr.second)) {
+            json[attr.first] = *v;
+        } else abort();
+    }
+    return json;
+}
+
+std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::string & name)
+{
+    auto i = attrs.find(name);
+    if (i == attrs.end()) return {};
+    if (auto v = std::get_if<std::string>(&i->second))
+        return *v;
+    throw Error("input attribute '%s' is not a string", name);
+}
+
+std::string getStrAttr(const Attrs & attrs, const std::string & name)
+{
+    auto s = maybeGetStrAttr(attrs, name);
+    if (!s)
+        throw Error("input attribute '%s' is missing", name);
+    return *s;
+}
+
+std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name)
+{
+    auto i = attrs.find(name);
+    if (i == attrs.end()) return {};
+    if (auto v = std::get_if<int64_t>(&i->second))
+        return *v;
+    throw Error("input attribute '%s' is not a string", name);
+}
+
+int64_t getIntAttr(const Attrs & attrs, const std::string & name)
+{
+    auto s = maybeGetIntAttr(attrs, name);
+    if (!s)
+        throw Error("input attribute '%s' is missing", name);
+    return *s;
+}
+
+}

src/libstore/fetchers/attrs.hh

@@ -0,0 +1,26 @@
+#pragma once
+
+#include "types.hh"
+
+#include <variant>
+
+#include <nlohmann/json_fwd.hpp>
+
+namespace nix::fetchers {
+
+typedef std::variant<std::string, int64_t> Attr;
+typedef std::map<std::string, Attr> Attrs;
+
+Attrs jsonToAttrs(const nlohmann::json & json);
+
+nlohmann::json attrsToJson(const Attrs & attrs);
+
+std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::string & name);
+
+std::string getStrAttr(const Attrs & attrs, const std::string & name);
+
+std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name);
+
+int64_t getIntAttr(const Attrs & attrs, const std::string & name);
+
+}

src/libstore/fetchers/cache.cc

@@ -0,0 +1,109 @@
+#include "fetchers/cache.hh"
+#include "sqlite.hh"
+#include "sync.hh"
+#include "store-api.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix::fetchers {
+
+static const char * schema = R"sql(
+
+create table if not exists Cache (
+    input     text not null,
+    info      text not null,
+    path      text not null,
+    immutable integer not null,
+    timestamp integer not null,
+    primary key (input)
+);
+)sql";
+
+struct CacheImpl : Cache
+{
+    struct State
+    {
+        SQLite db;
+        SQLiteStmt add, lookup;
+    };
+
+    Sync<State> _state;
+
+    CacheImpl()
+    {
+        auto state(_state.lock());
+
+        auto dbPath = getCacheDir() + "/nix/fetcher-cache-v1.sqlite";
+        createDirs(dirOf(dbPath));
+
+        state->db = SQLite(dbPath);
+        state->db.isCache();
+        state->db.exec(schema);
+
+        state->add.create(state->db,
+            "insert or replace into Cache(input, info, path, immutable, timestamp) values (?, ?, ?, ?, ?)");
+
+        state->lookup.create(state->db,
+            "select info, path, immutable, timestamp from Cache where input = ?");
+    }
+
+    void add(
+        ref<Store> store,
+        const Attrs & inAttrs,
+        const Attrs & infoAttrs,
+        const StorePath & storePath,
+        bool immutable) override
+    {
+        _state.lock()->add.use()
+            (attrsToJson(inAttrs).dump())
+            (attrsToJson(infoAttrs).dump())
+            (store->printStorePath(storePath))
+            (immutable)
+            (time(0)).exec();
+    }
+
+    std::optional<std::pair<Attrs, StorePath>> lookup(
+        ref<Store> store,
+        const Attrs & inAttrs) override
+    {
+        auto state(_state.lock());
+
+        auto inAttrsJson = attrsToJson(inAttrs).dump();
+
+        auto stmt(state->lookup.use()(inAttrsJson));
+        if (!stmt.next()) {
+            debug("did not find cache entry for '%s'", inAttrsJson);
+            return {};
+        }
+
+        auto infoJson = stmt.getStr(0);
+        auto storePath = store->parseStorePath(stmt.getStr(1));
+        auto immutable = stmt.getInt(2) != 0;
+        auto timestamp = stmt.getInt(3);
+
+        if (!immutable && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0))) {
+            debug("ignoring expired cache entry '%s'", inAttrsJson);
+            return {};
+        }
+
+        store->addTempRoot(storePath);
+        if (!store->isValidPath(storePath)) {
+            // FIXME: we could try to substitute 'storePath'.
+            debug("ignoring disappeared cache entry '%s'", inAttrsJson);
+            return {};
+        }
+
+        debug("using cache entry '%s' -> '%s', '%s'",
+            inAttrsJson, infoJson, store->printStorePath(storePath));
+
+        return {{jsonToAttrs(nlohmann::json::parse(infoJson)), std::move(storePath)}};
+    }
+};
+
+ref<Cache> getCache()
+{
+    static auto cache = std::make_shared<CacheImpl>();
+    return ref<Cache>(cache);
+}
+
+}

src/libstore/fetchers/cache.hh

@@ -0,0 +1,24 @@
+#pragma once
+
+#include "types.hh"
+#include "fetchers/fetchers.hh"
+
+namespace nix::fetchers {
+
+struct Cache
+{
+    virtual void add(
+        ref<Store> store,
+        const Attrs & inAttrs,
+        const Attrs & infoAttrs,
+        const StorePath & storePath,
+        bool immutable) = 0;
+
+    virtual std::optional<std::pair<Attrs, StorePath>> lookup(
+        ref<Store> store,
+        const Attrs & inAttrs) = 0;
+};
+
+ref<Cache> getCache();
+
+}

src/libstore/fetchers/fetchers.cc

@@ -28,7 +28,7 @@ std::unique_ptr<Input> inputFromURL(const std::string & url)
     return inputFromURL(parseURL(url));
 }
 
-std::unique_ptr<Input> inputFromAttrs(const Input::Attrs & attrs)
+std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs)
 {
     for (auto & inputScheme : *inputSchemes) {
         auto res = inputScheme->inputFromAttrs(attrs);
@@ -42,36 +42,7 @@ std::unique_ptr<Input> inputFromAttrs(const Input::Attrs & attrs)
     throw Error("input '%s' is unsupported", attrsToJson(attrs));
 }
 
-Input::Attrs jsonToAttrs(const nlohmann::json & json)
-{
-    fetchers::Input::Attrs attrs;
-
-    for (auto & i : json.items()) {
-        if (i.value().is_number())
-            attrs.emplace(i.key(), i.value().get<int64_t>());
-        else if (i.value().is_string())
-            attrs.emplace(i.key(), i.value().get<std::string>());
-        else
-            throw Error("unsupported input attribute type in lock file");
-    }
-
-    return attrs;
-}
-
-nlohmann::json attrsToJson(const fetchers::Input::Attrs & attrs)
-{
-    nlohmann::json json;
-    for (auto & attr : attrs) {
-        if (auto v = std::get_if<int64_t>(&attr.second)) {
-            json[attr.first] = *v;
-        } else if (auto v = std::get_if<std::string>(&attr.second)) {
-            json[attr.first] = *v;
-        } else abort();
-    }
-    return json;
-}
-
-Input::Attrs Input::toAttrs() const
+Attrs Input::toAttrs() const
 {
     auto attrs = toAttrsInternal();
     if (narHash)
@@ -80,23 +51,6 @@ Input::Attrs Input::toAttrs() const
     return attrs;
 }
 
-std::optional<std::string> maybeGetStrAttr(const Input::Attrs & attrs, const std::string & name)
-{
-    auto i = attrs.find(name);
-    if (i == attrs.end()) return {};
-    if (auto v = std::get_if<std::string>(&i->second))
-        return *v;
-    throw Error("input attribute '%s' is not a string", name);
-}
-
-std::string getStrAttr(const Input::Attrs & attrs, const std::string & name)
-{
-    auto s = maybeGetStrAttr(attrs, name);
-    if (!s)
-        throw Error("input attribute '%s' is missing", name);
-    return *s;
-}
-
 std::pair<Tree, std::shared_ptr<const Input>> Input::fetchTree(ref<Store> store) const
 {
     auto [tree, input] = fetchTreeInternal(store);

src/libstore/fetchers/fetchers.hh

@@ -4,11 +4,9 @@
 #include "hash.hh"
 #include "path.hh"
 #include "tree-info.hh"
+#include "attrs.hh"
 
 #include <memory>
-#include <variant>
-
-#include <nlohmann/json_fwd.hpp>
 
 namespace nix { class Store; }
 
@@ -49,9 +47,6 @@ struct Input : std::enable_shared_from_this<Input>
 
     virtual std::string to_string() const = 0;
 
-    typedef std::variant<std::string, int64_t> Attr;
-    typedef std::map<std::string, Attr> Attrs;
-
     Attrs toAttrs() const;
 
     std::pair<Tree, std::shared_ptr<const Input>> fetchTree(ref<Store> store) const;
@@ -87,23 +82,15 @@ struct InputScheme
 
     virtual std::unique_ptr<Input> inputFromURL(const ParsedURL & url) = 0;
 
-    virtual std::unique_ptr<Input> inputFromAttrs(const Input::Attrs & attrs) = 0;
+    virtual std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) = 0;
 };
 
 std::unique_ptr<Input> inputFromURL(const ParsedURL & url);
 
 std::unique_ptr<Input> inputFromURL(const std::string & url);
 
-std::unique_ptr<Input> inputFromAttrs(const Input::Attrs & attrs);
+std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs);
 
 void registerInputScheme(std::unique_ptr<InputScheme> && fetcher);
 
-Input::Attrs jsonToAttrs(const nlohmann::json & json);
-
-nlohmann::json attrsToJson(const Input::Attrs & attrs);
-
-std::optional<std::string> maybeGetStrAttr(const Input::Attrs & attrs, const std::string & name);
-
-std::string getStrAttr(const Input::Attrs & attrs, const std::string & name);
-
 }

src/libstore/fetchers/git.cc

@@ -420,7 +420,7 @@ struct GitInputScheme : InputScheme
         if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4);
         url2.query.clear();
 
-        Input::Attrs attrs;
+        Attrs attrs;
         attrs.emplace("type", "git");
 
         for (auto &[name, value] : url.query) {
@@ -435,7 +435,7 @@ struct GitInputScheme : InputScheme
         return inputFromAttrs(attrs);
     }
 
-    std::unique_ptr<Input> inputFromAttrs(const Input::Attrs & attrs) override
+    std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
     {
         if (maybeGetStrAttr(attrs, "type") != "git") return {};
 

src/libstore/fetchers/github.cc

@@ -1,8 +1,9 @@
-#include "fetchers.hh"
 #include "download.hh"
+#include "fetchers/cache.hh"
+#include "fetchers/fetchers.hh"
+#include "fetchers/parse.hh"
+#include "fetchers/regex.hh"
 #include "globals.hh"
-#include "parse.hh"
-#include "regex.hh"
 #include "store-api.hh"
 
 #include <nlohmann/json.hpp>
@@ -72,17 +73,36 @@ struct GitHubInput : Input
     std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
     {
         auto rev = this->rev;
+        auto ref = this->ref.value_or("master");
 
-    #if 0
-        if (rev) {
-            if (auto gitInfo = lookupGitInfo(store, "source", *rev))
-                return *gitInfo;
+        Attrs mutableAttrs({
+            {"type", "github"},
+            {"owner", owner},
+            {"repo", repo},
+            {"ref", ref},
+        });
+
+        if (!rev) {
+            if (auto res = getCache()->lookup(store, mutableAttrs)) {
+                auto input = std::make_shared<GitHubInput>(*this);
+                input->ref = {};
+                input->rev = Hash(getStrAttr(res->first, "rev"), htSHA1);
+                return {
+                    Tree{
+                        .actualPath = store->toRealPath(res->second),
+                        .storePath = std::move(res->second),
+                        .info = TreeInfo {
+                            .lastModified = getIntAttr(res->first, "lastModified"),
+                        },
+                    },
+                    input
+                };
+            }
         }
-    #endif
 
         if (!rev) {
             auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s",
-                owner, repo, ref ? *ref : "master");
+                owner, repo, ref);
             CachedDownloadRequest request(url);
             request.ttl = rev ? 1000000000 : settings.tarballTtl;
             auto result = getDownloader()->downloadCached(store, request);
@@ -91,6 +111,28 @@ struct GitHubInput : Input
             debug("HEAD revision for '%s' is %s", url, rev->gitRev());
         }
 
+        auto input = std::make_shared<GitHubInput>(*this);
+        input->ref = {};
+        input->rev = *rev;
+
+        Attrs immutableAttrs({
+            {"type", "git-tarball"},
+            {"rev", rev->gitRev()},
+        });
+
+        if (auto res = getCache()->lookup(store, immutableAttrs)) {
+            return {
+                Tree{
+                    .actualPath = store->toRealPath(res->second),
+                    .storePath = std::move(res->second),
+                    .info = TreeInfo {
+                        .lastModified = getIntAttr(res->first, "lastModified"),
+                    },
+                },
+                input
+            };
+        }
+
         // FIXME: use regular /archive URLs instead? api.github.com
         // might have stricter rate limits.
 
@@ -118,14 +160,25 @@ struct GitHubInput : Input
             },
         };
 
-    #if 0
-        // FIXME: this can overwrite a cache file that contains a revCount.
-        cacheGitInfo("source", gitInfo);
-    #endif
+        Attrs infoAttrs({
+            {"rev", rev->gitRev()},
+            {"lastModified", *result.info.lastModified}
+        });
 
-        auto input = std::make_shared<GitHubInput>(*this);
-        input->ref = {};
-        input->rev = *rev;
+        if (!this->rev)
+            getCache()->add(
+                store,
+                mutableAttrs,
+                infoAttrs,
+                result.storePath,
+                false);
+
+        getCache()->add(
+            store,
+            immutableAttrs,
+            infoAttrs,
+            result.storePath,
+            true);
 
         return {std::move(result), input};
     }
@@ -189,7 +242,7 @@ struct GitHubInputScheme : InputScheme
         return input;
     }
 
-    std::unique_ptr<Input> inputFromAttrs(const Input::Attrs & attrs) override
+    std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
     {
         if (maybeGetStrAttr(attrs, "type") != "github") return {};
 

src/libstore/fetchers/indirect.cc

@@ -120,7 +120,7 @@ struct IndirectInputScheme : InputScheme
         return input;
     }
 
-    std::unique_ptr<Input> inputFromAttrs(const Input::Attrs & attrs) override
+    std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
     {
         if (maybeGetStrAttr(attrs, "type") != "indirect") return {};
 

src/libstore/fetchers/mercurial.cc

@@ -1,5 +1,6 @@
-#include "fetchers.hh"
-#include "parse.hh"
+#include "fetchers/fetchers.hh"
+#include "fetchers/cache.hh"
+#include "fetchers/parse.hh"
 #include "globals.hh"
 #include "tarfile.hh"
 #include "store-api.hh"
@@ -7,8 +8,6 @@
 
 #include <sys/time.h>
 
-#include <nlohmann/json.hpp>
-
 using namespace std::string_literals;
 
 namespace nix::fetchers {
@@ -163,51 +162,80 @@ struct MercurialInput : Input
 
         if (!input->ref) input->ref = "default";
 
-        Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(Base32, false));
+        auto getImmutableAttrs = [&]()
+        {
+            return Attrs({
+                {"type", "hg"},
+                {"name", name},
+                {"rev", input->rev->gitRev()},
+            });
+        };
+
+        auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
+            -> std::pair<Tree, std::shared_ptr<const Input>>
+        {
+            input->rev = Hash(getStrAttr(infoAttrs, "rev"), htSHA1);
+            assert(!rev || rev == input->rev);
+            return {
+                Tree{
+                    .actualPath = store->toRealPath(storePath),
+                    .storePath = std::move(storePath),
+                    .info = TreeInfo {
+                        .revCount = getIntAttr(infoAttrs, "revCount"),
+                    },
+                },
+                input
+            };
+        };
+
+        if (input->rev) {
+            if (auto res = getCache()->lookup(store, getImmutableAttrs()))
+                return makeResult(res->first, std::move(res->second));
+        }
 
         assert(input->rev || input->ref);
         auto revOrRef = input->rev ? input->rev->gitRev() : *input->ref;
 
-        Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir, hashString(htSHA512, revOrRef).to_string(Base32, false));
+        Attrs mutableAttrs({
+            {"type", "hg"},
+            {"name", name},
+            {"url", actualUrl},
+            {"ref", *input->ref},
+        });
 
-        /* If we haven't pulled this repo less than ‘tarball-ttl’ seconds,
-           do so now. */
-        time_t now = time(0);
-        struct stat st;
-        if (stat(stampFile.c_str(), &st) != 0 ||
-            (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now)
+        if (auto res = getCache()->lookup(store, mutableAttrs))
+            return makeResult(res->first, std::move(res->second));
+
+        Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(Base32, false));
+
+        /* If this is a commit hash that we already have, we don't
+           have to pull again. */
+        if (!(input->rev
+                && pathExists(cacheDir)
+                && runProgram(
+                    RunOptions("hg", { "log", "-R", cacheDir, "-r", input->rev->gitRev(), "--template", "1" })
+                    .killStderr(true)).second == "1"))
         {
-            /* Except that if this is a commit hash that we already have,
-               we don't have to pull again. */
-            if (!(input->rev
-                    && pathExists(cacheDir)
-                    && runProgram(
-                        RunOptions("hg", { "log", "-R", cacheDir, "-r", input->rev->gitRev(), "--template", "1" })
-                        .killStderr(true)).second == "1"))
-            {
-                Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl));
+            Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl));
 
-                if (pathExists(cacheDir)) {
-                    try {
-                        runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl });
-                    }
-                    catch (ExecError & e) {
-                        string transJournal = cacheDir + "/.hg/store/journal";
-                        /* hg throws "abandoned transaction" error only if this file exists */
-                        if (pathExists(transJournal)) {
-                            runProgram("hg", true, { "recover", "-R", cacheDir });
-                            runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl });
-                        } else {
-                            throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status)));
-                        }
-                    }
-                } else {
-                    createDirs(dirOf(cacheDir));
-                    runProgram("hg", true, { "clone", "--noupdate", "--", actualUrl, cacheDir });
+            if (pathExists(cacheDir)) {
+                try {
+                    runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl });
                 }
+                catch (ExecError & e) {
+                    string transJournal = cacheDir + "/.hg/store/journal";
+                    /* hg throws "abandoned transaction" error only if this file exists */
+                    if (pathExists(transJournal)) {
+                        runProgram("hg", true, { "recover", "-R", cacheDir });
+                        runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl });
+                    } else {
+                        throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status)));
+                    }
+                }
+            } else {
+                createDirs(dirOf(cacheDir));
+                runProgram("hg", true, { "clone", "--noupdate", "--", actualUrl, cacheDir });
             }
-
-            writeFile(stampFile, "");
         }
 
         auto tokens = tokenizeString<std::vector<std::string>>(
@@ -218,33 +246,8 @@ struct MercurialInput : Input
         auto revCount = std::stoull(tokens[1]);
         input->ref = tokens[2];
 
-        std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + input->rev->gitRev()).to_string(Base32, false);
-        Path storeLink = fmt("%s/.hg/%s.link", cacheDir, storeLinkName);
-
-        try {
-            auto json = nlohmann::json::parse(readFile(storeLink));
-
-            assert(json["name"] == name && json["rev"] == input->rev->gitRev());
-
-            auto storePath = store->parseStorePath((std::string) json["storePath"]);
-
-            if (store->isValidPath(storePath)) {
-                printTalkative("using cached Mercurial store path '%s'", store->printStorePath(storePath));
-                return {
-                    Tree {
-                        .actualPath = store->printStorePath(storePath),
-                        .storePath = std::move(storePath),
-                        .info = TreeInfo {
-                            .revCount = revCount,
-                        },
-                    },
-                    input
-                };
-            }
-
-        } catch (SysError & e) {
-            if (e.errNo != ENOENT) throw;
-        }
+        if (auto res = getCache()->lookup(store, getImmutableAttrs()))
+            return makeResult(res->first, std::move(res->second));
 
         Path tmpDir = createTempDir();
         AutoDelete delTmpDir(tmpDir, true);
@@ -255,26 +258,27 @@ struct MercurialInput : Input
 
         auto storePath = store->addToStore(name, tmpDir);
 
-        nlohmann::json json;
-        json["storePath"] = store->printStorePath(storePath);
-        json["uri"] = actualUrl;
-        json["name"] = name;
-        json["branch"] = *input->ref;
-        json["rev"] = input->rev->gitRev();
-        json["revCount"] = revCount;
+        Attrs infoAttrs({
+            {"rev", input->rev->gitRev()},
+            {"revCount", revCount},
+        });
 
-        writeFile(storeLink, json.dump());
+        if (!this->rev)
+            getCache()->add(
+                store,
+                mutableAttrs,
+                infoAttrs,
+                storePath,
+                false);
 
-        return {
-            Tree {
-                .actualPath = store->printStorePath(storePath),
-                .storePath = std::move(storePath),
-                .info = TreeInfo {
-                    .revCount = revCount
-                }
-            },
-            input
-        };
+        getCache()->add(
+            store,
+            getImmutableAttrs(),
+            infoAttrs,
+            storePath,
+            true);
+
+        return makeResult(infoAttrs, std::move(storePath));
     }
 };
 
@@ -291,7 +295,7 @@ struct MercurialInputScheme : InputScheme
         url2.scheme = std::string(url2.scheme, 3);
         url2.query.clear();
 
-        Input::Attrs attrs;
+        Attrs attrs;
         attrs.emplace("type", "hg");
 
         for (auto &[name, value] : url.query) {
@@ -306,7 +310,7 @@ struct MercurialInputScheme : InputScheme
         return inputFromAttrs(attrs);
     }
 
-    std::unique_ptr<Input> inputFromAttrs(const Input::Attrs & attrs) override
+    std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
     {
         if (maybeGetStrAttr(attrs, "type") != "hg") return {};
 

src/libstore/fetchers/registry.cc

@@ -36,7 +36,7 @@ std::shared_ptr<Registry> Registry::read(
     else if (version == 2) {
         for (auto & i : json["flakes"]) {
             auto toAttrs = jsonToAttrs(i["to"]);
-            Input::Attrs extraAttrs;
+            Attrs extraAttrs;
             auto j = toAttrs.find("dir");
             if (j != toAttrs.end()) {
                 extraAttrs.insert(*j);
@@ -80,7 +80,7 @@ void Registry::write(const Path & path)
 void Registry::add(
     const std::shared_ptr<const Input> & from,
     const std::shared_ptr<const Input> & to,
-    const Input::Attrs & extraAttrs)
+    const Attrs & extraAttrs)
 {
     entries.emplace_back(from, to, extraAttrs);
 }
@@ -116,7 +116,7 @@ std::shared_ptr<Registry> getFlagRegistry()
 void overrideRegistry(
     const std::shared_ptr<const Input> & from,
     const std::shared_ptr<const Input> & to,
-    const Input::Attrs & extraAttrs)
+    const Attrs & extraAttrs)
 {
     flagRegistry->add(from, to, extraAttrs);
 }
@@ -148,11 +148,11 @@ Registries getRegistries(ref<Store> store)
     return registries;
 }
 
-std::pair<std::shared_ptr<const Input>, Input::Attrs> lookupInRegistries(
+std::pair<std::shared_ptr<const Input>, Attrs> lookupInRegistries(
     ref<Store> store,
     std::shared_ptr<const Input> input)
 {
-    Input::Attrs extraAttrs;
+    Attrs extraAttrs;
     int n = 0;
 
  restart:

src/libstore/fetchers/registry.hh

@@ -21,7 +21,7 @@ struct Registry
         std::tuple<
             std::shared_ptr<const Input>, // from
             std::shared_ptr<const Input>, // to
-            Input::Attrs // extra attributes
+            Attrs // extra attributes
             >
         > entries;
 
@@ -37,7 +37,7 @@ struct Registry
     void add(
         const std::shared_ptr<const Input> & from,
         const std::shared_ptr<const Input> & to,
-        const Input::Attrs & extraAttrs);
+        const Attrs & extraAttrs);
 
     void remove(const std::shared_ptr<const Input> & input);
 };
@@ -53,9 +53,9 @@ Registries getRegistries(ref<Store> store);
 void overrideRegistry(
     const std::shared_ptr<const Input> & from,
     const std::shared_ptr<const Input> & to,
-    const Input::Attrs & extraAttrs);
+    const Attrs & extraAttrs);
 
-std::pair<std::shared_ptr<const Input>, Input::Attrs> lookupInRegistries(
+std::pair<std::shared_ptr<const Input>, Attrs> lookupInRegistries(
     ref<Store> store,
     std::shared_ptr<const Input> input);
 

src/libstore/fetchers/tarball.cc

@@ -109,7 +109,7 @@ struct TarballInputScheme : InputScheme
         return input;
     }
 
-    std::unique_ptr<Input> inputFromAttrs(const Input::Attrs & attrs) override
+    std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
     {
         if (maybeGetStrAttr(attrs, "type") != "tarball") return {};
 

src/nix/flake.cc

@@ -507,7 +507,7 @@ struct CmdFlakeAdd : MixEvalArgs, Command
     {
         auto fromRef = parseFlakeRef(fromUrl);
         auto toRef = parseFlakeRef(toUrl);
-        fetchers::Input::Attrs extraAttrs;
+        fetchers::Attrs extraAttrs;
         if (toRef.subdir != "") extraAttrs["dir"] = toRef.subdir;
         auto userRegistry = fetchers::getUserRegistry();
         userRegistry->remove(fromRef.input);
@@ -558,7 +558,7 @@ struct CmdFlakePin : virtual Args, EvalCommand
         auto userRegistry = fetchers::getUserRegistry();
         userRegistry->remove(ref.input);
         auto [tree, resolved] = ref.resolve(store).input->fetchTree(store);
-        fetchers::Input::Attrs extraAttrs;
+        fetchers::Attrs extraAttrs;
         if (ref.subdir != "") extraAttrs["dir"] = ref.subdir;
         userRegistry->add(ref.input, resolved, extraAttrs);
     }

tests/fetchMercurial.sh

@@ -9,7 +9,7 @@ clearStore
 
 repo=$TEST_ROOT/hg
 
-rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix/hg
+rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix
 
 hg init $repo
 echo '[ui]' >> $repo/.hg/hgrc
@@ -50,13 +50,13 @@ path2=$(nix eval --impure --raw --expr "(builtins.fetchMercurial file://$repo).o
 [[ $(nix eval --impure --raw --expr "(builtins.fetchMercurial file://$repo).rev") = $rev2 ]]
 
 # But with TTL 0, it should fail.
-(! nix eval --impure --tarball-ttl 0 --expr "builtins.fetchMercurial file://$repo")
+(! nix eval --impure --refresh --expr "builtins.fetchMercurial file://$repo")
 
 # Fetching with a explicit hash should succeed.
-path2=$(nix eval --tarball-ttl 0 --raw --expr "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev2\"; }).outPath")
+path2=$(nix eval --refresh --raw --expr "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev2\"; }).outPath")
 [[ $path = $path2 ]]
 
-path2=$(nix eval --tarball-ttl 0 --raw --expr "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev1\"; }).outPath")
+path2=$(nix eval --refresh --raw --expr "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev1\"; }).outPath")
 [[ $(cat $path2/hello) = utrecht ]]
 
 mv ${repo}-tmp $repo
@@ -89,5 +89,5 @@ path3=$(nix eval --impure --raw --expr "(builtins.fetchMercurial { url = $repo;
 # Committing should not affect the store path.
 hg commit --cwd $repo -m 'Bla3'
 
-path4=$(nix eval --impure --tarball-ttl 0 --raw --expr "(builtins.fetchMercurial file://$repo).outPath")
+path4=$(nix eval --impure --refresh --raw --expr "(builtins.fetchMercurial file://$repo).outPath")
 [[ $path2 = $path4 ]]