From 2a4e4f6a6e021481f0e92b7d3006345e68e77684 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Mar 2020 20:54:36 +0100 Subject: [PATCH] Unified fetcher caching system --- src/libexpr/common-eval-args.cc | 2 +- src/libexpr/flake/flake.cc | 6 +- src/libexpr/flake/flakeref.cc | 4 +- src/libexpr/flake/flakeref.hh | 4 +- src/libexpr/primops/fetchTree.cc | 2 +- src/libstore/fetchers/attrs.cc | 71 ++++++++++++ src/libstore/fetchers/attrs.hh | 26 +++++ src/libstore/fetchers/cache.cc | 109 ++++++++++++++++++ src/libstore/fetchers/cache.hh | 24 ++++ src/libstore/fetchers/fetchers.cc | 50 +------- src/libstore/fetchers/fetchers.hh | 19 +-- src/libstore/fetchers/git.cc | 4 +- src/libstore/fetchers/github.cc | 87 +++++++++++--- src/libstore/fetchers/indirect.cc | 2 +- src/libstore/fetchers/mercurial.cc | 178 +++++++++++++++-------------- src/libstore/fetchers/registry.cc | 10 +- src/libstore/fetchers/registry.hh | 8 +- src/libstore/fetchers/tarball.cc | 2 +- src/nix/flake.cc | 4 +- tests/fetchMercurial.sh | 10 +- 20 files changed, 425 insertions(+), 197 deletions(-) create mode 100644 src/libstore/fetchers/attrs.cc create mode 100644 src/libstore/fetchers/attrs.hh create mode 100644 src/libstore/fetchers/cache.cc create mode 100644 src/libstore/fetchers/cache.hh diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index fc60954b4..59a2b9c89 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -44,7 +44,7 @@ MixEvalArgs::MixEvalArgs() .handler([&](std::vector ss) { auto from = parseFlakeRef(ss[0], absPath(".")); auto to = parseFlakeRef(ss[1], absPath(".")); - fetchers::Input::Attrs extraAttrs; + fetchers::Attrs extraAttrs; if (to.subdir != "") extraAttrs["dir"] = to.subdir; fetchers::overrideRegistry(from.input, to.input, extraAttrs); }); diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index c58b0eea9..f79abb4ee 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -132,10 +132,10 @@ static FlakeInput parseFlakeInput(EvalState & state, auto sFlake = state.symbols.create("flake"); auto sFollows = state.symbols.create("follows"); - fetchers::Input::Attrs attrs; + fetchers::Attrs attrs; std::optional url; - for (Attr attr : *(value->attrs)) { + for (nix::Attr attr : *(value->attrs)) { try { if (attr.name == sUrl || attr.name == sUri) { expectType(state, tString, *attr.value, *attr.pos); @@ -188,7 +188,7 @@ static std::map parseFlakeInputs( expectType(state, tAttrs, *value, pos); - for (Attr & inputAttr : *(*value).attrs) { + for (nix::Attr & inputAttr : *(*value).attrs) { inputs.emplace(inputAttr.name, parseFlakeInput(state, inputAttr.name, diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index e8a2abb5d..f97679dd9 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -19,7 +19,7 @@ std::string FlakeRef::to_string() const return input->to_string(); } -fetchers::Input::Attrs FlakeRef::toAttrs() const +fetchers::Attrs FlakeRef::toAttrs() const { auto attrs = input->toAttrs(); if (subdir != "") @@ -168,7 +168,7 @@ std::optional> maybeParseFlakeRefWithFragment( } } -FlakeRef FlakeRef::fromAttrs(const fetchers::Input::Attrs & attrs) +FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs) { auto attrs2(attrs); attrs2.erase("dir"); diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index 5acf43957..d23a8f601 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -29,11 +29,11 @@ struct FlakeRef // FIXME: change to operator <<. std::string to_string() const; - fetchers::Input::Attrs toAttrs() const; + fetchers::Attrs toAttrs() const; FlakeRef resolve(ref store) const; - static FlakeRef fromAttrs(const fetchers::Input::Attrs & attrs); + static FlakeRef fromAttrs(const fetchers::Attrs & attrs); std::pair fetchTree(ref store) const; }; diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 47667a1b8..a9becddd9 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -52,7 +52,7 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V if (args[0]->type == tAttrs) { state.forceAttrs(*args[0], pos); - fetchers::Input::Attrs attrs; + fetchers::Attrs attrs; for (auto & attr : *args[0]->attrs) { state.forceValue(*attr.value); diff --git a/src/libstore/fetchers/attrs.cc b/src/libstore/fetchers/attrs.cc new file mode 100644 index 000000000..83b6d9164 --- /dev/null +++ b/src/libstore/fetchers/attrs.cc @@ -0,0 +1,71 @@ +#include "attrs.hh" +#include "fetchers.hh" + +#include + +namespace nix::fetchers { + +Attrs jsonToAttrs(const nlohmann::json & json) +{ + Attrs attrs; + + for (auto & i : json.items()) { + if (i.value().is_number()) + attrs.emplace(i.key(), i.value().get()); + else if (i.value().is_string()) + attrs.emplace(i.key(), i.value().get()); + else + throw Error("unsupported input attribute type in lock file"); + } + + return attrs; +} + +nlohmann::json attrsToJson(const Attrs & attrs) +{ + nlohmann::json json; + for (auto & attr : attrs) { + if (auto v = std::get_if(&attr.second)) { + json[attr.first] = *v; + } else if (auto v = std::get_if(&attr.second)) { + json[attr.first] = *v; + } else abort(); + } + return json; +} + +std::optional maybeGetStrAttr(const Attrs & attrs, const std::string & name) +{ + auto i = attrs.find(name); + if (i == attrs.end()) return {}; + if (auto v = std::get_if(&i->second)) + return *v; + throw Error("input attribute '%s' is not a string", name); +} + +std::string getStrAttr(const Attrs & attrs, const std::string & name) +{ + auto s = maybeGetStrAttr(attrs, name); + if (!s) + throw Error("input attribute '%s' is missing", name); + return *s; +} + +std::optional maybeGetIntAttr(const Attrs & attrs, const std::string & name) +{ + auto i = attrs.find(name); + if (i == attrs.end()) return {}; + if (auto v = std::get_if(&i->second)) + return *v; + throw Error("input attribute '%s' is not a string", name); +} + +int64_t getIntAttr(const Attrs & attrs, const std::string & name) +{ + auto s = maybeGetIntAttr(attrs, name); + if (!s) + throw Error("input attribute '%s' is missing", name); + return *s; +} + +} diff --git a/src/libstore/fetchers/attrs.hh b/src/libstore/fetchers/attrs.hh new file mode 100644 index 000000000..b7f98d71b --- /dev/null +++ b/src/libstore/fetchers/attrs.hh @@ -0,0 +1,26 @@ +#pragma once + +#include "types.hh" + +#include + +#include + +namespace nix::fetchers { + +typedef std::variant Attr; +typedef std::map Attrs; + +Attrs jsonToAttrs(const nlohmann::json & json); + +nlohmann::json attrsToJson(const Attrs & attrs); + +std::optional maybeGetStrAttr(const Attrs & attrs, const std::string & name); + +std::string getStrAttr(const Attrs & attrs, const std::string & name); + +std::optional maybeGetIntAttr(const Attrs & attrs, const std::string & name); + +int64_t getIntAttr(const Attrs & attrs, const std::string & name); + +} diff --git a/src/libstore/fetchers/cache.cc b/src/libstore/fetchers/cache.cc new file mode 100644 index 000000000..4c88b64c5 --- /dev/null +++ b/src/libstore/fetchers/cache.cc @@ -0,0 +1,109 @@ +#include "fetchers/cache.hh" +#include "sqlite.hh" +#include "sync.hh" +#include "store-api.hh" + +#include + +namespace nix::fetchers { + +static const char * schema = R"sql( + +create table if not exists Cache ( + input text not null, + info text not null, + path text not null, + immutable integer not null, + timestamp integer not null, + primary key (input) +); +)sql"; + +struct CacheImpl : Cache +{ + struct State + { + SQLite db; + SQLiteStmt add, lookup; + }; + + Sync _state; + + CacheImpl() + { + auto state(_state.lock()); + + auto dbPath = getCacheDir() + "/nix/fetcher-cache-v1.sqlite"; + createDirs(dirOf(dbPath)); + + state->db = SQLite(dbPath); + state->db.isCache(); + state->db.exec(schema); + + state->add.create(state->db, + "insert or replace into Cache(input, info, path, immutable, timestamp) values (?, ?, ?, ?, ?)"); + + state->lookup.create(state->db, + "select info, path, immutable, timestamp from Cache where input = ?"); + } + + void add( + ref store, + const Attrs & inAttrs, + const Attrs & infoAttrs, + const StorePath & storePath, + bool immutable) override + { + _state.lock()->add.use() + (attrsToJson(inAttrs).dump()) + (attrsToJson(infoAttrs).dump()) + (store->printStorePath(storePath)) + (immutable) + (time(0)).exec(); + } + + std::optional> lookup( + ref store, + const Attrs & inAttrs) override + { + auto state(_state.lock()); + + auto inAttrsJson = attrsToJson(inAttrs).dump(); + + auto stmt(state->lookup.use()(inAttrsJson)); + if (!stmt.next()) { + debug("did not find cache entry for '%s'", inAttrsJson); + return {}; + } + + auto infoJson = stmt.getStr(0); + auto storePath = store->parseStorePath(stmt.getStr(1)); + auto immutable = stmt.getInt(2) != 0; + auto timestamp = stmt.getInt(3); + + if (!immutable && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0))) { + debug("ignoring expired cache entry '%s'", inAttrsJson); + return {}; + } + + store->addTempRoot(storePath); + if (!store->isValidPath(storePath)) { + // FIXME: we could try to substitute 'storePath'. + debug("ignoring disappeared cache entry '%s'", inAttrsJson); + return {}; + } + + debug("using cache entry '%s' -> '%s', '%s'", + inAttrsJson, infoJson, store->printStorePath(storePath)); + + return {{jsonToAttrs(nlohmann::json::parse(infoJson)), std::move(storePath)}}; + } +}; + +ref getCache() +{ + static auto cache = std::make_shared(); + return ref(cache); +} + +} diff --git a/src/libstore/fetchers/cache.hh b/src/libstore/fetchers/cache.hh new file mode 100644 index 000000000..ba2d30629 --- /dev/null +++ b/src/libstore/fetchers/cache.hh @@ -0,0 +1,24 @@ +#pragma once + +#include "types.hh" +#include "fetchers/fetchers.hh" + +namespace nix::fetchers { + +struct Cache +{ + virtual void add( + ref store, + const Attrs & inAttrs, + const Attrs & infoAttrs, + const StorePath & storePath, + bool immutable) = 0; + + virtual std::optional> lookup( + ref store, + const Attrs & inAttrs) = 0; +}; + +ref getCache(); + +} diff --git a/src/libstore/fetchers/fetchers.cc b/src/libstore/fetchers/fetchers.cc index 0cc6f1c91..25827ab7c 100644 --- a/src/libstore/fetchers/fetchers.cc +++ b/src/libstore/fetchers/fetchers.cc @@ -28,7 +28,7 @@ std::unique_ptr inputFromURL(const std::string & url) return inputFromURL(parseURL(url)); } -std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) +std::unique_ptr inputFromAttrs(const Attrs & attrs) { for (auto & inputScheme : *inputSchemes) { auto res = inputScheme->inputFromAttrs(attrs); @@ -42,36 +42,7 @@ std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) throw Error("input '%s' is unsupported", attrsToJson(attrs)); } -Input::Attrs jsonToAttrs(const nlohmann::json & json) -{ - fetchers::Input::Attrs attrs; - - for (auto & i : json.items()) { - if (i.value().is_number()) - attrs.emplace(i.key(), i.value().get()); - else if (i.value().is_string()) - attrs.emplace(i.key(), i.value().get()); - else - throw Error("unsupported input attribute type in lock file"); - } - - return attrs; -} - -nlohmann::json attrsToJson(const fetchers::Input::Attrs & attrs) -{ - nlohmann::json json; - for (auto & attr : attrs) { - if (auto v = std::get_if(&attr.second)) { - json[attr.first] = *v; - } else if (auto v = std::get_if(&attr.second)) { - json[attr.first] = *v; - } else abort(); - } - return json; -} - -Input::Attrs Input::toAttrs() const +Attrs Input::toAttrs() const { auto attrs = toAttrsInternal(); if (narHash) @@ -80,23 +51,6 @@ Input::Attrs Input::toAttrs() const return attrs; } -std::optional maybeGetStrAttr(const Input::Attrs & attrs, const std::string & name) -{ - auto i = attrs.find(name); - if (i == attrs.end()) return {}; - if (auto v = std::get_if(&i->second)) - return *v; - throw Error("input attribute '%s' is not a string", name); -} - -std::string getStrAttr(const Input::Attrs & attrs, const std::string & name) -{ - auto s = maybeGetStrAttr(attrs, name); - if (!s) - throw Error("input attribute '%s' is missing", name); - return *s; -} - std::pair> Input::fetchTree(ref store) const { auto [tree, input] = fetchTreeInternal(store); diff --git a/src/libstore/fetchers/fetchers.hh b/src/libstore/fetchers/fetchers.hh index 4202e8339..085a62f47 100644 --- a/src/libstore/fetchers/fetchers.hh +++ b/src/libstore/fetchers/fetchers.hh @@ -4,11 +4,9 @@ #include "hash.hh" #include "path.hh" #include "tree-info.hh" +#include "attrs.hh" #include -#include - -#include namespace nix { class Store; } @@ -49,9 +47,6 @@ struct Input : std::enable_shared_from_this virtual std::string to_string() const = 0; - typedef std::variant Attr; - typedef std::map Attrs; - Attrs toAttrs() const; std::pair> fetchTree(ref store) const; @@ -87,23 +82,15 @@ struct InputScheme virtual std::unique_ptr inputFromURL(const ParsedURL & url) = 0; - virtual std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) = 0; + virtual std::unique_ptr inputFromAttrs(const Attrs & attrs) = 0; }; std::unique_ptr inputFromURL(const ParsedURL & url); std::unique_ptr inputFromURL(const std::string & url); -std::unique_ptr inputFromAttrs(const Input::Attrs & attrs); +std::unique_ptr inputFromAttrs(const Attrs & attrs); void registerInputScheme(std::unique_ptr && fetcher); -Input::Attrs jsonToAttrs(const nlohmann::json & json); - -nlohmann::json attrsToJson(const Input::Attrs & attrs); - -std::optional maybeGetStrAttr(const Input::Attrs & attrs, const std::string & name); - -std::string getStrAttr(const Input::Attrs & attrs, const std::string & name); - } diff --git a/src/libstore/fetchers/git.cc b/src/libstore/fetchers/git.cc index 46ca187fe..179a5fb83 100644 --- a/src/libstore/fetchers/git.cc +++ b/src/libstore/fetchers/git.cc @@ -420,7 +420,7 @@ struct GitInputScheme : InputScheme if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4); url2.query.clear(); - Input::Attrs attrs; + Attrs attrs; attrs.emplace("type", "git"); for (auto &[name, value] : url.query) { @@ -435,7 +435,7 @@ struct GitInputScheme : InputScheme return inputFromAttrs(attrs); } - std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) override + std::unique_ptr inputFromAttrs(const Attrs & attrs) override { if (maybeGetStrAttr(attrs, "type") != "git") return {}; diff --git a/src/libstore/fetchers/github.cc b/src/libstore/fetchers/github.cc index 0a000e83f..1772b2828 100644 --- a/src/libstore/fetchers/github.cc +++ b/src/libstore/fetchers/github.cc @@ -1,8 +1,9 @@ -#include "fetchers.hh" #include "download.hh" +#include "fetchers/cache.hh" +#include "fetchers/fetchers.hh" +#include "fetchers/parse.hh" +#include "fetchers/regex.hh" #include "globals.hh" -#include "parse.hh" -#include "regex.hh" #include "store-api.hh" #include @@ -72,17 +73,36 @@ struct GitHubInput : Input std::pair> fetchTreeInternal(nix::ref store) const override { auto rev = this->rev; + auto ref = this->ref.value_or("master"); - #if 0 - if (rev) { - if (auto gitInfo = lookupGitInfo(store, "source", *rev)) - return *gitInfo; + Attrs mutableAttrs({ + {"type", "github"}, + {"owner", owner}, + {"repo", repo}, + {"ref", ref}, + }); + + if (!rev) { + if (auto res = getCache()->lookup(store, mutableAttrs)) { + auto input = std::make_shared(*this); + input->ref = {}; + input->rev = Hash(getStrAttr(res->first, "rev"), htSHA1); + return { + Tree{ + .actualPath = store->toRealPath(res->second), + .storePath = std::move(res->second), + .info = TreeInfo { + .lastModified = getIntAttr(res->first, "lastModified"), + }, + }, + input + }; + } } - #endif if (!rev) { auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s", - owner, repo, ref ? *ref : "master"); + owner, repo, ref); CachedDownloadRequest request(url); request.ttl = rev ? 1000000000 : settings.tarballTtl; auto result = getDownloader()->downloadCached(store, request); @@ -91,6 +111,28 @@ struct GitHubInput : Input debug("HEAD revision for '%s' is %s", url, rev->gitRev()); } + auto input = std::make_shared(*this); + input->ref = {}; + input->rev = *rev; + + Attrs immutableAttrs({ + {"type", "git-tarball"}, + {"rev", rev->gitRev()}, + }); + + if (auto res = getCache()->lookup(store, immutableAttrs)) { + return { + Tree{ + .actualPath = store->toRealPath(res->second), + .storePath = std::move(res->second), + .info = TreeInfo { + .lastModified = getIntAttr(res->first, "lastModified"), + }, + }, + input + }; + } + // FIXME: use regular /archive URLs instead? api.github.com // might have stricter rate limits. @@ -118,14 +160,25 @@ struct GitHubInput : Input }, }; - #if 0 - // FIXME: this can overwrite a cache file that contains a revCount. - cacheGitInfo("source", gitInfo); - #endif + Attrs infoAttrs({ + {"rev", rev->gitRev()}, + {"lastModified", *result.info.lastModified} + }); - auto input = std::make_shared(*this); - input->ref = {}; - input->rev = *rev; + if (!this->rev) + getCache()->add( + store, + mutableAttrs, + infoAttrs, + result.storePath, + false); + + getCache()->add( + store, + immutableAttrs, + infoAttrs, + result.storePath, + true); return {std::move(result), input}; } @@ -189,7 +242,7 @@ struct GitHubInputScheme : InputScheme return input; } - std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) override + std::unique_ptr inputFromAttrs(const Attrs & attrs) override { if (maybeGetStrAttr(attrs, "type") != "github") return {}; diff --git a/src/libstore/fetchers/indirect.cc b/src/libstore/fetchers/indirect.cc index 016f5fb39..963abd85f 100644 --- a/src/libstore/fetchers/indirect.cc +++ b/src/libstore/fetchers/indirect.cc @@ -120,7 +120,7 @@ struct IndirectInputScheme : InputScheme return input; } - std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) override + std::unique_ptr inputFromAttrs(const Attrs & attrs) override { if (maybeGetStrAttr(attrs, "type") != "indirect") return {}; diff --git a/src/libstore/fetchers/mercurial.cc b/src/libstore/fetchers/mercurial.cc index 6ab0add1d..a9c86f1b4 100644 --- a/src/libstore/fetchers/mercurial.cc +++ b/src/libstore/fetchers/mercurial.cc @@ -1,5 +1,6 @@ -#include "fetchers.hh" -#include "parse.hh" +#include "fetchers/fetchers.hh" +#include "fetchers/cache.hh" +#include "fetchers/parse.hh" #include "globals.hh" #include "tarfile.hh" #include "store-api.hh" @@ -7,8 +8,6 @@ #include -#include - using namespace std::string_literals; namespace nix::fetchers { @@ -163,51 +162,80 @@ struct MercurialInput : Input if (!input->ref) input->ref = "default"; - Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(Base32, false)); + auto getImmutableAttrs = [&]() + { + return Attrs({ + {"type", "hg"}, + {"name", name}, + {"rev", input->rev->gitRev()}, + }); + }; + + auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) + -> std::pair> + { + input->rev = Hash(getStrAttr(infoAttrs, "rev"), htSHA1); + assert(!rev || rev == input->rev); + return { + Tree{ + .actualPath = store->toRealPath(storePath), + .storePath = std::move(storePath), + .info = TreeInfo { + .revCount = getIntAttr(infoAttrs, "revCount"), + }, + }, + input + }; + }; + + if (input->rev) { + if (auto res = getCache()->lookup(store, getImmutableAttrs())) + return makeResult(res->first, std::move(res->second)); + } assert(input->rev || input->ref); auto revOrRef = input->rev ? input->rev->gitRev() : *input->ref; - Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir, hashString(htSHA512, revOrRef).to_string(Base32, false)); + Attrs mutableAttrs({ + {"type", "hg"}, + {"name", name}, + {"url", actualUrl}, + {"ref", *input->ref}, + }); - /* If we haven't pulled this repo less than ‘tarball-ttl’ seconds, - do so now. */ - time_t now = time(0); - struct stat st; - if (stat(stampFile.c_str(), &st) != 0 || - (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now) + if (auto res = getCache()->lookup(store, mutableAttrs)) + return makeResult(res->first, std::move(res->second)); + + Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(Base32, false)); + + /* If this is a commit hash that we already have, we don't + have to pull again. */ + if (!(input->rev + && pathExists(cacheDir) + && runProgram( + RunOptions("hg", { "log", "-R", cacheDir, "-r", input->rev->gitRev(), "--template", "1" }) + .killStderr(true)).second == "1")) { - /* Except that if this is a commit hash that we already have, - we don't have to pull again. */ - if (!(input->rev - && pathExists(cacheDir) - && runProgram( - RunOptions("hg", { "log", "-R", cacheDir, "-r", input->rev->gitRev(), "--template", "1" }) - .killStderr(true)).second == "1")) - { - Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl)); + Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl)); - if (pathExists(cacheDir)) { - try { - runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl }); - } - catch (ExecError & e) { - string transJournal = cacheDir + "/.hg/store/journal"; - /* hg throws "abandoned transaction" error only if this file exists */ - if (pathExists(transJournal)) { - runProgram("hg", true, { "recover", "-R", cacheDir }); - runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl }); - } else { - throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status))); - } - } - } else { - createDirs(dirOf(cacheDir)); - runProgram("hg", true, { "clone", "--noupdate", "--", actualUrl, cacheDir }); + if (pathExists(cacheDir)) { + try { + runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl }); } + catch (ExecError & e) { + string transJournal = cacheDir + "/.hg/store/journal"; + /* hg throws "abandoned transaction" error only if this file exists */ + if (pathExists(transJournal)) { + runProgram("hg", true, { "recover", "-R", cacheDir }); + runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl }); + } else { + throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status))); + } + } + } else { + createDirs(dirOf(cacheDir)); + runProgram("hg", true, { "clone", "--noupdate", "--", actualUrl, cacheDir }); } - - writeFile(stampFile, ""); } auto tokens = tokenizeString>( @@ -218,33 +246,8 @@ struct MercurialInput : Input auto revCount = std::stoull(tokens[1]); input->ref = tokens[2]; - std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + input->rev->gitRev()).to_string(Base32, false); - Path storeLink = fmt("%s/.hg/%s.link", cacheDir, storeLinkName); - - try { - auto json = nlohmann::json::parse(readFile(storeLink)); - - assert(json["name"] == name && json["rev"] == input->rev->gitRev()); - - auto storePath = store->parseStorePath((std::string) json["storePath"]); - - if (store->isValidPath(storePath)) { - printTalkative("using cached Mercurial store path '%s'", store->printStorePath(storePath)); - return { - Tree { - .actualPath = store->printStorePath(storePath), - .storePath = std::move(storePath), - .info = TreeInfo { - .revCount = revCount, - }, - }, - input - }; - } - - } catch (SysError & e) { - if (e.errNo != ENOENT) throw; - } + if (auto res = getCache()->lookup(store, getImmutableAttrs())) + return makeResult(res->first, std::move(res->second)); Path tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir, true); @@ -255,26 +258,27 @@ struct MercurialInput : Input auto storePath = store->addToStore(name, tmpDir); - nlohmann::json json; - json["storePath"] = store->printStorePath(storePath); - json["uri"] = actualUrl; - json["name"] = name; - json["branch"] = *input->ref; - json["rev"] = input->rev->gitRev(); - json["revCount"] = revCount; + Attrs infoAttrs({ + {"rev", input->rev->gitRev()}, + {"revCount", revCount}, + }); - writeFile(storeLink, json.dump()); + if (!this->rev) + getCache()->add( + store, + mutableAttrs, + infoAttrs, + storePath, + false); - return { - Tree { - .actualPath = store->printStorePath(storePath), - .storePath = std::move(storePath), - .info = TreeInfo { - .revCount = revCount - } - }, - input - }; + getCache()->add( + store, + getImmutableAttrs(), + infoAttrs, + storePath, + true); + + return makeResult(infoAttrs, std::move(storePath)); } }; @@ -291,7 +295,7 @@ struct MercurialInputScheme : InputScheme url2.scheme = std::string(url2.scheme, 3); url2.query.clear(); - Input::Attrs attrs; + Attrs attrs; attrs.emplace("type", "hg"); for (auto &[name, value] : url.query) { @@ -306,7 +310,7 @@ struct MercurialInputScheme : InputScheme return inputFromAttrs(attrs); } - std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) override + std::unique_ptr inputFromAttrs(const Attrs & attrs) override { if (maybeGetStrAttr(attrs, "type") != "hg") return {}; diff --git a/src/libstore/fetchers/registry.cc b/src/libstore/fetchers/registry.cc index 721af0c9b..acbed2109 100644 --- a/src/libstore/fetchers/registry.cc +++ b/src/libstore/fetchers/registry.cc @@ -36,7 +36,7 @@ std::shared_ptr Registry::read( else if (version == 2) { for (auto & i : json["flakes"]) { auto toAttrs = jsonToAttrs(i["to"]); - Input::Attrs extraAttrs; + Attrs extraAttrs; auto j = toAttrs.find("dir"); if (j != toAttrs.end()) { extraAttrs.insert(*j); @@ -80,7 +80,7 @@ void Registry::write(const Path & path) void Registry::add( const std::shared_ptr & from, const std::shared_ptr & to, - const Input::Attrs & extraAttrs) + const Attrs & extraAttrs) { entries.emplace_back(from, to, extraAttrs); } @@ -116,7 +116,7 @@ std::shared_ptr getFlagRegistry() void overrideRegistry( const std::shared_ptr & from, const std::shared_ptr & to, - const Input::Attrs & extraAttrs) + const Attrs & extraAttrs) { flagRegistry->add(from, to, extraAttrs); } @@ -148,11 +148,11 @@ Registries getRegistries(ref store) return registries; } -std::pair, Input::Attrs> lookupInRegistries( +std::pair, Attrs> lookupInRegistries( ref store, std::shared_ptr input) { - Input::Attrs extraAttrs; + Attrs extraAttrs; int n = 0; restart: diff --git a/src/libstore/fetchers/registry.hh b/src/libstore/fetchers/registry.hh index 6063f51d6..d2eb7749b 100644 --- a/src/libstore/fetchers/registry.hh +++ b/src/libstore/fetchers/registry.hh @@ -21,7 +21,7 @@ struct Registry std::tuple< std::shared_ptr, // from std::shared_ptr, // to - Input::Attrs // extra attributes + Attrs // extra attributes > > entries; @@ -37,7 +37,7 @@ struct Registry void add( const std::shared_ptr & from, const std::shared_ptr & to, - const Input::Attrs & extraAttrs); + const Attrs & extraAttrs); void remove(const std::shared_ptr & input); }; @@ -53,9 +53,9 @@ Registries getRegistries(ref store); void overrideRegistry( const std::shared_ptr & from, const std::shared_ptr & to, - const Input::Attrs & extraAttrs); + const Attrs & extraAttrs); -std::pair, Input::Attrs> lookupInRegistries( +std::pair, Attrs> lookupInRegistries( ref store, std::shared_ptr input); diff --git a/src/libstore/fetchers/tarball.cc b/src/libstore/fetchers/tarball.cc index 7c0b6690d..360befd31 100644 --- a/src/libstore/fetchers/tarball.cc +++ b/src/libstore/fetchers/tarball.cc @@ -109,7 +109,7 @@ struct TarballInputScheme : InputScheme return input; } - std::unique_ptr inputFromAttrs(const Input::Attrs & attrs) override + std::unique_ptr inputFromAttrs(const Attrs & attrs) override { if (maybeGetStrAttr(attrs, "type") != "tarball") return {}; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 82357aef8..317d1bc18 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -507,7 +507,7 @@ struct CmdFlakeAdd : MixEvalArgs, Command { auto fromRef = parseFlakeRef(fromUrl); auto toRef = parseFlakeRef(toUrl); - fetchers::Input::Attrs extraAttrs; + fetchers::Attrs extraAttrs; if (toRef.subdir != "") extraAttrs["dir"] = toRef.subdir; auto userRegistry = fetchers::getUserRegistry(); userRegistry->remove(fromRef.input); @@ -558,7 +558,7 @@ struct CmdFlakePin : virtual Args, EvalCommand auto userRegistry = fetchers::getUserRegistry(); userRegistry->remove(ref.input); auto [tree, resolved] = ref.resolve(store).input->fetchTree(store); - fetchers::Input::Attrs extraAttrs; + fetchers::Attrs extraAttrs; if (ref.subdir != "") extraAttrs["dir"] = ref.subdir; userRegistry->add(ref.input, resolved, extraAttrs); } diff --git a/tests/fetchMercurial.sh b/tests/fetchMercurial.sh index 6c11e06ba..af8ef8d5b 100644 --- a/tests/fetchMercurial.sh +++ b/tests/fetchMercurial.sh @@ -9,7 +9,7 @@ clearStore repo=$TEST_ROOT/hg -rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix/hg +rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix hg init $repo echo '[ui]' >> $repo/.hg/hgrc @@ -50,13 +50,13 @@ path2=$(nix eval --impure --raw --expr "(builtins.fetchMercurial file://$repo).o [[ $(nix eval --impure --raw --expr "(builtins.fetchMercurial file://$repo).rev") = $rev2 ]] # But with TTL 0, it should fail. -(! nix eval --impure --tarball-ttl 0 --expr "builtins.fetchMercurial file://$repo") +(! nix eval --impure --refresh --expr "builtins.fetchMercurial file://$repo") # Fetching with a explicit hash should succeed. -path2=$(nix eval --tarball-ttl 0 --raw --expr "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev2\"; }).outPath") +path2=$(nix eval --refresh --raw --expr "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev2\"; }).outPath") [[ $path = $path2 ]] -path2=$(nix eval --tarball-ttl 0 --raw --expr "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev1\"; }).outPath") +path2=$(nix eval --refresh --raw --expr "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev1\"; }).outPath") [[ $(cat $path2/hello) = utrecht ]] mv ${repo}-tmp $repo @@ -89,5 +89,5 @@ path3=$(nix eval --impure --raw --expr "(builtins.fetchMercurial { url = $repo; # Committing should not affect the store path. hg commit --cwd $repo -m 'Bla3' -path4=$(nix eval --impure --tarball-ttl 0 --raw --expr "(builtins.fetchMercurial file://$repo).outPath") +path4=$(nix eval --impure --refresh --raw --expr "(builtins.fetchMercurial file://$repo).outPath") [[ $path2 = $path4 ]]