Unified fetcher caching system

This commit is contained in:
Eelco Dolstra 2020-03-17 20:54:36 +01:00
parent fbcb897e21
commit 2a4e4f6a6e
No known key found for this signature in database
GPG key ID: 8170B4726D7198DE
20 changed files with 425 additions and 197 deletions

View file

@ -44,7 +44,7 @@ MixEvalArgs::MixEvalArgs()
.handler([&](std::vector<std::string> ss) { .handler([&](std::vector<std::string> ss) {
auto from = parseFlakeRef(ss[0], absPath(".")); auto from = parseFlakeRef(ss[0], absPath("."));
auto to = parseFlakeRef(ss[1], absPath(".")); auto to = parseFlakeRef(ss[1], absPath("."));
fetchers::Input::Attrs extraAttrs; fetchers::Attrs extraAttrs;
if (to.subdir != "") extraAttrs["dir"] = to.subdir; if (to.subdir != "") extraAttrs["dir"] = to.subdir;
fetchers::overrideRegistry(from.input, to.input, extraAttrs); fetchers::overrideRegistry(from.input, to.input, extraAttrs);
}); });

View file

@ -132,10 +132,10 @@ static FlakeInput parseFlakeInput(EvalState & state,
auto sFlake = state.symbols.create("flake"); auto sFlake = state.symbols.create("flake");
auto sFollows = state.symbols.create("follows"); auto sFollows = state.symbols.create("follows");
fetchers::Input::Attrs attrs; fetchers::Attrs attrs;
std::optional<std::string> url; std::optional<std::string> url;
for (Attr attr : *(value->attrs)) { for (nix::Attr attr : *(value->attrs)) {
try { try {
if (attr.name == sUrl || attr.name == sUri) { if (attr.name == sUrl || attr.name == sUri) {
expectType(state, tString, *attr.value, *attr.pos); expectType(state, tString, *attr.value, *attr.pos);
@ -188,7 +188,7 @@ static std::map<FlakeId, FlakeInput> parseFlakeInputs(
expectType(state, tAttrs, *value, pos); expectType(state, tAttrs, *value, pos);
for (Attr & inputAttr : *(*value).attrs) { for (nix::Attr & inputAttr : *(*value).attrs) {
inputs.emplace(inputAttr.name, inputs.emplace(inputAttr.name,
parseFlakeInput(state, parseFlakeInput(state,
inputAttr.name, inputAttr.name,

View file

@ -19,7 +19,7 @@ std::string FlakeRef::to_string() const
return input->to_string(); return input->to_string();
} }
fetchers::Input::Attrs FlakeRef::toAttrs() const fetchers::Attrs FlakeRef::toAttrs() const
{ {
auto attrs = input->toAttrs(); auto attrs = input->toAttrs();
if (subdir != "") if (subdir != "")
@ -168,7 +168,7 @@ std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment(
} }
} }
FlakeRef FlakeRef::fromAttrs(const fetchers::Input::Attrs & attrs) FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs)
{ {
auto attrs2(attrs); auto attrs2(attrs);
attrs2.erase("dir"); attrs2.erase("dir");

View file

@ -29,11 +29,11 @@ struct FlakeRef
// FIXME: change to operator <<. // FIXME: change to operator <<.
std::string to_string() const; std::string to_string() const;
fetchers::Input::Attrs toAttrs() const; fetchers::Attrs toAttrs() const;
FlakeRef resolve(ref<Store> store) const; FlakeRef resolve(ref<Store> store) const;
static FlakeRef fromAttrs(const fetchers::Input::Attrs & attrs); static FlakeRef fromAttrs(const fetchers::Attrs & attrs);
std::pair<fetchers::Tree, FlakeRef> fetchTree(ref<Store> store) const; std::pair<fetchers::Tree, FlakeRef> fetchTree(ref<Store> store) const;
}; };

View file

@ -52,7 +52,7 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
if (args[0]->type == tAttrs) { if (args[0]->type == tAttrs) {
state.forceAttrs(*args[0], pos); state.forceAttrs(*args[0], pos);
fetchers::Input::Attrs attrs; fetchers::Attrs attrs;
for (auto & attr : *args[0]->attrs) { for (auto & attr : *args[0]->attrs) {
state.forceValue(*attr.value); state.forceValue(*attr.value);

View file

@ -0,0 +1,71 @@
#include "attrs.hh"
#include "fetchers.hh"
#include <nlohmann/json.hpp>
namespace nix::fetchers {
Attrs jsonToAttrs(const nlohmann::json & json)
{
Attrs attrs;
for (auto & i : json.items()) {
if (i.value().is_number())
attrs.emplace(i.key(), i.value().get<int64_t>());
else if (i.value().is_string())
attrs.emplace(i.key(), i.value().get<std::string>());
else
throw Error("unsupported input attribute type in lock file");
}
return attrs;
}
nlohmann::json attrsToJson(const Attrs & attrs)
{
nlohmann::json json;
for (auto & attr : attrs) {
if (auto v = std::get_if<int64_t>(&attr.second)) {
json[attr.first] = *v;
} else if (auto v = std::get_if<std::string>(&attr.second)) {
json[attr.first] = *v;
} else abort();
}
return json;
}
std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::string & name)
{
auto i = attrs.find(name);
if (i == attrs.end()) return {};
if (auto v = std::get_if<std::string>(&i->second))
return *v;
throw Error("input attribute '%s' is not a string", name);
}
std::string getStrAttr(const Attrs & attrs, const std::string & name)
{
auto s = maybeGetStrAttr(attrs, name);
if (!s)
throw Error("input attribute '%s' is missing", name);
return *s;
}
std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name)
{
auto i = attrs.find(name);
if (i == attrs.end()) return {};
if (auto v = std::get_if<int64_t>(&i->second))
return *v;
throw Error("input attribute '%s' is not a string", name);
}
int64_t getIntAttr(const Attrs & attrs, const std::string & name)
{
auto s = maybeGetIntAttr(attrs, name);
if (!s)
throw Error("input attribute '%s' is missing", name);
return *s;
}
}

View file

@ -0,0 +1,26 @@
#pragma once
#include "types.hh"
#include <variant>
#include <nlohmann/json_fwd.hpp>
namespace nix::fetchers {
typedef std::variant<std::string, int64_t> Attr;
typedef std::map<std::string, Attr> Attrs;
Attrs jsonToAttrs(const nlohmann::json & json);
nlohmann::json attrsToJson(const Attrs & attrs);
std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::string & name);
std::string getStrAttr(const Attrs & attrs, const std::string & name);
std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name);
int64_t getIntAttr(const Attrs & attrs, const std::string & name);
}

View file

@ -0,0 +1,109 @@
#include "fetchers/cache.hh"
#include "sqlite.hh"
#include "sync.hh"
#include "store-api.hh"
#include <nlohmann/json.hpp>
namespace nix::fetchers {
static const char * schema = R"sql(
create table if not exists Cache (
input text not null,
info text not null,
path text not null,
immutable integer not null,
timestamp integer not null,
primary key (input)
);
)sql";
struct CacheImpl : Cache
{
struct State
{
SQLite db;
SQLiteStmt add, lookup;
};
Sync<State> _state;
CacheImpl()
{
auto state(_state.lock());
auto dbPath = getCacheDir() + "/nix/fetcher-cache-v1.sqlite";
createDirs(dirOf(dbPath));
state->db = SQLite(dbPath);
state->db.isCache();
state->db.exec(schema);
state->add.create(state->db,
"insert or replace into Cache(input, info, path, immutable, timestamp) values (?, ?, ?, ?, ?)");
state->lookup.create(state->db,
"select info, path, immutable, timestamp from Cache where input = ?");
}
void add(
ref<Store> store,
const Attrs & inAttrs,
const Attrs & infoAttrs,
const StorePath & storePath,
bool immutable) override
{
_state.lock()->add.use()
(attrsToJson(inAttrs).dump())
(attrsToJson(infoAttrs).dump())
(store->printStorePath(storePath))
(immutable)
(time(0)).exec();
}
std::optional<std::pair<Attrs, StorePath>> lookup(
ref<Store> store,
const Attrs & inAttrs) override
{
auto state(_state.lock());
auto inAttrsJson = attrsToJson(inAttrs).dump();
auto stmt(state->lookup.use()(inAttrsJson));
if (!stmt.next()) {
debug("did not find cache entry for '%s'", inAttrsJson);
return {};
}
auto infoJson = stmt.getStr(0);
auto storePath = store->parseStorePath(stmt.getStr(1));
auto immutable = stmt.getInt(2) != 0;
auto timestamp = stmt.getInt(3);
if (!immutable && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0))) {
debug("ignoring expired cache entry '%s'", inAttrsJson);
return {};
}
store->addTempRoot(storePath);
if (!store->isValidPath(storePath)) {
// FIXME: we could try to substitute 'storePath'.
debug("ignoring disappeared cache entry '%s'", inAttrsJson);
return {};
}
debug("using cache entry '%s' -> '%s', '%s'",
inAttrsJson, infoJson, store->printStorePath(storePath));
return {{jsonToAttrs(nlohmann::json::parse(infoJson)), std::move(storePath)}};
}
};
ref<Cache> getCache()
{
static auto cache = std::make_shared<CacheImpl>();
return ref<Cache>(cache);
}
}

View file

@ -0,0 +1,24 @@
#pragma once
#include "types.hh"
#include "fetchers/fetchers.hh"
namespace nix::fetchers {
struct Cache
{
virtual void add(
ref<Store> store,
const Attrs & inAttrs,
const Attrs & infoAttrs,
const StorePath & storePath,
bool immutable) = 0;
virtual std::optional<std::pair<Attrs, StorePath>> lookup(
ref<Store> store,
const Attrs & inAttrs) = 0;
};
ref<Cache> getCache();
}

View file

@ -28,7 +28,7 @@ std::unique_ptr<Input> inputFromURL(const std::string & url)
return inputFromURL(parseURL(url)); return inputFromURL(parseURL(url));
} }
std::unique_ptr<Input> inputFromAttrs(const Input::Attrs & attrs) std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs)
{ {
for (auto & inputScheme : *inputSchemes) { for (auto & inputScheme : *inputSchemes) {
auto res = inputScheme->inputFromAttrs(attrs); auto res = inputScheme->inputFromAttrs(attrs);
@ -42,36 +42,7 @@ std::unique_ptr<Input> inputFromAttrs(const Input::Attrs & attrs)
throw Error("input '%s' is unsupported", attrsToJson(attrs)); throw Error("input '%s' is unsupported", attrsToJson(attrs));
} }
Input::Attrs jsonToAttrs(const nlohmann::json & json) Attrs Input::toAttrs() const
{
fetchers::Input::Attrs attrs;
for (auto & i : json.items()) {
if (i.value().is_number())
attrs.emplace(i.key(), i.value().get<int64_t>());
else if (i.value().is_string())
attrs.emplace(i.key(), i.value().get<std::string>());
else
throw Error("unsupported input attribute type in lock file");
}
return attrs;
}
nlohmann::json attrsToJson(const fetchers::Input::Attrs & attrs)
{
nlohmann::json json;
for (auto & attr : attrs) {
if (auto v = std::get_if<int64_t>(&attr.second)) {
json[attr.first] = *v;
} else if (auto v = std::get_if<std::string>(&attr.second)) {
json[attr.first] = *v;
} else abort();
}
return json;
}
Input::Attrs Input::toAttrs() const
{ {
auto attrs = toAttrsInternal(); auto attrs = toAttrsInternal();
if (narHash) if (narHash)
@ -80,23 +51,6 @@ Input::Attrs Input::toAttrs() const
return attrs; return attrs;
} }
std::optional<std::string> maybeGetStrAttr(const Input::Attrs & attrs, const std::string & name)
{
auto i = attrs.find(name);
if (i == attrs.end()) return {};
if (auto v = std::get_if<std::string>(&i->second))
return *v;
throw Error("input attribute '%s' is not a string", name);
}
std::string getStrAttr(const Input::Attrs & attrs, const std::string & name)
{
auto s = maybeGetStrAttr(attrs, name);
if (!s)
throw Error("input attribute '%s' is missing", name);
return *s;
}
std::pair<Tree, std::shared_ptr<const Input>> Input::fetchTree(ref<Store> store) const std::pair<Tree, std::shared_ptr<const Input>> Input::fetchTree(ref<Store> store) const
{ {
auto [tree, input] = fetchTreeInternal(store); auto [tree, input] = fetchTreeInternal(store);

View file

@ -4,11 +4,9 @@
#include "hash.hh" #include "hash.hh"
#include "path.hh" #include "path.hh"
#include "tree-info.hh" #include "tree-info.hh"
#include "attrs.hh"
#include <memory> #include <memory>
#include <variant>
#include <nlohmann/json_fwd.hpp>
namespace nix { class Store; } namespace nix { class Store; }
@ -49,9 +47,6 @@ struct Input : std::enable_shared_from_this<Input>
virtual std::string to_string() const = 0; virtual std::string to_string() const = 0;
typedef std::variant<std::string, int64_t> Attr;
typedef std::map<std::string, Attr> Attrs;
Attrs toAttrs() const; Attrs toAttrs() const;
std::pair<Tree, std::shared_ptr<const Input>> fetchTree(ref<Store> store) const; std::pair<Tree, std::shared_ptr<const Input>> fetchTree(ref<Store> store) const;
@ -87,23 +82,15 @@ struct InputScheme
virtual std::unique_ptr<Input> inputFromURL(const ParsedURL & url) = 0; virtual std::unique_ptr<Input> inputFromURL(const ParsedURL & url) = 0;
virtual std::unique_ptr<Input> inputFromAttrs(const Input::Attrs & attrs) = 0; virtual std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) = 0;
}; };
std::unique_ptr<Input> inputFromURL(const ParsedURL & url); std::unique_ptr<Input> inputFromURL(const ParsedURL & url);
std::unique_ptr<Input> inputFromURL(const std::string & url); std::unique_ptr<Input> inputFromURL(const std::string & url);
std::unique_ptr<Input> inputFromAttrs(const Input::Attrs & attrs); std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs);
void registerInputScheme(std::unique_ptr<InputScheme> && fetcher); void registerInputScheme(std::unique_ptr<InputScheme> && fetcher);
Input::Attrs jsonToAttrs(const nlohmann::json & json);
nlohmann::json attrsToJson(const Input::Attrs & attrs);
std::optional<std::string> maybeGetStrAttr(const Input::Attrs & attrs, const std::string & name);
std::string getStrAttr(const Input::Attrs & attrs, const std::string & name);
} }

View file

@ -420,7 +420,7 @@ struct GitInputScheme : InputScheme
if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4); if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4);
url2.query.clear(); url2.query.clear();
Input::Attrs attrs; Attrs attrs;
attrs.emplace("type", "git"); attrs.emplace("type", "git");
for (auto &[name, value] : url.query) { for (auto &[name, value] : url.query) {
@ -435,7 +435,7 @@ struct GitInputScheme : InputScheme
return inputFromAttrs(attrs); return inputFromAttrs(attrs);
} }
std::unique_ptr<Input> inputFromAttrs(const Input::Attrs & attrs) override std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
{ {
if (maybeGetStrAttr(attrs, "type") != "git") return {}; if (maybeGetStrAttr(attrs, "type") != "git") return {};

View file

@ -1,8 +1,9 @@
#include "fetchers.hh"
#include "download.hh" #include "download.hh"
#include "fetchers/cache.hh"
#include "fetchers/fetchers.hh"
#include "fetchers/parse.hh"
#include "fetchers/regex.hh"
#include "globals.hh" #include "globals.hh"
#include "parse.hh"
#include "regex.hh"
#include "store-api.hh" #include "store-api.hh"
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
@ -72,17 +73,36 @@ struct GitHubInput : Input
std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
{ {
auto rev = this->rev; auto rev = this->rev;
auto ref = this->ref.value_or("master");
#if 0 Attrs mutableAttrs({
if (rev) { {"type", "github"},
if (auto gitInfo = lookupGitInfo(store, "source", *rev)) {"owner", owner},
return *gitInfo; {"repo", repo},
{"ref", ref},
});
if (!rev) {
if (auto res = getCache()->lookup(store, mutableAttrs)) {
auto input = std::make_shared<GitHubInput>(*this);
input->ref = {};
input->rev = Hash(getStrAttr(res->first, "rev"), htSHA1);
return {
Tree{
.actualPath = store->toRealPath(res->second),
.storePath = std::move(res->second),
.info = TreeInfo {
.lastModified = getIntAttr(res->first, "lastModified"),
},
},
input
};
}
} }
#endif
if (!rev) { if (!rev) {
auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s", auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s",
owner, repo, ref ? *ref : "master"); owner, repo, ref);
CachedDownloadRequest request(url); CachedDownloadRequest request(url);
request.ttl = rev ? 1000000000 : settings.tarballTtl; request.ttl = rev ? 1000000000 : settings.tarballTtl;
auto result = getDownloader()->downloadCached(store, request); auto result = getDownloader()->downloadCached(store, request);
@ -91,6 +111,28 @@ struct GitHubInput : Input
debug("HEAD revision for '%s' is %s", url, rev->gitRev()); debug("HEAD revision for '%s' is %s", url, rev->gitRev());
} }
auto input = std::make_shared<GitHubInput>(*this);
input->ref = {};
input->rev = *rev;
Attrs immutableAttrs({
{"type", "git-tarball"},
{"rev", rev->gitRev()},
});
if (auto res = getCache()->lookup(store, immutableAttrs)) {
return {
Tree{
.actualPath = store->toRealPath(res->second),
.storePath = std::move(res->second),
.info = TreeInfo {
.lastModified = getIntAttr(res->first, "lastModified"),
},
},
input
};
}
// FIXME: use regular /archive URLs instead? api.github.com // FIXME: use regular /archive URLs instead? api.github.com
// might have stricter rate limits. // might have stricter rate limits.
@ -118,14 +160,25 @@ struct GitHubInput : Input
}, },
}; };
#if 0 Attrs infoAttrs({
// FIXME: this can overwrite a cache file that contains a revCount. {"rev", rev->gitRev()},
cacheGitInfo("source", gitInfo); {"lastModified", *result.info.lastModified}
#endif });
auto input = std::make_shared<GitHubInput>(*this); if (!this->rev)
input->ref = {}; getCache()->add(
input->rev = *rev; store,
mutableAttrs,
infoAttrs,
result.storePath,
false);
getCache()->add(
store,
immutableAttrs,
infoAttrs,
result.storePath,
true);
return {std::move(result), input}; return {std::move(result), input};
} }
@ -189,7 +242,7 @@ struct GitHubInputScheme : InputScheme
return input; return input;
} }
std::unique_ptr<Input> inputFromAttrs(const Input::Attrs & attrs) override std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
{ {
if (maybeGetStrAttr(attrs, "type") != "github") return {}; if (maybeGetStrAttr(attrs, "type") != "github") return {};

View file

@ -120,7 +120,7 @@ struct IndirectInputScheme : InputScheme
return input; return input;
} }
std::unique_ptr<Input> inputFromAttrs(const Input::Attrs & attrs) override std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
{ {
if (maybeGetStrAttr(attrs, "type") != "indirect") return {}; if (maybeGetStrAttr(attrs, "type") != "indirect") return {};

View file

@ -1,5 +1,6 @@
#include "fetchers.hh" #include "fetchers/fetchers.hh"
#include "parse.hh" #include "fetchers/cache.hh"
#include "fetchers/parse.hh"
#include "globals.hh" #include "globals.hh"
#include "tarfile.hh" #include "tarfile.hh"
#include "store-api.hh" #include "store-api.hh"
@ -7,8 +8,6 @@
#include <sys/time.h> #include <sys/time.h>
#include <nlohmann/json.hpp>
using namespace std::string_literals; using namespace std::string_literals;
namespace nix::fetchers { namespace nix::fetchers {
@ -163,51 +162,80 @@ struct MercurialInput : Input
if (!input->ref) input->ref = "default"; if (!input->ref) input->ref = "default";
Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(Base32, false)); auto getImmutableAttrs = [&]()
{
return Attrs({
{"type", "hg"},
{"name", name},
{"rev", input->rev->gitRev()},
});
};
auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
-> std::pair<Tree, std::shared_ptr<const Input>>
{
input->rev = Hash(getStrAttr(infoAttrs, "rev"), htSHA1);
assert(!rev || rev == input->rev);
return {
Tree{
.actualPath = store->toRealPath(storePath),
.storePath = std::move(storePath),
.info = TreeInfo {
.revCount = getIntAttr(infoAttrs, "revCount"),
},
},
input
};
};
if (input->rev) {
if (auto res = getCache()->lookup(store, getImmutableAttrs()))
return makeResult(res->first, std::move(res->second));
}
assert(input->rev || input->ref); assert(input->rev || input->ref);
auto revOrRef = input->rev ? input->rev->gitRev() : *input->ref; auto revOrRef = input->rev ? input->rev->gitRev() : *input->ref;
Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir, hashString(htSHA512, revOrRef).to_string(Base32, false)); Attrs mutableAttrs({
{"type", "hg"},
{"name", name},
{"url", actualUrl},
{"ref", *input->ref},
});
/* If we haven't pulled this repo less than tarball-ttl seconds, if (auto res = getCache()->lookup(store, mutableAttrs))
do so now. */ return makeResult(res->first, std::move(res->second));
time_t now = time(0);
struct stat st; Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(Base32, false));
if (stat(stampFile.c_str(), &st) != 0 ||
(uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now) /* If this is a commit hash that we already have, we don't
have to pull again. */
if (!(input->rev
&& pathExists(cacheDir)
&& runProgram(
RunOptions("hg", { "log", "-R", cacheDir, "-r", input->rev->gitRev(), "--template", "1" })
.killStderr(true)).second == "1"))
{ {
/* Except that if this is a commit hash that we already have, Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl));
we don't have to pull again. */
if (!(input->rev
&& pathExists(cacheDir)
&& runProgram(
RunOptions("hg", { "log", "-R", cacheDir, "-r", input->rev->gitRev(), "--template", "1" })
.killStderr(true)).second == "1"))
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl));
if (pathExists(cacheDir)) { if (pathExists(cacheDir)) {
try { try {
runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl }); runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl });
}
catch (ExecError & e) {
string transJournal = cacheDir + "/.hg/store/journal";
/* hg throws "abandoned transaction" error only if this file exists */
if (pathExists(transJournal)) {
runProgram("hg", true, { "recover", "-R", cacheDir });
runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl });
} else {
throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status)));
}
}
} else {
createDirs(dirOf(cacheDir));
runProgram("hg", true, { "clone", "--noupdate", "--", actualUrl, cacheDir });
} }
catch (ExecError & e) {
string transJournal = cacheDir + "/.hg/store/journal";
/* hg throws "abandoned transaction" error only if this file exists */
if (pathExists(transJournal)) {
runProgram("hg", true, { "recover", "-R", cacheDir });
runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl });
} else {
throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status)));
}
}
} else {
createDirs(dirOf(cacheDir));
runProgram("hg", true, { "clone", "--noupdate", "--", actualUrl, cacheDir });
} }
writeFile(stampFile, "");
} }
auto tokens = tokenizeString<std::vector<std::string>>( auto tokens = tokenizeString<std::vector<std::string>>(
@ -218,33 +246,8 @@ struct MercurialInput : Input
auto revCount = std::stoull(tokens[1]); auto revCount = std::stoull(tokens[1]);
input->ref = tokens[2]; input->ref = tokens[2];
std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + input->rev->gitRev()).to_string(Base32, false); if (auto res = getCache()->lookup(store, getImmutableAttrs()))
Path storeLink = fmt("%s/.hg/%s.link", cacheDir, storeLinkName); return makeResult(res->first, std::move(res->second));
try {
auto json = nlohmann::json::parse(readFile(storeLink));
assert(json["name"] == name && json["rev"] == input->rev->gitRev());
auto storePath = store->parseStorePath((std::string) json["storePath"]);
if (store->isValidPath(storePath)) {
printTalkative("using cached Mercurial store path '%s'", store->printStorePath(storePath));
return {
Tree {
.actualPath = store->printStorePath(storePath),
.storePath = std::move(storePath),
.info = TreeInfo {
.revCount = revCount,
},
},
input
};
}
} catch (SysError & e) {
if (e.errNo != ENOENT) throw;
}
Path tmpDir = createTempDir(); Path tmpDir = createTempDir();
AutoDelete delTmpDir(tmpDir, true); AutoDelete delTmpDir(tmpDir, true);
@ -255,26 +258,27 @@ struct MercurialInput : Input
auto storePath = store->addToStore(name, tmpDir); auto storePath = store->addToStore(name, tmpDir);
nlohmann::json json; Attrs infoAttrs({
json["storePath"] = store->printStorePath(storePath); {"rev", input->rev->gitRev()},
json["uri"] = actualUrl; {"revCount", revCount},
json["name"] = name; });
json["branch"] = *input->ref;
json["rev"] = input->rev->gitRev();
json["revCount"] = revCount;
writeFile(storeLink, json.dump()); if (!this->rev)
getCache()->add(
store,
mutableAttrs,
infoAttrs,
storePath,
false);
return { getCache()->add(
Tree { store,
.actualPath = store->printStorePath(storePath), getImmutableAttrs(),
.storePath = std::move(storePath), infoAttrs,
.info = TreeInfo { storePath,
.revCount = revCount true);
}
}, return makeResult(infoAttrs, std::move(storePath));
input
};
} }
}; };
@ -291,7 +295,7 @@ struct MercurialInputScheme : InputScheme
url2.scheme = std::string(url2.scheme, 3); url2.scheme = std::string(url2.scheme, 3);
url2.query.clear(); url2.query.clear();
Input::Attrs attrs; Attrs attrs;
attrs.emplace("type", "hg"); attrs.emplace("type", "hg");
for (auto &[name, value] : url.query) { for (auto &[name, value] : url.query) {
@ -306,7 +310,7 @@ struct MercurialInputScheme : InputScheme
return inputFromAttrs(attrs); return inputFromAttrs(attrs);
} }
std::unique_ptr<Input> inputFromAttrs(const Input::Attrs & attrs) override std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
{ {
if (maybeGetStrAttr(attrs, "type") != "hg") return {}; if (maybeGetStrAttr(attrs, "type") != "hg") return {};

View file

@ -36,7 +36,7 @@ std::shared_ptr<Registry> Registry::read(
else if (version == 2) { else if (version == 2) {
for (auto & i : json["flakes"]) { for (auto & i : json["flakes"]) {
auto toAttrs = jsonToAttrs(i["to"]); auto toAttrs = jsonToAttrs(i["to"]);
Input::Attrs extraAttrs; Attrs extraAttrs;
auto j = toAttrs.find("dir"); auto j = toAttrs.find("dir");
if (j != toAttrs.end()) { if (j != toAttrs.end()) {
extraAttrs.insert(*j); extraAttrs.insert(*j);
@ -80,7 +80,7 @@ void Registry::write(const Path & path)
void Registry::add( void Registry::add(
const std::shared_ptr<const Input> & from, const std::shared_ptr<const Input> & from,
const std::shared_ptr<const Input> & to, const std::shared_ptr<const Input> & to,
const Input::Attrs & extraAttrs) const Attrs & extraAttrs)
{ {
entries.emplace_back(from, to, extraAttrs); entries.emplace_back(from, to, extraAttrs);
} }
@ -116,7 +116,7 @@ std::shared_ptr<Registry> getFlagRegistry()
void overrideRegistry( void overrideRegistry(
const std::shared_ptr<const Input> & from, const std::shared_ptr<const Input> & from,
const std::shared_ptr<const Input> & to, const std::shared_ptr<const Input> & to,
const Input::Attrs & extraAttrs) const Attrs & extraAttrs)
{ {
flagRegistry->add(from, to, extraAttrs); flagRegistry->add(from, to, extraAttrs);
} }
@ -148,11 +148,11 @@ Registries getRegistries(ref<Store> store)
return registries; return registries;
} }
std::pair<std::shared_ptr<const Input>, Input::Attrs> lookupInRegistries( std::pair<std::shared_ptr<const Input>, Attrs> lookupInRegistries(
ref<Store> store, ref<Store> store,
std::shared_ptr<const Input> input) std::shared_ptr<const Input> input)
{ {
Input::Attrs extraAttrs; Attrs extraAttrs;
int n = 0; int n = 0;
restart: restart:

View file

@ -21,7 +21,7 @@ struct Registry
std::tuple< std::tuple<
std::shared_ptr<const Input>, // from std::shared_ptr<const Input>, // from
std::shared_ptr<const Input>, // to std::shared_ptr<const Input>, // to
Input::Attrs // extra attributes Attrs // extra attributes
> >
> entries; > entries;
@ -37,7 +37,7 @@ struct Registry
void add( void add(
const std::shared_ptr<const Input> & from, const std::shared_ptr<const Input> & from,
const std::shared_ptr<const Input> & to, const std::shared_ptr<const Input> & to,
const Input::Attrs & extraAttrs); const Attrs & extraAttrs);
void remove(const std::shared_ptr<const Input> & input); void remove(const std::shared_ptr<const Input> & input);
}; };
@ -53,9 +53,9 @@ Registries getRegistries(ref<Store> store);
void overrideRegistry( void overrideRegistry(
const std::shared_ptr<const Input> & from, const std::shared_ptr<const Input> & from,
const std::shared_ptr<const Input> & to, const std::shared_ptr<const Input> & to,
const Input::Attrs & extraAttrs); const Attrs & extraAttrs);
std::pair<std::shared_ptr<const Input>, Input::Attrs> lookupInRegistries( std::pair<std::shared_ptr<const Input>, Attrs> lookupInRegistries(
ref<Store> store, ref<Store> store,
std::shared_ptr<const Input> input); std::shared_ptr<const Input> input);

View file

@ -109,7 +109,7 @@ struct TarballInputScheme : InputScheme
return input; return input;
} }
std::unique_ptr<Input> inputFromAttrs(const Input::Attrs & attrs) override std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
{ {
if (maybeGetStrAttr(attrs, "type") != "tarball") return {}; if (maybeGetStrAttr(attrs, "type") != "tarball") return {};

View file

@ -507,7 +507,7 @@ struct CmdFlakeAdd : MixEvalArgs, Command
{ {
auto fromRef = parseFlakeRef(fromUrl); auto fromRef = parseFlakeRef(fromUrl);
auto toRef = parseFlakeRef(toUrl); auto toRef = parseFlakeRef(toUrl);
fetchers::Input::Attrs extraAttrs; fetchers::Attrs extraAttrs;
if (toRef.subdir != "") extraAttrs["dir"] = toRef.subdir; if (toRef.subdir != "") extraAttrs["dir"] = toRef.subdir;
auto userRegistry = fetchers::getUserRegistry(); auto userRegistry = fetchers::getUserRegistry();
userRegistry->remove(fromRef.input); userRegistry->remove(fromRef.input);
@ -558,7 +558,7 @@ struct CmdFlakePin : virtual Args, EvalCommand
auto userRegistry = fetchers::getUserRegistry(); auto userRegistry = fetchers::getUserRegistry();
userRegistry->remove(ref.input); userRegistry->remove(ref.input);
auto [tree, resolved] = ref.resolve(store).input->fetchTree(store); auto [tree, resolved] = ref.resolve(store).input->fetchTree(store);
fetchers::Input::Attrs extraAttrs; fetchers::Attrs extraAttrs;
if (ref.subdir != "") extraAttrs["dir"] = ref.subdir; if (ref.subdir != "") extraAttrs["dir"] = ref.subdir;
userRegistry->add(ref.input, resolved, extraAttrs); userRegistry->add(ref.input, resolved, extraAttrs);
} }

View file

@ -9,7 +9,7 @@ clearStore
repo=$TEST_ROOT/hg repo=$TEST_ROOT/hg
rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix/hg rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix
hg init $repo hg init $repo
echo '[ui]' >> $repo/.hg/hgrc echo '[ui]' >> $repo/.hg/hgrc
@ -50,13 +50,13 @@ path2=$(nix eval --impure --raw --expr "(builtins.fetchMercurial file://$repo).o
[[ $(nix eval --impure --raw --expr "(builtins.fetchMercurial file://$repo).rev") = $rev2 ]] [[ $(nix eval --impure --raw --expr "(builtins.fetchMercurial file://$repo).rev") = $rev2 ]]
# But with TTL 0, it should fail. # But with TTL 0, it should fail.
(! nix eval --impure --tarball-ttl 0 --expr "builtins.fetchMercurial file://$repo") (! nix eval --impure --refresh --expr "builtins.fetchMercurial file://$repo")
# Fetching with a explicit hash should succeed. # Fetching with a explicit hash should succeed.
path2=$(nix eval --tarball-ttl 0 --raw --expr "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev2\"; }).outPath") path2=$(nix eval --refresh --raw --expr "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev2\"; }).outPath")
[[ $path = $path2 ]] [[ $path = $path2 ]]
path2=$(nix eval --tarball-ttl 0 --raw --expr "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev1\"; }).outPath") path2=$(nix eval --refresh --raw --expr "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev1\"; }).outPath")
[[ $(cat $path2/hello) = utrecht ]] [[ $(cat $path2/hello) = utrecht ]]
mv ${repo}-tmp $repo mv ${repo}-tmp $repo
@ -89,5 +89,5 @@ path3=$(nix eval --impure --raw --expr "(builtins.fetchMercurial { url = $repo;
# Committing should not affect the store path. # Committing should not affect the store path.
hg commit --cwd $repo -m 'Bla3' hg commit --cwd $repo -m 'Bla3'
path4=$(nix eval --impure --tarball-ttl 0 --raw --expr "(builtins.fetchMercurial file://$repo).outPath") path4=$(nix eval --impure --refresh --raw --expr "(builtins.fetchMercurial file://$repo).outPath")
[[ $path2 = $path4 ]] [[ $path2 = $path4 ]]