forked from lix-project/lix
Persistently cache InputAccessor::fetchToStore()
This avoids repeated copying of the same source tree between Nix invocations. It requires the accessor to have a "fingerprint" (e.g. a Git revision) that uniquely determines its contents.
This commit is contained in:
parent
f450c8773c
commit
99d5204baa
|
@ -108,6 +108,11 @@ Input Input::fromAttrs(Attrs && attrs)
|
|||
return std::move(*res);
|
||||
}
|
||||
|
||||
std::optional<std::string> Input::getFingerprint(ref<Store> store) const
|
||||
{
|
||||
return scheme ? scheme->getFingerprint(store, *this) : std::nullopt;
|
||||
}
|
||||
|
||||
ParsedURL Input::toURL() const
|
||||
{
|
||||
if (!scheme)
|
||||
|
|
|
@ -113,6 +113,12 @@ public:
|
|||
std::optional<Hash> getRev() const;
|
||||
std::optional<uint64_t> getRevCount() const;
|
||||
std::optional<time_t> getLastModified() const;
|
||||
|
||||
/**
|
||||
* For locked inputs, return a string that uniquely specifies the
|
||||
* content of the input (typically a commit hash or content hash).
|
||||
*/
|
||||
std::optional<std::string> getFingerprint(ref<Store> store) const;
|
||||
};
|
||||
|
||||
|
||||
|
@ -180,6 +186,9 @@ struct InputScheme
|
|||
|
||||
virtual bool isDirect(const Input & input) const
|
||||
{ return true; }
|
||||
|
||||
virtual std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const
|
||||
{ return std::nullopt; }
|
||||
};
|
||||
|
||||
void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);
|
||||
|
|
|
@ -700,10 +700,22 @@ struct GitInputScheme : InputScheme
|
|||
|
||||
auto repoInfo = getRepoInfo(input);
|
||||
|
||||
return
|
||||
auto [accessor, final] =
|
||||
input.getRef() || input.getRev() || !repoInfo.isLocal
|
||||
? getAccessorFromCommit(store, repoInfo, std::move(input))
|
||||
: getAccessorFromWorkdir(store, repoInfo, std::move(input));
|
||||
|
||||
accessor->fingerprint = final.getFingerprint(store);
|
||||
|
||||
return {accessor, std::move(final)};
|
||||
}
|
||||
|
||||
std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
|
||||
{
|
||||
if (auto rev = input.getRev())
|
||||
return rev->gitRev() + (getSubmodulesAttr(input) ? ";s" : "");
|
||||
else
|
||||
return std::nullopt;
|
||||
}
|
||||
};
|
||||
|
||||
|
|
|
@ -229,6 +229,14 @@ struct GitArchiveInputScheme : InputScheme
|
|||
{
|
||||
return Xp::Flakes;
|
||||
}
|
||||
|
||||
std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
|
||||
{
|
||||
if (auto rev = input.getRev())
|
||||
return rev->gitRev();
|
||||
else
|
||||
return std::nullopt;
|
||||
}
|
||||
};
|
||||
|
||||
struct GitHubInputScheme : GitArchiveInputScheme
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
#include "input-accessor.hh"
|
||||
#include "store-api.hh"
|
||||
#include "cache.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
|
@ -11,6 +12,30 @@ StorePath InputAccessor::fetchToStore(
|
|||
PathFilter * filter,
|
||||
RepairFlag repair)
|
||||
{
|
||||
// FIXME: add an optimisation for the case where the accessor is
|
||||
// an FSInputAccessor pointing to a store path.
|
||||
|
||||
std::optional<fetchers::Attrs> cacheKey;
|
||||
|
||||
if (!filter && fingerprint) {
|
||||
cacheKey = fetchers::Attrs{
|
||||
{"_what", "fetchToStore"},
|
||||
{"store", store->storeDir},
|
||||
{"name", std::string(name)},
|
||||
{"fingerprint", *fingerprint},
|
||||
{"method", (uint8_t) method},
|
||||
{"path", path.abs()}
|
||||
};
|
||||
if (auto res = fetchers::getCache()->lookup(*cacheKey)) {
|
||||
StorePath storePath(fetchers::getStrAttr(*res, "storePath"));
|
||||
if (store->isValidPath(storePath)) {
|
||||
debug("store path cache hit for '%s'", showPath(path));
|
||||
return storePath;
|
||||
}
|
||||
}
|
||||
} else
|
||||
debug("source path '%s' is uncacheable", showPath(path));
|
||||
|
||||
Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", showPath(path)));
|
||||
|
||||
auto source = sinkToSource([&](Sink & sink) {
|
||||
|
@ -25,6 +50,11 @@ StorePath InputAccessor::fetchToStore(
|
|||
? store->computeStorePathFromDump(*source, name, method, htSHA256).first
|
||||
: store->addToStoreFromDump(*source, name, method, htSHA256, repair);
|
||||
|
||||
if (cacheKey)
|
||||
fetchers::getCache()->upsert(
|
||||
*cacheKey,
|
||||
fetchers::Attrs{{"storePath", std::string(storePath.to_string())}});
|
||||
|
||||
return storePath;
|
||||
}
|
||||
|
||||
|
|
|
@ -18,6 +18,8 @@ class Store;
|
|||
|
||||
struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this<InputAccessor>
|
||||
{
|
||||
std::optional<std::string> fingerprint;
|
||||
|
||||
/**
|
||||
* Return the maximum last-modified time of the files in this
|
||||
* tree, if available.
|
||||
|
|
|
@ -339,6 +339,14 @@ struct MercurialInputScheme : InputScheme
|
|||
|
||||
return makeResult(infoAttrs, std::move(storePath));
|
||||
}
|
||||
|
||||
std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
|
||||
{
|
||||
if (auto rev = input.getRev())
|
||||
return rev->gitRev();
|
||||
else
|
||||
return std::nullopt;
|
||||
}
|
||||
};
|
||||
|
||||
static auto rMercurialInputScheme = OnStartup([] { registerInputScheme(std::make_unique<MercurialInputScheme>()); });
|
||||
|
|
|
@ -39,12 +39,12 @@ enum struct FileIngestionMethod : uint8_t {
|
|||
/**
|
||||
* Flat-file hashing. Directly ingest the contents of a single file
|
||||
*/
|
||||
Flat = false,
|
||||
Flat = 0,
|
||||
/**
|
||||
* Recursive (or NAR) hashing. Serializes the file-system object in Nix
|
||||
* Archive format and ingest that
|
||||
*/
|
||||
Recursive = true
|
||||
Recursive = 1
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
Loading…
Reference in a new issue