Persistently cache InputAccessor::fetchToStore()

This avoids repeated copying of the same source tree between Nix
invocations. It requires the accessor to have a "fingerprint" (e.g. a
Git revision) that uniquely determines its contents.
This commit is contained in:
Eelco Dolstra 2023-11-20 20:04:37 +01:00
parent f450c8773c
commit 99d5204baa
8 changed files with 77 additions and 3 deletions

View file

@ -108,6 +108,11 @@ Input Input::fromAttrs(Attrs && attrs)
return std::move(*res);
}
std::optional<std::string> Input::getFingerprint(ref<Store> store) const
{
return scheme ? scheme->getFingerprint(store, *this) : std::nullopt;
}
ParsedURL Input::toURL() const
{
if (!scheme)

View file

@ -113,6 +113,12 @@ public:
std::optional<Hash> getRev() const;
std::optional<uint64_t> getRevCount() const;
std::optional<time_t> getLastModified() const;
/**
* For locked inputs, return a string that uniquely specifies the
* content of the input (typically a commit hash or content hash).
*/
std::optional<std::string> getFingerprint(ref<Store> store) const;
};
@ -180,6 +186,9 @@ struct InputScheme
virtual bool isDirect(const Input & input) const
{ return true; }
virtual std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const
{ return std::nullopt; }
};
void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);

View file

@ -700,10 +700,22 @@ struct GitInputScheme : InputScheme
auto repoInfo = getRepoInfo(input);
return
auto [accessor, final] =
input.getRef() || input.getRev() || !repoInfo.isLocal
? getAccessorFromCommit(store, repoInfo, std::move(input))
: getAccessorFromWorkdir(store, repoInfo, std::move(input));
accessor->fingerprint = final.getFingerprint(store);
return {accessor, std::move(final)};
}
std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
{
if (auto rev = input.getRev())
return rev->gitRev() + (getSubmodulesAttr(input) ? ";s" : "");
else
return std::nullopt;
}
};

View file

@ -229,6 +229,14 @@ struct GitArchiveInputScheme : InputScheme
{
return Xp::Flakes;
}
std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
{
if (auto rev = input.getRev())
return rev->gitRev();
else
return std::nullopt;
}
};
struct GitHubInputScheme : GitArchiveInputScheme

View file

@ -1,5 +1,6 @@
#include "input-accessor.hh"
#include "store-api.hh"
#include "cache.hh"
namespace nix {
@ -11,6 +12,30 @@ StorePath InputAccessor::fetchToStore(
PathFilter * filter,
RepairFlag repair)
{
// FIXME: add an optimisation for the case where the accessor is
// an FSInputAccessor pointing to a store path.
std::optional<fetchers::Attrs> cacheKey;
if (!filter && fingerprint) {
cacheKey = fetchers::Attrs{
{"_what", "fetchToStore"},
{"store", store->storeDir},
{"name", std::string(name)},
{"fingerprint", *fingerprint},
{"method", (uint8_t) method},
{"path", path.abs()}
};
if (auto res = fetchers::getCache()->lookup(*cacheKey)) {
StorePath storePath(fetchers::getStrAttr(*res, "storePath"));
if (store->isValidPath(storePath)) {
debug("store path cache hit for '%s'", showPath(path));
return storePath;
}
}
} else
debug("source path '%s' is uncacheable", showPath(path));
Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", showPath(path)));
auto source = sinkToSource([&](Sink & sink) {
@ -25,6 +50,11 @@ StorePath InputAccessor::fetchToStore(
? store->computeStorePathFromDump(*source, name, method, htSHA256).first
: store->addToStoreFromDump(*source, name, method, htSHA256, repair);
if (cacheKey)
fetchers::getCache()->upsert(
*cacheKey,
fetchers::Attrs{{"storePath", std::string(storePath.to_string())}});
return storePath;
}

View file

@ -18,6 +18,8 @@ class Store;
struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this<InputAccessor>
{
std::optional<std::string> fingerprint;
/**
* Return the maximum last-modified time of the files in this
* tree, if available.

View file

@ -339,6 +339,14 @@ struct MercurialInputScheme : InputScheme
return makeResult(infoAttrs, std::move(storePath));
}
std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
{
if (auto rev = input.getRev())
return rev->gitRev();
else
return std::nullopt;
}
};
static auto rMercurialInputScheme = OnStartup([] { registerInputScheme(std::make_unique<MercurialInputScheme>()); });

View file

@ -39,12 +39,12 @@ enum struct FileIngestionMethod : uint8_t {
/**
* Flat-file hashing. Directly ingest the contents of a single file
*/
Flat = false,
Flat = 0,
/**
* Recursive (or NAR) hashing. Serializes the file-system object in Nix
* Archive format and ingest that
*/
Recursive = true
Recursive = 1
};
/**