lix-project/lix
18
47
Fork 24
Code Issues 333 Code Review (Gerrit) Projects 1 Releases Wiki Activity

Remove TreeInfo

Browse source 
The attributes previously stored in TreeInfo (narHash, revCount,
lastModified) are now stored in Input. This makes it less arbitrary
what attributes are stored where.

As a result, the lock file format has changed. An entry like

    "info": {
      "lastModified": 1585405475,
      "narHash": "sha256-bESW0n4KgPmZ0luxvwJ+UyATrC6iIltVCsGdLiphVeE="
    },
    "locked": {
      "owner": "NixOS",
      "repo": "nixpkgs",
      "rev": "b88ff468e9850410070d4e0ccd68c7011f15b2be",
      "type": "github"
    },

is now stored as

    "locked": {
      "owner": "NixOS",
      "repo": "nixpkgs",
      "rev": "b88ff468e9850410070d4e0ccd68c7011f15b2be",
      "type": "github",
      "lastModified": 1585405475,
      "narHash": "sha256-bESW0n4KgPmZ0luxvwJ+UyATrC6iIltVCsGdLiphVeE="
    },

The 'Input' class is now a dumb set of attributes. All the fetcher
implementations subclass InputScheme, not Input. This simplifies the
API.

Also, fix substitution of flake inputs. This was broken since lazy
flake fetching started using fetchTree internally.
This commit is contained in:
Eelco Dolstra 2020-05-30 00:44:11 +02:00
parent 5633c0975b
commit 950b46821f
30 changed files with 963 additions and 1145 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/libexpr/common-eval-args.cc
View file

@ -76,7 +76,7 @@ Path lookupFileArg(EvalState & state, string s)
if (isUri(s)) {
return state.store->toRealPath(
fetchers::downloadTarball(
state.store, resolveUri(s), "source", false).storePath);
state.store, resolveUri(s), "source", false).first.storePath);
} else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
Path p = s.substr(1, s.size() - 2);
return state.findFile(p);

2
src/libexpr/flake/call-flake.nix
View file

@ -11,7 +11,7 @@ let
sourceInfo =
if key == lockFile.root
then rootSrc
else fetchTree ({ inherit (node.info) narHash; } // removeAttrs node.locked ["dir"]);
else fetchTree (node.info or {} // removeAttrs node.locked ["dir"]);
subdir = if key == lockFile.root then rootSubdir else node.locked.dir or "";
flake = import (sourceInfo + (if subdir != "" then "/" else "") + subdir + "/flake.nix");
inputs = builtins.mapAttrs (inputName: key: allNodes.${key}) (node.inputs or {});

52
src/libexpr/flake/flake.cc
View file

@ -19,7 +19,7 @@ static FlakeRef maybeLookupFlake(
const FlakeRef & flakeRef,
bool allowLookup)
{
if (!flakeRef.input->isDirect()) {
if (!flakeRef.input.isDirect()) {
if (allowLookup)
return flakeRef.resolve(store);
else
@ -49,16 +49,15 @@ static FlakeRef lookupInFlakeCache(
static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree(
EvalState & state,
const FlakeRef & originalRef,
std::optional<TreeInfo> treeInfo,
bool allowLookup,
FlakeCache & flakeCache)
{
/* The tree may already be in the Nix store, or it could be
substituted (which is often faster than fetching from the
original source). So check that. */
if (treeInfo && originalRef.input->isDirect() && originalRef.input->isImmutable()) {
if (originalRef.input.isDirect() && originalRef.input.isImmutable() && originalRef.input.hasAllInfo()) {
try {
auto storePath = treeInfo->computeStorePath(*state.store);
auto storePath = originalRef.input.computeStorePath(*state.store);
state.store->ensurePath(storePath);
@ -74,7 +73,6 @@ static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree(
Tree {
.actualPath = actualPath,
.storePath = std::move(storePath),
.info = *treeInfo,
},
originalRef,
originalRef
@ -99,8 +97,7 @@ static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree(
if (state.allowedPaths)
state.allowedPaths->insert(tree.actualPath);
if (treeInfo)
assert(tree.storePath == treeInfo->computeStorePath(*state.store));
assert(!originalRef.input.getNarHash() || tree.storePath == originalRef.input.computeStorePath(*state.store));
return {std::move(tree), resolvedRef, lockedRef};
}
@ -202,12 +199,11 @@ static std::map<FlakeId, FlakeInput> parseFlakeInputs(
static Flake getFlake(
EvalState & state,
const FlakeRef & originalRef,
std::optional<TreeInfo> treeInfo,
bool allowLookup,
FlakeCache & flakeCache)
{
auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree(
state, originalRef, treeInfo, allowLookup, flakeCache);
state, originalRef, allowLookup, flakeCache);
// Guard against symlink attacks.
auto flakeFile = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir + "/flake.nix");
@ -278,7 +274,7 @@ static Flake getFlake(
Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup)
{
FlakeCache flakeCache;
return getFlake(state, originalRef, {}, allowLookup, flakeCache);
return getFlake(state, originalRef, allowLookup, flakeCache);
}
/* Compute an in-memory lock file for the specified top-level flake,
@ -292,7 +288,7 @@ LockedFlake lockFlake(
FlakeCache flakeCache;
auto flake = getFlake(state, topRef, {}, lockFlags.useRegistries, flakeCache);
auto flake = getFlake(state, topRef, lockFlags.useRegistries, flakeCache);
// FIXME: symlink attack
auto oldLockFile = LockFile::read(
@ -393,7 +389,7 @@ LockedFlake lockFlake(
didn't change and there is no override from a
higher level flake. */
auto childNode = std::make_shared<LockedNode>(
oldLock->lockedRef, oldLock->originalRef, oldLock->info, oldLock->isFlake);
oldLock->lockedRef, oldLock->originalRef, oldLock->isFlake);
node->inputs.insert_or_assign(id, childNode);
@ -409,7 +405,7 @@ LockedFlake lockFlake(
if (hasChildUpdate) {
auto inputFlake = getFlake(
state, oldLock->lockedRef, oldLock->info, false, flakeCache);
state, oldLock->lockedRef, false, flakeCache);
computeLocks(inputFlake.inputs, childNode, inputPath, oldLock);
} else {
/* No need to fetch this flake, we can be
@ -440,11 +436,11 @@ LockedFlake lockFlake(
/* We need to create a new lock file entry. So fetch
this input. */
if (!lockFlags.allowMutable && !input.ref.input->isImmutable())
if (!lockFlags.allowMutable && !input.ref.input.isImmutable())
throw Error("cannot update flake input '%s' in pure mode", inputPathS);
if (input.isFlake) {
auto inputFlake = getFlake(state, input.ref, {}, lockFlags.useRegistries, flakeCache);
auto inputFlake = getFlake(state, input.ref, lockFlags.useRegistries, flakeCache);
/* Note: in case of an --override-input, we use
the *original* ref (input2.ref) for the
@ -454,7 +450,7 @@ LockedFlake lockFlake(
file. That is, overrides are sticky unless you
use --no-write-lock-file. */
auto childNode = std::make_shared<LockedNode>(
inputFlake.lockedRef, input2.ref, inputFlake.sourceInfo->info);
inputFlake.lockedRef, input2.ref);
node->inputs.insert_or_assign(id, childNode);
@ -479,9 +475,9 @@ LockedFlake lockFlake(
else {
auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree(
state, input.ref, {}, lockFlags.useRegistries, flakeCache);
state, input.ref, lockFlags.useRegistries, flakeCache);
node->inputs.insert_or_assign(id,
std::make_shared<LockedNode>(lockedRef, input.ref, sourceInfo.info, false));
std::make_shared<LockedNode>(lockedRef, input.ref, false));
}
}
}
@ -534,7 +530,7 @@ LockedFlake lockFlake(
printInfo("inputs of flake '%s' changed:\n%s", topRef, chomp(diff));
if (lockFlags.writeLockFile) {
if (auto sourcePath = topRef.input->getSourcePath()) {
if (auto sourcePath = topRef.input.getSourcePath()) {
if (!newLockFile.isImmutable()) {
if (settings.warnDirty)
warn("will not write lock file of flake '%s' because it has a mutable input", topRef);
@ -555,7 +551,7 @@ LockedFlake lockFlake(
newLockFile.write(path);
topRef.input->markChangedFile(
topRef.input.markChangedFile(
(topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock",
lockFlags.commitLockFile
? std::optional<std::string>(fmt("%s: %s\n\nFlake input changes:\n\n%s",
@ -567,19 +563,19 @@ LockedFlake lockFlake(
also just clear the 'rev' field... */
auto prevLockedRef = flake.lockedRef;
FlakeCache dummyCache;
flake = getFlake(state, topRef, {}, lockFlags.useRegistries, dummyCache);
flake = getFlake(state, topRef, lockFlags.useRegistries, dummyCache);
if (lockFlags.commitLockFile &&
flake.lockedRef.input->getRev() &&
prevLockedRef.input->getRev() != flake.lockedRef.input->getRev())
warn("committed new revision '%s'", flake.lockedRef.input->getRev()->gitRev());
flake.lockedRef.input.getRev() &&
prevLockedRef.input.getRev() != flake.lockedRef.input.getRev())
warn("committed new revision '%s'", flake.lockedRef.input.getRev()->gitRev());
/* Make sure that we picked up the change,
i.e. the tree should usually be dirty
now. Corner case: we could have reverted from a
dirty to a clean tree! */
if (flake.lockedRef.input == prevLockedRef.input
&& !flake.lockedRef.input->isImmutable())
&& !flake.lockedRef.input.isImmutable())
throw Error("'%s' did not change after I updated its 'flake.lock' file; is 'flake.lock' under version control?", flake.originalRef);
}
} else
@ -625,7 +621,7 @@ static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Va
{
auto flakeRefS = state.forceStringNoCtx(*args[0], pos);
auto flakeRef = parseFlakeRef(flakeRefS, {}, true);
if (evalSettings.pureEval && !flakeRef.input->isImmutable())
if (evalSettings.pureEval && !flakeRef.input.isImmutable())
throw Error("cannot call 'getFlake' on mutable flake reference '%s', at %s (use --impure to override)", flakeRefS, pos);
callFlake(state,
@ -650,8 +646,8 @@ Fingerprint LockedFlake::getFingerprint() const
return hashString(htSHA256,
fmt("%s;%d;%d;%s",
flake.sourceInfo->storePath.to_string(),
flake.sourceInfo->info.revCount.value_or(0),
flake.sourceInfo->info.lastModified.value_or(0),
flake.lockedRef.input.getRevCount().value_or(0),
flake.lockedRef.input.getLastModified().value_or(0),
lockFile));
}

2
src/libexpr/flake/flake.hh
View file

@ -104,7 +104,7 @@ void callFlake(
void emitTreeAttrs(
EvalState & state,
const fetchers::Tree & tree,
std::shared_ptr<const fetchers::Input> input,
const fetchers::Input & input,
Value & v);
}

22
src/libexpr/flake/flakeref.cc
View file

@ -15,7 +15,7 @@ const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRege
std::string FlakeRef::to_string() const
{
auto url = input->toURL();
auto url = input.toURL();
if (subdir != "")
url.query.insert_or_assign("dir", subdir);
return url.to_string();
@ -23,7 +23,7 @@ std::string FlakeRef::to_string() const
fetchers::Attrs FlakeRef::toAttrs() const
{
auto attrs = input->toAttrs();
auto attrs = input.toAttrs();
if (subdir != "")
attrs.emplace("dir", subdir);
return attrs;
@ -37,13 +37,13 @@ std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef)
bool FlakeRef::operator ==(const FlakeRef & other) const
{
return *input == *other.input && subdir == other.subdir;
return input == other.input && subdir == other.subdir;
}
FlakeRef FlakeRef::resolve(ref<Store> store) const
{
auto [input2, extraAttrs] = lookupInRegistries(store, input);
return FlakeRef(input2, fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir));
return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir));
}
FlakeRef parseFlakeRef(
@ -98,7 +98,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
};
return std::make_pair(
FlakeRef(inputFromURL(parsedURL), ""),
FlakeRef(Input::fromURL(parsedURL), ""),
percentDecode(std::string(match[6])));
}
@ -143,7 +143,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
}
return std::make_pair(
FlakeRef(inputFromURL(parsedURL), get(parsedURL.query, "dir").value_or("")),
FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")),
fragment);
}
@ -155,7 +155,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
attrs.insert_or_assign("type", "path");
attrs.insert_or_assign("path", path);
return std::make_pair(FlakeRef(inputFromAttrs(attrs), ""), fragment);
return std::make_pair(FlakeRef(Input::fromAttrs(std::move(attrs)), ""), fragment);
}
else {
@ -163,7 +163,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
std::string fragment;
std::swap(fragment, parsedURL.fragment);
return std::make_pair(
FlakeRef(inputFromURL(parsedURL), get(parsedURL.query, "dir").value_or("")),
FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")),
fragment);
}
}
@ -183,14 +183,14 @@ FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs)
auto attrs2(attrs);
attrs2.erase("dir");
return FlakeRef(
fetchers::inputFromAttrs(attrs2),
fetchers::Input::fromAttrs(std::move(attrs2)),
fetchers::maybeGetStrAttr(attrs, "dir").value_or(""));
}
std::pair<fetchers::Tree, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const
{
auto [tree, lockedInput] = input->fetchTree(store);
return {std::move(tree), FlakeRef(lockedInput, subdir)};
auto [tree, lockedInput] = input.fetch(store);
return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)};
}
}

10
src/libexpr/flake/flakeref.hh
View file

@ -14,17 +14,15 @@ typedef std::string FlakeId;
struct FlakeRef
{
std::shared_ptr<const fetchers::Input> input;
fetchers::Input input;
Path subdir;
bool operator==(const FlakeRef & other) const;
FlakeRef(const std::shared_ptr<const fetchers::Input> & input, const Path & subdir)
: input(input), subdir(subdir)
{
assert(input);
}
FlakeRef(fetchers::Input && input, const Path & subdir)
: input(std::move(input)), subdir(subdir)
{ }
// FIXME: change to operator <<.
std::string to_string() const;

40
src/libexpr/flake/lockfile.cc
View file

@ -5,35 +5,40 @@
namespace nix::flake {
FlakeRef flakeRefFromJson(const nlohmann::json & json)
{
return FlakeRef::fromAttrs(jsonToAttrs(json));
}
FlakeRef getFlakeRef(
const nlohmann::json & json,
const char * attr)
const char * attr,
const char * info)
{
auto i = json.find(attr);
if (i != json.end())
return flakeRefFromJson(*i);
if (i != json.end()) {
auto attrs = jsonToAttrs(*i);
// FIXME: remove when we drop support for version 5.
if (info) {
auto j = json.find(info);
if (j != json.end()) {
for (auto k : jsonToAttrs(*j))
attrs.insert_or_assign(k.first, k.second);
}
}
return FlakeRef::fromAttrs(attrs);
}
throw Error("attribute '%s' missing in lock file", attr);
}
LockedNode::LockedNode(const nlohmann::json & json)
: lockedRef(getFlakeRef(json, "locked"))
, originalRef(getFlakeRef(json, "original"))
, info(TreeInfo::fromJson(json))
: lockedRef(getFlakeRef(json, "locked", "info"))
, originalRef(getFlakeRef(json, "original", nullptr))
, isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true)
{
if (!lockedRef.input->isImmutable())
throw Error("lockfile contains mutable flakeref '%s'", lockedRef);
if (!lockedRef.input.isImmutable())
throw Error("lockfile contains mutable lock '%s'", attrsToJson(lockedRef.input.toAttrs()));
}
StorePath LockedNode::computeStorePath(Store & store) const
{
return info.computeStorePath(store);
return lockedRef.input.computeStorePath(store);
}
std::shared_ptr<Node> Node::findInput(const InputPath & path)
@ -53,7 +58,7 @@ std::shared_ptr<Node> Node::findInput(const InputPath & path)
LockFile::LockFile(const nlohmann::json & json, const Path & path)
{
auto version = json.value("version", 0);
if (version != 5)
if (version < 5 || version > 6)
throw Error("lock file '%s' has unsupported version %d", path, version);
std::unordered_map<std::string, std::shared_ptr<Node>> nodeMap;
@ -119,7 +124,6 @@ nlohmann::json LockFile::toJson() const
if (auto lockedNode = std::dynamic_pointer_cast<const LockedNode>(node)) {
n["original"] = fetchers::attrsToJson(lockedNode->originalRef.toAttrs());
n["locked"] = fetchers::attrsToJson(lockedNode->lockedRef.toAttrs());
n["info"] = lockedNode->info.toJson();
if (!lockedNode->isFlake) n["flake"] = false;
}
@ -129,7 +133,7 @@ nlohmann::json LockFile::toJson() const
};
nlohmann::json json;
json["version"] = 5;
json["version"] = 6;
json["root"] = dumpNode("root", root);
json["nodes"] = std::move(nodes);
@ -176,7 +180,7 @@ bool LockFile::isImmutable() const
for (auto & i : nodes) {
if (i == root) continue;
auto lockedNode = std::dynamic_pointer_cast<const LockedNode>(i);
if (lockedNode && !lockedNode->lockedRef.input->isImmutable()) return false;
if (lockedNode && !lockedNode->lockedRef.input.isImmutable()) return false;
}
return true;

4
src/libexpr/flake/lockfile.hh
View file

@ -31,15 +31,13 @@ struct Node : std::enable_shared_from_this<Node>
struct LockedNode : Node
{
FlakeRef lockedRef, originalRef;
TreeInfo info;
bool isFlake = true;
LockedNode(
const FlakeRef & lockedRef,
const FlakeRef & originalRef,
const TreeInfo & info,
bool isFlake = true)
: lockedRef(lockedRef), originalRef(originalRef), info(info), isFlake(isFlake)
: lockedRef(lockedRef), originalRef(originalRef), isFlake(isFlake)
{ }
LockedNode(const nlohmann::json & json);

2
src/libexpr/parser.y
View file

@ -689,7 +689,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
if (isUri(elem.second)) {
try {
res = { true, store->toRealPath(fetchers::downloadTarball(
store, resolveUri(elem.second), "source", false).storePath) };
store, resolveUri(elem.second), "source", false).first.storePath) };
} catch (FileTransferError & e) {
printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second);
res = { false, "" };

10
src/libexpr/primops/fetchGit.cc
View file

@ -56,23 +56,23 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
if (ref) attrs.insert_or_assign("ref", *ref);
if (rev) attrs.insert_or_assign("rev", rev->gitRev());
if (fetchSubmodules) attrs.insert_or_assign("submodules", true);
auto input = fetchers::inputFromAttrs(attrs);
if (fetchSubmodules) attrs.insert_or_assign("submodules", fetchers::Explicit<bool>{true});
auto input = fetchers::Input::fromAttrs(std::move(attrs));
// FIXME: use name?
auto [tree, input2] = input->fetchTree(state.store);
auto [tree, input2] = input.fetch(state.store);
state.mkAttrs(v, 8);
auto storePath = state.store->printStorePath(tree.storePath);
mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
// Backward compatibility: set 'rev' to
// 0000000000000000000000000000000000000000 for a dirty tree.
auto rev2 = input2->getRev().value_or(Hash(htSHA1));
auto rev2 = input2.getRev().value_or(Hash(htSHA1));
mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev());
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev2.gitShortRev());
// Backward compatibility: set 'revCount' to 0 for a dirty tree.
mkInt(*state.allocAttr(v, state.symbols.create("revCount")),
tree.info.revCount.value_or(0));
input2.getRevCount().value_or(0));
mkBool(*state.allocAttr(v, state.symbols.create("submodules")), fetchSubmodules);
v.attrs->sort();

14
src/libexpr/primops/fetchMercurial.cc
View file

@ -59,23 +59,23 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
if (ref) attrs.insert_or_assign("ref", *ref);
if (rev) attrs.insert_or_assign("rev", rev->gitRev());
auto input = fetchers::inputFromAttrs(attrs);
auto input = fetchers::Input::fromAttrs(std::move(attrs));
// FIXME: use name
auto [tree, input2] = input->fetchTree(state.store);
auto [tree, input2] = input.fetch(state.store);
state.mkAttrs(v, 8);
auto storePath = state.store->printStorePath(tree.storePath);
mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
if (input2->getRef())
mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2->getRef());
if (input2.getRef())
mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2.getRef());
// Backward compatibility: set 'rev' to
// 0000000000000000000000000000000000000000 for a dirty tree.
auto rev2 = input2->getRev().value_or(Hash(htSHA1));
auto rev2 = input2.getRev().value_or(Hash(htSHA1));
mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev());
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(rev2.gitRev(), 0, 12));
if (tree.info.revCount)
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount);
if (auto revCount = input2.getRevCount())
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
v.attrs->sort();
if (state.allowedPaths)

71
src/libexpr/primops/fetchTree.cc
View file

@ -13,31 +13,36 @@ namespace nix {
void emitTreeAttrs(
EvalState & state,
const fetchers::Tree & tree,
std::shared_ptr<const fetchers::Input> input,
const fetchers::Input & input,
Value & v)
{
assert(input.isImmutable());
state.mkAttrs(v, 8);
auto storePath = state.store->printStorePath(tree.storePath);
mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
assert(tree.info.narHash);
mkString(*state.allocAttr(v, state.symbols.create("narHash")),
tree.info.narHash.to_string(SRI));
// FIXME: support arbitrary input attributes.
if (input->getRev()) {
mkString(*state.allocAttr(v, state.symbols.create("rev")), input->getRev()->gitRev());
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), input->getRev()->gitShortRev());
auto narHash = input.getNarHash();
assert(narHash);
mkString(*state.allocAttr(v, state.symbols.create("narHash")),
narHash->to_string(SRI));
if (auto rev = input.getRev()) {
mkString(*state.allocAttr(v, state.symbols.create("rev")), rev->gitRev());
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev->gitShortRev());
}
if (tree.info.revCount)
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount);
if (auto revCount = input.getRevCount())
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
if (tree.info.lastModified) {
mkInt(*state.allocAttr(v, state.symbols.create("lastModified")), *tree.info.lastModified);
if (auto lastModified = input.getLastModified()) {
mkInt(*state.allocAttr(v, state.symbols.create("lastModified")), *lastModified);
mkString(*state.allocAttr(v, state.symbols.create("lastModifiedDate")),
fmt("%s", std::put_time(std::gmtime(&*tree.info.lastModified), "%Y%m%d%H%M%S")));
fmt("%s", std::put_time(std::gmtime(&*lastModified), "%Y%m%d%H%M%S")));
}
v.attrs->sort();
@ -47,7 +52,7 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
{
settings.requireExperimentalFeature("flakes");
std::shared_ptr<const fetchers::Input> input;
fetchers::Input input;
PathSet context;
state.forceValue(*args[0]);
@ -62,7 +67,7 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
if (attr.value->type == tString)
attrs.emplace(attr.name, attr.value->string.s);
else if (attr.value->type == tBool)
attrs.emplace(attr.name, attr.value->boolean);
attrs.emplace(attr.name, fetchers::Explicit<bool>{attr.value->boolean});
else if (attr.value->type == tInt)
attrs.emplace(attr.name, attr.value->integer);
else
@ -73,18 +78,42 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
if (!attrs.count("type"))
throw Error("attribute 'type' is missing in call to 'fetchTree', at %s", pos);
input = fetchers::inputFromAttrs(attrs);
input = fetchers::Input::fromAttrs(std::move(attrs));
} else
input = fetchers::inputFromURL(state.coerceToString(pos, *args[0], context, false, false));
input = fetchers::Input::fromURL(state.coerceToString(pos, *args[0], context, false, false));
if (!evalSettings.pureEval && !input->isDirect())
if (!evalSettings.pureEval && !input.isDirect())
input = lookupInRegistries(state.store, input).first;
if (evalSettings.pureEval && !input->isImmutable())
if (evalSettings.pureEval && !input.isImmutable())
throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input, at %s", pos);
// FIXME: use fetchOrSubstituteTree
auto [tree, input2] = input->fetchTree(state.store);
/* The tree may already be in the Nix store, or it could be
substituted (which is often faster than fetching from the
original source). So check that. */
if (input.hasAllInfo()) {
auto storePath = input.computeStorePath(*state.store);
try {
state.store->ensurePath(storePath);
debug("using substituted/cached input '%s' in '%s'",
input.to_string(), state.store->printStorePath(storePath));
auto actualPath = state.store->toRealPath(storePath);
if (state.allowedPaths)
state.allowedPaths->insert(actualPath);
emitTreeAttrs(state, fetchers::Tree { .actualPath = actualPath, .storePath = std::move(storePath) }, input, v);
return;
} catch (Error & e) {
debug("substitution of input '%s' failed: %s", input.to_string(), e.what());
}
}
auto [tree, input2] = input.fetch(state.store);
if (state.allowedPaths)
state.allowedPaths->insert(tree.actualPath);
@ -137,7 +166,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
auto storePath =
unpack
? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath
? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).first.storePath
: fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath;
auto path = state.store->toRealPath(storePath);

14
src/libfetchers/attrs.cc
View file

@ -27,7 +27,7 @@ nlohmann::json attrsToJson(const Attrs & attrs)
{
nlohmann::json json;
for (auto & attr : attrs) {
if (auto v = std::get_if<int64_t>(&attr.second)) {
if (auto v = std::get_if<uint64_t>(&attr.second)) {
json[attr.first] = *v;
} else if (auto v = std::get_if<std::string>(&attr.second)) {
json[attr.first] = *v;
@ -55,16 +55,16 @@ std::string getStrAttr(const Attrs & attrs, const std::string & name)
return *s;
}
std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name)
std::optional<uint64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name)
{
auto i = attrs.find(name);
if (i == attrs.end()) return {};
if (auto v = std::get_if<int64_t>(&i->second))
if (auto v = std::get_if<uint64_t>(&i->second))
return *v;
throw Error("input attribute '%s' is not an integer", name);
}
int64_t getIntAttr(const Attrs & attrs, const std::string & name)
uint64_t getIntAttr(const Attrs & attrs, const std::string & name)
{
auto s = maybeGetIntAttr(attrs, name);
if (!s)
@ -76,8 +76,8 @@ std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & na
{
auto i = attrs.find(name);
if (i == attrs.end()) return {};
if (auto v = std::get_if<int64_t>(&i->second))
return *v;
if (auto v = std::get_if<Explicit<bool>>(&i->second))
return v->t;
throw Error("input attribute '%s' is not a Boolean", name);
}
@ -93,7 +93,7 @@ std::map<std::string, std::string> attrsToQuery(const Attrs & attrs)
{
std::map<std::string, std::string> query;
for (auto & attr : attrs) {
if (auto v = std::get_if<int64_t>(&attr.second)) {
if (auto v = std::get_if<uint64_t>(&attr.second)) {
query.insert_or_assign(attr.first, fmt("%d", *v));
} else if (auto v = std::get_if<std::string>(&attr.second)) {
query.insert_or_assign(attr.first, *v);

11
src/libfetchers/attrs.hh
View file

@ -13,9 +13,14 @@ namespace nix::fetchers {
template<typename T>
struct Explicit {
T t;
bool operator ==(const Explicit<T> & other) const
{
return t == other.t;
}
};
typedef std::variant<std::string, int64_t, Explicit<bool>> Attr;
typedef std::variant<std::string, uint64_t, Explicit<bool>> Attr;
typedef std::map<std::string, Attr> Attrs;
Attrs jsonToAttrs(const nlohmann::json & json);
@ -26,9 +31,9 @@ std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::strin
std::string getStrAttr(const Attrs & attrs, const std::string & name);
std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name);
std::optional<uint64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name);
int64_t getIntAttr(const Attrs & attrs, const std::string & name);
uint64_t getIntAttr(const Attrs & attrs, const std::string & name);
std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & name);

224
src/libfetchers/fetchers.cc
View file

@ -5,82 +5,234 @@
namespace nix::fetchers {
std::unique_ptr<std::vector<std::unique_ptr<InputScheme>>> inputSchemes = nullptr;
std::unique_ptr<std::vector<std::shared_ptr<InputScheme>>> inputSchemes = nullptr;
void registerInputScheme(std::unique_ptr<InputScheme> && inputScheme)
void registerInputScheme(std::shared_ptr<InputScheme> && inputScheme)
{
if (!inputSchemes) inputSchemes = std::make_unique<std::vector<std::unique_ptr<InputScheme>>>();
if (!inputSchemes) inputSchemes = std::make_unique<std::vector<std::shared_ptr<InputScheme>>>();
inputSchemes->push_back(std::move(inputScheme));
}
std::unique_ptr<Input> inputFromURL(const ParsedURL & url)
Input Input::fromURL(const std::string & url)
{
return fromURL(parseURL(url));
}
static void fixupInput(Input & input)
{
// Check common attributes.
input.getType();
input.getRef();
if (input.getRev())
input.immutable = true;
input.getRevCount();
input.getLastModified();
if (input.getNarHash())
input.immutable = true;
}
Input Input::fromURL(const ParsedURL & url)
{
for (auto & inputScheme : *inputSchemes) {
auto res = inputScheme->inputFromURL(url);
if (res) return res;
if (res) {
res->scheme = inputScheme;
fixupInput(*res);
return std::move(*res);
}
}
throw Error("input '%s' is unsupported", url.url);
}
std::unique_ptr<Input> inputFromURL(const std::string & url)
Input Input::fromAttrs(Attrs && attrs)
{
return inputFromURL(parseURL(url));
}
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs)
{
auto attrs2(attrs);
attrs2.erase("narHash");
for (auto & inputScheme : *inputSchemes) {
auto res = inputScheme->inputFromAttrs(attrs2);
auto res = inputScheme->inputFromAttrs(attrs);
if (res) {
if (auto narHash = maybeGetStrAttr(attrs, "narHash"))
// FIXME: require SRI hash.
res->narHash = Hash(*narHash);
return res;
res->scheme = inputScheme;
fixupInput(*res);
return std::move(*res);
}
}
throw Error("input '%s' is unsupported", attrsToJson(attrs));
Input input;
input.attrs = attrs;
fixupInput(input);
return input;
}
ParsedURL Input::toURL() const
{
if (!scheme)
throw Error("cannot show unsupported input '%s'", attrsToJson(attrs));
return scheme->toURL(*this);
}
std::string Input::to_string() const
{
return toURL().to_string();
}
Attrs Input::toAttrs() const
{
auto attrs = toAttrsInternal();
if (narHash)
attrs.emplace("narHash", narHash->to_string(SRI));
attrs.emplace("type", type());
return attrs;
}
std::pair<Tree, std::shared_ptr<const Input>> Input::fetchTree(ref<Store> store) const
bool Input::hasAllInfo() const
{
auto [tree, input] = fetchTreeInternal(store);
return getNarHash() && scheme && scheme->hasAllInfo(*this);
}
bool Input::operator ==(const Input & other) const
{
return attrs == other.attrs;
}
bool Input::contains(const Input & other) const
{
auto other2(other);
other2.attrs.erase("ref");
other2.attrs.erase("rev");
if (*this == other2) return true;
return false;
}
std::pair<Tree, Input> Input::fetch(ref<Store> store) const
{
if (!scheme)
throw Error("cannot fetch unsupported input '%s'", attrsToJson(toAttrs()));
auto [tree, input] = scheme->fetch(store, *this);
if (tree.actualPath == "")
tree.actualPath = store->toRealPath(tree.storePath);
if (!tree.info.narHash)
tree.info.narHash = store->queryPathInfo(tree.storePath)->narHash;
auto narHash = store->queryPathInfo(tree.storePath)->narHash;
input.attrs.insert_or_assign("narHash", narHash.to_string(SRI));
if (input->narHash)
assert(input->narHash == tree.info.narHash);
if (auto narHash2 = getNarHash()) {
if (narHash != *narHash2)
throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
to_string(), tree.actualPath, narHash2->to_string(SRI), narHash.to_string(SRI));
}
if (narHash && narHash != input->narHash)
throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
to_string(), tree.actualPath, narHash->to_string(SRI), input->narHash->to_string(SRI));
// FIXME: check lastModified, revCount
input.immutable = true;
assert(input.hasAllInfo());
return {std::move(tree), input};
}
std::shared_ptr<const Input> Input::applyOverrides(
Input Input::applyOverrides(
std::optional<std::string> ref,
std::optional<Hash> rev) const
{
if (!scheme) return *this;
return scheme->applyOverrides(*this, ref, rev);
}
void Input::clone(const Path & destDir) const
{
assert(scheme);
scheme->clone(*this, destDir);
}
std::optional<Path> Input::getSourcePath() const
{
assert(scheme);
return scheme->getSourcePath(*this);
}
void Input::markChangedFile(
std::string_view file,
std::optional<std::string> commitMsg) const
{
assert(scheme);
return scheme->markChangedFile(*this, file, commitMsg);
}
StorePath Input::computeStorePath(Store & store) const
{
auto narHash = getNarHash();
if (!narHash)
throw Error("cannot compute store path for mutable input '%s'", to_string());
return store.makeFixedOutputPath(true, *narHash, "source");
}
std::string Input::getType() const
{
return getStrAttr(attrs, "type");
}
std::optional<Hash> Input::getNarHash() const
{
if (auto s = maybeGetStrAttr(attrs, "narHash"))
// FIXME: require SRI hash.
return Hash(*s, htSHA256);
return {};
}
std::optional<std::string> Input::getRef() const
{
if (auto s = maybeGetStrAttr(attrs, "ref"))
return *s;
return {};
}
std::optional<Hash> Input::getRev() const
{
if (auto s = maybeGetStrAttr(attrs, "rev"))
return Hash(*s, htSHA1);
return {};
}
std::optional<uint64_t> Input::getRevCount() const
{
if (auto n = maybeGetIntAttr(attrs, "revCount"))
return *n;
return {};
}
std::optional<time_t> Input::getLastModified() const
{
if (auto n = maybeGetIntAttr(attrs, "lastModified"))
return *n;
return {};
}
ParsedURL InputScheme::toURL(const Input & input)
{
throw Error("don't know how to convert input '%s' to a URL", attrsToJson(input.attrs));
}
Input InputScheme::applyOverrides(
const Input & input,
std::optional<std::string> ref,
std::optional<Hash> rev)
{
if (ref)
throw Error("don't know how to apply '%s' to '%s'", *ref, to_string());
throw Error("don't know how to set branch/tag name of input '%s' to '%s'", input.to_string(), *ref);
if (rev)
throw Error("don't know how to apply '%s' to '%s'", rev->to_string(Base16, false), to_string());
return shared_from_this();
throw Error("don't know how to set revision of input '%s' to '%s'", input.to_string(), rev->gitRev());
return input;
}
std::optional<Path> InputScheme::getSourcePath(const Input & input)
{
return {};
}
void InputScheme::markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg)
{
assert(false);
}
void InputScheme::clone(const Input & input, const Path & destDir)
{
throw Error("do not know how to clone input '%s'", input.to_string());
}
}

116
src/libfetchers/fetchers.hh
View file

@ -3,7 +3,6 @@
#include "types.hh"
#include "hash.hh"
#include "path.hh"
#include "tree-info.hh"
#include "attrs.hh"
#include "url.hh"
@ -13,89 +12,100 @@ namespace nix { class Store; }
namespace nix::fetchers {
struct Input;
struct Tree
{
Path actualPath;
StorePath storePath;
TreeInfo info;
};
struct Input : std::enable_shared_from_this<Input>
struct InputScheme;
struct Input
{
std::optional<Hash> narHash; // FIXME: implement
friend class InputScheme;
virtual std::string type() const = 0;
std::shared_ptr<InputScheme> scheme; // note: can be null
Attrs attrs;
bool immutable = false;
bool direct = true;
virtual ~Input() { }
public:
static Input fromURL(const std::string & url);
virtual bool operator ==(const Input & other) const { return false; }
static Input fromURL(const ParsedURL & url);
/* Check whether this is a "direct" input, that is, not
one that goes through a registry. */
virtual bool isDirect() const { return true; }
static Input fromAttrs(Attrs && attrs);
/* Check whether this is an "immutable" input, that is,
one that contains a commit hash or content hash. */
virtual bool isImmutable() const { return (bool) narHash; }
ParsedURL toURL() const;
virtual bool contains(const Input & other) const { return false; }
virtual std::optional<std::string> getRef() const { return {}; }
virtual std::optional<Hash> getRev() const { return {}; }
virtual ParsedURL toURL() const = 0;
std::string to_string() const
{
return toURL().to_string();
}
std::string to_string() const;
Attrs toAttrs() const;
std::pair<Tree, std::shared_ptr<const Input>> fetchTree(ref<Store> store) const;
/* Check whether this is a "direct" input, that is, not
one that goes through a registry. */
bool isDirect() const { return direct; }
virtual std::shared_ptr<const Input> applyOverrides(
/* Check whether this is an "immutable" input, that is,
one that contains a commit hash or content hash. */
bool isImmutable() const { return immutable; }
bool hasAllInfo() const;
bool operator ==(const Input & other) const;
bool contains(const Input & other) const;
std::pair<Tree, Input> fetch(ref<Store> store) const;
Input applyOverrides(
std::optional<std::string> ref,
std::optional<Hash> rev) const;
virtual std::optional<Path> getSourcePath() const { return {}; }
void clone(const Path & destDir) const;
virtual void markChangedFile(
std::optional<Path> getSourcePath() const;
void markChangedFile(
std::string_view file,
std::optional<std::string> commitMsg) const
{ assert(false); }
std::optional<std::string> commitMsg) const;
virtual void clone(const Path & destDir) const
{
throw Error("do not know how to clone input '%s'", to_string());
}
StorePath computeStorePath(Store & store) const;
private:
virtual std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(ref<Store> store) const = 0;
virtual Attrs toAttrsInternal() const = 0;
// Convience functions for common attributes.
std::string getType() const;
std::optional<Hash> getNarHash() const;
std::optional<std::string> getRef() const;
std::optional<Hash> getRev() const;
std::optional<uint64_t> getRevCount() const;
std::optional<time_t> getLastModified() const;
};
struct InputScheme
{
virtual ~InputScheme() { }
virtual std::optional<Input> inputFromURL(const ParsedURL & url) = 0;
virtual std::unique_ptr<Input> inputFromURL(const ParsedURL & url) = 0;
virtual std::optional<Input> inputFromAttrs(const Attrs & attrs) = 0;
virtual std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) = 0;
virtual ParsedURL toURL(const Input & input);
virtual bool hasAllInfo(const Input & input) = 0;
virtual Input applyOverrides(
const Input & input,
std::optional<std::string> ref,
std::optional<Hash> rev);
virtual void clone(const Input & input, const Path & destDir);
virtual std::optional<Path> getSourcePath(const Input & input);
virtual void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg);
virtual std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) = 0;
};
std::unique_ptr<Input> inputFromURL(const ParsedURL & url);
std::unique_ptr<Input> inputFromURL(const std::string & url);
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs);
void registerInputScheme(std::unique_ptr<InputScheme> && fetcher);
void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);
struct DownloadFileResult
{
@ -110,7 +120,7 @@ DownloadFileResult downloadFile(
const std::string & name,
bool immutable);
Tree downloadTarball(
std::pair<Tree, time_t> downloadTarball(
ref<Store> store,
const std::string & url,
const std::string & name,

310
src/libfetchers/git.cc
View file

@ -22,110 +22,121 @@ static bool isNotDotGitDirectory(const Path & path)
return not std::regex_match(path, gitDirRegex);
}
struct GitInput : Input
struct GitInputScheme : InputScheme
{
ParsedURL url;
std::optional<std::string> ref;
std::optional<Hash> rev;
bool shallow = false;
bool submodules = false;
GitInput(const ParsedURL & url) : url(url)
{ }
std::string type() const override { return "git"; }
bool operator ==(const Input & other) const override
std::optional<Input> inputFromURL(const ParsedURL & url) override
{
auto other2 = dynamic_cast<const GitInput *>(&other);
return
other2
&& url == other2->url
&& rev == other2->rev
&& ref == other2->ref;
}
if (url.scheme != "git" &&
url.scheme != "git+http" &&
url.scheme != "git+https" &&
url.scheme != "git+ssh" &&
url.scheme != "git+file") return {};
bool isImmutable() const override
{
return (bool) rev || narHash;
}
auto url2(url);
if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4);
url2.query.clear();
std::optional<std::string> getRef() const override { return ref; }
std::optional<Hash> getRev() const override { return rev; }
ParsedURL toURL() const override
{
ParsedURL url2(url);
if (url2.scheme != "git") url2.scheme = "git+" + url2.scheme;
if (rev) url2.query.insert_or_assign("rev", rev->gitRev());
if (ref) url2.query.insert_or_assign("ref", *ref);
if (shallow) url2.query.insert_or_assign("shallow", "1");
return url2;
}
Attrs toAttrsInternal() const override
{
Attrs attrs;
attrs.emplace("url", url.to_string());
if (ref)
attrs.emplace("ref", *ref);
if (rev)
attrs.emplace("rev", rev->gitRev());
if (shallow)
attrs.emplace("shallow", true);
if (submodules)
attrs.emplace("submodules", true);
return attrs;
attrs.emplace("type", "git");
for (auto &[name, value] : url.query) {
if (name == "rev" || name == "ref")
attrs.emplace(name, value);
else
url2.query.emplace(name, value);
}
attrs.emplace("url", url2.to_string());
return inputFromAttrs(attrs);
}
void clone(const Path & destDir) const override
std::optional<Input> inputFromAttrs(const Attrs & attrs) override
{
auto [isLocal, actualUrl] = getActualUrl();
if (maybeGetStrAttr(attrs, "type") != "git") return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash")
throw Error("unsupported Git input attribute '%s'", name);
parseURL(getStrAttr(attrs, "url"));
maybeGetBoolAttr(attrs, "shallow");
maybeGetBoolAttr(attrs, "submodules");
if (auto ref = maybeGetStrAttr(attrs, "ref")) {
if (!std::regex_match(*ref, refRegex))
throw BadURL("invalid Git branch/tag name '%s'", *ref);
}
Input input;
input.attrs = attrs;
return input;
}
ParsedURL toURL(const Input & input) override
{
auto url = parseURL(getStrAttr(input.attrs, "url"));
if (url.scheme != "git") url.scheme = "git+" + url.scheme;
if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev());
if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref);
if (maybeGetBoolAttr(input.attrs, "shallow").value_or(false))
url.query.insert_or_assign("shallow", "1");
return url;
}
bool hasAllInfo(const Input & input) override
{
bool maybeDirty = !input.getRef();
bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
return
maybeGetIntAttr(input.attrs, "lastModified")
&& (shallow || maybeDirty || maybeGetIntAttr(input.attrs, "revCount"));
}
Input applyOverrides(
const Input & input,
std::optional<std::string> ref,
std::optional<Hash> rev) override
{
auto res(input);
if (rev) res.attrs.insert_or_assign("rev", rev->gitRev());
if (ref) res.attrs.insert_or_assign("ref", *ref);
if (!res.getRef() && res.getRev())
throw Error("Git input '%s' has a commit hash but no branch/tag name", res.to_string());
return res;
}
void clone(const Input & input, const Path & destDir) override
{
auto [isLocal, actualUrl] = getActualUrl(input);
Strings args = {"clone"};
args.push_back(actualUrl);
if (ref) {
if (auto ref = input.getRef()) {
args.push_back("--branch");
args.push_back(*ref);
}
if (rev) throw Error("cloning a specific revision is not implemented");
if (input.getRev()) throw Error("cloning a specific revision is not implemented");
args.push_back(destDir);
runProgram("git", true, args);
}
std::shared_ptr<const Input> applyOverrides(
std::optional<std::string> ref,
std::optional<Hash> rev) const override
std::optional<Path> getSourcePath(const Input & input) override
{
if (!ref && !rev) return shared_from_this();
auto res = std::make_shared<GitInput>(*this);
if (ref) res->ref = ref;
if (rev) res->rev = rev;
if (!res->ref && res->rev)
throw Error("Git input '%s' has a commit hash but no branch/tag name", res->to_string());
return res;
}
std::optional<Path> getSourcePath() const override
{
if (url.scheme == "file" && !ref && !rev)
auto url = parseURL(getStrAttr(input.attrs, "url"));
if (url.scheme == "file" && !input.getRef() && !input.getRev())
return url.path;
return {};
}
void markChangedFile(std::string_view file, std::optional<std::string> commitMsg) const override
void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override
{
auto sourcePath = getSourcePath();
auto sourcePath = getSourcePath(input);
assert(sourcePath);
runProgram("git", true,
@ -136,23 +147,25 @@ struct GitInput : Input
{ "-C", *sourcePath, "commit", std::string(file), "-m", *commitMsg });
}
std::pair<bool, std::string> getActualUrl() const
std::pair<bool, std::string> getActualUrl(const Input & input) const
{
// Don't clone file:// URIs (but otherwise treat them the
// same as remote URIs, i.e. don't use the working tree or
// HEAD).
static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing
auto url = parseURL(getStrAttr(input.attrs, "url"));
bool isLocal = url.scheme == "file" && !forceHttp;
return {isLocal, isLocal ? url.path : url.base};
}
std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
{
auto name = "source";
auto input = std::make_shared<GitInput>(*this);
Input input(_input);
assert(!rev || rev->type == htSHA1);
bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
std::string cacheType = "git";
if (shallow) cacheType += "-shallow";
@ -163,39 +176,38 @@ struct GitInput : Input
return Attrs({
{"type", cacheType},
{"name", name},
{"rev", input->rev->gitRev()},
{"rev", input.getRev()->gitRev()},
});
};
auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
-> std::pair<Tree, std::shared_ptr<const Input>>
-> std::pair<Tree, Input>
{
assert(input->rev);
assert(!rev || rev == input->rev);
assert(input.getRev());
assert(!_input.getRev() || _input.getRev() == input.getRev());
if (!shallow)
input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount"));
input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified"));
return {
Tree {
.actualPath = store->toRealPath(storePath),
.storePath = std::move(storePath),
.info = TreeInfo {
.revCount = shallow ? std::nullopt : std::optional(getIntAttr(infoAttrs, "revCount")),
.lastModified = getIntAttr(infoAttrs, "lastModified"),
},
},
input
};
};
if (rev) {
if (input.getRev()) {
if (auto res = getCache()->lookup(store, getImmutableAttrs()))
return makeResult(res->first, std::move(res->second));
}
auto [isLocal, actualUrl_] = getActualUrl();
auto [isLocal, actualUrl_] = getActualUrl(input);
auto actualUrl = actualUrl_; // work around clang bug
// If this is a local directory and no ref or revision is
// given, then allow the use of an unclean working tree.
if (!input->ref && !input->rev && isLocal) {
if (!input.getRef() && !input.getRev() && isLocal) {
bool clean = false;
/* Check whether this repo has any commits. There are
@ -254,35 +266,37 @@ struct GitInput : Input
auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter);
auto tree = Tree {
.actualPath = store->printStorePath(storePath),
.storePath = std::move(storePath),
.info = TreeInfo {
// FIXME: maybe we should use the timestamp of the last
// modified dirty file?
.lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0,
}
};
// FIXME: maybe we should use the timestamp of the last
// modified dirty file?
input.attrs.insert_or_assign(
"lastModified",
haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0);
return {std::move(tree), input};
return {
Tree {
.actualPath = store->printStorePath(storePath),
.storePath = std::move(storePath),
}, input
};
}
}
if (!input->ref) input->ref = isLocal ? readHead(actualUrl) : "master";
if (!input.getRef()) input.attrs.insert_or_assign("ref", isLocal ? readHead(actualUrl) : "master");
Attrs mutableAttrs({
{"type", cacheType},
{"name", name},
{"url", actualUrl},
{"ref", *input->ref},
{"ref", *input.getRef()},
});
Path repoDir;
if (isLocal) {
if (!input->rev)
input->rev = Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input->ref })), htSHA1);
if (!input.getRev())
input.attrs.insert_or_assign("rev",
Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input.getRef() })), htSHA1).gitRev());
repoDir = actualUrl;
@ -290,8 +304,8 @@ struct GitInput : Input
if (auto res = getCache()->lookup(store, mutableAttrs)) {
auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
if (!rev || rev == rev2) {
input->rev = rev2;
if (!input.getRev() || input.getRev() == rev2) {
input.attrs.insert_or_assign("rev", rev2.gitRev());
return makeResult(res->first, std::move(res->second));
}
}
@ -305,18 +319,18 @@ struct GitInput : Input
}
Path localRefFile =
input->ref->compare(0, 5, "refs/") == 0
? cacheDir + "/" + *input->ref
: cacheDir + "/refs/heads/" + *input->ref;
input.getRef()->compare(0, 5, "refs/") == 0
? cacheDir + "/" + *input.getRef()
: cacheDir + "/refs/heads/" + *input.getRef();
bool doFetch;
time_t now = time(0);
/* If a rev was specified, we need to fetch if it's not in the
repo. */
if (input->rev) {
if (input.getRev()) {
try {
runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input->rev->gitRev() });
runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input.getRev()->gitRev() });
doFetch = false;
} catch (ExecError & e) {
if (WIFEXITED(e.status)) {
@ -339,7 +353,7 @@ struct GitInput : Input
// FIXME: git stderr messes up our progress indicator, so
// we're using --quiet for now. Should process its stderr.
try {
runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", *input->ref, *input->ref) });
runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", *input.getRef(), *input.getRef()) });
} catch (Error & e) {
if (!pathExists(localRefFile)) throw;
warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl);
@ -354,8 +368,8 @@ struct GitInput : Input
utimes(localRefFile.c_str(), times);
}
if (!input->rev)
input->rev = Hash(chomp(readFile(localRefFile)), htSHA1);
if (!input.getRev())
input.attrs.insert_or_assign("rev", Hash(chomp(readFile(localRefFile)), htSHA1).gitRev());
}
bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true";
@ -365,7 +379,7 @@ struct GitInput : Input
// FIXME: check whether rev is an ancestor of ref.
printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl);
printTalkative("using revision %s of repo '%s'", input.getRev()->gitRev(), actualUrl);
/* Now that we know the ref, check again whether we have it in
the store. */
@ -387,7 +401,7 @@ struct GitInput : Input
runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force",
"--update-head-ok", "--", repoDir, "refs/*:refs/*" });
runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input->rev->gitRev() });
runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() });
runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", actualUrl });
runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" });
@ -396,7 +410,7 @@ struct GitInput : Input
// FIXME: should pipe this, or find some better way to extract a
// revision.
auto source = sinkToSource([&](Sink & sink) {
RunOptions gitOptions("git", { "-C", repoDir, "archive", input->rev->gitRev() });
RunOptions gitOptions("git", { "-C", repoDir, "archive", input.getRev()->gitRev() });
gitOptions.standardOut = &sink;
runProgram2(gitOptions);
});
@ -406,18 +420,18 @@ struct GitInput : Input
auto storePath = store->addToStore(name, tmpDir, true, htSHA256, filter);
auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() }));
auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input.getRev()->gitRev() }));
Attrs infoAttrs({
{"rev", input->rev->gitRev()},
{"rev", input.getRev()->gitRev()},
{"lastModified", lastModified},
});
if (!shallow)
infoAttrs.insert_or_assign("revCount",
std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() })));
std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input.getRev()->gitRev() })));
if (!this->rev)
if (!_input.getRev())
getCache()->add(
store,
mutableAttrs,
@ -436,60 +450,6 @@ struct GitInput : Input
}
};
struct GitInputScheme : InputScheme
{
std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
{
if (url.scheme != "git" &&
url.scheme != "git+http" &&
url.scheme != "git+https" &&
url.scheme != "git+ssh" &&
url.scheme != "git+file") return nullptr;
auto url2(url);
if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4);
url2.query.clear();
Attrs attrs;
attrs.emplace("type", "git");
for (auto &[name, value] : url.query) {
if (name == "rev" || name == "ref")
attrs.emplace(name, value);
else
url2.query.emplace(name, value);
}
attrs.emplace("url", url2.to_string());
return inputFromAttrs(attrs);
}
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
{
if (maybeGetStrAttr(attrs, "type") != "git") return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules")
throw Error("unsupported Git input attribute '%s'", name);
auto input = std::make_unique<GitInput>(parseURL(getStrAttr(attrs, "url")));
if (auto ref = maybeGetStrAttr(attrs, "ref")) {
if (!std::regex_match(*ref, refRegex))
throw BadURL("invalid Git branch/tag name '%s'", *ref);
input->ref = *ref;
}
if (auto rev = maybeGetStrAttr(attrs, "rev"))
input->rev = Hash(*rev, htSHA1);
input->shallow = maybeGetBoolAttr(attrs, "shallow").value_or(false);
input->submodules = maybeGetBoolAttr(attrs, "submodules").value_or(false);
return input;
}
};
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitInputScheme>()); });
}

303
src/libfetchers/github.cc
View file

@ -8,37 +8,80 @@
namespace nix::fetchers {
struct GitArchiveInput : Input
struct GitArchiveInputScheme : InputScheme
{
std::string owner;
std::string repo;
std::optional<std::string> ref;
std::optional<Hash> rev;
virtual std::string type() = 0;
virtual std::shared_ptr<GitArchiveInput> _clone() const = 0;
bool operator ==(const Input & other) const override
std::optional<Input> inputFromURL(const ParsedURL & url) override
{
auto other2 = dynamic_cast<const GitArchiveInput *>(&other);
return
other2
&& owner == other2->owner
&& repo == other2->repo
&& rev == other2->rev
&& ref == other2->ref;
if (url.scheme != type()) return {};
auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
std::optional<Hash> rev;
std::optional<std::string> ref;
if (path.size() == 2) {
} else if (path.size() == 3) {
if (std::regex_match(path[2], revRegex))
rev = Hash(path[2], htSHA1);
else if (std::regex_match(path[2], refRegex))
ref = path[2];
else
throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]);
} else
throw BadURL("URL '%s' is invalid", url.url);
for (auto &[name, value] : url.query) {
if (name == "rev") {
if (rev)
throw BadURL("URL '%s' contains multiple commit hashes", url.url);
rev = Hash(value, htSHA1);
}
else if (name == "ref") {
if (!std::regex_match(value, refRegex))
throw BadURL("URL '%s' contains an invalid branch/tag name", url.url);
if (ref)
throw BadURL("URL '%s' contains multiple branch/tag names", url.url);
ref = value;
}
}
if (ref && rev)
throw BadURL("URL '%s' contains both a commit hash and a branch/tag name", url.url);
Input input;
input.attrs.insert_or_assign("type", type());
input.attrs.insert_or_assign("owner", path[0]);
input.attrs.insert_or_assign("repo", path[1]);
if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
if (ref) input.attrs.insert_or_assign("ref", *ref);
return input;
}
bool isImmutable() const override
std::optional<Input> inputFromAttrs(const Attrs & attrs) override
{
return (bool) rev || narHash;
if (maybeGetStrAttr(attrs, "type") != type()) return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified")
throw Error("unsupported input attribute '%s'", name);
getStrAttr(attrs, "owner");
getStrAttr(attrs, "repo");
Input input;
input.attrs = attrs;
return input;
}
std::optional<std::string> getRef() const override { return ref; }
std::optional<Hash> getRev() const override { return rev; }
ParsedURL toURL() const override
ParsedURL toURL(const Input & input) override
{
auto owner = getStrAttr(input.attrs, "owner");
auto repo = getStrAttr(input.attrs, "repo");
auto ref = input.getRef();
auto rev = input.getRev();
auto path = owner + "/" + repo;
assert(!(ref && rev));
if (ref) path += "/" + *ref;
@ -49,32 +92,44 @@ struct GitArchiveInput : Input
};
}
Attrs toAttrsInternal() const override
bool hasAllInfo(const Input & input) override
{
Attrs attrs;
attrs.emplace("owner", owner);
attrs.emplace("repo", repo);
if (ref)
attrs.emplace("ref", *ref);
if (rev)
attrs.emplace("rev", rev->gitRev());
return attrs;
return input.getRev() && maybeGetIntAttr(input.attrs, "lastModified");
}
virtual Hash getRevFromRef(nix::ref<Store> store, std::string_view ref) const = 0;
virtual std::string getDownloadUrl() const = 0;
std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
Input applyOverrides(
const Input & _input,
std::optional<std::string> ref,
std::optional<Hash> rev) override
{
auto rev = this->rev;
auto ref = this->ref.value_or("master");
auto input(_input);
if (rev) {
input.attrs.insert_or_assign("rev", rev->gitRev());
input.attrs.erase("ref");
}
if (ref) {
if (input.getRev())
throw BadURL("input '%s' contains both a commit hash and a branch/tag name", input.to_string());
input.attrs.insert_or_assign("ref", *ref);
}
return input;
}
if (!rev) rev = getRevFromRef(store, ref);
virtual Hash getRevFromRef(nix::ref<Store> store, const Input & input) const = 0;
auto input = _clone();
input->ref = {};
input->rev = *rev;
virtual std::string getDownloadUrl(const Input & input) const = 0;
std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
{
Input input(_input);
if (!maybeGetStrAttr(input.attrs, "ref")) input.attrs.insert_or_assign("ref", "master");
auto rev = input.getRev();
if (!rev) rev = getRevFromRef(store, input);
input.attrs.erase("ref");
input.attrs.insert_or_assign("rev", rev->gitRev());
Attrs immutableAttrs({
{"type", "git-tarball"},
@ -82,131 +137,44 @@ struct GitArchiveInput : Input
});
if (auto res = getCache()->lookup(store, immutableAttrs)) {
input.attrs.insert_or_assign("lastModified", getIntAttr(res->first, "lastModified"));
return {
Tree{
.actualPath = store->toRealPath(res->second),
.storePath = std::move(res->second),
.info = TreeInfo {
.lastModified = getIntAttr(res->first, "lastModified"),
},
},
input
};
}
auto url = input->getDownloadUrl();
auto url = getDownloadUrl(input);
auto [tree, lastModified] = downloadTarball(store, url, "source", true);
auto tree = downloadTarball(store, url, "source", true);
input.attrs.insert_or_assign("lastModified", lastModified);
getCache()->add(
store,
immutableAttrs,
{
{"rev", rev->gitRev()},
{"lastModified", *tree.info.lastModified}
{"lastModified", lastModified}
},
tree.storePath,
true);
return {std::move(tree), input};
}
std::shared_ptr<const Input> applyOverrides(
std::optional<std::string> ref,
std::optional<Hash> rev) const override
{
if (!ref && !rev) return shared_from_this();
auto res = _clone();
if (ref) res->ref = ref;
if (rev) res->rev = rev;
return res;
}
};
struct GitArchiveInputScheme : InputScheme
struct GitHubInputScheme : GitArchiveInputScheme
{
std::string type;
std::string type() override { return "github"; }
GitArchiveInputScheme(std::string && type) : type(type)
{ }
virtual std::unique_ptr<GitArchiveInput> create() = 0;
std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
{
if (url.scheme != type) return nullptr;
auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
auto input = create();
if (path.size() == 2) {
} else if (path.size() == 3) {
if (std::regex_match(path[2], revRegex))
input->rev = Hash(path[2], htSHA1);
else if (std::regex_match(path[2], refRegex))
input->ref = path[2];
else
throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]);
} else
throw BadURL("URL '%s' is invalid", url.url);
for (auto &[name, value] : url.query) {
if (name == "rev") {
if (input->rev)
throw BadURL("URL '%s' contains multiple commit hashes", url.url);
input->rev = Hash(value, htSHA1);
}
else if (name == "ref") {
if (!std::regex_match(value, refRegex))
throw BadURL("URL '%s' contains an invalid branch/tag name", url.url);
if (input->ref)
throw BadURL("URL '%s' contains multiple branch/tag names", url.url);
input->ref = value;
}
}
if (input->ref && input->rev)
throw BadURL("URL '%s' contains both a commit hash and a branch/tag name", url.url);
input->owner = path[0];
input->repo = path[1];
return input;
}
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
{
if (maybeGetStrAttr(attrs, "type") != type) return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev")
throw Error("unsupported input attribute '%s'", name);
auto input = create();
input->owner = getStrAttr(attrs, "owner");
input->repo = getStrAttr(attrs, "repo");
input->ref = maybeGetStrAttr(attrs, "ref");
if (auto rev = maybeGetStrAttr(attrs, "rev"))
input->rev = Hash(*rev, htSHA1);
return input;
}
};
struct GitHubInput : GitArchiveInput
{
std::string type() const override { return "github"; }
std::shared_ptr<GitArchiveInput> _clone() const override
{ return std::make_shared<GitHubInput>(*this); }
Hash getRevFromRef(nix::ref<Store> store, std::string_view ref) const override
Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
{
auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s",
owner, repo, ref);
getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
auto json = nlohmann::json::parse(
readFile(
store->toRealPath(
@ -216,13 +184,14 @@ struct GitHubInput : GitArchiveInput
return rev;
}
std::string getDownloadUrl() const override
std::string getDownloadUrl(const Input & input) const override
{
// FIXME: use regular /archive URLs instead? api.github.com
// might have stricter rate limits.
auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s",
owner, repo, rev->to_string(Base16, false));
getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
input.getRev()->to_string(Base16, false));
std::string accessToken = settings.githubAccessToken.get();
if (accessToken != "")
@ -231,35 +200,23 @@ struct GitHubInput : GitArchiveInput
return url;
}
void clone(const Path & destDir) const override
void clone(const Input & input, const Path & destDir) override
{
std::shared_ptr<const Input> input = inputFromURL(fmt("git+ssh://git@github.com/%s/%s.git", owner, repo));
input = input->applyOverrides(ref.value_or("master"), rev);
input->clone(destDir);
Input::fromURL(fmt("git+ssh://git@github.com/%s/%s.git",
getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
.applyOverrides(input.getRef().value_or("master"), input.getRev())
.clone(destDir);
}
};
struct GitHubInputScheme : GitArchiveInputScheme
struct GitLabInputScheme : GitArchiveInputScheme
{
GitHubInputScheme() : GitArchiveInputScheme("github") { }
std::string type() override { return "gitlab"; }
std::unique_ptr<GitArchiveInput> create() override
{
return std::make_unique<GitHubInput>();
}
};
struct GitLabInput : GitArchiveInput
{
std::string type() const override { return "gitlab"; }
std::shared_ptr<GitArchiveInput> _clone() const override
{ return std::make_shared<GitLabInput>(*this); }
Hash getRevFromRef(nix::ref<Store> store, std::string_view ref) const override
Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
{
auto url = fmt("https://gitlab.com/api/v4/projects/%s%%2F%s/repository/branches/%s",
owner, repo, ref);
getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
auto json = nlohmann::json::parse(
readFile(
store->toRealPath(
@ -269,12 +226,13 @@ struct GitLabInput : GitArchiveInput
return rev;
}
std::string getDownloadUrl() const override
std::string getDownloadUrl(const Input & input) const override
{
// FIXME: This endpoint has a rate limit threshold of 5 requests per minute.
auto url = fmt("https://gitlab.com/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s",
owner, repo, rev->to_string(Base16, false));
getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
input.getRev()->to_string(Base16, false));
/* # FIXME: add privat token auth (`curl --header "PRIVATE-TOKEN: <your_access_token>"`)
std::string accessToken = settings.githubAccessToken.get();
@ -284,21 +242,12 @@ struct GitLabInput : GitArchiveInput
return url;
}
void clone(const Path & destDir) const override
void clone(const Input & input, const Path & destDir) override
{
std::shared_ptr<const Input> input = inputFromURL(fmt("git+ssh://git@gitlab.com/%s/%s.git", owner, repo));
input = input->applyOverrides(ref.value_or("master"), rev);
input->clone(destDir);
}
};
struct GitLabInputScheme : GitArchiveInputScheme
{
GitLabInputScheme() : GitArchiveInputScheme("gitlab") { }
std::unique_ptr<GitArchiveInput> create() override
{
return std::make_unique<GitLabInput>();
Input::fromURL(fmt("git+ssh://git@gitlab.com/%s/%s.git",
getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
.applyOverrides(input.getRef().value_or("master"), input.getRev())
.clone(destDir);
}
};

154
src/libfetchers/indirect.cc
View file

@ -4,135 +4,99 @@ namespace nix::fetchers {
std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
struct IndirectInput : Input
{
std::string id;
std::optional<Hash> rev;
std::optional<std::string> ref;
std::string type() const override { return "indirect"; }
bool operator ==(const Input & other) const override
{
auto other2 = dynamic_cast<const IndirectInput *>(&other);
return
other2
&& id == other2->id
&& rev == other2->rev
&& ref == other2->ref;
}
bool isDirect() const override
{
return false;
}
std::optional<std::string> getRef() const override { return ref; }
std::optional<Hash> getRev() const override { return rev; }
bool contains(const Input & other) const override
{
auto other2 = dynamic_cast<const IndirectInput *>(&other);
return
other2
&& id == other2->id
&& (!ref || ref == other2->ref)
&& (!rev || rev == other2->rev);
}
ParsedURL toURL() const override
{
ParsedURL url;
url.scheme = "flake";
url.path = id;
if (ref) { url.path += '/'; url.path += *ref; };
if (rev) { url.path += '/'; url.path += rev->gitRev(); };
return url;
}
Attrs toAttrsInternal() const override
{
Attrs attrs;
attrs.emplace("id", id);
if (ref)
attrs.emplace("ref", *ref);
if (rev)
attrs.emplace("rev", rev->gitRev());
return attrs;
}
std::shared_ptr<const Input> applyOverrides(
std::optional<std::string> ref,
std::optional<Hash> rev) const override
{
if (!ref && !rev) return shared_from_this();
auto res = std::make_shared<IndirectInput>(*this);
if (ref) res->ref = ref;
if (rev) res->rev = rev;
return res;
}
std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
{
throw Error("indirect input '%s' cannot be fetched directly", to_string());
}
};
struct IndirectInputScheme : InputScheme
{
std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
std::optional<Input> inputFromURL(const ParsedURL & url) override
{
if (url.scheme != "flake") return nullptr;
if (url.scheme != "flake") return {};
auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
auto input = std::make_unique<IndirectInput>();
std::optional<Hash> rev;
std::optional<std::string> ref;
if (path.size() == 1) {
} else if (path.size() == 2) {
if (std::regex_match(path[1], revRegex))
input->rev = Hash(path[1], htSHA1);
rev = Hash(path[1], htSHA1);
else if (std::regex_match(path[1], refRegex))
input->ref = path[1];
ref = path[1];
else
throw BadURL("in flake URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[1]);
} else if (path.size() == 3) {
if (!std::regex_match(path[1], refRegex))
throw BadURL("in flake URL '%s', '%s' is not a branch/tag name", url.url, path[1]);
input->ref = path[1];
ref = path[1];
if (!std::regex_match(path[2], revRegex))
throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]);
input->rev = Hash(path[2], htSHA1);
rev = Hash(path[2], htSHA1);
} else
throw BadURL("GitHub URL '%s' is invalid", url.url);
std::string id = path[0];
if (!std::regex_match(id, flakeRegex))
throw BadURL("'%s' is not a valid flake ID", id);
// FIXME: forbid query params?
input->id = path[0];
if (!std::regex_match(input->id, flakeRegex))
throw BadURL("'%s' is not a valid flake ID", input->id);
Input input;
input.direct = false;
input.attrs.insert_or_assign("type", "indirect");
input.attrs.insert_or_assign("id", id);
if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
if (ref) input.attrs.insert_or_assign("ref", *ref);
return input;
}
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
std::optional<Input> inputFromAttrs(const Attrs & attrs) override
{
if (maybeGetStrAttr(attrs, "type") != "indirect") return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "id" && name != "ref" && name != "rev")
if (name != "type" && name != "id" && name != "ref" && name != "rev" && name != "narHash")
throw Error("unsupported indirect input attribute '%s'", name);
auto input = std::make_unique<IndirectInput>();
input->id = getStrAttr(attrs, "id");
input->ref = maybeGetStrAttr(attrs, "ref");
if (auto rev = maybeGetStrAttr(attrs, "rev"))
input->rev = Hash(*rev, htSHA1);
auto id = getStrAttr(attrs, "id");
if (!std::regex_match(id, flakeRegex))
throw BadURL("'%s' is not a valid flake ID", id);
Input input;
input.direct = false;
input.attrs = attrs;
return input;
}
ParsedURL toURL(const Input & input) override
{
ParsedURL url;
url.scheme = "flake";
url.path = getStrAttr(input.attrs, "id");
if (auto ref = input.getRef()) { url.path += '/'; url.path += *ref; };
if (auto rev = input.getRev()) { url.path += '/'; url.path += rev->gitRev(); };
return url;
}
bool hasAllInfo(const Input & input) override
{
return false;
}
Input applyOverrides(
const Input & _input,
std::optional<std::string> ref,
std::optional<Hash> rev) override
{
auto input(_input);
if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
if (ref) input.attrs.insert_or_assign("ref", *ref);
return input;
}
std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
{
throw Error("indirect input '%s' cannot be fetched directly", input.to_string());
}
};
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<IndirectInputScheme>()); });

210
src/libfetchers/mercurial.cc
View file

@ -10,80 +10,92 @@ using namespace std::string_literals;
namespace nix::fetchers {
struct MercurialInput : Input
struct MercurialInputScheme : InputScheme
{
ParsedURL url;
std::optional<std::string> ref;
std::optional<Hash> rev;
MercurialInput(const ParsedURL & url) : url(url)
{ }
std::string type() const override { return "hg"; }
bool operator ==(const Input & other) const override
std::optional<Input> inputFromURL(const ParsedURL & url) override
{
auto other2 = dynamic_cast<const MercurialInput *>(&other);
return
other2
&& url == other2->url
&& rev == other2->rev
&& ref == other2->ref;
if (url.scheme != "hg+http" &&
url.scheme != "hg+https" &&
url.scheme != "hg+ssh" &&
url.scheme != "hg+file") return {};
auto url2(url);
url2.scheme = std::string(url2.scheme, 3);
url2.query.clear();
Attrs attrs;
attrs.emplace("type", "hg");
for (auto &[name, value] : url.query) {
if (name == "rev" || name == "ref")
attrs.emplace(name, value);
else
url2.query.emplace(name, value);
}
attrs.emplace("url", url2.to_string());
return inputFromAttrs(attrs);
}
bool isImmutable() const override
std::optional<Input> inputFromAttrs(const Attrs & attrs) override
{
return (bool) rev || narHash;
if (maybeGetStrAttr(attrs, "type") != "hg") return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash")
throw Error("unsupported Mercurial input attribute '%s'", name);
parseURL(getStrAttr(attrs, "url"));
if (auto ref = maybeGetStrAttr(attrs, "ref")) {
if (!std::regex_match(*ref, refRegex))
throw BadURL("invalid Mercurial branch/tag name '%s'", *ref);
}
Input input;
input.attrs = attrs;
return input;
}
std::optional<std::string> getRef() const override { return ref; }
std::optional<Hash> getRev() const override { return rev; }
ParsedURL toURL() const override
ParsedURL toURL(const Input & input) override
{
ParsedURL url2(url);
url2.scheme = "hg+" + url2.scheme;
if (rev) url2.query.insert_or_assign("rev", rev->gitRev());
if (ref) url2.query.insert_or_assign("ref", *ref);
auto url = parseURL(getStrAttr(input.attrs, "url"));
url.scheme = "hg+" + url.scheme;
if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev());
if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref);
return url;
}
Attrs toAttrsInternal() const override
bool hasAllInfo(const Input & input) override
{
Attrs attrs;
attrs.emplace("url", url.to_string());
if (ref)
attrs.emplace("ref", *ref);
if (rev)
attrs.emplace("rev", rev->gitRev());
return attrs;
// FIXME: ugly, need to distinguish between dirty and clean
// default trees.
return input.getRef() == "default" || maybeGetIntAttr(input.attrs, "revCount");
}
std::shared_ptr<const Input> applyOverrides(
Input applyOverrides(
const Input & input,
std::optional<std::string> ref,
std::optional<Hash> rev) const override
std::optional<Hash> rev) override
{
if (!ref && !rev) return shared_from_this();
auto res = std::make_shared<MercurialInput>(*this);
if (ref) res->ref = ref;
if (rev) res->rev = rev;
auto res(input);
if (rev) res.attrs.insert_or_assign("rev", rev->gitRev());
if (ref) res.attrs.insert_or_assign("ref", *ref);
return res;
}
std::optional<Path> getSourcePath() const
std::optional<Path> getSourcePath(const Input & input) override
{
if (url.scheme == "file" && !ref && !rev)
auto url = parseURL(getStrAttr(input.attrs, "url"));
if (url.scheme == "file" && !input.getRef() && !input.getRev())
return url.path;
return {};
}
void markChangedFile(std::string_view file, std::optional<std::string> commitMsg) const override
void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override
{
auto sourcePath = getSourcePath();
auto sourcePath = getSourcePath(input);
assert(sourcePath);
// FIXME: shut up if file is already tracked.
@ -95,26 +107,27 @@ struct MercurialInput : Input
{ "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg });
}
std::pair<bool, std::string> getActualUrl() const
std::pair<bool, std::string> getActualUrl(const Input & input) const
{
auto url = parseURL(getStrAttr(input.attrs, "url"));
bool isLocal = url.scheme == "file";
return {isLocal, isLocal ? url.path : url.base};
}
std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
{
auto name = "source";
auto input = std::make_shared<MercurialInput>(*this);
Input input(_input);
auto [isLocal, actualUrl_] = getActualUrl();
auto [isLocal, actualUrl_] = getActualUrl(input);
auto actualUrl = actualUrl_; // work around clang bug
// FIXME: return lastModified.
// FIXME: don't clone local repositories.
if (!input->ref && !input->rev && isLocal && pathExists(actualUrl + "/.hg")) {
if (!input.getRef() && !input.getRev() && isLocal && pathExists(actualUrl + "/.hg")) {
bool clean = runProgram("hg", true, { "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == "";
@ -129,7 +142,7 @@ struct MercurialInput : Input
if (settings.warnDirty)
warn("Mercurial tree '%s' is unclean", actualUrl);
input->ref = chomp(runProgram("hg", true, { "branch", "-R", actualUrl }));
input.attrs.insert_or_assign("ref", chomp(runProgram("hg", true, { "branch", "-R", actualUrl })));
auto files = tokenizeString<std::set<std::string>>(
runProgram("hg", true, { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
@ -158,53 +171,50 @@ struct MercurialInput : Input
}
}
if (!input->ref) input->ref = "default";
if (!input.getRef()) input.attrs.insert_or_assign("ref", "default");
auto getImmutableAttrs = [&]()
{
return Attrs({
{"type", "hg"},
{"name", name},
{"rev", input->rev->gitRev()},
{"rev", input.getRev()->gitRev()},
});
};
auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
-> std::pair<Tree, std::shared_ptr<const Input>>
-> std::pair<Tree, Input>
{
assert(input->rev);
assert(!rev || rev == input->rev);
assert(input.getRev());
assert(!_input.getRev() || _input.getRev() == input.getRev());
input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount"));
return {
Tree{
.actualPath = store->toRealPath(storePath),
.storePath = std::move(storePath),
.info = TreeInfo {
.revCount = getIntAttr(infoAttrs, "revCount"),
},
},
input
};
};
if (input->rev) {
if (input.getRev()) {
if (auto res = getCache()->lookup(store, getImmutableAttrs()))
return makeResult(res->first, std::move(res->second));
}
assert(input->rev || input->ref);
auto revOrRef = input->rev ? input->rev->gitRev() : *input->ref;
auto revOrRef = input.getRev() ? input.getRev()->gitRev() : *input.getRef();
Attrs mutableAttrs({
{"type", "hg"},
{"name", name},
{"url", actualUrl},
{"ref", *input->ref},
{"ref", *input.getRef()},
});
if (auto res = getCache()->lookup(store, mutableAttrs)) {
auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
if (!rev || rev == rev2) {
input->rev = rev2;
if (!input.getRev() || input.getRev() == rev2) {
input.attrs.insert_or_assign("rev", rev2.gitRev());
return makeResult(res->first, std::move(res->second));
}
}
@ -213,10 +223,10 @@ struct MercurialInput : Input
/* If this is a commit hash that we already have, we don't
have to pull again. */
if (!(input->rev
if (!(input.getRev()
&& pathExists(cacheDir)
&& runProgram(
RunOptions("hg", { "log", "-R", cacheDir, "-r", input->rev->gitRev(), "--template", "1" })
RunOptions("hg", { "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" })
.killStderr(true)).second == "1"))
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl));
@ -245,9 +255,9 @@ struct MercurialInput : Input
runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" }));
assert(tokens.size() == 3);
input->rev = Hash(tokens[0], htSHA1);
input.attrs.insert_or_assign("rev", Hash(tokens[0], htSHA1).gitRev());
auto revCount = std::stoull(tokens[1]);
input->ref = tokens[2];
input.attrs.insert_or_assign("ref", tokens[2]);
if (auto res = getCache()->lookup(store, getImmutableAttrs()))
return makeResult(res->first, std::move(res->second));
@ -255,18 +265,18 @@ struct MercurialInput : Input
Path tmpDir = createTempDir();
AutoDelete delTmpDir(tmpDir, true);
runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input->rev->gitRev(), tmpDir });
runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input.getRev()->gitRev(), tmpDir });
deletePath(tmpDir + "/.hg_archival.txt");
auto storePath = store->addToStore(name, tmpDir);
Attrs infoAttrs({
{"rev", input->rev->gitRev()},
{"rev", input.getRev()->gitRev()},
{"revCount", (int64_t) revCount},
});
if (!this->rev)
if (!_input.getRev())
getCache()->add(
store,
mutableAttrs,
@ -285,54 +295,6 @@ struct MercurialInput : Input
}
};
struct MercurialInputScheme : InputScheme
{
std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
{
if (url.scheme != "hg+http" &&
url.scheme != "hg+https" &&
url.scheme != "hg+ssh" &&
url.scheme != "hg+file") return nullptr;
auto url2(url);
url2.scheme = std::string(url2.scheme, 3);
url2.query.clear();
Attrs attrs;
attrs.emplace("type", "hg");
for (auto &[name, value] : url.query) {
if (name == "rev" || name == "ref")
attrs.emplace(name, value);
else
url2.query.emplace(name, value);
}
attrs.emplace("url", url2.to_string());
return inputFromAttrs(attrs);
}
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
{
if (maybeGetStrAttr(attrs, "type") != "hg") return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "url" && name != "ref" && name != "rev")
throw Error("unsupported Mercurial input attribute '%s'", name);
auto input = std::make_unique<MercurialInput>(parseURL(getStrAttr(attrs, "url")));
if (auto ref = maybeGetStrAttr(attrs, "ref")) {
if (!std::regex_match(*ref, refRegex))
throw BadURL("invalid Mercurial branch/tag name '%s'", *ref);
input->ref = *ref;
}
if (auto rev = maybeGetStrAttr(attrs, "rev"))
input->rev = Hash(*rev, htSHA1);
return input;
}
};
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<MercurialInputScheme>()); });
}

184
src/libfetchers/path.cc
View file

@ -3,76 +3,86 @@
namespace nix::fetchers {
struct PathInput : Input
struct PathInputScheme : InputScheme
{
Path path;
/* Allow the user to pass in "fake" tree info attributes. This is
useful for making a pinned tree work the same as the repository
from which is exported
(e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */
std::optional<Hash> rev;
std::optional<uint64_t> revCount;
std::optional<time_t> lastModified;
std::string type() const override { return "path"; }
std::optional<Hash> getRev() const override { return rev; }
bool operator ==(const Input & other) const override
std::optional<Input> inputFromURL(const ParsedURL & url) override
{
auto other2 = dynamic_cast<const PathInput *>(&other);
return
other2
&& path == other2->path
&& rev == other2->rev
&& revCount == other2->revCount
&& lastModified == other2->lastModified;
if (url.scheme != "path") return {};
if (url.authority && *url.authority != "")
throw Error("path URL '%s' should not have an authority ('%s')", url.url, *url.authority);
Input input;
input.attrs.insert_or_assign("type", "path");
input.attrs.insert_or_assign("path", url.path);
for (auto & [name, value] : url.query)
if (name == "rev" || name == "narHash")
input.attrs.insert_or_assign(name, value);
else if (name == "revCount" || name == "lastModified") {
uint64_t n;
if (!string2Int(value, n))
throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
input.attrs.insert_or_assign(name, n);
}
else
throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name);
return input;
}
bool isImmutable() const override
std::optional<Input> inputFromAttrs(const Attrs & attrs) override
{
return narHash || rev;
if (maybeGetStrAttr(attrs, "type") != "path") return {};
getStrAttr(attrs, "path");
for (auto & [name, value] : attrs)
/* Allow the user to pass in "fake" tree info
attributes. This is useful for making a pinned tree
work the same as the repository from which is exported
(e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */
if (name == "type" || name == "rev" || name == "revCount" || name == "lastModified" || name == "narHash" || name == "path")
// checked in Input::fromAttrs
;
else
throw Error("unsupported path input attribute '%s'", name);
Input input;
input.attrs = attrs;
return input;
}
ParsedURL toURL() const override
ParsedURL toURL(const Input & input) override
{
auto query = attrsToQuery(toAttrsInternal());
auto query = attrsToQuery(input.attrs);
query.erase("path");
query.erase("type");
return ParsedURL {
.scheme = "path",
.path = path,
.path = getStrAttr(input.attrs, "path"),
.query = query,
};
}
Attrs toAttrsInternal() const override
bool hasAllInfo(const Input & input) override
{
Attrs attrs;
attrs.emplace("path", path);
if (rev)
attrs.emplace("rev", rev->gitRev());
if (revCount)
attrs.emplace("revCount", *revCount);
if (lastModified)
attrs.emplace("lastModified", *lastModified);
if (!rev && narHash)
attrs.emplace("narHash", narHash->to_string(SRI));
return attrs;
return true;
}
std::optional<Path> getSourcePath() const override
std::optional<Path> getSourcePath(const Input & input) override
{
return path;
return getStrAttr(input.attrs, "path");
}
void markChangedFile(std::string_view file, std::optional<std::string> commitMsg) const override
void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override
{
// nothing to do
}
std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
{
auto input = std::make_shared<PathInput>(*this);
auto path = getStrAttr(input.attrs, "path");
// FIXME: check whether access to 'path' is allowed.
@ -85,83 +95,13 @@ struct PathInput : Input
// FIXME: try to substitute storePath.
storePath = store->addToStore("source", path);
input->narHash = store->queryPathInfo(*storePath)->narHash;
return
{
Tree {
.actualPath = store->toRealPath(*storePath),
.storePath = std::move(*storePath),
.info = TreeInfo {
.revCount = revCount,
.lastModified = lastModified
}
},
input
};
}
};
struct PathInputScheme : InputScheme
{
std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
{
if (url.scheme != "path") return nullptr;
auto input = std::make_unique<PathInput>();
input->path = url.path;
if (url.authority && *url.authority != "")
throw Error("path URL '%s' should not have an authority ('%s')", url.url, *url.authority);
for (auto & [name, value] : url.query)
if (name == "rev")
input->rev = Hash(value, htSHA1);
else if (name == "revCount") {
uint64_t revCount;
if (!string2Int(value, revCount))
throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
input->revCount = revCount;
}
else if (name == "lastModified") {
time_t lastModified;
if (!string2Int(value, lastModified))
throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
input->lastModified = lastModified;
}
else if (name == "narHash")
// FIXME: require SRI hash.
input->narHash = Hash(value);
else
throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name);
return input;
}
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
{
if (maybeGetStrAttr(attrs, "type") != "path") return {};
auto input = std::make_unique<PathInput>();
input->path = getStrAttr(attrs, "path");
for (auto & [name, value] : attrs)
if (name == "rev")
input->rev = Hash(getStrAttr(attrs, "rev"), htSHA1);
else if (name == "revCount")
input->revCount = getIntAttr(attrs, "revCount");
else if (name == "lastModified")
input->lastModified = getIntAttr(attrs, "lastModified");
else if (name == "narHash")
// FIXME: require SRI hash.
input->narHash = Hash(getStrAttr(attrs, "narHash"));
else if (name == "type" || name == "path")
;
else
throw Error("unsupported path input attribute '%s'", name);
return input;
return {
Tree {
.actualPath = store->toRealPath(*storePath),
.storePath = std::move(*storePath),
},
input
};
}
};

56
src/libfetchers/registry.cc
View file

@ -22,20 +22,7 @@ std::shared_ptr<Registry> Registry::read(
auto version = json.value("version", 0);
// FIXME: remove soon
if (version == 1) {
auto flakes = json["flakes"];
for (auto i = flakes.begin(); i != flakes.end(); ++i) {
auto url = i->value("url", i->value("uri", ""));
if (url.empty())
throw Error("flake registry '%s' lacks a 'url' attribute for entry '%s'",
path, i.key());
registry->entries.push_back(
{inputFromURL(i.key()), inputFromURL(url), {}});
}
}
else if (version == 2) {
if (version == 2) {
for (auto & i : json["flakes"]) {
auto toAttrs = jsonToAttrs(i["to"]);
Attrs extraAttrs;
@ -47,8 +34,8 @@ std::shared_ptr<Registry> Registry::read(
auto exact = i.find("exact");
registry->entries.push_back(
Entry {
.from = inputFromAttrs(jsonToAttrs(i["from"])),
.to = inputFromAttrs(toAttrs),
.from = Input::fromAttrs(jsonToAttrs(i["from"])),
.to = Input::fromAttrs(std::move(toAttrs)),
.extraAttrs = extraAttrs,
.exact = exact != i.end() && exact.value()
});
@ -72,8 +59,8 @@ void Registry::write(const Path & path)
nlohmann::json arr;
for (auto & entry : entries) {
nlohmann::json obj;
obj["from"] = attrsToJson(entry.from->toAttrs());
obj["to"] = attrsToJson(entry.to->toAttrs());
obj["from"] = attrsToJson(entry.from.toAttrs());
obj["to"] = attrsToJson(entry.to.toAttrs());
if (!entry.extraAttrs.empty())
obj["to"].update(attrsToJson(entry.extraAttrs));
if (entry.exact)
@ -90,8 +77,8 @@ void Registry::write(const Path & path)
}
void Registry::add(
const std::shared_ptr<const Input> & from,
const std::shared_ptr<const Input> & to,
const Input & from,
const Input & to,
const Attrs & extraAttrs)
{
entries.emplace_back(
@ -102,11 +89,11 @@ void Registry::add(
});
}
void Registry::remove(const std::shared_ptr<const Input> & input)
void Registry::remove(const Input & input)
{
// FIXME: use C++20 std::erase.
for (auto i = entries.begin(); i != entries.end(); )
if (*i->from == *input)
if (i->from == input)
i = entries.erase(i);
else
++i;
@ -145,8 +132,8 @@ std::shared_ptr<Registry> getFlagRegistry()
}
void overrideRegistry(
const std::shared_ptr<const Input> & from,
const std::shared_ptr<const Input> & to,
const Input & from,
const Input & to,
const Attrs & extraAttrs)
{
flagRegistry->add(from, to, extraAttrs);
@ -180,32 +167,33 @@ Registries getRegistries(ref<Store> store)
return registries;
}
std::pair<std::shared_ptr<const Input>, Attrs> lookupInRegistries(
std::pair<Input, Attrs> lookupInRegistries(
ref<Store> store,
std::shared_ptr<const Input> input)
const Input & _input)
{
Attrs extraAttrs;
int n = 0;
Input input(_input);
restart:
n++;
if (n > 100) throw Error("cycle detected in flake registr for '%s'", input);
if (n > 100) throw Error("cycle detected in flake registry for '%s'", input.to_string());
for (auto & registry : getRegistries(store)) {
// FIXME: O(n)
for (auto & entry : registry->entries) {
if (entry.exact) {
if (*entry.from == *input) {
if (entry.from == input) {
input = entry.to;
extraAttrs = entry.extraAttrs;
goto restart;
}
} else {
if (entry.from->contains(*input)) {
input = entry.to->applyOverrides(
!entry.from->getRef() && input->getRef() ? input->getRef() : std::optional<std::string>(),
!entry.from->getRev() && input->getRev() ? input->getRev() : std::optional<Hash>());
if (entry.from.contains(input)) {
input = entry.to.applyOverrides(
!entry.from.getRef() && input.getRef() ? input.getRef() : std::optional<std::string>(),
!entry.from.getRev() && input.getRev() ? input.getRev() : std::optional<Hash>());
extraAttrs = entry.extraAttrs;
goto restart;
}
@ -213,8 +201,8 @@ std::pair<std::shared_ptr<const Input>, Attrs> lookupInRegistries(
}
}
if (!input->isDirect())
throw Error("cannot find flake '%s' in the flake registries", input->to_string());
if (!input.isDirect())
throw Error("cannot find flake '%s' in the flake registries", input.to_string());
return {input, extraAttrs};
}

17
src/libfetchers/registry.hh
View file

@ -20,8 +20,7 @@ struct Registry
struct Entry
{
std::shared_ptr<const Input> from;
std::shared_ptr<const Input> to;
Input from, to;
Attrs extraAttrs;
bool exact = false;
};
@ -38,11 +37,11 @@ struct Registry
void write(const Path & path);
void add(
const std::shared_ptr<const Input> & from,
const std::shared_ptr<const Input> & to,
const Input & from,
const Input & to,
const Attrs & extraAttrs);
void remove(const std::shared_ptr<const Input> & input);
void remove(const Input & input);
};
typedef std::vector<std::shared_ptr<Registry>> Registries;
@ -54,12 +53,12 @@ Path getUserRegistryPath();
Registries getRegistries(ref<Store> store);
void overrideRegistry(
const std::shared_ptr<const Input> & from,
const std::shared_ptr<const Input> & to,
const Input & from,
const Input & to,
const Attrs & extraAttrs);
std::pair<std::shared_ptr<const Input>, Attrs> lookupInRegistries(
std::pair<Input, Attrs> lookupInRegistries(
ref<Store> store,
std::shared_ptr<const Input> input);
const Input & input);
}

143
src/libfetchers/tarball.cc
View file

@ -101,7 +101,7 @@ DownloadFileResult downloadFile(
};
}
Tree downloadTarball(
std::pair<Tree, time_t> downloadTarball(
ref<Store> store,
const std::string & url,
const std::string & name,
@ -116,12 +116,12 @@ Tree downloadTarball(
auto cached = getCache()->lookupExpired(store, inAttrs);
if (cached && !cached->expired)
return Tree {
.actualPath = store->toRealPath(cached->storePath),
.storePath = std::move(cached->storePath),
.info = TreeInfo {
.lastModified = getIntAttr(cached->infoAttrs, "lastModified"),
return {
Tree {
.actualPath = store->toRealPath(cached->storePath),
.storePath = std::move(cached->storePath),
},
getIntAttr(cached->infoAttrs, "lastModified")
};
auto res = downloadFile(store, url, name, immutable);
@ -156,118 +156,75 @@ Tree downloadTarball(
*unpackedStorePath,
immutable);
return Tree {
.actualPath = store->toRealPath(*unpackedStorePath),
.storePath = std::move(*unpackedStorePath),
.info = TreeInfo {
.lastModified = lastModified,
return {
Tree {
.actualPath = store->toRealPath(*unpackedStorePath),
.storePath = std::move(*unpackedStorePath),
},
lastModified,
};
}
struct TarballInput : Input
{
ParsedURL url;
std::optional<Hash> hash;
TarballInput(const ParsedURL & url) : url(url)
{ }
std::string type() const override { return "tarball"; }
bool operator ==(const Input & other) const override
{
auto other2 = dynamic_cast<const TarballInput *>(&other);
return
other2
&& to_string() == other2->to_string()
&& hash == other2->hash;
}
bool isImmutable() const override
{
return hash || narHash;
}
ParsedURL toURL() const override
{
auto url2(url);
// NAR hashes are preferred over file hashes since tar/zip files
// don't have a canonical representation.
if (narHash)
url2.query.insert_or_assign("narHash", narHash->to_string(SRI));
else if (hash)
url2.query.insert_or_assign("hash", hash->to_string(SRI));
return url2;
}
Attrs toAttrsInternal() const override
{
Attrs attrs;
attrs.emplace("url", url.to_string());
if (hash)
attrs.emplace("hash", hash->to_string(SRI));
return attrs;
}
std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
{
auto tree = downloadTarball(store, url.to_string(), "source", false);
auto input = std::make_shared<TarballInput>(*this);
input->narHash = store->queryPathInfo(tree.storePath)->narHash;
return {std::move(tree), input};
}
};
struct TarballInputScheme : InputScheme
{
std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
std::optional<Input> inputFromURL(const ParsedURL & url) override
{
if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return nullptr;
if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return {};
if (!hasSuffix(url.path, ".zip")
&& !hasSuffix(url.path, ".tar")
&& !hasSuffix(url.path, ".tar.gz")
&& !hasSuffix(url.path, ".tar.xz")
&& !hasSuffix(url.path, ".tar.bz2"))
return nullptr;
auto input = std::make_unique<TarballInput>(url);
auto hash = input->url.query.find("hash");
if (hash != input->url.query.end()) {
// FIXME: require SRI hash.
input->hash = Hash(hash->second);
input->url.query.erase(hash);
}
auto narHash = input->url.query.find("narHash");
if (narHash != input->url.query.end()) {
// FIXME: require SRI hash.
input->narHash = Hash(narHash->second);
input->url.query.erase(narHash);
}
return {};
Input input;
input.attrs.insert_or_assign("type", "tarball");
input.attrs.insert_or_assign("url", url.to_string());
auto narHash = url.query.find("narHash");
if (narHash != url.query.end())
input.attrs.insert_or_assign("narHash", narHash->second);
return input;
}
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
std::optional<Input> inputFromAttrs(const Attrs & attrs) override
{
if (maybeGetStrAttr(attrs, "type") != "tarball") return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "url" && name != "hash")
if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash")
throw Error("unsupported tarball input attribute '%s'", name);
auto input = std::make_unique<TarballInput>(parseURL(getStrAttr(attrs, "url")));
if (auto hash = maybeGetStrAttr(attrs, "hash"))
// FIXME: require SRI hash.
input->hash = Hash(*hash);
Input input;
input.attrs = attrs;
//input.immutable = (bool) maybeGetStrAttr(input.attrs, "hash");
return input;
}
ParsedURL toURL(const Input & input) override
{
auto url = parseURL(getStrAttr(input.attrs, "url"));
// NAR hashes are preferred over file hashes since tar/zip files
// don't have a canonical representation.
if (auto narHash = input.getNarHash())
url.query.insert_or_assign("narHash", narHash->to_string(SRI));
/*
else if (auto hash = maybeGetStrAttr(input.attrs, "hash"))
url.query.insert_or_assign("hash", Hash(*hash).to_string(SRI));
*/
return url;
}
bool hasAllInfo(const Input & input) override
{
return true;
}
std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
{
auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false).first;
return {std::move(tree), input};
}
};
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<TarballInputScheme>()); });

60
src/libfetchers/tree-info.cc
View file

@ -1,60 +0,0 @@
#include "tree-info.hh"
#include "store-api.hh"
#include <nlohmann/json.hpp>
namespace nix::fetchers {
StorePath TreeInfo::computeStorePath(Store & store) const
{
assert(narHash);
return store.makeFixedOutputPath(true, narHash, "source");
}
TreeInfo TreeInfo::fromJson(const nlohmann::json & json)
{
TreeInfo info;
auto i = json.find("info");
if (i != json.end()) {
const nlohmann::json & i2(*i);
auto j = i2.find("narHash");
if (j != i2.end())
info.narHash = Hash((std::string) *j);
else
throw Error("attribute 'narHash' missing in lock file");
j = i2.find("revCount");
if (j != i2.end())
info.revCount = *j;
j = i2.find("lastModified");
if (j != i2.end())
info.lastModified = *j;
return info;
}
i = json.find("narHash");
if (i != json.end()) {
info.narHash = Hash((std::string) *i);
return info;
}
throw Error("attribute 'info' missing in lock file");
}
nlohmann::json TreeInfo::toJson() const
{
nlohmann::json json;
assert(narHash);
json["narHash"] = narHash.to_string(SRI);
if (revCount)
json["revCount"] = *revCount;
if (lastModified)
json["lastModified"] = *lastModified;
return json;
}
}

33
src/libfetchers/tree-info.hh
View file

@ -1,33 +0,0 @@
#pragma once
#include "path.hh"
#include "hash.hh"
#include <nlohmann/json_fwd.hpp>
namespace nix { class Store; }
namespace nix::fetchers {
struct TreeInfo
{
Hash narHash;
std::optional<uint64_t> revCount;
std::optional<time_t> lastModified;
bool operator ==(const TreeInfo & other) const
{
return
narHash == other.narHash
&& revCount == other.revCount
&& lastModified == other.lastModified;
}
StorePath computeStorePath(Store & store) const;
static TreeInfo fromJson(const nlohmann::json & json);
nlohmann::json toJson() const;
};
}

32
src/nix/flake.cc
View file

@ -62,13 +62,13 @@ static void printFlakeInfo(const Store & store, const Flake & flake)
if (flake.description)
logger->stdout("Description: %s", *flake.description);
logger->stdout("Path: %s", store.printStorePath(flake.sourceInfo->storePath));
if (auto rev = flake.lockedRef.input->getRev())
if (auto rev = flake.lockedRef.input.getRev())
logger->stdout("Revision: %s", rev->to_string(Base16, false));
if (flake.sourceInfo->info.revCount)
logger->stdout("Revisions: %s", *flake.sourceInfo->info.revCount);
if (flake.sourceInfo->info.lastModified)
if (auto revCount = flake.lockedRef.input.getRevCount())
logger->stdout("Revisions: %s", *revCount);
if (auto lastModified = flake.lockedRef.input.getLastModified())
logger->stdout("Last modified: %s",
std::put_time(std::localtime(&*flake.sourceInfo->info.lastModified), "%F %T"));
std::put_time(std::localtime(&*lastModified), "%F %T"));
}
static nlohmann::json flakeToJson(const Store & store, const Flake & flake)
@ -82,13 +82,12 @@ static nlohmann::json flakeToJson(const Store & store, const Flake & flake)
j["resolved"] = attrsToJson(flake.resolvedRef.toAttrs());
j["url"] = flake.lockedRef.to_string(); // FIXME: rename to lockedUrl
j["locked"] = attrsToJson(flake.lockedRef.toAttrs());
j["info"] = flake.sourceInfo->info.toJson();
if (auto rev = flake.lockedRef.input->getRev())
if (auto rev = flake.lockedRef.input.getRev())
j["revision"] = rev->to_string(Base16, false);
if (flake.sourceInfo->info.revCount)
j["revCount"] = *flake.sourceInfo->info.revCount;
if (flake.sourceInfo->info.lastModified)
j["lastModified"] = *flake.sourceInfo->info.lastModified;
if (auto revCount = flake.lockedRef.input.getRevCount())
j["revCount"] = *revCount;
if (auto lastModified = flake.lockedRef.input.getLastModified())
j["lastModified"] = *lastModified;
j["path"] = store.printStorePath(flake.sourceInfo->storePath);
return j;
}
@ -172,7 +171,7 @@ struct CmdFlakeListInputs : FlakeCommand, MixJSON
logger->stdout("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s",
prefix + (last ? treeLast : treeConn), input.first,
lockedNode ? lockedNode->lockedRef : flake.flake.lockedRef);
if (firstVisit) recurse(*input.second, prefix + (last ? treeNull : treeLine));
}
};
@ -501,7 +500,7 @@ struct CmdFlakeClone : FlakeCommand
if (destDir.empty())
throw Error("missing flag '--dest'");
getFlakeRef().resolve(store).input->clone(destDir);
getFlakeRef().resolve(store).input.clone(destDir);
}
};
@ -563,9 +562,10 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun
auto lockedInput = std::dynamic_pointer_cast<const LockedNode>(input.second);
assert(lockedInput);
auto jsonObj3 = jsonObj2 ? jsonObj2->object(input.first) : std::optional<JSONObject>();
if (!dryRun)
lockedInput->lockedRef.input->fetchTree(store);
auto storePath = lockedInput->computeStorePath(*store);
auto storePath =
dryRun
? lockedInput->lockedRef.input.computeStorePath(*store)
: lockedInput->lockedRef.input.fetch(store).first.storePath;
if (jsonObj3)
jsonObj3->attr("path", store->printStorePath(storePath));
sources.insert(std::move(storePath));

2
src/nix/installables.cc
View file

@ -194,7 +194,7 @@ void EvalCommand::completeFlakeRef(std::string_view prefix)
/* Look for registry entries that match the prefix. */
for (auto & registry : fetchers::getRegistries(getStore())) {
for (auto & entry : registry->entries) {
auto from = entry.from->to_string();
auto from = entry.from.to_string();
if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) {
std::string from2(from, 6);
if (hasPrefix(from2, prefix))

2
src/nix/profile.cc
View file

@ -330,7 +330,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
for (size_t i = 0; i < manifest.elements.size(); ++i) {
auto & element(manifest.elements[i]);
if (element.source
&& !element.source->originalRef.input->isImmutable()
&& !element.source->originalRef.input.isImmutable()
&& matches(*store, element, i, matchers))
{
Activity act(*logger, lvlChatty, actUnknown,

6
src/nix/registry.cc
View file

@ -31,8 +31,8 @@ struct CmdRegistryList : StoreCommand
registry->type == Registry::User ? "user " :
registry->type == Registry::System ? "system" :
"global",
entry.from->to_string(),
entry.to->to_string());
entry.from.to_string(),
entry.to.to_string());
}
}
}
@ -107,7 +107,7 @@ struct CmdRegistryPin : virtual Args, EvalCommand
auto ref = parseFlakeRef(url);
auto userRegistry = fetchers::getUserRegistry();
userRegistry->remove(ref.input);
auto [tree, resolved] = ref.resolve(store).input->fetchTree(store);
auto [tree, resolved] = ref.resolve(store).input.fetch(store);
fetchers::Attrs extraAttrs;
if (ref.subdir != "") extraAttrs["dir"] = ref.subdir;
userRegistry->add(ref.input, resolved, extraAttrs);