Move fetchers from libstore to libfetchers

This commit is contained in:
Eelco Dolstra 2020-03-30 14:03:28 +02:00
parent 4ba4c7ff66
commit e0a0ae0467
35 changed files with 80 additions and 90 deletions

View file

@ -4,6 +4,7 @@ makefiles = \
nix-rust/local.mk \ nix-rust/local.mk \
src/libutil/local.mk \ src/libutil/local.mk \
src/libstore/local.mk \ src/libstore/local.mk \
src/libfetchers/local.mk \
src/libmain/local.mk \ src/libmain/local.mk \
src/libexpr/local.mk \ src/libexpr/local.mk \
src/nix/local.mk \ src/nix/local.mk \

View file

@ -6,7 +6,7 @@ dist-files += configure config.h.in perl/configure
clean-files += Makefile.config clean-files += Makefile.config
GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr -I src/nix -Wno-deprecated-declarations GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr -I src/nix -Wno-deprecated-declarations
$(foreach i, config.h $(wildcard src/lib*/*.hh), \ $(foreach i, config.h $(wildcard src/lib*/*.hh), \
$(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644)))

View file

@ -3,8 +3,8 @@
#include "download.hh" #include "download.hh"
#include "util.hh" #include "util.hh"
#include "eval.hh" #include "eval.hh"
#include "fetchers/registry.hh" #include "registry.hh"
#include "fetchers/fetchers.hh" #include "fetchers.hh"
#include "flake/flakeref.hh" #include "flake/flakeref.hh"
#include "store-api.hh" #include "store-api.hh"

View file

@ -3,7 +3,7 @@
#include "primops.hh" #include "primops.hh"
#include "eval-inline.hh" #include "eval-inline.hh"
#include "store-api.hh" #include "store-api.hh"
#include "fetchers/fetchers.hh" #include "fetchers.hh"
#include "finally.hh" #include "finally.hh"
namespace nix { namespace nix {

View file

@ -1,9 +1,8 @@
#include "flakeref.hh" #include "flakeref.hh"
#include "store-api.hh" #include "store-api.hh"
#include "fetchers/parse.hh" #include "url.hh"
#include "fetchers/fetchers.hh" #include "fetchers.hh"
#include "fetchers/registry.hh" #include "registry.hh"
#include "fetchers/regex.hh"
namespace nix { namespace nix {

View file

@ -2,7 +2,7 @@
#include "types.hh" #include "types.hh"
#include "hash.hh" #include "hash.hh"
#include "fetchers/fetchers.hh" #include "fetchers.hh"
#include <variant> #include <variant>

View file

@ -1,6 +1,5 @@
#include "lockfile.hh" #include "lockfile.hh"
#include "store-api.hh" #include "store-api.hh"
#include "fetchers/regex.hh"
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
@ -268,7 +267,7 @@ InputPath parseInputPath(std::string_view s)
InputPath path; InputPath path;
for (auto & elem : tokenizeString<std::vector<std::string>>(s, "/")) { for (auto & elem : tokenizeString<std::vector<std::string>>(s, "/")) {
if (!std::regex_match(elem, fetchers::flakeIdRegex)) if (!std::regex_match(elem, flakeIdRegex))
throw Error("invalid flake input path element '%s'", elem); throw Error("invalid flake input path element '%s'", elem);
path.push_back(elem); path.push_back(elem);
} }

View file

@ -11,7 +11,7 @@ libexpr_SOURCES := \
$(d)/lexer-tab.cc \ $(d)/lexer-tab.cc \
$(d)/parser-tab.cc $(d)/parser-tab.cc
libexpr_LIBS = libutil libstore libnixrust libexpr_LIBS = libutil libstore libfetchers libnixrust
libexpr_LDFLAGS = libexpr_LDFLAGS =
ifneq ($(OS), FreeBSD) ifneq ($(OS), FreeBSD)

View file

@ -545,7 +545,7 @@ formal
#include "eval.hh" #include "eval.hh"
#include "download.hh" #include "download.hh"
#include "fetchers/fetchers.hh" #include "fetchers.hh"
#include "store-api.hh" #include "store-api.hh"

View file

@ -2,8 +2,8 @@
#include "eval-inline.hh" #include "eval-inline.hh"
#include "store-api.hh" #include "store-api.hh"
#include "hash.hh" #include "hash.hh"
#include "fetchers/fetchers.hh" #include "fetchers.hh"
#include "fetchers/parse.hh" #include "url.hh"
namespace nix { namespace nix {
@ -48,14 +48,14 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
if (evalSettings.pureEval && !rev) if (evalSettings.pureEval && !rev)
throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
auto parsedUrl = fetchers::parseURL( auto parsedUrl = parseURL(
url.find("://") != std::string::npos url.find("://") != std::string::npos
? "git+" + url ? "git+" + url
: "git+file://" + url); : "git+file://" + url);
if (ref) parsedUrl.query.insert_or_assign("ref", *ref); if (ref) parsedUrl.query.insert_or_assign("ref", *ref);
if (rev) parsedUrl.query.insert_or_assign("rev", rev->gitRev()); if (rev) parsedUrl.query.insert_or_assign("rev", rev->gitRev());
// FIXME: use name // FIXME: use name
auto input = inputFromURL(parsedUrl); auto input = fetchers::inputFromURL(parsedUrl);
auto [tree, input2] = input->fetchTree(state.store); auto [tree, input2] = input->fetchTree(state.store);

View file

@ -1,9 +1,8 @@
#include "primops.hh" #include "primops.hh"
#include "eval-inline.hh" #include "eval-inline.hh"
#include "store-api.hh" #include "store-api.hh"
#include "fetchers/fetchers.hh" #include "fetchers.hh"
#include "fetchers/parse.hh" #include "url.hh"
#include "fetchers/regex.hh"
#include <regex> #include <regex>
@ -31,7 +30,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
// Ugly: unlike fetchGit, here the "rev" attribute can // Ugly: unlike fetchGit, here the "rev" attribute can
// be both a revision or a branch/tag name. // be both a revision or a branch/tag name.
auto value = state.forceStringNoCtx(*attr.value, *attr.pos); auto value = state.forceStringNoCtx(*attr.value, *attr.pos);
if (std::regex_match(value, fetchers::revRegex)) if (std::regex_match(value, revRegex))
rev = Hash(value, htSHA1); rev = Hash(value, htSHA1);
else else
ref = value; ref = value;
@ -55,14 +54,14 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
if (evalSettings.pureEval && !rev) if (evalSettings.pureEval && !rev)
throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision"); throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision");
auto parsedUrl = fetchers::parseURL( auto parsedUrl = parseURL(
url.find("://") != std::string::npos url.find("://") != std::string::npos
? "hg+" + url ? "hg+" + url
: "hg+file://" + url); : "hg+file://" + url);
if (rev) parsedUrl.query.insert_or_assign("rev", rev->gitRev()); if (rev) parsedUrl.query.insert_or_assign("rev", rev->gitRev());
if (ref) parsedUrl.query.insert_or_assign("ref", *ref); if (ref) parsedUrl.query.insert_or_assign("ref", *ref);
// FIXME: use name // FIXME: use name
auto input = inputFromURL(parsedUrl); auto input = fetchers::inputFromURL(parsedUrl);
auto [tree, input2] = input->fetchTree(state.store); auto [tree, input2] = input->fetchTree(state.store);

View file

@ -1,8 +1,8 @@
#include "primops.hh" #include "primops.hh"
#include "eval-inline.hh" #include "eval-inline.hh"
#include "store-api.hh" #include "store-api.hh"
#include "fetchers/fetchers.hh" #include "fetchers.hh"
#include "fetchers/registry.hh" #include "registry.hh"
#include "download.hh" #include "download.hh"
#include <ctime> #include <ctime>

View file

@ -1,4 +1,4 @@
#include "fetchers/cache.hh" #include "cache.hh"
#include "sqlite.hh" #include "sqlite.hh"
#include "sync.hh" #include "sync.hh"
#include "store-api.hh" #include "store-api.hh"

View file

@ -1,7 +1,6 @@
#pragma once #pragma once
#include "types.hh" #include "fetchers.hh"
#include "fetchers/fetchers.hh"
namespace nix::fetchers { namespace nix::fetchers {

View file

@ -1,5 +1,4 @@
#include "fetchers.hh" #include "fetchers.hh"
#include "parse.hh"
#include "store-api.hh" #include "store-api.hh"
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>

View file

@ -5,7 +5,7 @@
#include "path.hh" #include "path.hh"
#include "tree-info.hh" #include "tree-info.hh"
#include "attrs.hh" #include "attrs.hh"
#include "parse.hh" #include "url.hh"
#include <memory> #include <memory>

View file

@ -1,10 +1,8 @@
#include "fetchers/fetchers.hh" #include "fetchers.hh"
#include "fetchers/cache.hh" #include "cache.hh"
#include "fetchers/parse.hh"
#include "globals.hh" #include "globals.hh"
#include "tarfile.hh" #include "tarfile.hh"
#include "store-api.hh" #include "store-api.hh"
#include "regex.hh"
#include <sys/time.h> #include <sys/time.h>

View file

@ -1,8 +1,6 @@
#include "download.hh" #include "download.hh"
#include "fetchers/cache.hh" #include "cache.hh"
#include "fetchers/fetchers.hh" #include "fetchers.hh"
#include "fetchers/parse.hh"
#include "fetchers/regex.hh"
#include "globals.hh" #include "globals.hh"
#include "store-api.hh" #include "store-api.hh"

View file

@ -1,6 +1,4 @@
#include "fetchers.hh" #include "fetchers.hh"
#include "parse.hh"
#include "regex.hh"
namespace nix::fetchers { namespace nix::fetchers {

9
src/libfetchers/local.mk Normal file
View file

@ -0,0 +1,9 @@
libraries += libfetchers
libfetchers_NAME = libnixfetchers
libfetchers_DIR := $(d)
libfetchers_SOURCES := $(wildcard $(d)/*.cc)
libfetchers_LIBS = libutil libstore libnixrust

View file

@ -1,10 +1,8 @@
#include "fetchers/fetchers.hh" #include "fetchers.hh"
#include "fetchers/cache.hh" #include "cache.hh"
#include "fetchers/parse.hh"
#include "globals.hh" #include "globals.hh"
#include "tarfile.hh" #include "tarfile.hh"
#include "store-api.hh" #include "store-api.hh"
#include "regex.hh"
#include <sys/time.h> #include <sys/time.h>

View file

@ -1,5 +1,5 @@
#include "fetchers/registry.hh" #include "registry.hh"
#include "fetchers/fetchers.hh" #include "fetchers.hh"
#include "util.hh" #include "util.hh"
#include "globals.hh" #include "globals.hh"
#include "download.hh" #include "download.hh"

View file

@ -1,6 +1,5 @@
#include "fetchers/fetchers.hh" #include "fetchers.hh"
#include "fetchers/parse.hh" #include "cache.hh"
#include "fetchers/cache.hh"
#include "download.hh" #include "download.hh"
#include "globals.hh" #include "globals.hh"
#include "store-api.hh" #include "store-api.hh"

View file

@ -1,30 +0,0 @@
#pragma once
#include "types.hh"
namespace nix::fetchers {
struct ParsedURL
{
std::string url;
std::string base; // URL without query/fragment
std::string scheme;
std::optional<std::string> authority;
std::string path;
std::map<std::string, std::string> query;
std::string fragment;
std::string to_string() const;
bool operator ==(const ParsedURL & other) const;
};
MakeError(BadURL, Error);
std::string percentDecode(std::string_view in);
std::map<std::string, std::string> decodeQuery(const std::string & query);
ParsedURL parseURL(const std::string & url);
}

View file

@ -6,7 +6,7 @@
#include "thread-pool.hh" #include "thread-pool.hh"
#include "json.hh" #include "json.hh"
#include "derivations.hh" #include "derivations.hh"
#include "fetchers/parse.hh" #include "url.hh"
#include <future> #include <future>
@ -867,7 +867,7 @@ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_
Store::Params params; Store::Params params;
auto q = uri.find('?'); auto q = uri.find('?');
if (q != std::string::npos) { if (q != std::string::npos) {
params = fetchers::decodeQuery(uri.substr(q + 1)); params = decodeQuery(uri.substr(q + 1));
uri = uri_.substr(0, q); uri = uri_.substr(0, q);
} }
return {uri, params}; return {uri, params};

View file

@ -1,8 +1,7 @@
#include "parse.hh" #include "url.hh"
#include "util.hh" #include "util.hh"
#include "regex.hh"
namespace nix::fetchers { namespace nix {
std::regex refRegex(refRegexS, std::regex::ECMAScript); std::regex refRegex(refRegexS, std::regex::ECMAScript);
std::regex revRegex(revRegexS, std::regex::ECMAScript); std::regex revRegex(revRegexS, std::regex::ECMAScript);

View file

@ -1,8 +1,33 @@
#pragma once #pragma once
#include "types.hh"
#include <regex> #include <regex>
namespace nix::fetchers { namespace nix {
struct ParsedURL
{
std::string url;
std::string base; // URL without query/fragment
std::string scheme;
std::optional<std::string> authority;
std::string path;
std::map<std::string, std::string> query;
std::string fragment;
std::string to_string() const;
bool operator ==(const ParsedURL & other) const;
};
MakeError(BadURL, Error);
std::string percentDecode(std::string_view in);
std::map<std::string, std::string> decodeQuery(const std::string & query);
ParsedURL parseURL(const std::string & url);
// URI stuff. // URI stuff.
const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])"; const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])";

View file

@ -3,7 +3,7 @@
#include "download.hh" #include "download.hh"
#include "store-api.hh" #include "store-api.hh"
#include "legacy.hh" #include "legacy.hh"
#include "fetchers/fetchers.hh" #include "fetchers.hh"
#include <fcntl.h> #include <fcntl.h>
#include <regex> #include <regex>

View file

@ -9,8 +9,8 @@
#include "store-api.hh" #include "store-api.hh"
#include "derivations.hh" #include "derivations.hh"
#include "attr-path.hh" #include "attr-path.hh"
#include "fetchers/fetchers.hh" #include "fetchers.hh"
#include "fetchers/registry.hh" #include "registry.hh"
#include "json.hh" #include "json.hh"
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>

View file

@ -10,7 +10,7 @@
#include "shared.hh" #include "shared.hh"
#include "flake/flake.hh" #include "flake/flake.hh"
#include "flake/eval-cache.hh" #include "flake/eval-cache.hh"
#include "fetchers/parse.hh" #include "url.hh"
#include <regex> #include <regex>
#include <queue> #include <queue>

View file

@ -15,7 +15,7 @@ nix_SOURCES := \
$(wildcard src/nix-prefetch-url/*.cc) \ $(wildcard src/nix-prefetch-url/*.cc) \
$(wildcard src/nix-store/*.cc) \ $(wildcard src/nix-store/*.cc) \
nix_LIBS = libexpr libmain libstore libutil libnixrust nix_LIBS = libexpr libmain libfetchers libstore libutil libnixrust
nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system