Merge remote-tracking branch 'upstream/master' into ca-floating-upstream

This commit is contained in:
John Ericson 2020-09-22 04:15:55 +00:00
commit 3786a801c3
40 changed files with 189 additions and 126 deletions

View file

@ -51,6 +51,9 @@ Nix store is also printed.
result to the Nix store. The resulting hash can be used with
functions such as Nixpkgss `fetchzip` or `fetchFromGitHub`.
- `--executable`
Set the executable bit on the downloaded file.
- `--name` *name*
Override the name of the file in the Nix store. By default, this is
`hash-basename`, where *basename* is the last component of *url*.

View file

@ -3,16 +3,15 @@
"lowdown-src": {
"flake": false,
"locked": {
"lastModified": 1598296217,
"narHash": "sha256-ha7lyNY1d8m+osmDpPc9f/bfZ3ZC1IVIXwfyklSWg8I=",
"owner": "edolstra",
"lastModified": 1598695561,
"narHash": "sha256-gyH/5j+h/nWw0W8AcR2WKvNBUsiQ7QuxqSJNXAwV+8E=",
"owner": "kristapsdz",
"repo": "lowdown",
"rev": "c7a4e715af1e233080842db82d15b261cb74cb28",
"rev": "1705b4a26fbf065d9574dce47a94e8c7c79e052f",
"type": "github"
},
"original": {
"owner": "edolstra",
"ref": "no-structs-in-anonymous-unions",
"owner": "kristapsdz",
"repo": "lowdown",
"type": "github"
}

View file

@ -2,7 +2,7 @@
description = "The purely functional package manager";
inputs.nixpkgs.url = "nixpkgs/nixos-20.03-small";
inputs.lowdown-src = { url = "github:edolstra/lowdown/no-structs-in-anonymous-unions"; flake = false; };
inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; };
outputs = { self, nixpkgs, lowdown-src }:
@ -136,7 +136,7 @@
enableParallelBuilding = true;
makeFlags = "profiledir=$(out)/etc/profile.d";
makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1";
doCheck = true;
@ -334,9 +334,6 @@
# syntax-check generated dot files, it still requires some
# fonts. So provide those.
FONTCONFIG_FILE = texFunctions.fontsConf;
# To test building without precompiled headers.
makeFlagsArray = [ "PRECOMPILE_HEADERS=0" ];
};
# System tests.

View file

@ -4,6 +4,8 @@
<dict>
<key>EnvironmentVariables</key>
<dict>
<key>NIX_SSL_CERT_FILE</key>
<string>/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt</string>
<key>OBJC_DISABLE_INITIALIZE_FORK_SAFETY</key>
<string>YES</string>
</dict>

View file

@ -1,4 +1,4 @@
PRECOMPILE_HEADERS ?= 1
PRECOMPILE_HEADERS ?= 0
print-var-help += \
echo " PRECOMPILE_HEADERS ($(PRECOMPILE_HEADERS)): Whether to use precompiled headers to speed up the build";

View file

@ -356,6 +356,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
, sEpsilon(symbols.create(""))
, repair(NoRepair)
, store(store)
, regexCache(makeRegexCache())
, baseEnv(allocEnv(128))
, staticBaseEnv(false, 0)
{

View file

@ -6,7 +6,6 @@
#include "symbol-table.hh"
#include "config.hh"
#include <regex>
#include <map>
#include <optional>
#include <unordered_map>
@ -65,6 +64,11 @@ typedef std::list<SearchPathElem> SearchPath;
void initGC();
struct RegexCache;
std::shared_ptr<RegexCache> makeRegexCache();
class EvalState
{
public:
@ -120,7 +124,7 @@ private:
std::unordered_map<Path, Path> resolvedPaths;
/* Cache used by prim_match(). */
std::unordered_map<std::string, std::regex> regexCache;
std::shared_ptr<RegexCache> regexCache;
public:

View file

@ -1,6 +1,7 @@
#include "flakeref.hh"
#include "store-api.hh"
#include "url.hh"
#include "url-parts.hh"
#include "fetchers.hh"
#include "registry.hh"

View file

@ -1,5 +1,6 @@
#include "lockfile.hh"
#include "store-api.hh"
#include "url-parts.hh"
#include <nlohmann/json.hpp>

View file

@ -3085,17 +3085,25 @@ static RegisterPrimOp primop_hashString({
.fun = prim_hashString,
});
/* Match a regular expression against a string and return either
null or a list containing substring matches. */
struct RegexCache
{
std::unordered_map<std::string, std::regex> cache;
};
std::shared_ptr<RegexCache> makeRegexCache()
{
return std::make_shared<RegexCache>();
}
void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
auto re = state.forceStringNoCtx(*args[0], pos);
try {
auto regex = state.regexCache.find(re);
if (regex == state.regexCache.end())
regex = state.regexCache.emplace(re, std::regex(re, std::regex::extended)).first;
auto regex = state.regexCache->cache.find(re);
if (regex == state.regexCache->cache.end())
regex = state.regexCache->cache.emplace(re, std::regex(re, std::regex::extended)).first;
PathSet context;
const std::string str = state.forceString(*args[1], context, pos);

View file

@ -3,8 +3,7 @@
#include "store-api.hh"
#include "fetchers.hh"
#include "url.hh"
#include <regex>
#include "url-parts.hh"
namespace nix {

View file

@ -3,6 +3,7 @@
#include "globals.hh"
#include "tarfile.hh"
#include "store-api.hh"
#include "url-parts.hh"
#include <sys/time.h>

View file

@ -3,14 +3,15 @@
#include "fetchers.hh"
#include "globals.hh"
#include "store-api.hh"
#include "url-parts.hh"
#include <nlohmann/json.hpp>
namespace nix::fetchers {
// A github or gitlab url
const static std::string urlRegexS = "[a-zA-Z0-9.]*"; // FIXME: check
std::regex urlRegex(urlRegexS, std::regex::ECMAScript);
// A github or gitlab host
const static std::string hostRegexS = "[a-zA-Z0-9.]*"; // FIXME: check
std::regex hostRegex(hostRegexS, std::regex::ECMAScript);
struct GitArchiveInputScheme : InputScheme
{
@ -50,9 +51,9 @@ struct GitArchiveInputScheme : InputScheme
throw BadURL("URL '%s' contains multiple branch/tag names", url.url);
ref = value;
}
else if (name == "url") {
if (!std::regex_match(value, urlRegex))
throw BadURL("URL '%s' contains an invalid instance url", url.url);
else if (name == "host") {
if (!std::regex_match(value, hostRegex))
throw BadURL("URL '%s' contains an invalid instance host", url.url);
host_url = value;
}
// FIXME: barf on unsupported attributes
@ -67,7 +68,7 @@ struct GitArchiveInputScheme : InputScheme
input.attrs.insert_or_assign("repo", path[1]);
if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
if (ref) input.attrs.insert_or_assign("ref", *ref);
if (host_url) input.attrs.insert_or_assign("url", *host_url);
if (host_url) input.attrs.insert_or_assign("host", *host_url);
return input;
}
@ -77,7 +78,7 @@ struct GitArchiveInputScheme : InputScheme
if (maybeGetStrAttr(attrs, "type") != type()) return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified")
if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified" && name != "host")
throw Error("unsupported input attribute '%s'", name);
getStrAttr(attrs, "owner");
@ -210,7 +211,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
{
// FIXME: use regular /archive URLs instead? api.github.com
// might have stricter rate limits.
auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
auto host_url = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
auto url = fmt("https://api.%s/repos/%s/%s/tarball/%s", // FIXME: check if this is correct for self hosted instances
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
input.getRev()->to_string(Base16, false));
@ -236,7 +237,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
{
auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
auto host_url = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s",
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
auto json = nlohmann::json::parse(

View file

@ -1,4 +1,5 @@
#include "fetchers.hh"
#include "url-parts.hh"
namespace nix::fetchers {

View file

@ -3,6 +3,7 @@
#include "globals.hh"
#include "tarfile.hh"
#include "store-api.hh"
#include "url-parts.hh"
#include <sys/time.h>

View file

@ -11,6 +11,7 @@
#include "nar-accessor.hh"
#include "json.hh"
#include "thread-pool.hh"
#include "callback.hh"
#include <chrono>
#include <future>

View file

@ -17,6 +17,7 @@
#include "daemon.hh"
#include "worker-protocol.hh"
#include "topo-sort.hh"
#include "callback.hh"
#include <algorithm>
#include <iostream>

View file

@ -1,4 +1,5 @@
#include "store-api.hh"
#include "callback.hh"
namespace nix {

View file

@ -5,6 +5,7 @@
#include "s3.hh"
#include "compression.hh"
#include "finally.hh"
#include "callback.hh"
#ifdef ENABLE_S3
#include <aws/core/client/ClientConfiguration.h>

View file

@ -2,6 +2,7 @@
#include "util.hh"
#include "archive.hh"
#include "args.hh"
#include "abstract-setting-to-json.hh"
#include <algorithm>
#include <map>

View file

@ -2,7 +2,6 @@
#include "types.hh"
#include "config.hh"
#include "abstractsettingtojson.hh"
#include "util.hh"
#include <map>

View file

@ -2,6 +2,7 @@
#include "filetransfer.hh"
#include "globals.hh"
#include "nar-info-disk-cache.hh"
#include "callback.hh"
namespace nix {

View file

@ -6,6 +6,7 @@
#include "worker-protocol.hh"
#include "ssh.hh"
#include "derivations.hh"
#include "callback.hh"
namespace nix {

View file

@ -6,6 +6,7 @@
#include "derivations.hh"
#include "nar-info.hh"
#include "references.hh"
#include "callback.hh"
#include <iostream>
#include <algorithm>

View file

@ -5,7 +5,7 @@
#include "store-api.hh"
#include "thread-pool.hh"
#include "topo-sort.hh"
#include "callback.hh"
namespace nix {

View file

@ -1,10 +1,18 @@
#include "names.hh"
#include "util.hh"
#include <regex>
namespace nix {
struct Regex
{
std::regex regex;
};
DrvName::DrvName()
{
name = "";
@ -30,11 +38,18 @@ DrvName::DrvName(std::string_view s) : hits(0)
}
DrvName::~DrvName()
{ }
bool DrvName::matches(DrvName & n)
{
if (name != "*") {
if (!regex) regex = std::unique_ptr<std::regex>(new std::regex(name, std::regex::extended));
if (!std::regex_match(n.name, *regex)) return false;
if (!regex) {
regex = std::make_unique<Regex>();
regex->regex = std::regex(name, std::regex::extended);
}
if (!std::regex_match(n.name, regex->regex)) return false;
}
if (version != "" && version != n.version) return false;
return true;
@ -99,7 +114,7 @@ DrvNames drvNamesFromArgs(const Strings & opArgs)
{
DrvNames result;
for (auto & i : opArgs)
result.push_back(DrvName(i));
result.emplace_back(i);
return result;
}

View file

@ -3,10 +3,11 @@
#include <memory>
#include "types.hh"
#include <regex>
namespace nix {
struct Regex;
struct DrvName
{
string fullName;
@ -16,10 +17,12 @@ struct DrvName
DrvName();
DrvName(std::string_view s);
~DrvName();
bool matches(DrvName & n);
private:
std::unique_ptr<std::regex> regex;
std::unique_ptr<Regex> regex;
};
typedef list<DrvName> DrvNames;

View file

@ -10,6 +10,7 @@
#include "pool.hh"
#include "finally.hh"
#include "logging.hh"
#include "callback.hh"
#include <sys/types.h>
#include <sys/stat.h>

View file

@ -8,9 +8,7 @@
#include "json.hh"
#include "url.hh"
#include "archive.hh"
#include <future>
#include "callback.hh"
namespace nix {

46
src/libutil/callback.hh Normal file
View file

@ -0,0 +1,46 @@
#pragma once
#include <future>
#include <functional>
namespace nix {
/* A callback is a wrapper around a lambda that accepts a valid of
type T or an exception. (We abuse std::future<T> to pass the value or
exception.) */
template<typename T>
class Callback
{
std::function<void(std::future<T>)> fun;
std::atomic_flag done = ATOMIC_FLAG_INIT;
public:
Callback(std::function<void(std::future<T>)> fun) : fun(fun) { }
Callback(Callback && callback) : fun(std::move(callback.fun))
{
auto prev = callback.done.test_and_set();
if (prev) done.test_and_set();
}
void operator()(T && t) noexcept
{
auto prev = done.test_and_set();
assert(!prev);
std::promise<T> promise;
promise.set_value(std::move(t));
fun(promise.get_future());
}
void rethrow(const std::exception_ptr & exc = std::current_exception()) noexcept
{
auto prev = done.test_and_set();
assert(!prev);
std::promise<T> promise;
promise.set_exception(exc);
fun(promise.get_future());
}
};
}

View file

@ -1,6 +1,6 @@
#include "config.hh"
#include "args.hh"
#include "abstractsettingtojson.hh"
#include "abstract-setting-to-json.hh"
#include <nlohmann/json.hpp>

View file

@ -1,7 +1,6 @@
#pragma once
#include <boost/format.hpp>
#include <boost/algorithm/string/replace.hpp>
#include <string>
#include "ansicolor.hh"

44
src/libutil/url-parts.hh Normal file
View file

@ -0,0 +1,44 @@
#pragma once
#include <string>
#include <regex>
namespace nix {
// URI stuff.
const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])";
const static std::string schemeRegex = "(?:[a-z+.-]+)";
const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])";
const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])";
const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])";
const static std::string hostnameRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + ")*)";
const static std::string hostRegex = "(?:" + ipv6AddressRegex + "|" + hostnameRegex + ")";
const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|:)*)";
const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?";
const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])";
const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*";
const static std::string segmentRegex = "(?:" + pcharRegex + "+)";
const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)";
const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)";
// A Git ref (i.e. branch or tag name).
const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check
extern std::regex refRegex;
// Instead of defining what a good Git Ref is, we define what a bad Git Ref is
// This is because of the definition of a ref in refs.c in https://github.com/git/git
// See tests/fetchGitRefs.sh for the full definition
const static std::string badGitRefRegexS = "//|^[./]|/\\.|\\.\\.|[[:cntrl:][:space:]:?^~\[]|\\\\|\\*|\\.lock$|\\.lock/|@\\{|[/.]$|^@$|^$";
extern std::regex badGitRefRegex;
// A Git revision (a SHA-1 commit hash).
const static std::string revRegexS = "[0-9a-fA-F]{40}";
extern std::regex revRegex;
// A ref or revision, or a ref followed by a revision.
const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))";
const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*";
extern std::regex flakeIdRegex;
}

View file

@ -1,4 +1,5 @@
#include "url.hh"
#include "url-parts.hh"
#include "util.hh"
namespace nix {

View file

@ -2,8 +2,6 @@
#include "error.hh"
#include <regex>
namespace nix {
struct ParsedURL
@ -29,40 +27,4 @@ std::map<std::string, std::string> decodeQuery(const std::string & query);
ParsedURL parseURL(const std::string & url);
// URI stuff.
const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])";
const static std::string schemeRegex = "(?:[a-z+.-]+)";
const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])";
const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])";
const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])";
const static std::string hostnameRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + ")*)";
const static std::string hostRegex = "(?:" + ipv6AddressRegex + "|" + hostnameRegex + ")";
const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|:)*)";
const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?";
const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])";
const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*";
const static std::string segmentRegex = "(?:" + pcharRegex + "+)";
const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)";
const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)";
// A Git ref (i.e. branch or tag name).
const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check
extern std::regex refRegex;
// Instead of defining what a good Git Ref is, we define what a bad Git Ref is
// This is because of the definition of a ref in refs.c in https://github.com/git/git
// See tests/fetchGitRefs.sh for the full definition
const static std::string badGitRefRegexS = "//|^[./]|/\\.|\\.\\.|[[:cntrl:][:space:]:?^~\[]|\\\\|\\*|\\.lock$|\\.lock/|@\\{|[/.]$|^@$|^$";
extern std::regex badGitRefRegex;
// A Git revision (a SHA-1 commit hash).
const static std::string revRegexS = "[0-9a-fA-F]{40}";
extern std::regex revRegex;
// A ref or revision, or a ref followed by a revision.
const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))";
const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*";
extern std::regex flakeIdRegex;
}

View file

@ -12,13 +12,9 @@
#include <signal.h>
#include <functional>
#include <limits>
#include <cstdio>
#include <map>
#include <sstream>
#include <optional>
#include <future>
#include <iterator>
#ifndef HAVE_STRUCT_DIRENT_D_TYPE
#define DT_UNKNOWN 0
@ -480,43 +476,8 @@ std::optional<typename T::mapped_type> get(const T & map, const typename T::key_
}
/* A callback is a wrapper around a lambda that accepts a valid of
type T or an exception. (We abuse std::future<T> to pass the value or
exception.) */
template<typename T>
class Callback
{
std::function<void(std::future<T>)> fun;
std::atomic_flag done = ATOMIC_FLAG_INIT;
public:
Callback(std::function<void(std::future<T>)> fun) : fun(fun) { }
Callback(Callback && callback) : fun(std::move(callback.fun))
{
auto prev = callback.done.test_and_set();
if (prev) done.test_and_set();
}
void operator()(T && t) noexcept
{
auto prev = done.test_and_set();
assert(!prev);
std::promise<T> promise;
promise.set_value(std::move(t));
fun(promise.get_future());
}
void rethrow(const std::exception_ptr & exc = std::current_exception()) noexcept
{
auto prev = done.test_and_set();
assert(!prev);
std::promise<T> promise;
promise.set_exception(exc);
fun(promise.get_future());
}
};
class Callback;
/* Start a thread that handles various signals. Also block those signals

View file

@ -230,7 +230,7 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems,
{
DrvNames selectors = drvNamesFromArgs(args);
if (selectors.empty())
selectors.push_back(DrvName("*"));
selectors.emplace_back("*");
DrvInfos elems;
set<unsigned int> done;

View file

@ -57,6 +57,7 @@ static int _main(int argc, char * * argv)
bool fromExpr = false;
string attrPath;
bool unpack = false;
bool executable = false;
string name;
struct MyArgs : LegacyArgs, MixEvalArgs
@ -81,6 +82,8 @@ static int _main(int argc, char * * argv)
}
else if (*arg == "--unpack")
unpack = true;
else if (*arg == "--executable")
executable = true;
else if (*arg == "--name")
name = getArg(*arg, arg, end);
else if (*arg != "" && arg->at(0) == '-')
@ -175,7 +178,11 @@ static int _main(int argc, char * * argv)
/* Download the file. */
{
AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0600);
auto mode = 0600;
if (executable)
mode = 0700;
AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, mode);
if (!fd) throw SysError("creating temporary file '%s'", tmpFile);
FdSink sink(fd.get());
@ -201,7 +208,7 @@ static int _main(int argc, char * * argv)
tmpFile = unpacked;
}
const auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
const auto method = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
auto info = store->addToStoreSlow(name, tmpFile, method, ht, expectedHash);
storePath = info.path;

View file

@ -98,14 +98,14 @@ struct CmdBundle : InstallableCommand
if (!evalState->isDerivation(*vRes))
throw Error("the bundler '%s' does not produce a derivation", bundler.what());
auto attr1 = vRes->attrs->find(evalState->sDrvPath);
auto attr1 = vRes->attrs->get(evalState->sDrvPath);
if (!attr1)
throw Error("the bundler '%s' does not produce a derivation", bundler.what());
PathSet context2;
StorePath drvPath = store->parseStorePath(evalState->coerceToPath(*attr1->pos, *attr1->value, context2));
auto attr2 = vRes->attrs->find(evalState->sOutPath);
auto attr2 = vRes->attrs->get(evalState->sOutPath);
if (!attr2)
throw Error("the bundler '%s' does not produce a derivation", bundler.what());