From b7c02232b2c2c51b3b81389b247e400b7a115abc Mon Sep 17 00:00:00 2001 From: Marwan Aljubeh Date: Wed, 16 Sep 2020 17:56:43 +0100 Subject: [PATCH 01/13] Fix the nix-daemon Mac OS SSL CA cert Mac OS multi-user installations are currently broken because all requests made by nix-daemon to the binary cache fail with: ``` unable to download ... Problem with the SSL CA cert (path? access rights?) (77). ``` This change ensures that the nix-daemon knows where to find the SSL CA cert file. Fixes #2899 and #3261. --- misc/launchd/org.nixos.nix-daemon.plist.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in index 9f26296a9..c334639e2 100644 --- a/misc/launchd/org.nixos.nix-daemon.plist.in +++ b/misc/launchd/org.nixos.nix-daemon.plist.in @@ -4,6 +4,8 @@ EnvironmentVariables + NIX_SSL_CERT_FILE + /nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt OBJC_DISABLE_INITIALIZE_FORK_SAFETY YES From 958bf5712377f59622c59f05a84641aa1093fd32 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 18 Sep 2020 13:10:42 +0200 Subject: [PATCH 02/13] nix build: find() -> get() find() returns an iterator so "!attr" doesn't work. --- src/nix/bundle.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 241c8699b..fc41da9e4 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -98,14 +98,14 @@ struct CmdBundle : InstallableCommand if (!evalState->isDerivation(*vRes)) throw Error("the bundler '%s' does not produce a derivation", bundler.what()); - auto attr1 = vRes->attrs->find(evalState->sDrvPath); + auto attr1 = vRes->attrs->get(evalState->sDrvPath); if (!attr1) throw Error("the bundler '%s' does not produce a derivation", bundler.what()); PathSet context2; StorePath drvPath = store->parseStorePath(evalState->coerceToPath(*attr1->pos, *attr1->value, context2)); - auto attr2 = vRes->attrs->find(evalState->sOutPath); + auto attr2 = vRes->attrs->get(evalState->sOutPath); if (!attr2) throw Error("the bundler '%s' does not produce a derivation", bundler.what()); From 2bcf8cbe7a36ebcd2ad27b4df18fdb911d7ab224 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Fri, 18 Sep 2020 14:10:45 +0200 Subject: [PATCH 03/13] libfetchers/github: allow `url` attribute Since 108debef6f703aa3ca1f6d6c45428880624ec6f8 we allow a `url`-attribute for the `github`-fetcher to fetch tarballs from self-hosted `gitlab`/`github` instances. However it's not used when defining e.g. a flake-input foobar = { type = "github"; url = "gitlab.myserver"; /* ... */ } and breaks with an evaluation-error: error: --- Error --------------------------------------nix unsupported input attribute 'url' (use '--show-trace' to show detailed location information) This patch allows flake-inputs to be fetched from self-hosted instances as well. --- src/libfetchers/github.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 1cc0c5e2e..f510ae5f5 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -77,7 +77,7 @@ struct GitArchiveInputScheme : InputScheme if (maybeGetStrAttr(attrs, "type") != type()) return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified") + if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified" && name != "url") throw Error("unsupported input attribute '%s'", name); getStrAttr(attrs, "owner"); From 5fe375a8f1d60b49835b52df48686caddaa297a4 Mon Sep 17 00:00:00 2001 From: Bryan Richter Date: Fri, 18 Sep 2020 18:36:17 +0300 Subject: [PATCH 04/13] nix-prefetch-url: Add --executable flag pkgs.fetchurl supports an executable argument, which is especially nice when downloading a large executable. This patch adds the same option to nix-prefetch-url. I have tested this to work on the simple case of prefetching a little executable: 1. nix-prefetch-url --executable https://my/little/script 2. Paste the hash into a pkgs.fetchurl-based package, script-pkg.nix 3. Delete the output from the store to avoid any misidentified artifacts 4. Realise the package script-pkg.nix 5. Run the executable I repeated the above while using --name, as well. I suspect --executable would have no meaningful effect if combined with --unpack, but I have not tried it. --- doc/manual/src/command-ref/nix-prefetch-url.md | 3 +++ src/nix-prefetch-url/nix-prefetch-url.cc | 11 +++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/command-ref/nix-prefetch-url.md b/doc/manual/src/command-ref/nix-prefetch-url.md index 1cd1063cd..1307c7c37 100644 --- a/doc/manual/src/command-ref/nix-prefetch-url.md +++ b/doc/manual/src/command-ref/nix-prefetch-url.md @@ -51,6 +51,9 @@ Nix store is also printed. result to the Nix store. The resulting hash can be used with functions such as Nixpkgs’s `fetchzip` or `fetchFromGitHub`. + - `--executable` + Set the executable bit on the downloaded file. + - `--name` *name* Override the name of the file in the Nix store. By default, this is `hash-basename`, where *basename* is the last component of *url*. diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 1001f27af..377ae03a8 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -57,6 +57,7 @@ static int _main(int argc, char * * argv) bool fromExpr = false; string attrPath; bool unpack = false; + bool executable = false; string name; struct MyArgs : LegacyArgs, MixEvalArgs @@ -81,6 +82,8 @@ static int _main(int argc, char * * argv) } else if (*arg == "--unpack") unpack = true; + else if (*arg == "--executable") + executable = true; else if (*arg == "--name") name = getArg(*arg, arg, end); else if (*arg != "" && arg->at(0) == '-') @@ -175,7 +178,11 @@ static int _main(int argc, char * * argv) /* Download the file. */ { - AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0600); + auto mode = 0600; + if (executable) + mode = 0700; + + AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, mode); if (!fd) throw SysError("creating temporary file '%s'", tmpFile); FdSink sink(fd.get()); @@ -201,7 +208,7 @@ static int _main(int argc, char * * argv) tmpFile = unpacked; } - const auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + const auto method = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; auto info = store->addToStoreSlow(name, tmpFile, method, ht, expectedHash); storePath = info.path; From 9aa0dafe205899f350382b587e284c493b223cdd Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 21 Sep 2020 13:11:31 +0200 Subject: [PATCH 05/13] Update lowdown version Fix #4042 According to https://github.com/kristapsdz/lowdown/commit/8aef9e9290de22a10c14ae138257bc1c7fa8ba1f, we shouldn't need to use a fork anymore so we can switch back to upstream --- flake.lock | 11 +++++------ flake.nix | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/flake.lock b/flake.lock index f4368b170..822a73332 100644 --- a/flake.lock +++ b/flake.lock @@ -3,16 +3,15 @@ "lowdown-src": { "flake": false, "locked": { - "lastModified": 1598296217, - "narHash": "sha256-ha7lyNY1d8m+osmDpPc9f/bfZ3ZC1IVIXwfyklSWg8I=", - "owner": "edolstra", + "lastModified": 1598695561, + "narHash": "sha256-gyH/5j+h/nWw0W8AcR2WKvNBUsiQ7QuxqSJNXAwV+8E=", + "owner": "kristapsdz", "repo": "lowdown", - "rev": "c7a4e715af1e233080842db82d15b261cb74cb28", + "rev": "1705b4a26fbf065d9574dce47a94e8c7c79e052f", "type": "github" }, "original": { - "owner": "edolstra", - "ref": "no-structs-in-anonymous-unions", + "owner": "kristapsdz", "repo": "lowdown", "type": "github" } diff --git a/flake.nix b/flake.nix index a50533a29..1b9eb4c77 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ description = "The purely functional package manager"; inputs.nixpkgs.url = "nixpkgs/nixos-20.03-small"; - inputs.lowdown-src = { url = "github:edolstra/lowdown/no-structs-in-anonymous-unions"; flake = false; }; + inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; outputs = { self, nixpkgs, lowdown-src }: From d110fdd03f8860b2a1cd689187f8056b9e22af09 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Sep 2020 13:28:51 +0200 Subject: [PATCH 06/13] Disable precompiled headers in 'nix develop' They're still enabled in regular builds though. --- flake.nix | 5 +---- mk/precompiled-headers.mk | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/flake.nix b/flake.nix index 1b9eb4c77..0304557e8 100644 --- a/flake.nix +++ b/flake.nix @@ -136,7 +136,7 @@ enableParallelBuilding = true; - makeFlags = "profiledir=$(out)/etc/profile.d"; + makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1"; doCheck = true; @@ -334,9 +334,6 @@ # syntax-check generated dot files, it still requires some # fonts. So provide those. FONTCONFIG_FILE = texFunctions.fontsConf; - - # To test building without precompiled headers. - makeFlagsArray = [ "PRECOMPILE_HEADERS=0" ]; }; # System tests. diff --git a/mk/precompiled-headers.mk b/mk/precompiled-headers.mk index 500c99e4a..1fdb4b3a4 100644 --- a/mk/precompiled-headers.mk +++ b/mk/precompiled-headers.mk @@ -1,4 +1,4 @@ -PRECOMPILE_HEADERS ?= 1 +PRECOMPILE_HEADERS ?= 0 print-var-help += \ echo " PRECOMPILE_HEADERS ($(PRECOMPILE_HEADERS)): Whether to use precompiled headers to speed up the build"; From 56f1e0df0546c744421c7d1bd31b1a341796513c Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Mon, 21 Sep 2020 16:29:08 +0200 Subject: [PATCH 07/13] libfetchers/github: rename `url` to `host` --- src/libfetchers/github.cc | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index f510ae5f5..a4db5c5fa 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -8,9 +8,9 @@ namespace nix::fetchers { -// A github or gitlab url -const static std::string urlRegexS = "[a-zA-Z0-9.]*"; // FIXME: check -std::regex urlRegex(urlRegexS, std::regex::ECMAScript); +// A github or gitlab host +const static std::string hostRegexS = "[a-zA-Z0-9.]*"; // FIXME: check +std::regex hostRegex(hostRegexS, std::regex::ECMAScript); struct GitArchiveInputScheme : InputScheme { @@ -50,9 +50,9 @@ struct GitArchiveInputScheme : InputScheme throw BadURL("URL '%s' contains multiple branch/tag names", url.url); ref = value; } - else if (name == "url") { - if (!std::regex_match(value, urlRegex)) - throw BadURL("URL '%s' contains an invalid instance url", url.url); + else if (name == "host") { + if (!std::regex_match(value, hostRegex)) + throw BadURL("URL '%s' contains an invalid instance host", url.url); host_url = value; } // FIXME: barf on unsupported attributes @@ -67,7 +67,7 @@ struct GitArchiveInputScheme : InputScheme input.attrs.insert_or_assign("repo", path[1]); if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); if (ref) input.attrs.insert_or_assign("ref", *ref); - if (host_url) input.attrs.insert_or_assign("url", *host_url); + if (host_url) input.attrs.insert_or_assign("host", *host_url); return input; } @@ -77,7 +77,7 @@ struct GitArchiveInputScheme : InputScheme if (maybeGetStrAttr(attrs, "type") != type()) return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified" && name != "url") + if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified" && name != "host") throw Error("unsupported input attribute '%s'", name); getStrAttr(attrs, "owner"); @@ -210,7 +210,7 @@ struct GitHubInputScheme : GitArchiveInputScheme { // FIXME: use regular /archive URLs instead? api.github.com // might have stricter rate limits. - auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com"); + auto host_url = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); auto url = fmt("https://api.%s/repos/%s/%s/tarball/%s", // FIXME: check if this is correct for self hosted instances host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(Base16, false)); @@ -236,7 +236,7 @@ struct GitLabInputScheme : GitArchiveInputScheme Hash getRevFromRef(nix::ref store, const Input & input) const override { - auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com"); + auto host_url = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s", host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); auto json = nlohmann::json::parse( From e8e1d420f364afbfface61d3f03889e10e6066c9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Sep 2020 18:22:45 +0200 Subject: [PATCH 08/13] Don't include in header files This reduces compilation time by ~15 seconds (CPU time). Issue #4045. --- src/libexpr/eval.cc | 1 + src/libexpr/eval.hh | 8 +++-- src/libexpr/flake/flakeref.cc | 1 + src/libexpr/flake/lockfile.cc | 1 + src/libexpr/primops.cc | 18 ++++++++--- src/libexpr/primops/fetchMercurial.cc | 3 +- src/libfetchers/git.cc | 1 + src/libfetchers/github.cc | 1 + src/libfetchers/indirect.cc | 1 + src/libfetchers/mercurial.cc | 1 + src/libstore/names.cc | 21 +++++++++++-- src/libstore/names.hh | 7 +++-- src/libutil/url-parts.hh | 44 +++++++++++++++++++++++++++ src/libutil/url.cc | 1 + src/libutil/url.hh | 38 ----------------------- src/nix-env/nix-env.cc | 2 +- 16 files changed, 96 insertions(+), 53 deletions(-) create mode 100644 src/libutil/url-parts.hh diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 139067f20..883fc27a7 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -356,6 +356,7 @@ EvalState::EvalState(const Strings & _searchPath, ref store) , sEpsilon(symbols.create("")) , repair(NoRepair) , store(store) + , regexCache(makeRegexCache()) , baseEnv(allocEnv(128)) , staticBaseEnv(false, 0) { diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 80078d8a5..0e1f61baa 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -6,7 +6,6 @@ #include "symbol-table.hh" #include "config.hh" -#include #include #include #include @@ -65,6 +64,11 @@ typedef std::list SearchPath; void initGC(); +struct RegexCache; + +std::shared_ptr makeRegexCache(); + + class EvalState { public: @@ -120,7 +124,7 @@ private: std::unordered_map resolvedPaths; /* Cache used by prim_match(). */ - std::unordered_map regexCache; + std::shared_ptr regexCache; public: diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 6363446f6..d5c2ffe66 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -1,6 +1,7 @@ #include "flakeref.hh" #include "store-api.hh" #include "url.hh" +#include "url-parts.hh" #include "fetchers.hh" #include "registry.hh" diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index a74846944..78431f000 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -1,5 +1,6 @@ #include "lockfile.hh" #include "store-api.hh" +#include "url-parts.hh" #include diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 7e8526ea1..9cfe3f402 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3085,17 +3085,25 @@ static RegisterPrimOp primop_hashString({ .fun = prim_hashString, }); -/* Match a regular expression against a string and return either - ‘null’ or a list containing substring matches. */ +struct RegexCache +{ + std::unordered_map cache; +}; + +std::shared_ptr makeRegexCache() +{ + return std::make_shared(); +} + void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v) { auto re = state.forceStringNoCtx(*args[0], pos); try { - auto regex = state.regexCache.find(re); - if (regex == state.regexCache.end()) - regex = state.regexCache.emplace(re, std::regex(re, std::regex::extended)).first; + auto regex = state.regexCache->cache.find(re); + if (regex == state.regexCache->cache.end()) + regex = state.regexCache->cache.emplace(re, std::regex(re, std::regex::extended)).first; PathSet context; const std::string str = state.forceString(*args[1], context, pos); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index cef85cfef..1a064ed5c 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -3,8 +3,7 @@ #include "store-api.hh" #include "fetchers.hh" #include "url.hh" - -#include +#include "url-parts.hh" namespace nix { diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 5ca0f8521..ad7638d73 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -3,6 +3,7 @@ #include "globals.hh" #include "tarfile.hh" #include "store-api.hh" +#include "url-parts.hh" #include diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index a4db5c5fa..1737658a7 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -3,6 +3,7 @@ #include "fetchers.hh" #include "globals.hh" #include "store-api.hh" +#include "url-parts.hh" #include diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index b981d4d8e..74332ae3d 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -1,4 +1,5 @@ #include "fetchers.hh" +#include "url-parts.hh" namespace nix::fetchers { diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 3e76ffc4d..d80c2ea7a 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -3,6 +3,7 @@ #include "globals.hh" #include "tarfile.hh" #include "store-api.hh" +#include "url-parts.hh" #include diff --git a/src/libstore/names.cc b/src/libstore/names.cc index d1c8a6101..41e28dc99 100644 --- a/src/libstore/names.cc +++ b/src/libstore/names.cc @@ -1,10 +1,18 @@ #include "names.hh" #include "util.hh" +#include + namespace nix { +struct Regex +{ + std::regex regex; +}; + + DrvName::DrvName() { name = ""; @@ -30,11 +38,18 @@ DrvName::DrvName(std::string_view s) : hits(0) } +DrvName::~DrvName() +{ } + + bool DrvName::matches(DrvName & n) { if (name != "*") { - if (!regex) regex = std::unique_ptr(new std::regex(name, std::regex::extended)); - if (!std::regex_match(n.name, *regex)) return false; + if (!regex) { + regex = std::make_unique(); + regex->regex = std::regex(name, std::regex::extended); + } + if (!std::regex_match(n.name, regex->regex)) return false; } if (version != "" && version != n.version) return false; return true; @@ -99,7 +114,7 @@ DrvNames drvNamesFromArgs(const Strings & opArgs) { DrvNames result; for (auto & i : opArgs) - result.push_back(DrvName(i)); + result.emplace_back(i); return result; } diff --git a/src/libstore/names.hh b/src/libstore/names.hh index 00e14b8c7..bc62aac93 100644 --- a/src/libstore/names.hh +++ b/src/libstore/names.hh @@ -3,10 +3,11 @@ #include #include "types.hh" -#include namespace nix { +struct Regex; + struct DrvName { string fullName; @@ -16,10 +17,12 @@ struct DrvName DrvName(); DrvName(std::string_view s); + ~DrvName(); + bool matches(DrvName & n); private: - std::unique_ptr regex; + std::unique_ptr regex; }; typedef list DrvNames; diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh new file mode 100644 index 000000000..64e06cfbc --- /dev/null +++ b/src/libutil/url-parts.hh @@ -0,0 +1,44 @@ +#pragma once + +#include +#include + +namespace nix { + +// URI stuff. +const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])"; +const static std::string schemeRegex = "(?:[a-z+.-]+)"; +const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])"; +const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])"; +const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])"; +const static std::string hostnameRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + ")*)"; +const static std::string hostRegex = "(?:" + ipv6AddressRegex + "|" + hostnameRegex + ")"; +const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|:)*)"; +const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?"; +const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])"; +const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*"; +const static std::string segmentRegex = "(?:" + pcharRegex + "+)"; +const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)"; +const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)"; + +// A Git ref (i.e. branch or tag name). +const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check +extern std::regex refRegex; + +// Instead of defining what a good Git Ref is, we define what a bad Git Ref is +// This is because of the definition of a ref in refs.c in https://github.com/git/git +// See tests/fetchGitRefs.sh for the full definition +const static std::string badGitRefRegexS = "//|^[./]|/\\.|\\.\\.|[[:cntrl:][:space:]:?^~\[]|\\\\|\\*|\\.lock$|\\.lock/|@\\{|[/.]$|^@$|^$"; +extern std::regex badGitRefRegex; + +// A Git revision (a SHA-1 commit hash). +const static std::string revRegexS = "[0-9a-fA-F]{40}"; +extern std::regex revRegex; + +// A ref or revision, or a ref followed by a revision. +const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))"; + +const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*"; +extern std::regex flakeIdRegex; + +} diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 88c09eef9..c1bab866c 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -1,4 +1,5 @@ #include "url.hh" +#include "url-parts.hh" #include "util.hh" namespace nix { diff --git a/src/libutil/url.hh b/src/libutil/url.hh index 1f716ba10..6e77142e3 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -2,8 +2,6 @@ #include "error.hh" -#include - namespace nix { struct ParsedURL @@ -29,40 +27,4 @@ std::map decodeQuery(const std::string & query); ParsedURL parseURL(const std::string & url); -// URI stuff. -const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])"; -const static std::string schemeRegex = "(?:[a-z+.-]+)"; -const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])"; -const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])"; -const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])"; -const static std::string hostnameRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + ")*)"; -const static std::string hostRegex = "(?:" + ipv6AddressRegex + "|" + hostnameRegex + ")"; -const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|:)*)"; -const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?"; -const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])"; -const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*"; -const static std::string segmentRegex = "(?:" + pcharRegex + "+)"; -const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)"; -const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)"; - -// A Git ref (i.e. branch or tag name). -const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check -extern std::regex refRegex; - -// Instead of defining what a good Git Ref is, we define what a bad Git Ref is -// This is because of the definition of a ref in refs.c in https://github.com/git/git -// See tests/fetchGitRefs.sh for the full definition -const static std::string badGitRefRegexS = "//|^[./]|/\\.|\\.\\.|[[:cntrl:][:space:]:?^~\[]|\\\\|\\*|\\.lock$|\\.lock/|@\\{|[/.]$|^@$|^$"; -extern std::regex badGitRefRegex; - -// A Git revision (a SHA-1 commit hash). -const static std::string revRegexS = "[0-9a-fA-F]{40}"; -extern std::regex revRegex; - -// A ref or revision, or a ref followed by a revision. -const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))"; - -const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*"; -extern std::regex flakeIdRegex; - } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index e5a433ac0..3e7c453fb 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -230,7 +230,7 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems, { DrvNames selectors = drvNamesFromArgs(args); if (selectors.empty()) - selectors.push_back(DrvName("*")); + selectors.emplace_back("*"); DrvInfos elems; set done; From d51ba430473368b29abfd1a8c4655da74b3a780c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Sep 2020 18:40:11 +0200 Subject: [PATCH 09/13] Move Callback into its own header This gets rid of the inclusion of in util.hh, cutting compilation time by ~20s (CPU time). Issue #4045. --- src/libstore/binary-cache-store.cc | 1 + src/libstore/build.cc | 1 + src/libstore/dummy-store.cc | 1 + src/libstore/filetransfer.cc | 1 + src/libstore/http-binary-cache-store.cc | 1 + src/libstore/legacy-ssh-store.cc | 1 + src/libstore/local-store.cc | 1 + src/libstore/misc.cc | 2 +- src/libstore/remote-store.cc | 1 + src/libstore/store-api.cc | 4 +-- src/libutil/callback.hh | 46 +++++++++++++++++++++++++ src/libutil/util.hh | 38 +------------------- 12 files changed, 57 insertions(+), 41 deletions(-) create mode 100644 src/libutil/callback.hh diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 34f844a18..ebc0bd6a4 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -11,6 +11,7 @@ #include "nar-accessor.hh" #include "json.hh" #include "thread-pool.hh" +#include "callback.hh" #include #include diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 6e55f83d5..0b51d90ea 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -17,6 +17,7 @@ #include "daemon.hh" #include "worker-protocol.hh" #include "topo-sort.hh" +#include "callback.hh" #include #include diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 128832e60..49641c2ac 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -1,4 +1,5 @@ #include "store-api.hh" +#include "callback.hh" namespace nix { diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 4149f8155..6241b5e00 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -5,6 +5,7 @@ #include "s3.hh" #include "compression.hh" #include "finally.hh" +#include "callback.hh" #ifdef ENABLE_S3 #include diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index f4ab15a10..86be7c006 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -2,6 +2,7 @@ #include "filetransfer.hh" #include "globals.hh" #include "nar-info-disk-cache.hh" +#include "callback.hh" namespace nix { diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index e9478c1d5..5af75669a 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -6,6 +6,7 @@ #include "worker-protocol.hh" #include "ssh.hh" #include "derivations.hh" +#include "callback.hh" namespace nix { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c618203f0..94ff34cb8 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -6,6 +6,7 @@ #include "derivations.hh" #include "nar-info.hh" #include "references.hh" +#include "callback.hh" #include #include diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index da3981696..ad4dccef9 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -5,7 +5,7 @@ #include "store-api.hh" #include "thread-pool.hh" #include "topo-sort.hh" - +#include "callback.hh" namespace nix { diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index e92b94975..27535f1d0 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -10,6 +10,7 @@ #include "pool.hh" #include "finally.hh" #include "logging.hh" +#include "callback.hh" #include #include diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 2d5077ed0..1bbc74db8 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -8,9 +8,7 @@ #include "json.hh" #include "url.hh" #include "archive.hh" - -#include - +#include "callback.hh" namespace nix { diff --git a/src/libutil/callback.hh b/src/libutil/callback.hh new file mode 100644 index 000000000..ef31794be --- /dev/null +++ b/src/libutil/callback.hh @@ -0,0 +1,46 @@ +#pragma once + +#include +#include + +namespace nix { + +/* A callback is a wrapper around a lambda that accepts a valid of + type T or an exception. (We abuse std::future to pass the value or + exception.) */ +template +class Callback +{ + std::function)> fun; + std::atomic_flag done = ATOMIC_FLAG_INIT; + +public: + + Callback(std::function)> fun) : fun(fun) { } + + Callback(Callback && callback) : fun(std::move(callback.fun)) + { + auto prev = callback.done.test_and_set(); + if (prev) done.test_and_set(); + } + + void operator()(T && t) noexcept + { + auto prev = done.test_and_set(); + assert(!prev); + std::promise promise; + promise.set_value(std::move(t)); + fun(promise.get_future()); + } + + void rethrow(const std::exception_ptr & exc = std::current_exception()) noexcept + { + auto prev = done.test_and_set(); + assert(!prev); + std::promise promise; + promise.set_exception(exc); + fun(promise.get_future()); + } +}; + +} diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 082e26375..91e0a1543 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -17,7 +17,6 @@ #include #include #include -#include #include #ifndef HAVE_STRUCT_DIRENT_D_TYPE @@ -480,43 +479,8 @@ std::optional get(const T & map, const typename T::key_ } -/* A callback is a wrapper around a lambda that accepts a valid of - type T or an exception. (We abuse std::future to pass the value or - exception.) */ template -class Callback -{ - std::function)> fun; - std::atomic_flag done = ATOMIC_FLAG_INIT; - -public: - - Callback(std::function)> fun) : fun(fun) { } - - Callback(Callback && callback) : fun(std::move(callback.fun)) - { - auto prev = callback.done.test_and_set(); - if (prev) done.test_and_set(); - } - - void operator()(T && t) noexcept - { - auto prev = done.test_and_set(); - assert(!prev); - std::promise promise; - promise.set_value(std::move(t)); - fun(promise.get_future()); - } - - void rethrow(const std::exception_ptr & exc = std::current_exception()) noexcept - { - auto prev = done.test_and_set(); - assert(!prev); - std::promise promise; - promise.set_exception(exc); - fun(promise.get_future()); - } -}; +class Callback; /* Start a thread that handles various signals. Also block those signals From 340ca382c4de5863f0ecb5c67bae5461ea20c60b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Sep 2020 18:47:18 +0200 Subject: [PATCH 10/13] Don't include nlohmann/json.hpp in globals.hh This reduces compilation time by 207s. Issue #4045. --- src/libstore/globals.cc | 1 + src/libstore/globals.hh | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 491c664db..7b841d925 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -2,6 +2,7 @@ #include "util.hh" #include "archive.hh" #include "args.hh" +#include "abstractsettingtojson.hh" #include #include diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 02721285a..8a2d3ff75 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -2,7 +2,6 @@ #include "types.hh" #include "config.hh" -#include "abstractsettingtojson.hh" #include "util.hh" #include From 0716adaa8b23855bd5e1a59f4115ec933b6a9205 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Sep 2020 18:49:43 +0200 Subject: [PATCH 11/13] abstractsettingtojson.hh -> abstract-setting-to-json.hh --- src/libstore/globals.cc | 2 +- .../{abstractsettingtojson.hh => abstract-setting-to-json.hh} | 0 src/libutil/config.cc | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename src/libutil/{abstractsettingtojson.hh => abstract-setting-to-json.hh} (100%) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 7b841d925..0beb9b2b7 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -2,7 +2,7 @@ #include "util.hh" #include "archive.hh" #include "args.hh" -#include "abstractsettingtojson.hh" +#include "abstract-setting-to-json.hh" #include #include diff --git a/src/libutil/abstractsettingtojson.hh b/src/libutil/abstract-setting-to-json.hh similarity index 100% rename from src/libutil/abstractsettingtojson.hh rename to src/libutil/abstract-setting-to-json.hh diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 309d23b40..5e6a211df 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -1,6 +1,6 @@ #include "config.hh" #include "args.hh" -#include "abstractsettingtojson.hh" +#include "abstract-setting-to-json.hh" #include From 557d2427ee7c21b32e691e1c44f682b667673a70 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Sep 2020 18:57:02 +0200 Subject: [PATCH 12/13] Random header cleanup --- src/libutil/util.hh | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 91e0a1543..b8e201203 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -12,12 +12,9 @@ #include #include -#include -#include #include #include #include -#include #ifndef HAVE_STRUCT_DIRENT_D_TYPE #define DT_UNKNOWN 0 From ecc8672aa007af045d77434b495ca09541e9fee3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Sep 2020 19:06:23 +0200 Subject: [PATCH 13/13] fmt.hh: Don't include boost/algorithm/string/replace.hpp This cuts compilation time by ~49s. Issue #4045. --- src/libutil/fmt.hh | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index a39de041f..6e69bdce2 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -1,7 +1,6 @@ #pragma once #include -#include #include #include "ansicolor.hh"