0
0
Fork 0
forked from lix-project/lix

Merge remote-tracking branch 'upstream/master' into trustless-remote-builder-simple

This commit is contained in:
John Ericson 2023-04-17 09:27:17 -04:00
commit b1343e8ad1
38 changed files with 523 additions and 239 deletions

View file

@ -184,7 +184,7 @@ fi
# Look for OpenSSL, a required dependency. FIXME: this is only (maybe)
# used by S3BinaryCacheStore.
PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"])
PKG_CHECK_MODULES([OPENSSL], [libcrypto >= 1.1.1], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"])
# Look for libarchive.

View file

@ -10,7 +10,9 @@ let
result = ''
> **Warning** \
> This program is **experimental** and its interface is subject to change.
> This program is
> [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command)
> and its interface is subject to change.
# Name
@ -29,19 +31,18 @@ let
showSynopsis = command: args:
let
showArgument = arg: "*${arg.label}*" + (if arg ? arity then "" else "...");
showArgument = arg: "*${arg.label}*" + optionalString (! arg ? arity) "...";
arguments = concatStringsSep " " (map showArgument args);
in ''
`${command}` [*option*...] ${arguments}
'';
maybeSubcommands = if details ? commands && details.commands != {}
then ''
maybeSubcommands = optionalString (details ? commands && details.commands != {})
''
where *subcommand* is one of the following:
${subcommands}
''
else "";
'';
subcommands = if length categories > 1
then listCategories
@ -63,12 +64,11 @@ let
* [`${command} ${name}`](./${appendName filename name}.md) - ${subcmd.description}
'';
maybeDocumentation =
if details ? doc
then replaceStrings ["@stores@"] [storeDocs] details.doc
else "";
maybeDocumentation = optionalString
(details ? doc)
(replaceStrings ["@stores@"] [storeDocs] details.doc);
maybeOptions = if details.flags == {} then "" else ''
maybeOptions = optionalString (details.flags != {}) ''
# Options
${showOptions details.flags toplevel.flags}
@ -78,15 +78,19 @@ let
let
allOptions = options // commonOptions;
showCategory = cat: ''
${if cat != "" then "**${cat}:**" else ""}
${optionalString (cat != "") "**${cat}:**"}
${listOptions (filterAttrs (n: v: v.category == cat) allOptions)}
'';
listOptions = opts: concatStringsSep "\n" (attrValues (mapAttrs showOption opts));
showOption = name: option:
let
shortName = if option ? shortName then "/ `-${option.shortName}`" else "";
labels = if option ? labels then (concatStringsSep " " (map (s: "*${s}*") option.labels)) else "";
shortName = optionalString
(option ? shortName)
("/ `-${option.shortName}`");
labels = optionalString
(option ? labels)
(concatStringsSep " " (map (s: "*${s}*") option.labels));
in trim ''
- `--${name}` ${shortName} ${labels}

View file

@ -1,6 +1,6 @@
# Experimental Commands
This section lists experimental commands.
This section lists [experimental commands](@docroot@/contributing/experimental-features.md#xp-feature-nix-command).
> **Warning**
>

View file

@ -225,3 +225,9 @@
[string]: ./language/values.md#type-string
[path]: ./language/values.md#type-path
[attribute name]: ./language/values.md#attribute-set
- [experimental feature]{#gloss-experimental-feature}\
Not yet stabilized functionality guarded by named experimental feature flags.
These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting.
See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/contributing/experimental-features.md).

View file

@ -208,12 +208,26 @@ Derivations can declare some infrequently used optional attributes.
about converting to and from base-32 notation.)
- [`__contentAddressed`]{#adv-attr-__contentAddressed}
If this **experimental** attribute is set to true, then the derivation
> **Warning**
> This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md).
>
> To use this attribute, you must enable the
> [`ca-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) experimental feature.
> For example, in [nix.conf](../command-ref/conf-file.md) you could add:
>
> ```
> extra-experimental-features = ca-derivations
> ```
If this attribute is set to `true`, then the derivation
outputs will be stored in a content-addressed location rather than the
traditional input-addressed one.
This only has an effect if the `ca-derivations` experimental feature is enabled.
Setting this attribute also requires setting `outputHashMode` and `outputHashAlgo` like for *fixed-output derivations* (see above).
Setting this attribute also requires setting
[`outputHashMode`](#adv-attr-outputHashMode)
and
[`outputHashAlgo`](#adv-attr-outputHashAlgo)
like for *fixed-output derivations* (see above).
- [`passAsFile`]{#adv-attr-passAsFile}\
A list of names of attributes that should be passed via files rather
@ -307,9 +321,11 @@ Derivations can declare some infrequently used optional attributes.
- [`unsafeDiscardReferences`]{#adv-attr-unsafeDiscardReferences}\
> **Warning**
> This is an experimental feature.
> This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md).
>
> To enable it, add the following to [nix.conf](../command-ref/conf-file.md):
> To use this attribute, you must enable the
> [`discard-references`](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) experimental feature.
> For example, in [nix.conf](../command-ref/conf-file.md) you could add:
>
> ```
> extra-experimental-features = discard-references

View file

@ -19,7 +19,7 @@ to subsequent chapters.
channel:
```console
$ nix-env -qaP
$ nix-env --query --available --attr-path
nixpkgs.docbook_xml_dtd_43 docbook-xml-4.3
nixpkgs.docbook_xml_dtd_45 docbook-xml-4.5
nixpkgs.firefox firefox-33.0.2
@ -31,7 +31,7 @@ to subsequent chapters.
1. Install some packages from the channel:
```console
$ nix-env -iA nixpkgs.hello
$ nix-env --install --attr nixpkgs.hello
```
This should download pre-built packages; it should not build them
@ -49,13 +49,13 @@ to subsequent chapters.
1. Uninstall a package:
```console
$ nix-env -e hello
$ nix-env --uninstall hello
```
1. You can also test a package without installing it:
```console
$ nix-shell -p hello
$ nix-shell --packages hello
```
This builds or downloads GNU Hello and its dependencies, then drops
@ -76,7 +76,7 @@ to subsequent chapters.
```console
$ nix-channel --update nixpkgs
$ nix-env -u '*'
$ nix-env --upgrade '*'
```
The latter command will upgrade each installed package for which
@ -95,5 +95,5 @@ to subsequent chapters.
them:
```console
$ nix-collect-garbage -d
$ nix-collect-garbage --delete-old
```

View file

@ -42,7 +42,9 @@ rec {
filterAttrs = pred: set:
listToAttrs (concatMap (name: let v = set.${name}; in if pred name v then [(nameValuePair name v)] else []) (attrNames set));
showSetting = { useAnchors }: name: { description, documentDefault, defaultValue, aliases, value }:
optionalString = cond: string: if cond then string else "";
showSetting = { useAnchors }: name: { description, documentDefault, defaultValue, aliases, value, experimentalFeature }:
let
result = squash ''
- ${if useAnchors
@ -52,10 +54,28 @@ rec {
${indent " " body}
'';
experimentalFeatureNote = optionalString (experimentalFeature != null) ''
> **Warning**
> This setting is part of an
> [experimental feature](@docroot@/contributing/experimental-features.md).
To change this setting, you need to make sure the corresponding experimental feature,
[`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}),
is enabled.
For example, include the following in [`nix.conf`](#):
```
extra-experimental-features = ${experimentalFeature}
${name} = ...
```
'';
# separate body to cleanly handle indentation
body = ''
${description}
${experimentalFeatureNote}
**Default:** ${showDefault documentDefault defaultValue}
${showAliases aliases}
@ -74,7 +94,7 @@ rec {
else "*machine-specific*";
showAliases = aliases:
if aliases == [] then "" else
optionalString (aliases != [])
"**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}";
in result;

View file

@ -27,8 +27,6 @@ static ref<Store> store()
if (!_store) {
try {
initLibStore();
loadConfFile();
settings.lockCPU = false;
_store = openStore();
} catch (Error & e) {
croak("%s", e.what());

View file

@ -40,6 +40,7 @@ extern "C" {
#include "markdown.hh"
#include "local-fs-store.hh"
#include "progress-bar.hh"
#include "print.hh"
#if HAVE_BOEHMGC
#define GC_INCLUDE_NEW
@ -425,6 +426,7 @@ StringSet NixRepl::completePrefix(const std::string & prefix)
}
// FIXME: DRY and match or use the parser
static bool isVarName(std::string_view s)
{
if (s.size() == 0) return false;
@ -894,17 +896,6 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
}
std::ostream & printStringValue(std::ostream & str, const char * string) {
str << "\"";
for (const char * i = string; *i; i++)
if (*i == '\"' || *i == '\\') str << "\\" << *i;
else if (*i == '\n') str << "\\n";
else if (*i == '\r') str << "\\r";
else if (*i == '\t') str << "\\t";
else str << *i;
str << "\"";
return str;
}
// FIXME: lot of cut&paste from Nix's eval.cc.
@ -922,12 +913,14 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
break;
case nBool:
str << ANSI_CYAN << (v.boolean ? "true" : "false") << ANSI_NORMAL;
str << ANSI_CYAN;
printLiteralBool(str, v.boolean);
str << ANSI_NORMAL;
break;
case nString:
str << ANSI_WARNING;
printStringValue(str, v.string.s);
printLiteralString(str, v.string.s);
str << ANSI_NORMAL;
break;
@ -964,10 +957,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
sorted.emplace(state->symbols[i.name], i.value);
for (auto & i : sorted) {
if (isVarName(i.first))
str << i.first;
else
printStringValue(str, i.first.c_str());
printAttributeName(str, i.first);
str << " = ";
if (seen.count(i.second))
str << "«repeated»";

View file

@ -9,6 +9,7 @@
#include "filetransfer.hh"
#include "function-trace.hh"
#include "profiles.hh"
#include "print.hh"
#include <algorithm>
#include <chrono>
@ -104,18 +105,10 @@ void Value::print(const SymbolTable & symbols, std::ostream & str,
str << integer;
break;
case tBool:
str << (boolean ? "true" : "false");
printLiteralBool(str, boolean);
break;
case tString:
str << "\"";
for (const char * i = string.s; *i; i++)
if (*i == '\"' || *i == '\\') str << "\\" << *i;
else if (*i == '\n') str << "\\n";
else if (*i == '\r') str << "\\r";
else if (*i == '\t') str << "\\t";
else if (*i == '$' && *(i+1) == '{') str << "\\" << *i;
else str << *i;
str << "\"";
printLiteralString(str, string.s);
break;
case tPath:
str << path; // !!! escaping?

View file

@ -3,6 +3,7 @@
#include "eval.hh"
#include "symbol-table.hh"
#include "util.hh"
#include "print.hh"
#include <cstdlib>
@ -60,45 +61,12 @@ Pos::operator std::shared_ptr<AbstractPos>() const
return pos;
}
/* Displaying abstract syntax trees. */
static void showString(std::ostream & str, std::string_view s)
{
str << '"';
for (auto c : s)
if (c == '"' || c == '\\' || c == '$') str << "\\" << c;
else if (c == '\n') str << "\\n";
else if (c == '\r') str << "\\r";
else if (c == '\t') str << "\\t";
else str << c;
str << '"';
}
// FIXME: remove, because *symbols* are abstract and do not have a single
// textual representation; see printIdentifier()
std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol)
{
std::string_view s = symbol;
if (s.empty())
str << "\"\"";
else if (s == "if") // FIXME: handle other keywords
str << '"' << s << '"';
else {
char c = s[0];
if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) {
showString(str, s);
return str;
}
for (auto c : s)
if (!((c >= 'a' && c <= 'z') ||
(c >= 'A' && c <= 'Z') ||
(c >= '0' && c <= '9') ||
c == '_' || c == '\'' || c == '-')) {
showString(str, s);
return str;
}
str << s;
}
return str;
return printIdentifier(str, s);
}
void Expr::show(const SymbolTable & symbols, std::ostream & str) const
@ -118,7 +86,7 @@ void ExprFloat::show(const SymbolTable & symbols, std::ostream & str) const
void ExprString::show(const SymbolTable & symbols, std::ostream & str) const
{
showString(str, s);
printLiteralString(str, s);
}
void ExprPath::show(const SymbolTable & symbols, std::ostream & str) const

78
src/libexpr/print.cc Normal file
View file

@ -0,0 +1,78 @@
#include "print.hh"
namespace nix {
std::ostream &
printLiteralString(std::ostream & str, const std::string_view string)
{
str << "\"";
for (auto i = string.begin(); i != string.end(); ++i) {
if (*i == '\"' || *i == '\\') str << "\\" << *i;
else if (*i == '\n') str << "\\n";
else if (*i == '\r') str << "\\r";
else if (*i == '\t') str << "\\t";
else if (*i == '$' && *(i+1) == '{') str << "\\" << *i;
else str << *i;
}
str << "\"";
return str;
}
std::ostream &
printLiteralBool(std::ostream & str, bool boolean)
{
str << (boolean ? "true" : "false");
return str;
}
std::ostream &
printIdentifier(std::ostream & str, std::string_view s) {
if (s.empty())
str << "\"\"";
else if (s == "if") // FIXME: handle other keywords
str << '"' << s << '"';
else {
char c = s[0];
if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) {
printLiteralString(str, s);
return str;
}
for (auto c : s)
if (!((c >= 'a' && c <= 'z') ||
(c >= 'A' && c <= 'Z') ||
(c >= '0' && c <= '9') ||
c == '_' || c == '\'' || c == '-')) {
printLiteralString(str, s);
return str;
}
str << s;
}
return str;
}
// FIXME: keywords
static bool isVarName(std::string_view s)
{
if (s.size() == 0) return false;
char c = s[0];
if ((c >= '0' && c <= '9') || c == '-' || c == '\'') return false;
for (auto & i : s)
if (!((i >= 'a' && i <= 'z') ||
(i >= 'A' && i <= 'Z') ||
(i >= '0' && i <= '9') ||
i == '_' || i == '-' || i == '\''))
return false;
return true;
}
std::ostream &
printAttributeName(std::ostream & str, std::string_view name) {
if (isVarName(name))
str << name;
else
printLiteralString(str, name);
return str;
}
}

48
src/libexpr/print.hh Normal file
View file

@ -0,0 +1,48 @@
#pragma once
/**
* @file
* @brief Common printing functions for the Nix language
*
* While most types come with their own methods for printing, they share some
* functions that are placed here.
*/
#include <iostream>
namespace nix {
/**
* Print a string as a Nix string literal.
*
* Quotes and fairly minimal escaping are added.
*
* @param s The logical string
*/
std::ostream & printLiteralString(std::ostream & o, std::string_view s);
inline std::ostream & printLiteralString(std::ostream & o, const char * s) {
return printLiteralString(o, std::string_view(s));
}
inline std::ostream & printLiteralString(std::ostream & o, const std::string & s) {
return printLiteralString(o, std::string_view(s));
}
/** Print `true` or `false`. */
std::ostream & printLiteralBool(std::ostream & o, bool b);
/**
* Print a string as an attribute name in the Nix expression language syntax.
*
* Prints a quoted string if necessary.
*/
std::ostream & printAttributeName(std::ostream & o, std::string_view s);
/**
* Print a string as an identifier in the Nix expression language syntax.
*
* FIXME: "identifier" is ambiguous. Identifiers do not have a single
* textual representation. They can be used in variable references,
* let bindings, left-hand sides or attribute names in a select
* expression, or something else entirely, like JSON. Use one of the
* `print*` functions instead.
*/
std::ostream & printIdentifier(std::ostream & o, std::string_view s);
}

View file

@ -10,7 +10,6 @@
#include <cctype>
#include <exception>
#include <iostream>
#include <mutex>
#include <cstdlib>
#include <sys/time.h>
@ -20,16 +19,9 @@
#ifdef __linux__
#include <features.h>
#endif
#ifdef __GLIBC__
#include <gnu/lib-names.h>
#include <nss.h>
#include <dlfcn.h>
#endif
#include <openssl/crypto.h>
#include <sodium.h>
namespace nix {
@ -115,57 +107,6 @@ std::string getArg(const std::string & opt,
return *i;
}
#if OPENSSL_VERSION_NUMBER < 0x10101000L
/* OpenSSL is not thread-safe by default - it will randomly crash
unless the user supplies a mutex locking function. So let's do
that. */
static std::vector<std::mutex> opensslLocks;
static void opensslLockCallback(int mode, int type, const char * file, int line)
{
if (mode & CRYPTO_LOCK)
opensslLocks[type].lock();
else
opensslLocks[type].unlock();
}
#endif
static std::once_flag dns_resolve_flag;
static void preloadNSS() {
/* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of
one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already
been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to
load its lookup libraries in the parent before any child gets a chance to. */
std::call_once(dns_resolve_flag, []() {
#ifdef __GLIBC__
/* On linux, glibc will run every lookup through the nss layer.
* That means every lookup goes, by default, through nscd, which acts as a local
* cache.
* Because we run builds in a sandbox, we also remove access to nscd otherwise
* lookups would leak into the sandbox.
*
* But now we have a new problem, we need to make sure the nss_dns backend that
* does the dns lookups when nscd is not available is loaded or available.
*
* We can't make it available without leaking nix's environment, so instead we'll
* load the backend, and configure nss so it does not try to run dns lookups
* through nscd.
*
* This is technically only used for builtins:fetch* functions so we only care
* about dns.
*
* All other platforms are unaffected.
*/
if (!dlopen(LIBNSS_DNS_SO, RTLD_NOW))
warn("unable to load nss_dns backend");
// FIXME: get hosts entry from nsswitch.conf.
__nss_configure_lookup("hosts", "files dns");
#endif
});
}
static void sigHandler(int signo) { }
@ -177,16 +118,7 @@ void initNix()
std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf));
#endif
#if OPENSSL_VERSION_NUMBER < 0x10101000L
/* Initialise OpenSSL locking. */
opensslLocks = std::vector<std::mutex>(CRYPTO_num_locks());
CRYPTO_set_locking_callback(opensslLockCallback);
#endif
if (sodium_init() == -1)
throw Error("could not initialise libsodium");
loadConfFile();
initLibStore();
startSignalHandlerThread();
@ -223,7 +155,10 @@ void initNix()
if (sigaction(SIGTRAP, &act, 0)) throw SysError("handling SIGTRAP");
#endif
/* Register a SIGSEGV handler to detect stack overflows. */
/* Register a SIGSEGV handler to detect stack overflows.
Why not initLibExpr()? initGC() is essentially that, but
detectStackOverflow is not an instance of the init function concept, as
it may have to be invoked more than once per process. */
detectStackOverflow();
/* There is no privacy in the Nix system ;-) At least not for
@ -236,16 +171,6 @@ void initNix()
gettimeofday(&tv, 0);
srandom(tv.tv_usec);
/* On macOS, don't use the per-session TMPDIR (as set e.g. by
sshd). This breaks build users because they don't have access
to the TMPDIR, in particular in nix-store --serve. */
#if __APPLE__
if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/"))
unsetenv("TMPDIR");
#endif
preloadNSS();
initLibStore();
}

View file

@ -313,6 +313,15 @@ Derivation parseDerivation(const Store & store, std::string && s, std::string_vi
}
/**
* Print a derivation string literal to an `std::string`.
*
* This syntax does not generalize to the expression language, which needs to
* escape `$`.
*
* @param res Where to print to
* @param s Which logical string to print
*/
static void printString(std::string & res, std::string_view s)
{
boost::container::small_vector<char, 64 * 1024> buffer;

View file

@ -62,15 +62,31 @@ std::string DerivedPath::Opaque::to_string(const Store & store) const
std::string DerivedPath::Built::to_string(const Store & store) const
{
return store.printStorePath(drvPath)
+ "!"
+ '^'
+ outputs.to_string();
}
std::string DerivedPath::Built::to_string_legacy(const Store & store) const
{
return store.printStorePath(drvPath)
+ '!'
+ outputs.to_string();
}
std::string DerivedPath::to_string(const Store & store) const
{
return std::visit(
[&](const auto & req) { return req.to_string(store); },
this->raw());
return std::visit(overloaded {
[&](const DerivedPath::Built & req) { return req.to_string(store); },
[&](const DerivedPath::Opaque & req) { return req.to_string(store); },
}, this->raw());
}
std::string DerivedPath::to_string_legacy(const Store & store) const
{
return std::visit(overloaded {
[&](const DerivedPath::Built & req) { return req.to_string_legacy(store); },
[&](const DerivedPath::Opaque & req) { return req.to_string(store); },
}, this->raw());
}
@ -87,14 +103,24 @@ DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_vi
};
}
DerivedPath DerivedPath::parse(const Store & store, std::string_view s)
static inline DerivedPath parseWith(const Store & store, std::string_view s, std::string_view separator)
{
size_t n = s.find("!");
size_t n = s.find(separator);
return n == s.npos
? (DerivedPath) DerivedPath::Opaque::parse(store, s)
: (DerivedPath) DerivedPath::Built::parse(store, s.substr(0, n), s.substr(n + 1));
}
DerivedPath DerivedPath::parse(const Store & store, std::string_view s)
{
return parseWith(store, s, "^");
}
DerivedPath DerivedPath::parseLegacy(const Store & store, std::string_view s)
{
return parseWith(store, s, "!");
}
RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const
{
RealisedPath::Set res;

View file

@ -48,8 +48,18 @@ struct DerivedPathBuilt {
StorePath drvPath;
OutputsSpec outputs;
/**
* Uses `^` as the separator
*/
std::string to_string(const Store & store) const;
static DerivedPathBuilt parse(const Store & store, std::string_view, std::string_view);
/**
* Uses `!` as the separator
*/
std::string to_string_legacy(const Store & store) const;
/**
* The caller splits on the separator, so it works for both variants.
*/
static DerivedPathBuilt parse(const Store & store, std::string_view drvPath, std::string_view outputs);
nlohmann::json toJSON(ref<Store> store) const;
GENERATE_CMP(DerivedPathBuilt, me->drvPath, me->outputs);
@ -81,8 +91,22 @@ struct DerivedPath : _DerivedPathRaw {
return static_cast<const Raw &>(*this);
}
/**
* Uses `^` as the separator
*/
std::string to_string(const Store & store) const;
/**
* Uses `!` as the separator
*/
std::string to_string_legacy(const Store & store) const;
/**
* Uses `^` as the separator
*/
static DerivedPath parse(const Store & store, std::string_view);
/**
* Uses `!` as the separator
*/
static DerivedPath parseLegacy(const Store & store, std::string_view);
};
/**

View file

@ -71,6 +71,9 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store
void queryRealisationUncached(const DrvOutput &,
Callback<std::shared_ptr<const Realisation>> callback) noexcept override
{ callback(nullptr); }
virtual ref<FSAccessor> getFSAccessor() override
{ unsupported("getFSAccessor"); }
};
static RegisterStoreImplementation<DummyStore, DummyStoreConfig> regDummyStore;

View file

@ -7,12 +7,20 @@
#include <algorithm>
#include <map>
#include <mutex>
#include <thread>
#include <dlfcn.h>
#include <sys/utsname.h>
#include <nlohmann/json.hpp>
#include <sodium/core.h>
#ifdef __GLIBC__
#include <gnu/lib-names.h>
#include <nss.h>
#include <dlfcn.h>
#endif
namespace nix {
@ -41,7 +49,6 @@ Settings::Settings()
, nixDaemonSocketFile(canonPath(getEnvNonEmpty("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH)))
{
buildUsersGroup = getuid() == 0 ? "nixbld" : "";
lockCPU = getEnv("NIX_AFFINITY_HACK") == "1";
allowSymlinkedStore = getEnv("NIX_IGNORE_SYMLINK_STORE") == "1";
auto sslOverride = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or(""));
@ -281,6 +288,42 @@ void initPlugins()
settings.pluginFiles.pluginsLoaded = true;
}
static void preloadNSS()
{
/* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of
one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already
been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to
load its lookup libraries in the parent before any child gets a chance to. */
static std::once_flag dns_resolve_flag;
std::call_once(dns_resolve_flag, []() {
#ifdef __GLIBC__
/* On linux, glibc will run every lookup through the nss layer.
* That means every lookup goes, by default, through nscd, which acts as a local
* cache.
* Because we run builds in a sandbox, we also remove access to nscd otherwise
* lookups would leak into the sandbox.
*
* But now we have a new problem, we need to make sure the nss_dns backend that
* does the dns lookups when nscd is not available is loaded or available.
*
* We can't make it available without leaking nix's environment, so instead we'll
* load the backend, and configure nss so it does not try to run dns lookups
* through nscd.
*
* This is technically only used for builtins:fetch* functions so we only care
* about dns.
*
* All other platforms are unaffected.
*/
if (!dlopen(LIBNSS_DNS_SO, RTLD_NOW))
warn("unable to load nss_dns backend");
// FIXME: get hosts entry from nsswitch.conf.
__nss_configure_lookup("hosts", "files dns");
#endif
});
}
static bool initLibStoreDone = false;
void assertLibStoreInitialized() {
@ -291,6 +334,24 @@ void assertLibStoreInitialized() {
}
void initLibStore() {
initLibUtil();
if (sodium_init() == -1)
throw Error("could not initialise libsodium");
loadConfFile();
preloadNSS();
/* On macOS, don't use the per-session TMPDIR (as set e.g. by
sshd). This breaks build users because they don't have access
to the TMPDIR, in particular in nix-store --serve. */
#if __APPLE__
if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/"))
unsetenv("TMPDIR");
#endif
initLibStoreDone = true;
}

View file

@ -328,16 +328,6 @@ public:
users in `build-users-group`.
UIDs are allocated starting at 872415232 (0x34000000) on Linux and 56930 on macOS.
> **Warning**
> This is an experimental feature.
To enable it, add the following to [`nix.conf`](#):
```
extra-experimental-features = auto-allocate-uids
auto-allocate-uids = true
```
)"};
Setting<uint32_t> startId{this,
@ -367,16 +357,6 @@ public:
Cgroups are required and enabled automatically for derivations
that require the `uid-range` system feature.
> **Warning**
> This is an experimental feature.
To enable it, add the following to [`nix.conf`](#):
```
extra-experimental-features = cgroups
use-cgroups = true
```
)"};
#endif
@ -478,11 +458,6 @@ public:
)",
{"env-keep-derivations"}};
/**
* Whether to lock the Nix client and worker to the same CPU.
*/
bool lockCPU;
Setting<SandboxMode> sandboxMode{
this,
#if __linux__

View file

@ -342,6 +342,9 @@ public:
void ensurePath(const StorePath & path) override
{ unsupported("ensurePath"); }
virtual ref<FSAccessor> getFSAccessor() override
{ unsupported("getFSAccessor"); }
void computeFSClosure(const StorePathSet & paths,
StorePathSet & out, bool flipDirection = false,
bool includeOutputs = false, bool includeDerivers = false) override

View file

@ -90,12 +90,12 @@ void write(const Store & store, Sink & out, const ContentAddress & ca)
DerivedPath read(const Store & store, Source & from, Phantom<DerivedPath> _)
{
auto s = readString(from);
return DerivedPath::parse(store, s);
return DerivedPath::parseLegacy(store, s);
}
void write(const Store & store, Sink & out, const DerivedPath & req)
{
out << req.to_string(store);
out << req.to_string_legacy(store);
}

View file

@ -239,14 +239,11 @@ SQLiteTxn::~SQLiteTxn()
}
}
void handleSQLiteBusy(const SQLiteBusy & e)
void handleSQLiteBusy(const SQLiteBusy & e, time_t & nextWarning)
{
static std::atomic<time_t> lastWarned{0};
time_t now = time(0);
if (now > lastWarned + 10) {
lastWarned = now;
if (now > nextWarning) {
nextWarning = now + 10;
logWarning({
.msg = hintfmt(e.what())
});

View file

@ -139,7 +139,7 @@ protected:
MakeError(SQLiteBusy, SQLiteError);
void handleSQLiteBusy(const SQLiteBusy & e);
void handleSQLiteBusy(const SQLiteBusy & e, time_t & nextWarning);
/**
* Convenience function for retrying a SQLite transaction when the
@ -148,11 +148,13 @@ void handleSQLiteBusy(const SQLiteBusy & e);
template<typename T, typename F>
T retrySQLite(F && fun)
{
time_t nextWarning = time(0) + 1;
while (true) {
try {
return fun();
} catch (SQLiteBusy & e) {
handleSQLiteBusy(e);
handleSQLiteBusy(e, nextWarning);
}
}
}

View file

@ -678,8 +678,7 @@ public:
/**
* @return An object to access files in the Nix store.
*/
virtual ref<FSAccessor> getFSAccessor()
{ unsupported("getFSAccessor"); }
virtual ref<FSAccessor> getFSAccessor() = 0;
/**
* Repair the contents of the given path by redownloading it using

View file

@ -51,6 +51,14 @@ TEST_F(DerivedPathTest, force_init)
{
}
RC_GTEST_FIXTURE_PROP(
DerivedPathTest,
prop_legacy_round_rip,
(const DerivedPath & o))
{
RC_ASSERT(o == DerivedPath::parseLegacy(*store, o.to_string_legacy(*store)));
}
RC_GTEST_FIXTURE_PROP(
DerivedPathTest,
prop_round_rip,

View file

@ -23,7 +23,7 @@ struct ChunkedCompressionSink : CompressionSink
{
uint8_t outbuf[32 * 1024];
void write(std::string_view data) override
void writeUnbuffered(std::string_view data) override
{
const size_t CHUNK_SIZE = sizeof(outbuf) << 2;
while (!data.empty()) {
@ -103,7 +103,7 @@ struct ArchiveCompressionSink : CompressionSink
throw Error(reason, archive_error_string(this->archive));
}
void write(std::string_view data) override
void writeUnbuffered(std::string_view data) override
{
ssize_t result = archive_write_data(archive, data.data(), data.length());
if (result <= 0) check(result);
@ -136,7 +136,7 @@ struct NoneSink : CompressionSink
warn("requested compression level '%d' not supported by compression method 'none'", level);
}
void finish() override { flush(); }
void write(std::string_view data) override { nextSink(data); }
void writeUnbuffered(std::string_view data) override { nextSink(data); }
};
struct BrotliDecompressionSink : ChunkedCompressionSink

View file

@ -12,7 +12,7 @@ namespace nix {
struct CompressionSink : BufferedSink, FinishSink
{
using BufferedSink::operator ();
using BufferedSink::write;
using BufferedSink::writeUnbuffered;
using FinishSink::finish;
};

View file

@ -191,6 +191,10 @@ std::map<std::string, nlohmann::json> AbstractSetting::toJSONObject()
std::map<std::string, nlohmann::json> obj;
obj.emplace("description", description);
obj.emplace("aliases", aliases);
if (experimentalFeature)
obj.emplace("experimentalFeature", *experimentalFeature);
else
obj.emplace("experimentalFeature", nullptr);
return obj;
}

View file

@ -1,6 +1,7 @@
#include <iostream>
#include <cstring>
#include <openssl/crypto.h>
#include <openssl/md5.h>
#include <openssl/sha.h>
@ -16,7 +17,6 @@
namespace nix {
static size_t regularHashSize(HashType type) {
switch (type) {
case htMD5: return md5HashSize;
@ -343,7 +343,7 @@ HashSink::~HashSink()
delete ctx;
}
void HashSink::write(std::string_view data)
void HashSink::writeUnbuffered(std::string_view data)
{
bytes += data.size();
update(ht, *ctx, data);

View file

@ -197,7 +197,7 @@ public:
HashSink(HashType ht);
HashSink(const HashSink & h);
~HashSink();
void write(std::string_view data) override;
void writeUnbuffered(std::string_view data) override;
HashResult finish() override;
HashResult currentHash();
};

View file

@ -20,7 +20,7 @@ void BufferedSink::operator () (std::string_view data)
buffer size. */
if (bufPos + data.size() >= bufSize) {
flush();
write(data);
writeUnbuffered(data);
break;
}
/* Otherwise, copy the bytes to the buffer. Flush the buffer
@ -38,7 +38,7 @@ void BufferedSink::flush()
if (bufPos == 0) return;
size_t n = bufPos;
bufPos = 0; // don't trigger the assert() in ~BufferedSink()
write({buffer.get(), n});
writeUnbuffered({buffer.get(), n});
}
@ -48,7 +48,7 @@ FdSink::~FdSink()
}
void FdSink::write(std::string_view data)
void FdSink::writeUnbuffered(std::string_view data)
{
written += data.size();
try {

View file

@ -53,7 +53,9 @@ struct BufferedSink : virtual Sink
void flush();
virtual void write(std::string_view data) = 0;
protected:
virtual void writeUnbuffered(std::string_view data) = 0;
};
@ -133,7 +135,7 @@ struct FdSink : BufferedSink
~FdSink();
void write(std::string_view data) override;
void writeUnbuffered(std::string_view data) override;
bool good() override;
@ -520,7 +522,7 @@ struct FramedSink : nix::BufferedSink
}
}
void write(std::string_view data) override
void writeUnbuffered(std::string_view data) override
{
/* Don't send more data if the remote has
encountered an error. */

View file

@ -156,12 +156,54 @@ namespace nix {
}
TEST(Config, toJSONOnNonEmptyConfig) {
using nlohmann::literals::operator "" _json;
Config config;
std::map<std::string, Config::SettingInfo> settings;
Setting<std::string> setting{&config, "", "name-of-the-setting", "description"};
Setting<std::string> setting{
&config,
"",
"name-of-the-setting",
"description",
};
setting.assign("value");
ASSERT_EQ(config.toJSON().dump(), R"#({"name-of-the-setting":{"aliases":[],"defaultValue":"","description":"description\n","documentDefault":true,"value":"value"}})#");
ASSERT_EQ(config.toJSON(),
R"#({
"name-of-the-setting": {
"aliases": [],
"defaultValue": "",
"description": "description\n",
"documentDefault": true,
"value": "value",
"experimentalFeature": null
}
})#"_json);
}
TEST(Config, toJSONOnNonEmptyConfigWithExperimentalSetting) {
using nlohmann::literals::operator "" _json;
Config config;
Setting<std::string> setting{
&config,
"",
"name-of-the-setting",
"description",
{},
true,
Xp::Flakes,
};
setting.assign("value");
ASSERT_EQ(config.toJSON(),
R"#({
"name-of-the-setting": {
"aliases": [],
"defaultValue": "",
"description": "description\n",
"documentDefault": true,
"value": "value",
"experimentalFeature": "flakes"
}
})#"_json);
}
TEST(Config, setSettingAlias) {

View file

@ -47,6 +47,9 @@ extern char * * environ __attribute__((weak));
namespace nix {
void initLibUtil() {
}
std::optional<std::string> getEnv(const std::string & key)
{
char * value = getenv(key.c_str());
@ -1744,13 +1747,39 @@ void triggerInterrupt()
}
static sigset_t savedSignalMask;
static bool savedSignalMaskIsSet = false;
void setChildSignalMask(sigset_t * sigs)
{
assert(sigs); // C style function, but think of sigs as a reference
#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE
sigemptyset(&savedSignalMask);
// There's no "assign" or "copy" function, so we rely on (math) idempotence
// of the or operator: a or a = a.
sigorset(&savedSignalMask, sigs, sigs);
#else
// Without sigorset, our best bet is to assume that sigset_t is a type that
// can be assigned directly, such as is the case for a sigset_t defined as
// an integer type.
savedSignalMask = *sigs;
#endif
savedSignalMaskIsSet = true;
}
void saveSignalMask() {
if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask))
throw SysError("querying signal mask");
savedSignalMaskIsSet = true;
}
void startSignalHandlerThread()
{
updateWindowSize();
if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask))
throw SysError("querying signal mask");
saveSignalMask();
sigset_t set;
sigemptyset(&set);
@ -1767,6 +1796,20 @@ void startSignalHandlerThread()
static void restoreSignals()
{
// If startSignalHandlerThread wasn't called, that means we're not running
// in a proper libmain process, but a process that presumably manages its
// own signal handlers. Such a process should call either
// - initNix(), to be a proper libmain process
// - startSignalHandlerThread(), to resemble libmain regarding signal
// handling only
// - saveSignalMask(), for processes that define their own signal handling
// thread
// TODO: Warn about this? Have a default signal mask? The latter depends on
// whether we should generally inherit signal masks from the caller.
// I don't know what the larger unix ecosystem expects from us here.
if (!savedSignalMaskIsSet)
return;
if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr))
throw SysError("restoring signals");
}

View file

@ -32,6 +32,7 @@ namespace nix {
struct Sink;
struct Source;
void initLibUtil();
/**
* The system for which Nix is compiled.
@ -445,6 +446,8 @@ void setStackSize(size_t stackSize);
/**
* Restore the original inherited Unix process context (such as signal
* masks, stack size).
* See startSignalHandlerThread(), saveSignalMask().
*/
void restoreProcessContext(bool restoreMounts = true);
@ -814,9 +817,26 @@ class Callback;
/**
* Start a thread that handles various signals. Also block those signals
* on the current thread (and thus any threads created by it).
* Saves the signal mask before changing the mask to block those signals.
* See saveSignalMask().
*/
void startSignalHandlerThread();
/**
* Saves the signal mask, which is the signal mask that nix will restore
* before creating child processes.
* See setChildSignalMask() to set an arbitrary signal mask instead of the
* current mask.
*/
void saveSignalMask();
/**
* Sets the signal mask. Like saveSignalMask() but for a signal set that doesn't
* necessarily match the current thread's mask.
* See saveSignalMask() to set the saved mask to the current mask.
*/
void setChildSignalMask(sigset_t *sigs);
struct InterruptCallback
{
virtual ~InterruptCallback() { };

View file

@ -48,12 +48,17 @@ manual](https://nixos.org/manual/nix/stable/).
# Installables
> **Warning** \
> Installables are part of the unstable
> [`nix-command` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-nix-command),
> and subject to change without notice.
Many `nix` subcommands operate on one or more *installables*.
These are command line arguments that represent something that can be realised in the Nix store.
The following types of installable are supported by most commands:
- [Flake output attribute](#flake-output-attribute)
- [Flake output attribute](#flake-output-attribute) (experimental)
- [Store path](#store-path)
- [Nix file](#nix-file), optionally qualified by an attribute path
- [Nix expression](#nix-expression), optionally qualified by an attribute path
@ -63,6 +68,13 @@ That is, Nix will operate on the default flake output attribute of the flake in
### Flake output attribute
> **Warning** \
> Flake output attribute installables depend on both the
> [`flakes`](@docroot@/contributing/experimental-features.md#xp-feature-flakes)
> and
> [`nix-command`](@docroot@/contributing/experimental-features.md#xp-feature-nix-command)
> experimental features, and subject to change without notice.
Example: `nixpkgs#hello`
These have the form *flakeref*[`#`*attrpath*], where *flakeref* is a

View file

@ -79,6 +79,14 @@ testReplResponse '
"result: ${a}"
' "result: 2"
# check dollar escaping https://github.com/NixOS/nix/issues/4909
# note the escaped \,
# \\
# because the second argument is a regex
testReplResponse '
"$" + "{hi}"
' '"\\${hi}"'
testReplResponse '
drvPath
' '".*-simple.drv"' \