lunaphied/lix
0
0
Fork 0
forked from lix-project/lix
Code Pull requests Activity

Merge remote-tracking branch 'upstream/master' into ca-drv-exotic

Browse source
This commit is contained in:
John Ericson 2021-10-08 23:59:15 +00:00
parent edf67e1508 01e9f046a8
commit 195daa8299
40 changed files with 617 additions and 306 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -40,6 +40,7 @@ perl/Makefile.config
# /src/libstore/
*.gen.*
/src/libstore/tests/libstore-tests
# /src/libutil/
/src/libutil/tests/libutil-tests

2
.version
View file

@ -1 +1 @@
2.4
2.5

1
Makefile
View file

@ -4,6 +4,7 @@ makefiles = \
src/libutil/local.mk \
src/libutil/tests/local.mk \
src/libstore/local.mk \
src/libstore/tests/local.mk \
src/libfetchers/local.mk \
src/libmain/local.mk \
src/libexpr/local.mk \

8
doc/manual/generate-manpage.nix
View file

@ -1,4 +1,4 @@
command:
{ command, renderLinks ? false }:
with builtins;
with import ./utils.nix;
@ -20,7 +20,11 @@ let
categories = sort (x: y: x.id < y.id) (unique (map (cmd: cmd.category) (attrValues def.commands)));
listCommands = cmds:
concatStrings (map (name:
"* [`${command} ${name}`](./${appendName filename name}.md) - ${cmds.${name}.description}\n")
"* "
+ (if renderLinks
then "[`${command} ${name}`](./${appendName filename name}.md)"
else "`${command} ${name}`")
+ " - ${cmds.${name}.description}\n")
(attrNames cmds));
in
"where *subcommand* is one of the following:\n\n"

2
doc/manual/local.mk
View file

@ -44,7 +44,7 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli
$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix
@rm -rf $@
$(trace-gen) $(nix-eval) --write-to $@ --expr 'import doc/manual/generate-manpage.nix (builtins.readFile $<)'
$(trace-gen) $(nix-eval) --write-to $@ --expr 'import doc/manual/generate-manpage.nix { command = builtins.readFile $<; renderLinks = true; }'
$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix
@cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp

2
doc/manual/src/installation/env-variables.md
View file

@ -40,7 +40,7 @@ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt
> **Note**
>
> You must not add the export and then do the install, as the Nix
> installer will detect the presense of Nix configuration, and abort.
> installer will detect the presence of Nix configuration, and abort.
## `NIX_SSL_CERT_FILE` with macOS and the Nix daemon

2
doc/manual/src/release-notes/rl-2.5.md Normal file
View file

@ -0,0 +1,2 @@
# Release 2.5 (2021-XX-XX)

6
flake.lock
View file

@ -3,11 +3,11 @@
"lowdown-src": {
"flake": false,
"locked": {
"lastModified": 1632468475,
"narHash": "sha256-NNOm9CbdA8cuwbvaBHslGbPTiU6bh1Ao+MpEPx4rSGo=",
"lastModified": 1633514407,
"narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=",
"owner": "kristapsdz",
"repo": "lowdown",
"rev": "6bd668af3fd098bdd07a1bedd399564141e275da",
"rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8",
"type": "github"
},
"original": {

29
flake.nix
View file

@ -78,7 +78,7 @@
# Tests
buildPackages.git
buildPackages.mercurial
buildPackages.mercurial # FIXME: remove? only needed for tests
buildPackages.jq
]
++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)];
@ -126,8 +126,7 @@
''
mkdir -p $out/nix-support
# Converts /nix/store/50p3qk8kka9dl6wyq40vydq945k0j3kv-nix-2.4pre20201102_550e11f/bin/nix
# To 50p3qk8kka9dl6wyq40vydq945k0j3kv/bin/nix
# Converts /nix/store/50p3qk8k...-nix-2.4pre20201102_550e11f/bin/nix to 50p3qk8k.../bin/nix.
tarballPath() {
# Remove the store prefix
local path=''${1#${builtins.storeDir}/}
@ -153,13 +152,15 @@
echo "file installer $out/install" >> $out/nix-support/hydra-build-products
'';
testNixVersions = pkgs: client: daemon: with commonDeps pkgs; pkgs.stdenv.mkDerivation {
testNixVersions = pkgs: client: daemon: with commonDeps pkgs; with pkgs.lib; pkgs.stdenv.mkDerivation {
NIX_DAEMON_PACKAGE = daemon;
NIX_CLIENT_PACKAGE = client;
# Must keep this name short as OSX has a rather strict limit on the
# socket path length, and this name appears in the path of the
# nix-daemon socket used in the tests
name = "nix-tests";
name =
"nix-tests"
+ optionalString
(versionAtLeast daemon.version "2.4pre20211005" &&
versionAtLeast client.version "2.4pre20211005")
"-${client.version}-against-${daemon.version}";
inherit version;
src = self;
@ -443,6 +444,12 @@
inherit (self) overlay;
};
tests.nssPreload = (import ./tests/nss-preload.nix rec {
system = "x86_64-linux";
inherit nixpkgs;
inherit (self) overlay;
});
tests.githubFlakes = (import ./tests/github-flakes.nix rec {
system = "x86_64-linux";
inherit nixpkgs;
@ -490,7 +497,11 @@
let pkgs = nixpkgsFor.${system}; in
pkgs.runCommand "install-tests" {
againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix;
againstCurrentUnstable = testNixVersions pkgs pkgs.nix pkgs.nixUnstable;
againstCurrentUnstable =
# FIXME: temporarily disable this on macOS because of #3605.
if system == "x86_64-linux"
then testNixVersions pkgs pkgs.nix pkgs.nixUnstable
else null;
# Disabled because the latest stable version doesn't handle
# `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work
# againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable;

63
maintainers/upload-release.pl
View file

@ -12,6 +12,7 @@ use LWP::UserAgent;
use Net::Amazon::S3;
my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n";
my $nixRev = $ARGV[1] or die; # FIXME
my $releasesBucketName = "nix-releases";
my $channelsBucketName = "nix-channels";
@ -19,6 +20,8 @@ my $nixpkgsDir = "/home/eelco/Dev/nixpkgs-pristine";
my $TMPDIR = $ENV{'TMPDIR'} // "/tmp";
my $isLatest = ($ENV{'IS_LATEST'} // "") eq "1";
# FIXME: cut&paste from nixos-channel-scripts.
sub fetch {
my ($url, $type) = @_;
@ -33,14 +36,15 @@ sub fetch {
}
my $evalUrl = "https://hydra.nixos.org/eval/$evalId";
my $evalInfo = decode_json(fetch($evalUrl, 'application/json'));
#my $evalInfo = decode_json(fetch($evalUrl, 'application/json'));
#print Dumper($evalInfo);
my $nixRev = $evalInfo->{jobsetevalinputs}->{nix}->{revision} or die;
#my $nixRev = $evalInfo->{jobsetevalinputs}->{nix}->{revision} or die;
my $tarballInfo = decode_json(fetch("$evalUrl/job/tarball", 'application/json'));
my $buildInfo = decode_json(fetch("$evalUrl/job/build.x86_64-linux", 'application/json'));
#print Dumper($buildInfo);
my $releaseName = $tarballInfo->{releasename};
my $releaseName = $buildInfo->{nixname};
$releaseName =~ /nix-(.*)$/ or die;
my $version = $1;
@ -104,8 +108,6 @@ sub downloadFile {
return $sha256_expected;
}
downloadFile("tarball", "2"); # .tar.bz2
my $tarballHash = downloadFile("tarball", "3"); # .tar.xz
downloadFile("binaryTarball.i686-linux", "1");
downloadFile("binaryTarball.x86_64-linux", "1");
downloadFile("binaryTarball.aarch64-linux", "1");
@ -134,42 +136,43 @@ for my $fn (glob "$tmpDir/*") {
}
}
exit if $version =~ /pre/;
# Update nix-fallback-paths.nix.
system("cd $nixpkgsDir && git pull") == 0 or die;
if ($isLatest) {
system("cd $nixpkgsDir && git pull") == 0 or die;
sub getStorePath {
my ($jobName) = @_;
my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json'));
for my $product (values %{$buildInfo->{buildproducts}}) {
next unless $product->{type} eq "nix-build";
next if $product->{path} =~ /[a-z]+$/;
return $product->{path};
sub getStorePath {
my ($jobName) = @_;
my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json'));
for my $product (values %{$buildInfo->{buildproducts}}) {
next unless $product->{type} eq "nix-build";
next if $product->{path} =~ /[a-z]+$/;
return $product->{path};
}
die;
}
die;
write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix",
"{\n" .
" x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" .
" i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" .
" aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" .
" x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" .
" aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" .
"}\n");
system("cd $nixpkgsDir && git commit -a -m 'nix-fallback-paths.nix: Update to $version'") == 0 or die;
}
write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix",
"{\n" .
" x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" .
" i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" .
" aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" .
" x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" .
" aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" .
"}\n");
system("cd $nixpkgsDir && git commit -a -m 'nix-fallback-paths.nix: Update to $version'") == 0 or die;
# Update the "latest" symlink.
$channelsBucket->add_key(
"nix-latest/install", "",
{ "x-amz-website-redirect-location" => "https://releases.nixos.org/$releaseDir/install" })
or die $channelsBucket->err . ": " . $channelsBucket->errstr;
or die $channelsBucket->err . ": " . $channelsBucket->errstr
if $isLatest;
# Tag the release in Git.
chdir("/home/eelco/Dev/nix-pristine") or die;
system("git remote update origin") == 0 or die;
system("git tag --force --sign $version $nixRev -m 'Tagging release $version'") == 0 or die;
system("git push --tags") == 0 or die;
system("git push --force-with-lease origin $nixRev:refs/heads/latest-release") == 0 or die;
system("git push --force-with-lease origin $nixRev:refs/heads/latest-release") == 0 or die if $isLatest;

4
mk/libraries.mk
View file

@ -91,7 +91,7 @@ define build-library
$(1)_PATH := $$(_d)/$$($(1)_NAME).$(SO_EXT)
$$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/
$$(trace-ld) $(CXX) -o $$(abspath $$@) -shared $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$($(1)_LDFLAGS_UNINSTALLED) $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED)
$$(trace-ld) $(CXX) -o $$(abspath $$@) -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$($(1)_LDFLAGS_UNINSTALLED)
ifndef HOST_DARWIN
$(1)_LDFLAGS_USE += -Wl,-rpath,$$(abspath $$(_d))
@ -105,7 +105,7 @@ define build-library
$$(eval $$(call create-dir, $$($(1)_INSTALL_DIR)))
$$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/
$$(trace-ld) $(CXX) -o $$@ -shared $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED)
$$(trace-ld) $(CXX) -o $$@ -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED))
$(1)_LDFLAGS_USE_INSTALLED += -L$$(DESTDIR)$$($(1)_INSTALL_DIR) -l$$(patsubst lib%,%,$$(strip $$($(1)_NAME)))
ifndef HOST_DARWIN

4
mk/programs.mk
View file

@ -32,7 +32,7 @@ define build-program
$$(eval $$(call create-dir, $$(_d)))
$$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/
$$(trace-ld) $(CXX) -o $$@ $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS)
$$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE))
$(1)_INSTALL_DIR ?= $$(bindir)
@ -49,7 +49,7 @@ define build-program
_libs_final := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_INSTALL_PATH))
$(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/
$$(trace-ld) $(CXX) -o $$@ $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS)
$$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED))
else

4
mk/tests.mk
View file

@ -13,3 +13,7 @@ define run-install-test
endef
.PHONY: check installcheck
print-top-help += \
echo " check: Run unit tests"; \
echo " installcheck: Run functional tests";

25
src/libexpr/eval.cc
View file

@ -445,12 +445,12 @@ EvalState::EvalState(
StorePathSet closure;
store->computeFSClosure(store->toStorePath(r.second).first, closure);
for (auto & path : closure)
allowedPaths->insert(store->printStorePath(path));
allowPath(path);
} catch (InvalidPath &) {
allowedPaths->insert(r.second);
allowPath(r.second);
}
} else
allowedPaths->insert(r.second);
allowPath(r.second);
}
}
@ -482,6 +482,18 @@ void EvalState::requireExperimentalFeatureOnEvaluation(
}
}
void EvalState::allowPath(const Path & path)
{
if (allowedPaths)
allowedPaths->insert(path);
}
void EvalState::allowPath(const StorePath & storePath)
{
if (allowedPaths)
allowedPaths->insert(store->toRealPath(storePath));
}
Path EvalState::checkSourcePath(const Path & path_)
{
if (!allowedPaths) return path_;
@ -1316,13 +1328,13 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
auto size =
(lambda.arg.empty() ? 0 : 1) +
(lambda.matchAttrs ? lambda.formals->formals.size() : 0);
(lambda.hasFormals() ? lambda.formals->formals.size() : 0);
Env & env2(allocEnv(size));
env2.up = fun.lambda.env;
size_t displ = 0;
if (!lambda.matchAttrs)
if (!lambda.hasFormals())
env2.values[displ++] = &arg;
else {
@ -1402,7 +1414,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
}
}
if (!fun.isLambda() || !fun.lambda.fun->matchAttrs) {
if (!fun.isLambda() || !fun.lambda.fun->hasFormals()) {
res = fun;
return;
}
@ -1889,6 +1901,7 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path)
? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first
: store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair);
dstPath = store->printStorePath(p);
allowPath(p);
srcToStore.insert_or_assign(path, std::move(p));
printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath);
}

9
src/libexpr/eval.hh
View file

@ -150,6 +150,15 @@ public:
SearchPath getSearchPath() { return searchPath; }
/* Allow access to a path. */
void allowPath(const Path & path);
/* Allow access to a store path. Note that this gets remapped to
the real store path if `store` is a chroot store. */
void allowPath(const StorePath & storePath);
/* Check whether access to a path is allowed and throw an error if
not. Otherwise return the canonicalised path. */
Path checkSourcePath(const Path & path);
void checkURI(const std::string & uri);

5
src/libexpr/flake/flake.cc
View file

@ -64,8 +64,7 @@ static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree(
debug("got tree '%s' from '%s'",
state.store->printStorePath(tree.storePath), lockedRef);
if (state.allowedPaths)
state.allowedPaths->insert(tree.actualPath);
state.allowPath(tree.storePath);
assert(!originalRef.input.getNarHash() || tree.storePath == originalRef.input.computeStorePath(*state.store));
@ -231,7 +230,7 @@ static Flake getFlake(
if (auto outputs = vInfo.attrs->get(sOutputs)) {
expectType(state, nFunction, *outputs->value, *outputs->pos);
if (outputs->value->isLambda() && outputs->value->lambda.fun->matchAttrs) {
if (outputs->value->isLambda() && outputs->value->lambda.fun->hasFormals()) {
for (auto & formal : outputs->value->lambda.fun->formals->formals) {
if (formal.name != state.sSelf)
flake.inputs.emplace(formal.name, FlakeInput {

4
src/libexpr/nixexpr.cc
View file

@ -124,7 +124,7 @@ void ExprList::show(std::ostream & str) const
void ExprLambda::show(std::ostream & str) const
{
str << "(";
if (matchAttrs) {
if (hasFormals()) {
str << "{ ";
bool first = true;
for (auto & i : formals->formals) {
@ -348,7 +348,7 @@ void ExprLambda::bindVars(const StaticEnv & env)
if (!arg.empty()) newEnv.vars[arg] = displ++;
if (matchAttrs) {
if (hasFormals()) {
for (auto & i : formals->formals)
newEnv.vars[i.name] = displ++;

6
src/libexpr/nixexpr.hh
View file

@ -233,11 +233,10 @@ struct ExprLambda : Expr
Pos pos;
Symbol name;
Symbol arg;
bool matchAttrs;
Formals * formals;
Expr * body;
ExprLambda(const Pos & pos, const Symbol & arg, bool matchAttrs, Formals * formals, Expr * body)
: pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body)
ExprLambda(const Pos & pos, const Symbol & arg, Formals * formals, Expr * body)
: pos(pos), arg(arg), formals(formals), body(body)
{
if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end())
throw ParseError({
@ -247,6 +246,7 @@ struct ExprLambda : Expr
};
void setName(Symbol & name);
string showNamePos() const;
inline bool hasFormals() const { return formals != nullptr; }
COMMON_METHODS
};

8
src/libexpr/parser.y
View file

@ -324,13 +324,13 @@ expr: expr_function;
expr_function
: ID ':' expr_function
{ $$ = new ExprLambda(CUR_POS, data->symbols.create($1), false, 0, $3); }
{ $$ = new ExprLambda(CUR_POS, data->symbols.create($1), 0, $3); }
| '{' formals '}' ':' expr_function
{ $$ = new ExprLambda(CUR_POS, data->symbols.create(""), true, $2, $5); }
{ $$ = new ExprLambda(CUR_POS, data->symbols.create(""), $2, $5); }
| '{' formals '}' '@' ID ':' expr_function
{ $$ = new ExprLambda(CUR_POS, data->symbols.create($5), true, $2, $7); }
{ $$ = new ExprLambda(CUR_POS, data->symbols.create($5), $2, $7); }
| ID '@' '{' formals '}' ':' expr_function
{ $$ = new ExprLambda(CUR_POS, data->symbols.create($1), true, $4, $7); }
{ $$ = new ExprLambda(CUR_POS, data->symbols.create($1), $4, $7); }
| ASSERT expr ';' expr_function
{ $$ = new ExprAssert(CUR_POS, $2, $4); }
| WITH expr ';' expr_function

156
src/libexpr/primops.cc
View file

@ -56,13 +56,9 @@ void EvalState::realiseContext(const PathSet & context)
"cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled",
store->printStorePath(drvs.begin()->drvPath));
/* For performance, prefetch all substitute info. */
StorePathSet willBuild, willSubstitute, unknown;
uint64_t downloadSize, narSize;
/* Build/substitute the context. */
std::vector<DerivedPath> buildReqs;
for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d });
store->queryMissing(buildReqs, willBuild, willSubstitute, unknown, downloadSize, narSize);
store->buildPaths(buildReqs);
/* Add the output of this derivations to the allowed
@ -1851,56 +1847,87 @@ static RegisterPrimOp primop_toFile({
.fun = prim_toFile,
});
static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_,
Value * filterFun, FileIngestionMethod method, const std::optional<Hash> expectedHash, Value & v)
static void addPath(
EvalState & state,
const Pos & pos,
const string & name,
Path path,
Value * filterFun,
FileIngestionMethod method,
const std::optional<Hash> expectedHash,
Value & v,
const PathSet & context)
{
const auto path = evalSettings.pureEval && expectedHash ?
path_ :
state.checkSourcePath(path_);
PathFilter filter = filterFun ? ([&](const Path & path) {
auto st = lstat(path);
try {
// FIXME: handle CA derivation outputs (where path needs to
// be rewritten to the actual output).
state.realiseContext(context);
/* Call the filter function. The first argument is the path,
the second is a string indicating the type of the file. */
Value arg1;
mkString(arg1, path);
if (state.store->isInStore(path)) {
auto [storePath, subPath] = state.store->toStorePath(path);
auto info = state.store->queryPathInfo(storePath);
if (!info->references.empty())
throw EvalError("store path '%s' is not allowed to have references",
state.store->printStorePath(storePath));
path = state.store->toRealPath(storePath) + subPath;
}
Value fun2;
state.callFunction(*filterFun, arg1, fun2, noPos);
path = evalSettings.pureEval && expectedHash
? path
: state.checkSourcePath(path);
Value arg2;
mkString(arg2,
S_ISREG(st.st_mode) ? "regular" :
S_ISDIR(st.st_mode) ? "directory" :
S_ISLNK(st.st_mode) ? "symlink" :
"unknown" /* not supported, will fail! */);
PathFilter filter = filterFun ? ([&](const Path & path) {
auto st = lstat(path);
Value res;
state.callFunction(fun2, arg2, res, noPos);
/* Call the filter function. The first argument is the path,
the second is a string indicating the type of the file. */
Value arg1;
mkString(arg1, path);
return state.forceBool(res, pos);
}) : defaultPathFilter;
Value fun2;
state.callFunction(*filterFun, arg1, fun2, noPos);
std::optional<StorePath> expectedStorePath;
if (expectedHash)
expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo {
{
.method = method,
.hash = *expectedHash,
},
{},
});
Path dstPath;
if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
dstPath = state.store->printStorePath(settings.readOnlyMode
? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first
: state.store->addToStore(name, path, method, htSHA256, filter, state.repair));
if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath))
throw Error("store path mismatch in (possibly filtered) path added from '%s'", path);
} else
dstPath = state.store->printStorePath(*expectedStorePath);
Value arg2;
mkString(arg2,
S_ISREG(st.st_mode) ? "regular" :
S_ISDIR(st.st_mode) ? "directory" :
S_ISLNK(st.st_mode) ? "symlink" :
"unknown" /* not supported, will fail! */);
mkString(v, dstPath, {dstPath});
Value res;
state.callFunction(fun2, arg2, res, noPos);
return state.forceBool(res, pos);
}) : defaultPathFilter;
std::optional<StorePath> expectedStorePath;
if (expectedHash)
expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo {
{
.method = method,
.hash = *expectedHash,
},
{},
});
Path dstPath;
if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
dstPath = state.store->printStorePath(settings.readOnlyMode
? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first
: state.store->addToStore(name, path, method, htSHA256, filter, state.repair));
if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath))
throw Error("store path mismatch in (possibly filtered) path added from '%s'", path);
} else
dstPath = state.store->printStorePath(*expectedStorePath);
mkString(v, dstPath, {dstPath});
state.allowPath(dstPath);
} catch (Error & e) {
e.addTrace(pos, "while adding path '%s'", path);
throw;
}
}
@ -1908,11 +1935,6 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
{
PathSet context;
Path path = state.coerceToPath(pos, *args[1], context);
if (!context.empty())
throw EvalError({
.msg = hintfmt("string '%1%' cannot refer to other paths", path),
.errPos = pos
});
state.forceValue(*args[0], pos);
if (args[0]->type() != nFunction)
@ -1923,13 +1945,26 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
.errPos = pos
});
addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v);
addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v, context);
}
static RegisterPrimOp primop_filterSource({
.name = "__filterSource",
.args = {"e1", "e2"},
.doc = R"(
> **Warning**
>
> `filterSource` should not be used to filter store paths. Since
> `filterSource` uses the name of the input directory while naming
> the output directory, doing so will produce a directory name in
> the form of `<hash2>-<hash>-<name>`, where `<hash>-<name>` is
> the name of the input directory. Since `<hash>` depends on the
> unfiltered directory, the name of the output directory will
> indirectly depend on files that are filtered out by the
> function. This will trigger a rebuild even when a filtered out
> file is changed. Use `builtins.path` instead, which allows
> specifying the name of the output directory.
This function allows you to copy sources into the Nix store while
filtering certain files. For instance, suppose that you want to use
the directory `source-dir` as an input to a Nix expression, e.g.
@ -1976,18 +2011,13 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
Value * filterFun = nullptr;
auto method = FileIngestionMethod::Recursive;
std::optional<Hash> expectedHash;
PathSet context;
for (auto & attr : *args[0]->attrs) {
const string & n(attr.name);
if (n == "path") {
PathSet context;
if (n == "path")
path = state.coerceToPath(*attr.pos, *attr.value, context);
if (!context.empty())
throw EvalError({
.msg = hintfmt("string '%1%' cannot refer to other paths", path),
.errPos = *attr.pos
});
} else if (attr.name == state.sName)
else if (attr.name == state.sName)
name = state.forceStringNoCtx(*attr.value, *attr.pos);
else if (n == "filter") {
state.forceValue(*attr.value, pos);
@ -2010,7 +2040,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
if (name.empty())
name = baseNameOf(path);
addPath(state, pos, name, path, filterFun, method, expectedHash, v);
addPath(state, pos, name, path, filterFun, method, expectedHash, v, context);
}
static RegisterPrimOp primop_path({
@ -2375,7 +2405,7 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args
.errPos = pos
});
if (!args[0]->lambda.fun->matchAttrs) {
if (!args[0]->lambda.fun->hasFormals()) {
state.mkAttrs(v, 0);
return;
}

3
src/libexpr/primops/fetchMercurial.cc
View file

@ -84,8 +84,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
v.attrs->sort();
if (state.allowedPaths)
state.allowedPaths->insert(tree.actualPath);
state.allowPath(tree.storePath);
}
static RegisterPrimOp r_fetchMercurial("fetchMercurial", 1, prim_fetchMercurial);

72
src/libexpr/primops/fetchTree.cc
View file

@ -66,7 +66,7 @@ void emitTreeAttrs(
v.attrs->sort();
}
std::string fixURI(std::string uri, EvalState &state, const std::string & defaultScheme = "file")
std::string fixURI(std::string uri, EvalState & state, const std::string & defaultScheme = "file")
{
state.checkURI(uri);
return uri.find("://") != std::string::npos ? uri : defaultScheme + "://" + uri;
@ -81,23 +81,17 @@ std::string fixURIForGit(std::string uri, EvalState & state)
return fixURI(uri, state);
}
void addURI(EvalState &state, fetchers::Attrs &attrs, Symbol name, std::string v)
{
string n(name);
attrs.emplace(name, n == "url" ? fixURI(v, state) : v);
}
struct FetchTreeParams {
bool emptyRevFallback = false;
bool allowNameArgument = false;
};
static void fetchTree(
EvalState &state,
const Pos &pos,
Value **args,
Value &v,
const std::optional<std::string> type,
EvalState & state,
const Pos & pos,
Value * * args,
Value & v,
std::optional<std::string> type,
const FetchTreeParams & params = FetchTreeParams{}
) {
fetchers::Input input;
@ -110,17 +104,33 @@ static void fetchTree(
fetchers::Attrs attrs;
if (auto aType = args[0]->attrs->get(state.sType)) {
if (type)
throw Error({
.msg = hintfmt("unexpected attribute 'type'"),
.errPos = pos
});
type = state.forceStringNoCtx(*aType->value, *aType->pos);
} else if (!type)
throw Error({
.msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
.errPos = pos
});
attrs.emplace("type", type.value());
for (auto & attr : *args[0]->attrs) {
if (attr.name == state.sType) continue;
state.forceValue(*attr.value);
if (attr.value->type() == nPath || attr.value->type() == nString)
addURI(
state,
attrs,
attr.name,
state.coerceToString(*attr.pos, *attr.value, context, false, false)
);
else if (attr.value->type() == nString)
addURI(state, attrs, attr.name, attr.value->string.s);
if (attr.value->type() == nPath || attr.value->type() == nString) {
auto s = state.coerceToString(*attr.pos, *attr.value, context, false, false);
attrs.emplace(attr.name,
attr.name == "url"
? type == "git"
? fixURIForGit(s, state)
: fixURI(s, state)
: s);
}
else if (attr.value->type() == nBool)
attrs.emplace(attr.name, Explicit<bool>{attr.value->boolean});
else if (attr.value->type() == nInt)
@ -130,15 +140,6 @@ static void fetchTree(
attr.name, showType(*attr.value));
}
if (type)
attrs.emplace("type", type.value());
if (!attrs.count("type"))
throw Error({
.msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
.errPos = pos
});
if (!params.allowNameArgument)
if (auto nameIter = attrs.find("name"); nameIter != attrs.end())
throw Error({
@ -146,7 +147,6 @@ static void fetchTree(
.errPos = pos
});
input = fetchers::Input::fromAttrs(std::move(attrs));
} else {
auto url = state.coerceToString(pos, *args[0], context, false, false);
@ -169,8 +169,7 @@ static void fetchTree(
auto [tree, input2] = input.fetch(state.store);
if (state.allowedPaths)
state.allowedPaths->insert(tree.actualPath);
state.allowPath(tree.storePath);
emitTreeAttrs(state, tree, input2, v, params.emptyRevFallback, false);
}
@ -234,19 +233,16 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).first.storePath
: fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath;
auto realPath = state.store->toRealPath(storePath);
if (expectedHash) {
auto hash = unpack
? state.store->queryPathInfo(storePath)->narHash
: hashFile(htSHA256, realPath);
: hashFile(htSHA256, state.store->toRealPath(storePath));
if (hash != *expectedHash)
throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s",
*url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true));
}
if (state.allowedPaths)
state.allowedPaths->insert(realPath);
state.allowPath(storePath);
auto path = state.store->printStorePath(storePath);
mkString(v, path, PathSet({path}));

2
src/libexpr/value-to-xml.cc
View file

@ -135,7 +135,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
if (location) posToXML(xmlAttrs, v.lambda.fun->pos);
XMLOpenElement _(doc, "function", xmlAttrs);
if (v.lambda.fun->matchAttrs) {
if (v.lambda.fun->hasFormals()) {
XMLAttrs attrs;
if (!v.lambda.fun->arg.empty()) attrs["name"] = v.lambda.fun->arg;
if (v.lambda.fun->formals->ellipsis) attrs["ellipsis"] = "1";

14
src/libstore/build/local-derivation-goal.cc
View file

@ -711,6 +711,7 @@ void LocalDerivationGoal::startBuilder()
if (!builderOut.readSide)
throw SysError("opening pseudoterminal master");
// FIXME: not thread-safe, use ptsname_r
std::string slaveName(ptsname(builderOut.readSide.get()));
if (buildUser) {
@ -754,7 +755,6 @@ void LocalDerivationGoal::startBuilder()
result.startTime = time(0);
/* Fork a child to build the package. */
ProcessOptions options;
#if __linux__
if (useChroot) {
@ -797,8 +797,6 @@ void LocalDerivationGoal::startBuilder()
userNamespaceSync.create();
options.allowVfork = false;
Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces";
static bool userNamespacesEnabled =
pathExists(maxUserNamespaces)
@ -856,7 +854,7 @@ void LocalDerivationGoal::startBuilder()
writeFull(builderOut.writeSide.get(),
fmt("%d %d\n", usingUserNamespace, child));
_exit(0);
}, options);
});
int res = helper.wait();
if (res != 0 && settings.sandboxFallback) {
@ -920,10 +918,9 @@ void LocalDerivationGoal::startBuilder()
#endif
{
fallback:
options.allowVfork = !buildUser && !drv->isBuiltin();
pid = startProcess([&]() {
runChild();
}, options);
});
}
/* parent */
@ -2140,8 +2137,7 @@ void LocalDerivationGoal::registerOutputs()
/* Pass blank Sink as we are not ready to hash data at this stage. */
NullSink blank;
auto references = worker.store.parseStorePathSet(
scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths)));
auto references = scanForReferences(blank, actualPath, referenceablePaths);
outputReferencesIfUnregistered.insert_or_assign(
outputName,
@ -2208,7 +2204,7 @@ void LocalDerivationGoal::registerOutputs()
auto rewriteOutput = [&]() {
/* Apply hash rewriting if necessary. */
if (!outputRewrites.empty()) {
warn("rewriting hashes in '%1%'; cross fingers", actualPath);
debug("rewriting hashes in '%1%'; cross fingers", actualPath);
/* FIXME: this is in-memory. */
StringSink sink;

2
src/libstore/build/worker.cc
View file

@ -239,7 +239,7 @@ void Worker::run(const Goals & _topGoals)
}
}
/* Call queryMissing() efficiently query substitutes. */
/* Call queryMissing() to efficiently query substitutes. */
StorePathSet willBuild, willSubstitute, unknown;
uint64_t downloadSize, narSize;
store.queryMissing(topPaths, willBuild, willSubstitute, unknown, downloadSize, narSize);

89
src/libstore/references.cc
View file

@ -11,11 +11,13 @@
namespace nix {
static unsigned int refLength = 32; /* characters */
static size_t refLength = 32; /* characters */
static void search(const unsigned char * s, size_t len,
StringSet & hashes, StringSet & seen)
static void search(
std::string_view s,
StringSet & hashes,
StringSet & seen)
{
static std::once_flag initialised;
static bool isBase32[256];
@ -25,7 +27,7 @@ static void search(const unsigned char * s, size_t len,
isBase32[(unsigned char) base32Chars[i]] = true;
});
for (size_t i = 0; i + refLength <= len; ) {
for (size_t i = 0; i + refLength <= s.size(); ) {
int j;
bool match = true;
for (j = refLength - 1; j >= 0; --j)
@ -35,7 +37,7 @@ static void search(const unsigned char * s, size_t len,
break;
}
if (!match) continue;
string ref((const char *) s + i, refLength);
std::string ref(s.substr(i, refLength));
if (hashes.erase(ref)) {
debug(format("found reference to '%1%' at offset '%2%'")
% ref % i);
@ -46,69 +48,60 @@ static void search(const unsigned char * s, size_t len,
}
struct RefScanSink : Sink
void RefScanSink::operator () (std::string_view data)
{
StringSet hashes;
StringSet seen;
/* It's possible that a reference spans the previous and current
fragment, so search in the concatenation of the tail of the
previous fragment and the start of the current fragment. */
auto s = tail;
s.append(data.data(), refLength);
search(s, hashes, seen);
string tail;
search(data, hashes, seen);
RefScanSink() { }
void operator () (std::string_view data) override
{
/* It's possible that a reference spans the previous and current
fragment, so search in the concatenation of the tail of the
previous fragment and the start of the current fragment. */
string s = tail + std::string(data, 0, refLength);
search((const unsigned char *) s.data(), s.size(), hashes, seen);
search((const unsigned char *) data.data(), data.size(), hashes, seen);
size_t tailLen = data.size() <= refLength ? data.size() : refLength;
tail = std::string(tail, tail.size() < refLength - tailLen ? 0 : tail.size() - (refLength - tailLen));
tail.append({data.data() + data.size() - tailLen, tailLen});
}
};
auto tailLen = std::min(data.size(), refLength);
auto rest = refLength - tailLen;
if (rest < tail.size())
tail = tail.substr(tail.size() - rest);
tail.append(data.data() + data.size() - tailLen, tailLen);
}
std::pair<PathSet, HashResult> scanForReferences(const string & path,
const PathSet & refs)
std::pair<StorePathSet, HashResult> scanForReferences(
const string & path,
const StorePathSet & refs)
{
HashSink hashSink { htSHA256 };
auto found = scanForReferences(hashSink, path, refs);
auto hash = hashSink.finish();
return std::pair<PathSet, HashResult>(found, hash);
return std::pair<StorePathSet, HashResult>(found, hash);
}
PathSet scanForReferences(Sink & toTee,
const string & path, const PathSet & refs)
StorePathSet scanForReferences(
Sink & toTee,
const Path & path,
const StorePathSet & refs)
{
RefScanSink refsSink;
TeeSink sink { refsSink, toTee };
std::map<string, Path> backMap;
StringSet hashes;
std::map<std::string, StorePath> backMap;
for (auto & i : refs) {
auto baseName = std::string(baseNameOf(i));
string::size_type pos = baseName.find('-');
if (pos == string::npos)
throw Error("bad reference '%1%'", i);
string s = string(baseName, 0, pos);
assert(s.size() == refLength);
assert(backMap.find(s) == backMap.end());
// parseHash(htSHA256, s);
refsSink.hashes.insert(s);
backMap[s] = i;
std::string hashPart(i.hashPart());
auto inserted = backMap.emplace(hashPart, i).second;
assert(inserted);
hashes.insert(hashPart);
}
/* Look for the hashes in the NAR dump of the path. */
RefScanSink refsSink(std::move(hashes));
TeeSink sink { refsSink, toTee };
dumpPath(path, sink);
/* Map the hashes found back to their store paths. */
PathSet found;
for (auto & i : refsSink.seen) {
std::map<string, Path>::iterator j;
if ((j = backMap.find(i)) == backMap.end()) abort();
StorePathSet found;
for (auto & i : refsSink.getResult()) {
auto j = backMap.find(i);
assert(j != backMap.end());
found.insert(j->second);
}

24
src/libstore/references.hh
View file

@ -1,13 +1,31 @@
#pragma once
#include "types.hh"
#include "hash.hh"
#include "path.hh"
namespace nix {
std::pair<PathSet, HashResult> scanForReferences(const Path & path, const PathSet & refs);
std::pair<StorePathSet, HashResult> scanForReferences(const Path & path, const StorePathSet & refs);
PathSet scanForReferences(Sink & toTee, const Path & path, const PathSet & refs);
StorePathSet scanForReferences(Sink & toTee, const Path & path, const StorePathSet & refs);
class RefScanSink : public Sink
{
StringSet hashes;
StringSet seen;
std::string tail;
public:
RefScanSink(StringSet && hashes) : hashes(hashes)
{ }
StringSet & getResult()
{ return seen; }
void operator () (std::string_view data) override;
};
struct RewritingSink : Sink
{

15
src/libstore/tests/local.mk Normal file
View file

@ -0,0 +1,15 @@
check: libstore-tests_RUN
programs += libstore-tests
libstore-tests_DIR := $(d)
libstore-tests_INSTALL_DIR :=
libstore-tests_SOURCES := $(wildcard $(d)/*.cc)
libstore-tests_CXXFLAGS += -I src/libstore -I src/libutil
libstore-tests_LIBS = libstore libutil
libstore-tests_LDFLAGS := $(GTEST_LIBS)

45
src/libstore/tests/references.cc Normal file
View file

@ -0,0 +1,45 @@
#include "references.hh"
#include <gtest/gtest.h>
namespace nix {
TEST(references, scan)
{
std::string hash1 = "dc04vv14dak1c1r48qa0m23vr9jy8sm0";
std::string hash2 = "zc842j0rz61mjsp3h3wp5ly71ak6qgdn";
{
RefScanSink scanner(StringSet{hash1});
auto s = "foobar";
scanner(s);
ASSERT_EQ(scanner.getResult(), StringSet{});
}
{
RefScanSink scanner(StringSet{hash1});
auto s = "foobar" + hash1 + "xyzzy";
scanner(s);
ASSERT_EQ(scanner.getResult(), StringSet{hash1});
}
{
RefScanSink scanner(StringSet{hash1, hash2});
auto s = "foobar" + hash1 + "xyzzy" + hash2;
scanner(((std::string_view) s).substr(0, 10));
scanner(((std::string_view) s).substr(10, 5));
scanner(((std::string_view) s).substr(15, 5));
scanner(((std::string_view) s).substr(20));
ASSERT_EQ(scanner.getResult(), StringSet({hash1, hash2}));
}
{
RefScanSink scanner(StringSet{hash1, hash2});
auto s = "foobar" + hash1 + "xyzzy" + hash2;
for (auto & i : s)
scanner(std::string(1, i));
ASSERT_EQ(scanner.getResult(), StringSet({hash1, hash2}));
}
}
}

11
src/libstore/uds-remote-store.cc
View file

@ -65,16 +65,7 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
throw SysError("cannot create Unix domain socket");
closeOnExec(conn->fd.get());
string socketPath = path ? *path : settings.nixDaemonSocketFile;
struct sockaddr_un addr;
addr.sun_family = AF_UNIX;
if (socketPath.size() + 1 >= sizeof(addr.sun_path))
throw Error("socket path '%1%' is too long", socketPath);
strcpy(addr.sun_path, socketPath.c_str());
if (::connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1)
throw SysError("cannot connect to daemon at '%1%'", socketPath);
nix::connect(conn->fd.get(), path ? *path : settings.nixDaemonSocketFile);
conn->from.fd = conn->fd.get();
conn->to.fd = conn->fd.get();

4
src/libutil/archive.cc
View file

@ -42,7 +42,7 @@ static string caseHackSuffix = "~nix~case~hack~";
PathFilter defaultPathFilter = [](const Path &) { return true; };
static void dumpContents(const Path & path, size_t size,
static void dumpContents(const Path & path, off_t size,
Sink & sink)
{
sink << "contents" << size;
@ -76,7 +76,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
sink << "type" << "regular";
if (st.st_mode & S_IXUSR)
sink << "executable" << "";
dumpContents(path, (size_t) st.st_size, sink);
dumpContents(path, st.st_size, sink);
}
else if (S_ISDIR(st.st_mode)) {

81
src/libutil/util.cc
View file

@ -903,7 +903,7 @@ int Pid::wait()
return status;
}
if (errno != EINTR)
throw SysError("cannot get child exit status");
throw SysError("cannot get exit status of PID %d", pid);
checkInterrupt();
}
}
@ -939,9 +939,6 @@ void killUser(uid_t uid)
users to which the current process can send signals. So we
fork a process, switch to uid, and send a mass kill. */
ProcessOptions options;
options.allowVfork = false;
Pid pid = startProcess([&]() {
if (setuid(uid) == -1)
@ -964,7 +961,7 @@ void killUser(uid_t uid)
}
_exit(0);
}, options);
});
int status = pid.wait();
if (status != 0)
@ -1085,8 +1082,7 @@ void runProgram2(const RunOptions & options)
// vfork implies that the environment of the main process and the fork will
// be shared (technically this is undefined, but in practice that's the
// case), so we can't use it if we alter the environment
if (options.environment)
processOptions.allowVfork = false;
processOptions.allowVfork = !options.environment;
/* Fork. */
Pid pid = startProcess([&]() {
@ -1686,16 +1682,7 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode)
closeOnExec(fdSocket.get());
struct sockaddr_un addr;
addr.sun_family = AF_UNIX;
if (path.size() + 1 >= sizeof(addr.sun_path))
throw Error("socket path '%1%' is too long", path);
strcpy(addr.sun_path, path.c_str());
unlink(path.c_str());
if (bind(fdSocket.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1)
throw SysError("cannot bind to socket '%1%'", path);
bind(fdSocket.get(), path);
if (chmod(path.c_str(), mode) == -1)
throw SysError("changing permissions on '%1%'", path);
@ -1707,6 +1694,66 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode)
}
void bind(int fd, const std::string & path)
{
unlink(path.c_str());
struct sockaddr_un addr;
addr.sun_family = AF_UNIX;
if (path.size() + 1 >= sizeof(addr.sun_path)) {
Pid pid = startProcess([&]() {
auto dir = dirOf(path);
if (chdir(dir.c_str()) == -1)
throw SysError("chdir to '%s' failed", dir);
std::string base(baseNameOf(path));
if (base.size() + 1 >= sizeof(addr.sun_path))
throw Error("socket path '%s' is too long", base);
strcpy(addr.sun_path, base.c_str());
if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
throw SysError("cannot bind to socket '%s'", path);
_exit(0);
});
int status = pid.wait();
if (status != 0)
throw Error("cannot bind to socket '%s'", path);
} else {
strcpy(addr.sun_path, path.c_str());
if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
throw SysError("cannot bind to socket '%s'", path);
}
}
void connect(int fd, const std::string & path)
{
struct sockaddr_un addr;
addr.sun_family = AF_UNIX;
if (path.size() + 1 >= sizeof(addr.sun_path)) {
Pid pid = startProcess([&]() {
auto dir = dirOf(path);
if (chdir(dir.c_str()) == -1)
throw SysError("chdir to '%s' failed", dir);
std::string base(baseNameOf(path));
if (base.size() + 1 >= sizeof(addr.sun_path))
throw Error("socket path '%s' is too long", base);
strcpy(addr.sun_path, base.c_str());
if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
throw SysError("cannot connect to socket at '%s'", path);
_exit(0);
});
int status = pid.wait();
if (status != 0)
throw Error("cannot connect to socket at '%s'", path);
} else {
strcpy(addr.sun_path, path.c_str());
if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
throw SysError("cannot connect to socket at '%s'", path);
}
}
string showBytes(uint64_t bytes)
{
return fmt("%.2f MiB", bytes / (1024.0 * 1024.0));

10
src/libutil/util.hh
View file

@ -259,10 +259,10 @@ void killUser(uid_t uid);
pid to the caller. */
struct ProcessOptions
{
string errorPrefix = "error: ";
string errorPrefix = "";
bool dieWithParent = true;
bool runExitHandlers = false;
bool allowVfork = true;
bool allowVfork = false;
};
pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = ProcessOptions());
@ -574,6 +574,12 @@ void commonChildInit(Pipe & logPipe);
/* Create a Unix domain socket in listen mode. */
AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode);
/* Bind a Unix domain socket to a path. */
void bind(int fd, const std::string & path);
/* Connect to a Unix domain socket. */
void connect(int fd, const std::string & path);
// A Rust/Python-like enumerate() iterator adapter.
// Borrowed from http://reedbeta.com/blog/python-like-enumerate-in-cpp17.

6
src/nix/daemon.cc
View file

@ -156,9 +156,6 @@ static void daemonLoop()
if (chdir("/") == -1)
throw SysError("cannot change current directory");
// Get rid of children automatically; don't let them become zombies.
setSigChldAction(true);
AutoCloseFD fdSocket;
// Handle socket-based activation by systemd.
@ -176,6 +173,9 @@ static void daemonLoop()
fdSocket = createUnixDomainSocket(settings.nixDaemonSocketFile, 0666);
}
// Get rid of children automatically; don't let them become zombies.
setSigChldAction(true);
// Loop accepting connections.
while (1) {

6
src/nix/flake.cc
View file

@ -346,10 +346,10 @@ struct CmdFlakeCheck : FlakeCommand
auto checkOverlay = [&](const std::string & attrPath, Value & v, const Pos & pos) {
try {
state->forceValue(v, pos);
if (!v.isLambda() || v.lambda.fun->matchAttrs || std::string(v.lambda.fun->arg) != "final")
if (!v.isLambda() || v.lambda.fun->hasFormals() || std::string(v.lambda.fun->arg) != "final")
throw Error("overlay does not take an argument named 'final'");
auto body = dynamic_cast<ExprLambda *>(v.lambda.fun->body);
if (!body || body->matchAttrs || std::string(body->arg) != "prev")
if (!body || body->hasFormals() || std::string(body->arg) != "prev")
throw Error("overlay does not take an argument named 'prev'");
// FIXME: if we have a 'nixpkgs' input, use it to
// evaluate the overlay.
@ -363,7 +363,7 @@ struct CmdFlakeCheck : FlakeCommand
try {
state->forceValue(v, pos);
if (v.isLambda()) {
if (!v.lambda.fun->matchAttrs || !v.lambda.fun->formals->ellipsis)
if (!v.lambda.fun->hasFormals() || !v.lambda.fun->formals->ellipsis)
throw Error("module must match an open attribute set ('{ config, ... }')");
} else if (v.type() == nAttrs) {
for (auto & attr : *v.attrs)

2
src/nix/flake.md
View file

@ -225,7 +225,7 @@ Currently the `type` attribute can be one of the following:
[flake:]<flake-id>(/<rev-or-ref>(/rev)?)?
```
These perform a lookup of `<flake-id>` in the flake registry. or
These perform a lookup of `<flake-id>` in the flake registry. For
example, `nixpkgs` and `nixpkgs/release-20.09` are indirect flake
references. The specified `rev` and/or `ref` are merged with the
entry in the registry; see [nix registry](./nix3-registry.md) for

7
src/nix/main.cc
View file

@ -187,11 +187,14 @@ static void showHelp(std::vector<std::string> subcommand, MultiCommand & topleve
, "/"),
*vUtils);
auto vJson = state.allocValue();
auto vArgs = state.allocValue();
state.mkAttrs(*vArgs, 16);
auto vJson = state.allocAttr(*vArgs, state.symbols.create("command"));
mkString(*vJson, toplevel.toJSON().dump());
vArgs->attrs->sort();
auto vRes = state.allocValue();
state.callFunction(*vGenerateManpage, *vJson, *vRes, noPos);
state.callFunction(*vGenerateManpage, *vArgs, *vRes, noPos);
auto attr = vRes->attrs->get(state.symbols.create(mdName + ".md"));
if (!attr)

6
src/nix/profile.cc
View file

@ -98,10 +98,8 @@ struct ProfileManifest
else if (pathExists(profile + "/manifest.nix")) {
// FIXME: needed because of pure mode; ugly.
if (state.allowedPaths) {
state.allowedPaths->insert(state.store->followLinksToStore(profile));
state.allowedPaths->insert(state.store->followLinksToStore(profile + "/manifest.nix"));
}
state.allowPath(state.store->followLinksToStore(profile));
state.allowPath(state.store->followLinksToStore(profile + "/manifest.nix"));
auto drvInfos = queryInstalled(state, state.store->followLinksToStore(profile));

56
tests/flakes.sh
View file

@ -5,11 +5,6 @@ if [[ -z $(type -p git) ]]; then
exit 99
fi
if [[ -z $(type -p hg) ]]; then
echo "Mercurial not installed; skipping flake tests"
exit 99
fi
clearStore
rm -rf $TEST_HOME/.cache $TEST_HOME/.config
@ -266,6 +261,8 @@ cat > $flake3Dir/flake.nix <<EOF
mkDerivation {
inherit system;
name = "fnord";
dummy = builtins.readFile (builtins.path { name = "source"; path = ./.; filter = path: type: baseNameOf path == "config.nix"; } + "/config.nix");
dummy2 = builtins.readFile (builtins.path { name = "source"; path = inputs.flake1; filter = path: type: baseNameOf path == "simple.nix"; } + "/simple.nix");
buildCommand = ''
cat \${inputs.nonFlake}/README.md > \$out
'';
@ -579,45 +576,52 @@ nix build -o $TEST_ROOT/result git+file://$flakeGitBare
# Test Mercurial flakes.
rm -rf $flake5Dir
hg init $flake5Dir
mkdir $flake5Dir
cat > $flake5Dir/flake.nix <<EOF
{
outputs = { self, flake1 }: {
defaultPackage.$system = flake1.defaultPackage.$system;
expr = assert builtins.pathExists ./flake.lock; 123;
};
}
EOF
hg add $flake5Dir/flake.nix
hg commit --config ui.username=foobar@example.org $flake5Dir -m 'Initial commit'
if [[ -n $(type -p hg) ]]; then
hg init $flake5Dir
nix build -o $TEST_ROOT/result hg+file://$flake5Dir
[[ -e $TEST_ROOT/result/hello ]]
hg add $flake5Dir/flake.nix
hg commit --config ui.username=foobar@example.org $flake5Dir -m 'Initial commit'
(! nix flake metadata --json hg+file://$flake5Dir | jq -e -r .revision)
nix build -o $TEST_ROOT/result hg+file://$flake5Dir
[[ -e $TEST_ROOT/result/hello ]]
nix eval hg+file://$flake5Dir#expr
(! nix flake metadata --json hg+file://$flake5Dir | jq -e -r .revision)
nix eval hg+file://$flake5Dir#expr
nix eval hg+file://$flake5Dir#expr
(! nix eval hg+file://$flake5Dir#expr --no-allow-dirty)
nix eval hg+file://$flake5Dir#expr
(! nix flake metadata --json hg+file://$flake5Dir | jq -e -r .revision)
(! nix eval hg+file://$flake5Dir#expr --no-allow-dirty)
hg commit --config ui.username=foobar@example.org $flake5Dir -m 'Add lock file'
(! nix flake metadata --json hg+file://$flake5Dir | jq -e -r .revision)
nix flake metadata --json hg+file://$flake5Dir --refresh | jq -e -r .revision
nix flake metadata --json hg+file://$flake5Dir
[[ $(nix flake metadata --json hg+file://$flake5Dir | jq -e -r .revCount) = 1 ]]
hg commit --config ui.username=foobar@example.org $flake5Dir -m 'Add lock file'
nix build -o $TEST_ROOT/result hg+file://$flake5Dir --no-registries --no-allow-dirty
nix build -o $TEST_ROOT/result hg+file://$flake5Dir --no-use-registries --no-allow-dirty
nix flake metadata --json hg+file://$flake5Dir --refresh | jq -e -r .revision
nix flake metadata --json hg+file://$flake5Dir
[[ $(nix flake metadata --json hg+file://$flake5Dir | jq -e -r .revCount) = 1 ]]
# Test tarball flakes
tar cfz $TEST_ROOT/flake.tar.gz -C $TEST_ROOT --exclude .hg flake5
nix build -o $TEST_ROOT/result hg+file://$flake5Dir --no-registries --no-allow-dirty
nix build -o $TEST_ROOT/result hg+file://$flake5Dir --no-use-registries --no-allow-dirty
fi
# Test path flakes.
rm -rf $flake5Dir/.hg $flake5Dir/flake.lock
nix flake lock path://$flake5Dir
# Test tarball flakes.
tar cfz $TEST_ROOT/flake.tar.gz -C $TEST_ROOT flake5
nix build -o $TEST_ROOT/result file://$TEST_ROOT/flake.tar.gz
@ -632,8 +636,8 @@ nix build -o $TEST_ROOT/result "file://$TEST_ROOT/flake.tar.gz?narHash=sha256-qQ
# Test --override-input.
git -C $flake3Dir reset --hard
nix flake lock $flake3Dir --override-input flake2/flake1 flake5 -vvvvv
[[ $(jq .nodes.flake1_2.locked.url $flake3Dir/flake.lock) =~ flake5 ]]
nix flake lock $flake3Dir --override-input flake2/flake1 file://$TEST_ROOT/flake.tar.gz -vvvvv
[[ $(jq .nodes.flake1_2.locked.url $flake3Dir/flake.lock) =~ flake.tar.gz ]]
nix flake lock $flake3Dir --override-input flake2/flake1 flake1
[[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) =~ $hash2 ]]

123
tests/nss-preload.nix Normal file
View file

@ -0,0 +1,123 @@
{ nixpkgs, system, overlay }:
with import (nixpkgs + "/nixos/lib/testing-python.nix") {
inherit system;
extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ];
};
makeTest (
rec {
name = "nss-preload";
nodes = {
http_dns = { lib, pkgs, config, ... }: {
networking.firewall.enable = false;
networking.interfaces.eth1.ipv6.addresses = lib.mkForce [
{ address = "fd21::1"; prefixLength = 64; }
];
networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
{ address = "192.168.0.1"; prefixLength = 24; }
];
services.unbound = {
enable = true;
enableRootTrustAnchor = false;
settings = {
server = {
interface = [ "192.168.0.1" "fd21::1" "::1" "127.0.0.1" ];
access-control = [ "192.168.0.0/24 allow" "fd21::/64 allow" "::1 allow" "127.0.0.0/8 allow" ];
local-data = [
''"example.com. IN A 192.168.0.1"''
''"example.com. IN AAAA fd21::1"''
''"tarballs.nixos.org. IN A 192.168.0.1"''
''"tarballs.nixos.org. IN AAAA fd21::1"''
];
};
};
};
services.nginx = {
enable = true;
virtualHosts."example.com" = {
root = pkgs.runCommand "testdir" {} ''
mkdir "$out"
echo hello world > "$out/index.html"
'';
};
};
};
# client consumes a remote resolver
client = { lib, nodes, pkgs, ... }: {
networking.useDHCP = false;
networking.nameservers = [
(lib.head nodes.http_dns.config.networking.interfaces.eth1.ipv6.addresses).address
(lib.head nodes.http_dns.config.networking.interfaces.eth1.ipv4.addresses).address
];
networking.interfaces.eth1.ipv6.addresses = [
{ address = "fd21::10"; prefixLength = 64; }
];
networking.interfaces.eth1.ipv4.addresses = [
{ address = "192.168.0.10"; prefixLength = 24; }
];
nix.sandboxPaths = lib.mkForce [];
nix.binaryCaches = lib.mkForce [];
nix.useSandbox = lib.mkForce true;
};
};
nix-fetch = pkgs.writeText "fetch.nix" ''
derivation {
# This derivation is an copy from what is available over at
# nix.git:corepkgs/fetchurl.nix
builder = "builtin:fetchurl";
# We're going to fetch data from the http_dns instance created before
# we expect the content to be the same as the content available there.
# ```
# $ nix-hash --type sha256 --to-base32 $(echo "hello world" | sha256sum | cut -d " " -f 1)
# 0ix4jahrkll5zg01wandq78jw3ab30q4nscph67rniqg5x7r0j59
# ```
outputHash = "0ix4jahrkll5zg01wandq78jw3ab30q4nscph67rniqg5x7r0j59";
outputHashAlgo = "sha256";
outputHashMode = "flat";
name = "example.com";
url = "http://example.com";
unpack = false;
executable = false;
system = "builtin";
preferLocalBuild = true;
impureEnvVars = [
"http_proxy" "https_proxy" "ftp_proxy" "all_proxy" "no_proxy"
];
urls = [ "http://example.com" ];
}
'';
testScript = { nodes, ... }: ''
http_dns.wait_for_unit("nginx")
http_dns.wait_for_open_port(80)
http_dns.wait_for_unit("unbound")
http_dns.wait_for_open_port(53)
client.start()
client.wait_for_unit('multi-user.target')
with subtest("can fetch data from a remote server outside sandbox"):
client.succeed("nix --version >&2")
client.succeed("curl -vvv http://example.com/index.html >&2")
with subtest("nix-build can lookup dns and fetch data"):
client.succeed("""
nix-build ${nix-fetch} >&2
""")
'';
})