lix-project/lix
20
55
Fork 31
Code Issues 347 Code Review (Gerrit) Projects 3 Releases Wiki Activity

Merge remote-tracking branch 'upstream/master' into trustless-remote-builder-simple

Browse source
This commit is contained in:
John Ericson 2020-12-20 14:02:12 +00:00
parent bdc7720227 ec3e202832
commit bd96403da6
193 changed files with 7026 additions and 5613 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/test.yml vendored
View file

@ -12,6 +12,6 @@ jobs:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- uses: cachix/install-nix-action@v11
- uses: cachix/install-nix-action@v12
#- run: nix flake check
- run: nix-build -A checks.$(if [[ `uname` = Linux ]]; then echo x86_64-linux; else echo x86_64-darwin; fi)

4
.gitignore vendored
View file

@ -18,13 +18,13 @@ perl/Makefile.config
/doc/manual/nix.json
/doc/manual/conf-file.json
/doc/manual/builtins.json
/doc/manual/src/command-ref/nix.md
/doc/manual/src/SUMMARY.md
/doc/manual/src/command-ref/new-cli
/doc/manual/src/command-ref/conf-file.md
/doc/manual/src/expressions/builtins.md
# /scripts/
/scripts/nix-profile.sh
/scripts/nix-copy-closure
/scripts/nix-reduce-build
/scripts/nix-http-export.cgi
/scripts/nix-profile-daemon.sh

2
.version
View file

@ -1 +1 @@
3.0
2.4

500
config/config.guess vendored
View file

@ -1,8 +1,8 @@
#! /bin/sh
# Attempt to guess a canonical system name.
# Copyright 1992-2018 Free Software Foundation, Inc.
# Copyright 1992-2020 Free Software Foundation, Inc.
timestamp='2018-08-02'
timestamp='2020-11-19'
# This file is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
@ -27,12 +27,12 @@ timestamp='2018-08-02'
# Originally written by Per Bothner; maintained since 2000 by Ben Elliston.
#
# You can get the latest version of this script from:
# https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess
# https://git.savannah.gnu.org/cgit/config.git/plain/config.guess
#
# Please send patches to <config-patches@gnu.org>.
me=`echo "$0" | sed -e 's,.*/,,'`
me=$(echo "$0" | sed -e 's,.*/,,')
usage="\
Usage: $0 [OPTION]
@ -50,7 +50,7 @@ version="\
GNU config.guess ($timestamp)
Originally written by Per Bothner.
Copyright 1992-2018 Free Software Foundation, Inc.
Copyright 1992-2020 Free Software Foundation, Inc.
This is free software; see the source for copying conditions. There is NO
warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE."
@ -96,13 +96,14 @@ fi
tmp=
# shellcheck disable=SC2172
trap 'test -z "$tmp" || rm -fr "$tmp"' 1 2 13 15
trap 'exitcode=$?; test -z "$tmp" || rm -fr "$tmp"; exit $exitcode' 0
trap 'test -z "$tmp" || rm -fr "$tmp"' 0 1 2 13 15
set_cc_for_build() {
# prevent multiple calls if $tmp is already set
test "$tmp" && return 0
: "${TMPDIR=/tmp}"
# shellcheck disable=SC2039
{ tmp=`(umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null` && test -n "$tmp" && test -d "$tmp" ; } ||
{ tmp=$( (umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null) && test -n "$tmp" && test -d "$tmp" ; } ||
{ test -n "$RANDOM" && tmp=$TMPDIR/cg$$-$RANDOM && (umask 077 && mkdir "$tmp" 2>/dev/null) ; } ||
{ tmp=$TMPDIR/cg-$$ && (umask 077 && mkdir "$tmp" 2>/dev/null) && echo "Warning: creating insecure temp directory" >&2 ; } ||
{ echo "$me: cannot create a temporary directory in $TMPDIR" >&2 ; exit 1 ; }
@ -130,16 +131,14 @@ if test -f /.attbin/uname ; then
PATH=$PATH:/.attbin ; export PATH
fi
UNAME_MACHINE=`(uname -m) 2>/dev/null` || UNAME_MACHINE=unknown
UNAME_RELEASE=`(uname -r) 2>/dev/null` || UNAME_RELEASE=unknown
UNAME_SYSTEM=`(uname -s) 2>/dev/null` || UNAME_SYSTEM=unknown
UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown
UNAME_MACHINE=$( (uname -m) 2>/dev/null) || UNAME_MACHINE=unknown
UNAME_RELEASE=$( (uname -r) 2>/dev/null) || UNAME_RELEASE=unknown
UNAME_SYSTEM=$( (uname -s) 2>/dev/null) || UNAME_SYSTEM=unknown
UNAME_VERSION=$( (uname -v) 2>/dev/null) || UNAME_VERSION=unknown
case "$UNAME_SYSTEM" in
Linux|GNU|GNU/*)
# If the system lacks a compiler, then just pick glibc.
# We could probably try harder.
LIBC=gnu
LIBC=unknown
set_cc_for_build
cat <<-EOF > "$dummy.c"
@ -148,18 +147,30 @@ Linux|GNU|GNU/*)
LIBC=uclibc
#elif defined(__dietlibc__)
LIBC=dietlibc
#else
#elif defined(__GLIBC__)
LIBC=gnu
#else
#include <stdarg.h>
/* First heuristic to detect musl libc. */
#ifdef __DEFINED_va_list
LIBC=musl
#endif
#endif
EOF
eval "`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^LIBC' | sed 's, ,,g'`"
eval "$($CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^LIBC' | sed 's, ,,g')"
# If ldd exists, use it to detect musl libc.
if command -v ldd >/dev/null && \
ldd --version 2>&1 | grep -q ^musl
then
# Second heuristic to detect musl libc.
if [ "$LIBC" = unknown ] &&
command -v ldd >/dev/null &&
ldd --version 2>&1 | grep -q ^musl; then
LIBC=musl
fi
# If the system lacks a compiler, then just pick glibc.
# We could probably try harder.
if [ "$LIBC" = unknown ]; then
LIBC=gnu
fi
;;
esac
@ -178,19 +189,20 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in
# Note: NetBSD doesn't particularly care about the vendor
# portion of the name. We always set it to "unknown".
sysctl="sysctl -n hw.machine_arch"
UNAME_MACHINE_ARCH=`(uname -p 2>/dev/null || \
UNAME_MACHINE_ARCH=$( (uname -p 2>/dev/null || \
"/sbin/$sysctl" 2>/dev/null || \
"/usr/sbin/$sysctl" 2>/dev/null || \
echo unknown)`
echo unknown))
case "$UNAME_MACHINE_ARCH" in
aarch64eb) machine=aarch64_be-unknown ;;
armeb) machine=armeb-unknown ;;
arm*) machine=arm-unknown ;;
sh3el) machine=shl-unknown ;;
sh3eb) machine=sh-unknown ;;
sh5el) machine=sh5le-unknown ;;
earmv*)
arch=`echo "$UNAME_MACHINE_ARCH" | sed -e 's,^e\(armv[0-9]\).*$,\1,'`
endian=`echo "$UNAME_MACHINE_ARCH" | sed -ne 's,^.*\(eb\)$,\1,p'`
arch=$(echo "$UNAME_MACHINE_ARCH" | sed -e 's,^e\(armv[0-9]\).*$,\1,')
endian=$(echo "$UNAME_MACHINE_ARCH" | sed -ne 's,^.*\(eb\)$,\1,p')
machine="${arch}${endian}"-unknown
;;
*) machine="$UNAME_MACHINE_ARCH"-unknown ;;
@ -221,7 +233,7 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in
case "$UNAME_MACHINE_ARCH" in
earm*)
expr='s/^earmv[0-9]/-eabi/;s/eb$//'
abi=`echo "$UNAME_MACHINE_ARCH" | sed -e "$expr"`
abi=$(echo "$UNAME_MACHINE_ARCH" | sed -e "$expr")
;;
esac
# The OS release
@ -234,7 +246,7 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in
release='-gnu'
;;
*)
release=`echo "$UNAME_RELEASE" | sed -e 's/[-_].*//' | cut -d. -f1,2`
release=$(echo "$UNAME_RELEASE" | sed -e 's/[-_].*//' | cut -d. -f1,2)
;;
esac
# Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM:
@ -243,15 +255,15 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in
echo "$machine-${os}${release}${abi-}"
exit ;;
*:Bitrig:*:*)
UNAME_MACHINE_ARCH=`arch | sed 's/Bitrig.//'`
UNAME_MACHINE_ARCH=$(arch | sed 's/Bitrig.//')
echo "$UNAME_MACHINE_ARCH"-unknown-bitrig"$UNAME_RELEASE"
exit ;;
*:OpenBSD:*:*)
UNAME_MACHINE_ARCH=`arch | sed 's/OpenBSD.//'`
UNAME_MACHINE_ARCH=$(arch | sed 's/OpenBSD.//')
echo "$UNAME_MACHINE_ARCH"-unknown-openbsd"$UNAME_RELEASE"
exit ;;
*:LibertyBSD:*:*)
UNAME_MACHINE_ARCH=`arch | sed 's/^.*BSD\.//'`
UNAME_MACHINE_ARCH=$(arch | sed 's/^.*BSD\.//')
echo "$UNAME_MACHINE_ARCH"-unknown-libertybsd"$UNAME_RELEASE"
exit ;;
*:MidnightBSD:*:*)
@ -263,6 +275,9 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in
*:SolidBSD:*:*)
echo "$UNAME_MACHINE"-unknown-solidbsd"$UNAME_RELEASE"
exit ;;
*:OS108:*:*)
echo "$UNAME_MACHINE"-unknown-os108_"$UNAME_RELEASE"
exit ;;
macppc:MirBSD:*:*)
echo powerpc-unknown-mirbsd"$UNAME_RELEASE"
exit ;;
@ -272,6 +287,9 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in
*:Sortix:*:*)
echo "$UNAME_MACHINE"-unknown-sortix
exit ;;
*:Twizzler:*:*)
echo "$UNAME_MACHINE"-unknown-twizzler
exit ;;
*:Redox:*:*)
echo "$UNAME_MACHINE"-unknown-redox
exit ;;
@ -281,17 +299,17 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in
alpha:OSF1:*:*)
case $UNAME_RELEASE in
*4.0)
UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $3}'`
UNAME_RELEASE=$(/usr/sbin/sizer -v | awk '{print $3}')
;;
*5.*)
UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $4}'`
UNAME_RELEASE=$(/usr/sbin/sizer -v | awk '{print $4}')
;;
esac
# According to Compaq, /usr/sbin/psrinfo has been available on
# OSF/1 and Tru64 systems produced since 1995. I hope that
# covers most systems running today. This code pipes the CPU
# types through head -n 1, so we only detect the type of CPU 0.
ALPHA_CPU_TYPE=`/usr/sbin/psrinfo -v | sed -n -e 's/^ The alpha \(.*\) processor.*$/\1/p' | head -n 1`
ALPHA_CPU_TYPE=$(/usr/sbin/psrinfo -v | sed -n -e 's/^ The alpha \(.*\) processor.*$/\1/p' | head -n 1)
case "$ALPHA_CPU_TYPE" in
"EV4 (21064)")
UNAME_MACHINE=alpha ;;
@ -329,7 +347,7 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in
# A Tn.n version is a released field test version.
# A Xn.n version is an unreleased experimental baselevel.
# 1.2 uses "1.2" for uname -r.
echo "$UNAME_MACHINE"-dec-osf"`echo "$UNAME_RELEASE" | sed -e 's/^[PVTX]//' | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz`"
echo "$UNAME_MACHINE"-dec-osf"$(echo "$UNAME_RELEASE" | sed -e 's/^[PVTX]//' | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz)"
# Reset EXIT trap before exiting to avoid spurious non-zero exit code.
exitcode=$?
trap '' 0
@ -363,7 +381,7 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in
exit ;;
Pyramid*:OSx*:*:* | MIS*:OSx*:*:* | MIS*:SMP_DC-OSx*:*:*)
# akee@wpdis03.wpafb.af.mil (Earle F. Ake) contributed MIS and NILE.
if test "`(/bin/universe) 2>/dev/null`" = att ; then
if test "$( (/bin/universe) 2>/dev/null)" = att ; then
echo pyramid-pyramid-sysv3
else
echo pyramid-pyramid-bsd
@ -376,54 +394,59 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in
echo sparc-icl-nx6
exit ;;
DRS?6000:UNIX_SV:4.2*:7* | DRS?6000:isis:4.2*:7*)
case `/usr/bin/uname -p` in
case $(/usr/bin/uname -p) in
sparc) echo sparc-icl-nx7; exit ;;
esac ;;
s390x:SunOS:*:*)
echo "$UNAME_MACHINE"-ibm-solaris2"`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'`"
echo "$UNAME_MACHINE"-ibm-solaris2"$(echo "$UNAME_RELEASE" | sed -e 's/[^.]*//')"
exit ;;
sun4H:SunOS:5.*:*)
echo sparc-hal-solaris2"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`"
echo sparc-hal-solaris2"$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*//')"
exit ;;
sun4*:SunOS:5.*:* | tadpole*:SunOS:5.*:*)
echo sparc-sun-solaris2"`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'`"
echo sparc-sun-solaris2"$(echo "$UNAME_RELEASE" | sed -e 's/[^.]*//')"
exit ;;
i86pc:AuroraUX:5.*:* | i86xen:AuroraUX:5.*:*)
echo i386-pc-auroraux"$UNAME_RELEASE"
exit ;;
i86pc:SunOS:5.*:* | i86xen:SunOS:5.*:*)
UNAME_REL="`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'`"
case `isainfo -b` in
32)
echo i386-pc-solaris2"$UNAME_REL"
;;
64)
echo x86_64-pc-solaris2"$UNAME_REL"
;;
esac
set_cc_for_build
SUN_ARCH=i386
# If there is a compiler, see if it is configured for 64-bit objects.
# Note that the Sun cc does not turn __LP64__ into 1 like gcc does.
# This test works for both compilers.
if test "$CC_FOR_BUILD" != no_compiler_found; then
if (echo '#ifdef __amd64'; echo IS_64BIT_ARCH; echo '#endif') | \
(CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \
grep IS_64BIT_ARCH >/dev/null
then
SUN_ARCH=x86_64
fi
fi
echo "$SUN_ARCH"-pc-solaris2"$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*//')"
exit ;;
sun4*:SunOS:6*:*)
# According to config.sub, this is the proper way to canonicalize
# SunOS6. Hard to guess exactly what SunOS6 will be like, but
# it's likely to be more like Solaris than SunOS4.
echo sparc-sun-solaris3"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`"
echo sparc-sun-solaris3"$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*//')"
exit ;;
sun4*:SunOS:*:*)
case "`/usr/bin/arch -k`" in
case "$(/usr/bin/arch -k)" in
Series*|S4*)
UNAME_RELEASE=`uname -v`
UNAME_RELEASE=$(uname -v)
;;
esac
# Japanese Language versions have a version number like `4.1.3-JL'.
echo sparc-sun-sunos"`echo "$UNAME_RELEASE"|sed -e 's/-/_/'`"
echo sparc-sun-sunos"$(echo "$UNAME_RELEASE"|sed -e 's/-/_/')"
exit ;;
sun3*:SunOS:*:*)
echo m68k-sun-sunos"$UNAME_RELEASE"
exit ;;
sun*:*:4.2BSD:*)
UNAME_RELEASE=`(sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null`
UNAME_RELEASE=$( (sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null)
test "x$UNAME_RELEASE" = x && UNAME_RELEASE=3
case "`/bin/arch`" in
case "$(/bin/arch)" in
sun3)
echo m68k-sun-sunos"$UNAME_RELEASE"
;;
@ -503,8 +526,8 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in
}
EOF
$CC_FOR_BUILD -o "$dummy" "$dummy.c" &&
dummyarg=`echo "$UNAME_RELEASE" | sed -n 's/\([0-9]*\).*/\1/p'` &&
SYSTEM_NAME=`"$dummy" "$dummyarg"` &&
dummyarg=$(echo "$UNAME_RELEASE" | sed -n 's/\([0-9]*\).*/\1/p') &&
SYSTEM_NAME=$("$dummy" "$dummyarg") &&
{ echo "$SYSTEM_NAME"; exit; }
echo mips-mips-riscos"$UNAME_RELEASE"
exit ;;
@ -531,11 +554,11 @@ EOF
exit ;;
AViiON:dgux:*:*)
# DG/UX returns AViiON for all architectures
UNAME_PROCESSOR=`/usr/bin/uname -p`
if [ "$UNAME_PROCESSOR" = mc88100 ] || [ "$UNAME_PROCESSOR" = mc88110 ]
UNAME_PROCESSOR=$(/usr/bin/uname -p)
if test "$UNAME_PROCESSOR" = mc88100 || test "$UNAME_PROCESSOR" = mc88110
then
if [ "$TARGET_BINARY_INTERFACE"x = m88kdguxelfx ] || \
[ "$TARGET_BINARY_INTERFACE"x = x ]
if test "$TARGET_BINARY_INTERFACE"x = m88kdguxelfx || \
test "$TARGET_BINARY_INTERFACE"x = x
then
echo m88k-dg-dgux"$UNAME_RELEASE"
else
@ -559,17 +582,17 @@ EOF
echo m68k-tektronix-bsd
exit ;;
*:IRIX*:*:*)
echo mips-sgi-irix"`echo "$UNAME_RELEASE"|sed -e 's/-/_/g'`"
echo mips-sgi-irix"$(echo "$UNAME_RELEASE"|sed -e 's/-/_/g')"
exit ;;
????????:AIX?:[12].1:2) # AIX 2.2.1 or AIX 2.1.1 is RT/PC AIX.
echo romp-ibm-aix # uname -m gives an 8 hex-code CPU id
exit ;; # Note that: echo "'`uname -s`'" gives 'AIX '
exit ;; # Note that: echo "'$(uname -s)'" gives 'AIX '
i*86:AIX:*:*)
echo i386-ibm-aix
exit ;;
ia64:AIX:*:*)
if [ -x /usr/bin/oslevel ] ; then
IBM_REV=`/usr/bin/oslevel`
if test -x /usr/bin/oslevel ; then
IBM_REV=$(/usr/bin/oslevel)
else
IBM_REV="$UNAME_VERSION.$UNAME_RELEASE"
fi
@ -589,7 +612,7 @@ EOF
exit(0);
}
EOF
if $CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=`"$dummy"`
if $CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=$("$dummy")
then
echo "$SYSTEM_NAME"
else
@ -602,15 +625,15 @@ EOF
fi
exit ;;
*:AIX:*:[4567])
IBM_CPU_ID=`/usr/sbin/lsdev -C -c processor -S available | sed 1q | awk '{ print $1 }'`
IBM_CPU_ID=$(/usr/sbin/lsdev -C -c processor -S available | sed 1q | awk '{ print $1 }')
if /usr/sbin/lsattr -El "$IBM_CPU_ID" | grep ' POWER' >/dev/null 2>&1; then
IBM_ARCH=rs6000
else
IBM_ARCH=powerpc
fi
if [ -x /usr/bin/lslpp ] ; then
IBM_REV=`/usr/bin/lslpp -Lqc bos.rte.libc |
awk -F: '{ print $3 }' | sed s/[0-9]*$/0/`
if test -x /usr/bin/lslpp ; then
IBM_REV=$(/usr/bin/lslpp -Lqc bos.rte.libc |
awk -F: '{ print $3 }' | sed s/[0-9]*$/0/)
else
IBM_REV="$UNAME_VERSION.$UNAME_RELEASE"
fi
@ -638,14 +661,14 @@ EOF
echo m68k-hp-bsd4.4
exit ;;
9000/[34678]??:HP-UX:*:*)
HPUX_REV=`echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//'`
HPUX_REV=$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//')
case "$UNAME_MACHINE" in
9000/31?) HP_ARCH=m68000 ;;
9000/[34]??) HP_ARCH=m68k ;;
9000/[678][0-9][0-9])
if [ -x /usr/bin/getconf ]; then
sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null`
sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null`
if test -x /usr/bin/getconf; then
sc_cpu_version=$(/usr/bin/getconf SC_CPU_VERSION 2>/dev/null)
sc_kernel_bits=$(/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null)
case "$sc_cpu_version" in
523) HP_ARCH=hppa1.0 ;; # CPU_PA_RISC1_0
528) HP_ARCH=hppa1.1 ;; # CPU_PA_RISC1_1
@ -657,7 +680,7 @@ EOF
esac ;;
esac
fi
if [ "$HP_ARCH" = "" ]; then
if test "$HP_ARCH" = ""; then
set_cc_for_build
sed 's/^ //' << EOF > "$dummy.c"
@ -692,11 +715,11 @@ EOF
exit (0);
}
EOF
(CCOPTS="" $CC_FOR_BUILD -o "$dummy" "$dummy.c" 2>/dev/null) && HP_ARCH=`"$dummy"`
(CCOPTS="" $CC_FOR_BUILD -o "$dummy" "$dummy.c" 2>/dev/null) && HP_ARCH=$("$dummy")
test -z "$HP_ARCH" && HP_ARCH=hppa
fi ;;
esac
if [ "$HP_ARCH" = hppa2.0w ]
if test "$HP_ARCH" = hppa2.0w
then
set_cc_for_build
@ -720,7 +743,7 @@ EOF
echo "$HP_ARCH"-hp-hpux"$HPUX_REV"
exit ;;
ia64:HP-UX:*:*)
HPUX_REV=`echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//'`
HPUX_REV=$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//')
echo ia64-hp-hpux"$HPUX_REV"
exit ;;
3050*:HI-UX:*:*)
@ -750,7 +773,7 @@ EOF
exit (0);
}
EOF
$CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=`"$dummy"` &&
$CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=$("$dummy") &&
{ echo "$SYSTEM_NAME"; exit; }
echo unknown-hitachi-hiuxwe2
exit ;;
@ -770,7 +793,7 @@ EOF
echo hppa1.0-hp-osf
exit ;;
i*86:OSF1:*:*)
if [ -x /usr/sbin/sysversion ] ; then
if test -x /usr/sbin/sysversion ; then
echo "$UNAME_MACHINE"-unknown-osf1mk
else
echo "$UNAME_MACHINE"-unknown-osf1
@ -819,14 +842,14 @@ EOF
echo craynv-cray-unicosmp"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/'
exit ;;
F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*)
FUJITSU_PROC=`uname -m | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz`
FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'`
FUJITSU_REL=`echo "$UNAME_RELEASE" | sed -e 's/ /_/'`
FUJITSU_PROC=$(uname -m | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz)
FUJITSU_SYS=$(uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///')
FUJITSU_REL=$(echo "$UNAME_RELEASE" | sed -e 's/ /_/')
echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}"
exit ;;
5000:UNIX_System_V:4.*:*)
FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'`
FUJITSU_REL=`echo "$UNAME_RELEASE" | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/ /_/'`
FUJITSU_SYS=$(uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///')
FUJITSU_REL=$(echo "$UNAME_RELEASE" | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/ /_/')
echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}"
exit ;;
i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*)
@ -838,26 +861,26 @@ EOF
*:BSD/OS:*:*)
echo "$UNAME_MACHINE"-unknown-bsdi"$UNAME_RELEASE"
exit ;;
arm*:FreeBSD:*:*)
UNAME_PROCESSOR=`uname -p`
arm:FreeBSD:*:*)
UNAME_PROCESSOR=$(uname -p)
set_cc_for_build
if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \
| grep -q __ARM_PCS_VFP
then
echo "${UNAME_PROCESSOR}"-unknown-freebsd"`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`"-gnueabi
echo "${UNAME_PROCESSOR}"-unknown-freebsd"$(echo ${UNAME_RELEASE}|sed -e 's/[-(].*//')"-gnueabi
else
echo "${UNAME_PROCESSOR}"-unknown-freebsd"`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`"-gnueabihf
echo "${UNAME_PROCESSOR}"-unknown-freebsd"$(echo ${UNAME_RELEASE}|sed -e 's/[-(].*//')"-gnueabihf
fi
exit ;;
*:FreeBSD:*:*)
UNAME_PROCESSOR=`/usr/bin/uname -p`
UNAME_PROCESSOR=$(/usr/bin/uname -p)
case "$UNAME_PROCESSOR" in
amd64)
UNAME_PROCESSOR=x86_64 ;;
i386)
UNAME_PROCESSOR=i586 ;;
esac
echo "$UNAME_PROCESSOR"-unknown-freebsd"`echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`"
echo "$UNAME_PROCESSOR"-unknown-freebsd"$(echo "$UNAME_RELEASE"|sed -e 's/[-(].*//')"
exit ;;
i*:CYGWIN*:*)
echo "$UNAME_MACHINE"-pc-cygwin
@ -890,18 +913,18 @@ EOF
echo "$UNAME_MACHINE"-pc-uwin
exit ;;
amd64:CYGWIN*:*:* | x86_64:CYGWIN*:*:*)
echo x86_64-unknown-cygwin
echo x86_64-pc-cygwin
exit ;;
prep*:SunOS:5.*:*)
echo powerpcle-unknown-solaris2"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`"
echo powerpcle-unknown-solaris2"$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*//')"
exit ;;
*:GNU:*:*)
# the GNU system
echo "`echo "$UNAME_MACHINE"|sed -e 's,[-/].*$,,'`-unknown-$LIBC`echo "$UNAME_RELEASE"|sed -e 's,/.*$,,'`"
echo "$(echo "$UNAME_MACHINE"|sed -e 's,[-/].*$,,')-unknown-$LIBC$(echo "$UNAME_RELEASE"|sed -e 's,/.*$,,')"
exit ;;
*:GNU/*:*:*)
# other systems with GNU libc and userland
echo "$UNAME_MACHINE-unknown-`echo "$UNAME_SYSTEM" | sed 's,^[^/]*/,,' | tr "[:upper:]" "[:lower:]"``echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`-$LIBC"
echo "$UNAME_MACHINE-unknown-$(echo "$UNAME_SYSTEM" | sed 's,^[^/]*/,,' | tr "[:upper:]" "[:lower:]")$(echo "$UNAME_RELEASE"|sed -e 's/[-(].*//')-$LIBC"
exit ;;
*:Minix:*:*)
echo "$UNAME_MACHINE"-unknown-minix
@ -914,7 +937,7 @@ EOF
echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"
exit ;;
alpha:Linux:*:*)
case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in
case $(sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' /proc/cpuinfo 2>/dev/null) in
EV5) UNAME_MACHINE=alphaev5 ;;
EV56) UNAME_MACHINE=alphaev56 ;;
PCA56) UNAME_MACHINE=alphapca56 ;;
@ -981,22 +1004,50 @@ EOF
exit ;;
mips:Linux:*:* | mips64:Linux:*:*)
set_cc_for_build
IS_GLIBC=0
test x"${LIBC}" = xgnu && IS_GLIBC=1
sed 's/^ //' << EOF > "$dummy.c"
#undef CPU
#undef ${UNAME_MACHINE}
#undef ${UNAME_MACHINE}el
#undef mips
#undef mipsel
#undef mips64
#undef mips64el
#if ${IS_GLIBC} && defined(_ABI64)
LIBCABI=gnuabi64
#else
#if ${IS_GLIBC} && defined(_ABIN32)
LIBCABI=gnuabin32
#else
LIBCABI=${LIBC}
#endif
#endif
#if ${IS_GLIBC} && defined(__mips64) && defined(__mips_isa_rev) && __mips_isa_rev>=6
CPU=mipsisa64r6
#else
#if ${IS_GLIBC} && !defined(__mips64) && defined(__mips_isa_rev) && __mips_isa_rev>=6
CPU=mipsisa32r6
#else
#if defined(__mips64)
CPU=mips64
#else
CPU=mips
#endif
#endif
#endif
#if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL)
CPU=${UNAME_MACHINE}el
MIPS_ENDIAN=el
#else
#if defined(__MIPSEB__) || defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB)
CPU=${UNAME_MACHINE}
MIPS_ENDIAN=
#else
CPU=
MIPS_ENDIAN=
#endif
#endif
EOF
eval "`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^CPU'`"
test "x$CPU" != x && { echo "$CPU-unknown-linux-$LIBC"; exit; }
eval "$($CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^CPU\|^MIPS_ENDIAN\|^LIBCABI')"
test "x$CPU" != x && { echo "$CPU${MIPS_ENDIAN}-unknown-linux-$LIBCABI"; exit; }
;;
mips64el:Linux:*:*)
echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"
@ -1015,7 +1066,7 @@ EOF
exit ;;
parisc:Linux:*:* | hppa:Linux:*:*)
# Look for CPU level
case `grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2` in
case $(grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2) in
PA7*) echo hppa1.1-unknown-linux-"$LIBC" ;;
PA8*) echo hppa2.0-unknown-linux-"$LIBC" ;;
*) echo hppa-unknown-linux-"$LIBC" ;;
@ -1055,7 +1106,17 @@ EOF
echo "$UNAME_MACHINE"-dec-linux-"$LIBC"
exit ;;
x86_64:Linux:*:*)
echo "$UNAME_MACHINE"-pc-linux-"$LIBC"
set_cc_for_build
LIBCABI=$LIBC
if test "$CC_FOR_BUILD" != no_compiler_found; then
if (echo '#ifdef __ILP32__'; echo IS_X32; echo '#endif') | \
(CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \
grep IS_X32 >/dev/null
then
LIBCABI="$LIBC"x32
fi
fi
echo "$UNAME_MACHINE"-pc-linux-"$LIBCABI"
exit ;;
xtensa*:Linux:*:*)
echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"
@ -1095,7 +1156,7 @@ EOF
echo "$UNAME_MACHINE"-pc-msdosdjgpp
exit ;;
i*86:*:4.*:*)
UNAME_REL=`echo "$UNAME_RELEASE" | sed 's/\/MP$//'`
UNAME_REL=$(echo "$UNAME_RELEASE" | sed 's/\/MP$//')
if grep Novell /usr/include/link.h >/dev/null 2>/dev/null; then
echo "$UNAME_MACHINE"-univel-sysv"$UNAME_REL"
else
@ -1104,19 +1165,19 @@ EOF
exit ;;
i*86:*:5:[678]*)
# UnixWare 7.x, OpenUNIX and OpenServer 6.
case `/bin/uname -X | grep "^Machine"` in
case $(/bin/uname -X | grep "^Machine") in
*486*) UNAME_MACHINE=i486 ;;
*Pentium) UNAME_MACHINE=i586 ;;
*Pent*|*Celeron) UNAME_MACHINE=i686 ;;
esac
echo "$UNAME_MACHINE-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}{$UNAME_VERSION}"
echo "$UNAME_MACHINE-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}${UNAME_VERSION}"
exit ;;
i*86:*:3.2:*)
if test -f /usr/options/cb.name; then
UNAME_REL=`sed -n 's/.*Version //p' </usr/options/cb.name`
UNAME_REL=$(sed -n 's/.*Version //p' </usr/options/cb.name)
echo "$UNAME_MACHINE"-pc-isc"$UNAME_REL"
elif /bin/uname -X 2>/dev/null >/dev/null ; then
UNAME_REL=`(/bin/uname -X|grep Release|sed -e 's/.*= //')`
UNAME_REL=$( (/bin/uname -X|grep Release|sed -e 's/.*= //'))
(/bin/uname -X|grep i80486 >/dev/null) && UNAME_MACHINE=i486
(/bin/uname -X|grep '^Machine.*Pentium' >/dev/null) \
&& UNAME_MACHINE=i586
@ -1166,7 +1227,7 @@ EOF
3[345]??:*:4.0:3.0 | 3[34]??A:*:4.0:3.0 | 3[34]??,*:*:4.0:3.0 | 3[34]??/*:*:4.0:3.0 | 4400:*:4.0:3.0 | 4850:*:4.0:3.0 | SKA40:*:4.0:3.0 | SDS2:*:4.0:3.0 | SHG2:*:4.0:3.0 | S7501*:*:4.0:3.0)
OS_REL=''
test -r /etc/.relid \
&& OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid`
&& OS_REL=.$(sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid)
/bin/uname -p 2>/dev/null | grep 86 >/dev/null \
&& { echo i486-ncr-sysv4.3"$OS_REL"; exit; }
/bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \
@ -1177,7 +1238,7 @@ EOF
NCR*:*:4.2:* | MPRAS*:*:4.2:*)
OS_REL='.3'
test -r /etc/.relid \
&& OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid`
&& OS_REL=.$(sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid)
/bin/uname -p 2>/dev/null | grep 86 >/dev/null \
&& { echo i486-ncr-sysv4.3"$OS_REL"; exit; }
/bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \
@ -1210,7 +1271,7 @@ EOF
exit ;;
*:SINIX-*:*:*)
if uname -p 2>/dev/null >/dev/null ; then
UNAME_MACHINE=`(uname -p) 2>/dev/null`
UNAME_MACHINE=$( (uname -p) 2>/dev/null)
echo "$UNAME_MACHINE"-sni-sysv4
else
echo ns32k-sni-sysv
@ -1244,7 +1305,7 @@ EOF
echo mips-sony-newsos6
exit ;;
R[34]000:*System_V*:*:* | R4000:UNIX_SYSV:*:* | R*000:UNIX_SV:*:*)
if [ -d /usr/nec ]; then
if test -d /usr/nec; then
echo mips-nec-sysv"$UNAME_RELEASE"
else
echo mips-unknown-sysv"$UNAME_RELEASE"
@ -1292,14 +1353,24 @@ EOF
*:Rhapsody:*:*)
echo "$UNAME_MACHINE"-apple-rhapsody"$UNAME_RELEASE"
exit ;;
arm64:Darwin:*:*)
echo aarch64-apple-darwin"$UNAME_RELEASE"
exit ;;
*:Darwin:*:*)
UNAME_PROCESSOR=`uname -p` || UNAME_PROCESSOR=unknown
UNAME_PROCESSOR=$(uname -p)
case $UNAME_PROCESSOR in
unknown) UNAME_PROCESSOR=powerpc ;;
esac
if command -v xcode-select > /dev/null 2> /dev/null && \
! xcode-select --print-path > /dev/null 2> /dev/null ; then
# Avoid executing cc if there is no toolchain installed as
# cc will be a stub that puts up a graphical alert
# prompting the user to install developer tools.
CC_FOR_BUILD=no_compiler_found
else
set_cc_for_build
if test "$UNAME_PROCESSOR" = unknown ; then
UNAME_PROCESSOR=powerpc
fi
if test "`echo "$UNAME_RELEASE" | sed -e 's/\..*//'`" -le 10 ; then
if [ "$CC_FOR_BUILD" != no_compiler_found ]; then
if test "$CC_FOR_BUILD" != no_compiler_found; then
if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \
(CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \
grep IS_64BIT_ARCH >/dev/null
@ -1316,20 +1387,14 @@ EOF
then
UNAME_PROCESSOR=powerpc
fi
fi
elif test "$UNAME_PROCESSOR" = i386 ; then
# Avoid executing cc on OS X 10.9, as it ships with a stub
# that puts up a graphical alert prompting to install
# developer tools. Any system running Mac OS X 10.7 or
# later (Darwin 11 and later) is required to have a 64-bit
# processor. This is not true of the ARM version of Darwin
# that Apple uses in portable devices.
UNAME_PROCESSOR=x86_64
# uname -m returns i386 or x86_64
UNAME_PROCESSOR=$UNAME_MACHINE
fi
echo "$UNAME_PROCESSOR"-apple-darwin"$UNAME_RELEASE"
exit ;;
*:procnto*:*:* | *:QNX:[0123456789]*:*)
UNAME_PROCESSOR=`uname -p`
UNAME_PROCESSOR=$(uname -p)
if test "$UNAME_PROCESSOR" = x86; then
UNAME_PROCESSOR=i386
UNAME_MACHINE=pc
@ -1397,10 +1462,10 @@ EOF
echo mips-sei-seiux"$UNAME_RELEASE"
exit ;;
*:DragonFly:*:*)
echo "$UNAME_MACHINE"-unknown-dragonfly"`echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`"
echo "$UNAME_MACHINE"-unknown-dragonfly"$(echo "$UNAME_RELEASE"|sed -e 's/[-(].*//')"
exit ;;
*:*VMS:*:*)
UNAME_MACHINE=`(uname -p) 2>/dev/null`
UNAME_MACHINE=$( (uname -p) 2>/dev/null)
case "$UNAME_MACHINE" in
A*) echo alpha-dec-vms ; exit ;;
I*) echo ia64-dec-vms ; exit ;;
@ -1410,7 +1475,7 @@ EOF
echo i386-pc-xenix
exit ;;
i*86:skyos:*:*)
echo "$UNAME_MACHINE"-pc-skyos"`echo "$UNAME_RELEASE" | sed -e 's/ .*$//'`"
echo "$UNAME_MACHINE"-pc-skyos"$(echo "$UNAME_RELEASE" | sed -e 's/ .*$//')"
exit ;;
i*86:rdos:*:*)
echo "$UNAME_MACHINE"-pc-rdos
@ -1424,8 +1489,148 @@ EOF
amd64:Isilon\ OneFS:*:*)
echo x86_64-unknown-onefs
exit ;;
*:Unleashed:*:*)
echo "$UNAME_MACHINE"-unknown-unleashed"$UNAME_RELEASE"
exit ;;
esac
# No uname command or uname output not recognized.
set_cc_for_build
cat > "$dummy.c" <<EOF
#ifdef _SEQUENT_
#include <sys/types.h>
#include <sys/utsname.h>
#endif
#if defined(ultrix) || defined(_ultrix) || defined(__ultrix) || defined(__ultrix__)
#if defined (vax) || defined (__vax) || defined (__vax__) || defined(mips) || defined(__mips) || defined(__mips__) || defined(MIPS) || defined(__MIPS__)
#include <signal.h>
#if defined(_SIZE_T_) || defined(SIGLOST)
#include <sys/utsname.h>
#endif
#endif
#endif
main ()
{
#if defined (sony)
#if defined (MIPSEB)
/* BFD wants "bsd" instead of "newsos". Perhaps BFD should be changed,
I don't know.... */
printf ("mips-sony-bsd\n"); exit (0);
#else
#include <sys/param.h>
printf ("m68k-sony-newsos%s\n",
#ifdef NEWSOS4
"4"
#else
""
#endif
); exit (0);
#endif
#endif
#if defined (NeXT)
#if !defined (__ARCHITECTURE__)
#define __ARCHITECTURE__ "m68k"
#endif
int version;
version=$( (hostinfo | sed -n 's/.*NeXT Mach \([0-9]*\).*/\1/p') 2>/dev/null);
if (version < 4)
printf ("%s-next-nextstep%d\n", __ARCHITECTURE__, version);
else
printf ("%s-next-openstep%d\n", __ARCHITECTURE__, version);
exit (0);
#endif
#if defined (MULTIMAX) || defined (n16)
#if defined (UMAXV)
printf ("ns32k-encore-sysv\n"); exit (0);
#else
#if defined (CMU)
printf ("ns32k-encore-mach\n"); exit (0);
#else
printf ("ns32k-encore-bsd\n"); exit (0);
#endif
#endif
#endif
#if defined (__386BSD__)
printf ("i386-pc-bsd\n"); exit (0);
#endif
#if defined (sequent)
#if defined (i386)
printf ("i386-sequent-dynix\n"); exit (0);
#endif
#if defined (ns32000)
printf ("ns32k-sequent-dynix\n"); exit (0);
#endif
#endif
#if defined (_SEQUENT_)
struct utsname un;
uname(&un);
if (strncmp(un.version, "V2", 2) == 0) {
printf ("i386-sequent-ptx2\n"); exit (0);
}
if (strncmp(un.version, "V1", 2) == 0) { /* XXX is V1 correct? */
printf ("i386-sequent-ptx1\n"); exit (0);
}
printf ("i386-sequent-ptx\n"); exit (0);
#endif
#if defined (vax)
#if !defined (ultrix)
#include <sys/param.h>
#if defined (BSD)
#if BSD == 43
printf ("vax-dec-bsd4.3\n"); exit (0);
#else
#if BSD == 199006
printf ("vax-dec-bsd4.3reno\n"); exit (0);
#else
printf ("vax-dec-bsd\n"); exit (0);
#endif
#endif
#else
printf ("vax-dec-bsd\n"); exit (0);
#endif
#else
#if defined(_SIZE_T_) || defined(SIGLOST)
struct utsname un;
uname (&un);
printf ("vax-dec-ultrix%s\n", un.release); exit (0);
#else
printf ("vax-dec-ultrix\n"); exit (0);
#endif
#endif
#endif
#if defined(ultrix) || defined(_ultrix) || defined(__ultrix) || defined(__ultrix__)
#if defined(mips) || defined(__mips) || defined(__mips__) || defined(MIPS) || defined(__MIPS__)
#if defined(_SIZE_T_) || defined(SIGLOST)
struct utsname *un;
uname (&un);
printf ("mips-dec-ultrix%s\n", un.release); exit (0);
#else
printf ("mips-dec-ultrix\n"); exit (0);
#endif
#endif
#endif
#if defined (alliant) && defined (i860)
printf ("i860-alliant-bsd\n"); exit (0);
#endif
exit (1);
}
EOF
$CC_FOR_BUILD -o "$dummy" "$dummy.c" 2>/dev/null && SYSTEM_NAME=$($dummy) &&
{ echo "$SYSTEM_NAME"; exit; }
# Apollos put the system type in the environment.
test -d /usr/apollo && { echo "$ISP-apollo-$SYSTYPE"; exit; }
echo "$0: unable to guess system type" >&2
case "$UNAME_MACHINE:$UNAME_SYSTEM" in
@ -1445,9 +1650,15 @@ This script (version $timestamp), has failed to recognize the
operating system you are using. If your script is old, overwrite *all*
copies of config.guess and config.sub with the latest versions from:
https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess
https://git.savannah.gnu.org/cgit/config.git/plain/config.guess
and
https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub
https://git.savannah.gnu.org/cgit/config.git/plain/config.sub
EOF
year=$(echo $timestamp | sed 's,-.*,,')
# shellcheck disable=SC2003
if test "$(expr "$(date +%Y)" - "$year")" -lt 3 ; then
cat >&2 <<EOF
If $0 has already been updated, send the following data and any
information you think might be pertinent to config-patches@gnu.org to
@ -1455,26 +1666,27 @@ provide the necessary information to handle your system.
config.guess timestamp = $timestamp
uname -m = `(uname -m) 2>/dev/null || echo unknown`
uname -r = `(uname -r) 2>/dev/null || echo unknown`
uname -s = `(uname -s) 2>/dev/null || echo unknown`
uname -v = `(uname -v) 2>/dev/null || echo unknown`
uname -m = $( (uname -m) 2>/dev/null || echo unknown)
uname -r = $( (uname -r) 2>/dev/null || echo unknown)
uname -s = $( (uname -s) 2>/dev/null || echo unknown)
uname -v = $( (uname -v) 2>/dev/null || echo unknown)
/usr/bin/uname -p = `(/usr/bin/uname -p) 2>/dev/null`
/bin/uname -X = `(/bin/uname -X) 2>/dev/null`
/usr/bin/uname -p = $( (/usr/bin/uname -p) 2>/dev/null)
/bin/uname -X = $( (/bin/uname -X) 2>/dev/null)
hostinfo = `(hostinfo) 2>/dev/null`
/bin/universe = `(/bin/universe) 2>/dev/null`
/usr/bin/arch -k = `(/usr/bin/arch -k) 2>/dev/null`
/bin/arch = `(/bin/arch) 2>/dev/null`
/usr/bin/oslevel = `(/usr/bin/oslevel) 2>/dev/null`
/usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null`
hostinfo = $( (hostinfo) 2>/dev/null)
/bin/universe = $( (/bin/universe) 2>/dev/null)
/usr/bin/arch -k = $( (/usr/bin/arch -k) 2>/dev/null)
/bin/arch = $( (/bin/arch) 2>/dev/null)
/usr/bin/oslevel = $( (/usr/bin/oslevel) 2>/dev/null)
/usr/convex/getsysinfo = $( (/usr/convex/getsysinfo) 2>/dev/null)
UNAME_MACHINE = "$UNAME_MACHINE"
UNAME_RELEASE = "$UNAME_RELEASE"
UNAME_SYSTEM = "$UNAME_SYSTEM"
UNAME_VERSION = "$UNAME_VERSION"
EOF
fi
exit 1

1834
config/config.sub vendored
View file

File diff suppressed because it is too large Load diff

5
configure.ac
View file

@ -179,6 +179,10 @@ AC_CHECK_HEADERS([bzlib.h], [true],
[AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])])
# Checks for libarchive
PKG_CHECK_MODULES([LIBARCHIVE], [libarchive >= 3.1.2], [CXXFLAGS="$LIBARCHIVE_CFLAGS $CXXFLAGS"])
# Workaround until https://github.com/libarchive/libarchive/issues/1446 is fixed
if test "$shared" != yes; then
LIBARCHIVE_LIBS+=' -lz'
fi
# Look for SQLite, a required dependency.
PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CXXFLAGS"])
@ -251,6 +255,7 @@ if test -n "$enable_s3"; then
declare -a aws_version_tokens=($(printf '#include <aws/core/VersionConfig.h>\nAWS_SDK_VERSION_STRING' | $CPP $CPPFLAGS - | grep -v '^#.*' | sed 's/"//g' | tr '.' ' '))
AC_DEFINE_UNQUOTED([AWS_VERSION_MAJOR], ${aws_version_tokens@<:@0@:>@}, [Major version of aws-sdk-cpp.])
AC_DEFINE_UNQUOTED([AWS_VERSION_MINOR], ${aws_version_tokens@<:@1@:>@}, [Minor version of aws-sdk-cpp.])
AC_DEFINE_UNQUOTED([AWS_VERSION_PATCH], ${aws_version_tokens@<:@2@:>@}, [Patch version of aws-sdk-cpp.])
fi

2
corepkgs/fetchurl.nix
View file

@ -1,6 +1,6 @@
{ system ? "" # obsolete
, url
, hash ? "" # an SRI ash
, hash ? "" # an SRI hash
# Legacy hash specification
, md5 ? "", sha1 ? "", sha256 ? "", sha512 ? ""

1236
doc/manual/command-ref/conf-file.xml
View file

File diff suppressed because it is too large Load diff

59
doc/manual/generate-manpage.nix
View file

@ -1,33 +1,40 @@
command:
with builtins;
with import ./utils.nix;
let
showCommand =
{ command, section, def }:
"${section} Name\n\n"
{ command, def, filename }:
"# Name\n\n"
+ "`${command}` - ${def.description}\n\n"
+ "${section} Synopsis\n\n"
+ "# Synopsis\n\n"
+ showSynopsis { inherit command; args = def.args; }
+ (if def ? doc
then "${section} Description\n\n" + def.doc + "\n\n"
else "")
+ (let s = showFlags def.flags; in
if s != ""
then "${section} Flags\n\n${s}"
+ (if def.commands or {} != {}
then
"where *subcommand* is one of the following:\n\n"
# FIXME: group by category
+ concatStrings (map (name:
"* [`${command} ${name}`](./${appendName filename name}.md) - ${def.commands.${name}.description}\n")
(attrNames def.commands))
+ "\n"
else "")
+ (if def.examples or [] != []
then
"${section} Examples\n\n"
"# Examples\n\n"
+ concatStrings (map ({ description, command }: "${description}\n\n```console\n${command}\n```\n\n") def.examples)
else "")
+ (if def.commands or [] != []
then concatStrings (
map (name:
"# Subcommand `${command} ${name}`\n\n"
+ showCommand { command = command + " " + name; section = "##"; def = def.commands.${name}; })
(attrNames def.commands))
else "");
+ (if def ? doc
then def.doc + "\n\n"
else "")
+ (let s = showFlags def.flags; in
if s != ""
then "# Flags\n\n${s}"
else "")
;
appendName = filename: name: (if filename == "nix" then "nix3" else filename) + "-" + name;
showFlags = flags:
concatStrings
@ -48,8 +55,20 @@ let
"`${command}` [*flags*...] ${concatStringsSep " "
(map (arg: "*${arg.label}*" + (if arg ? arity then "" else "...")) args)}\n\n";
processCommand = { command, def, filename }:
[ { name = filename + ".md"; value = showCommand { inherit command def filename; }; inherit command; } ]
++ concatMap
(name: processCommand {
filename = appendName filename name;
command = command + " " + name;
def = def.commands.${name};
})
(attrNames def.commands or {});
in
command:
showCommand { command = "nix"; section = "#"; def = command; }
let
manpages = processCommand { filename = "nix"; command = "nix"; def = command; };
summary = concatStrings (map (manpage: " - [${manpage.command}](command-ref/new-cli/${manpage.name})\n") manpages);
in
(listToAttrs manpages) // { "SUMMARY.md" = summary; }

45
doc/manual/local.mk
View file

@ -4,7 +4,7 @@ MANUAL_SRCS := $(call rwildcard, $(d)/src, *.md)
# Generate man pages.
man-pages := $(foreach n, \
nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 nix.1 \
nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 \
nix-collect-garbage.1 \
nix-prefetch-url.1 nix-channel.1 \
nix-hash.1 nix-copy-closure.1 \
@ -13,9 +13,14 @@ man-pages := $(foreach n, \
clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8
dist-files += $(man-pages)
# Provide a dummy environment for nix, so that it will not access files outside the macOS sandbox.
dummy-env = env -i \
HOME=/dummy \
NIX_CONF_DIR=/dummy \
NIX_SSL_CERT_FILE=/dummy/no-ca-bundle.crt \
NIX_STATE_DIR=/dummy
nix-eval = $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw --expr
nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw
$(d)/%.1: $(d)/src/command-ref/%.md
@printf "Title: %s\n\n" "$$(basename $@ .1)" > $^.tmp
@ -35,37 +40,51 @@ $(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md
$(trace-gen) lowdown -sT man $^.tmp -o $@
@rm $^.tmp
$(d)/src/command-ref/nix.md: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix
$(trace-gen) $(nix-eval) 'import doc/manual/generate-manpage.nix (builtins.fromJSON (builtins.readFile $<))' > $@.tmp
$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli
$(trace-gen) cat doc/manual/src/SUMMARY.md.in | while IFS= read line; do if [[ $$line = @manpages@ ]]; then cat doc/manual/src/command-ref/new-cli/SUMMARY.md; else echo "$$line"; fi; done > $@.tmp
@mv $@.tmp $@
$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix
@rm -rf $@
$(trace-gen) $(nix-eval) --write-to $@ --expr 'import doc/manual/generate-manpage.nix (builtins.fromJSON (builtins.readFile $<))'
$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix
@cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp
$(trace-gen) $(nix-eval) 'import doc/manual/generate-options.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp
$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-options.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp
@mv $@.tmp $@
$(d)/nix.json: $(bindir)/nix
$(trace-gen) $(bindir)/nix __dump-args > $@.tmp
$(trace-gen) $(dummy-env) $(bindir)/nix __dump-args > $@.tmp
@mv $@.tmp $@
$(d)/conf-file.json: $(bindir)/nix
$(trace-gen) env -i NIX_CONF_DIR=/dummy HOME=/dummy NIX_SSL_CERT_FILE=/dummy/no-ca-bundle.crt $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp
$(trace-gen) $(dummy-env) $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp
@mv $@.tmp $@
$(d)/src/expressions/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/expressions/builtins-prefix.md $(bindir)/nix
@cat doc/manual/src/expressions/builtins-prefix.md > $@.tmp
$(trace-gen) $(nix-eval) 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp
$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp
@mv $@.tmp $@
$(d)/builtins.json: $(bindir)/nix
$(trace-gen) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-builtins > $@.tmp
mv $@.tmp $@
$(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-builtins > $@.tmp
@mv $@.tmp $@
# Generate the HTML manual.
install: $(docdir)/manual/index.html
$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/custom.css $(d)/src/command-ref/nix.md $(d)/src/command-ref/conf-file.md $(d)/src/expressions/builtins.md
$(trace-gen) mdbook build doc/manual -d $(docdir)/manual
# Generate 'nix' manpages.
install: $(d)/src/command-ref/new-cli
$(trace-gen) for i in doc/manual/src/command-ref/new-cli/*.md; do \
name=$$(basename $$i .md); \
if [[ $$name = SUMMARY ]]; then continue; fi; \
printf "Title: %s\n\n" "$$name" > $$i.tmp; \
cat $$i >> $$i.tmp; \
lowdown -sT man $$i.tmp -o $(mandir)/man1/$$name.1; \
done
$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/expressions/builtins.md
$(trace-gen) RUST_LOG=warn mdbook build doc/manual -d $(docdir)/manual
@cp doc/manual/highlight.pack.js $(docdir)/manual/highlight.js
endif

6
doc/manual/src/SUMMARY.md → doc/manual/src/SUMMARY.md.in
View file

@ -62,11 +62,13 @@
- [nix-instantiate](command-ref/nix-instantiate.md)
- [nix-prefetch-url](command-ref/nix-prefetch-url.md)
- [Experimental Commands](command-ref/experimental-commands.md)
- [nix](command-ref/nix.md)
@manpages@
- [Files](command-ref/files.md)
- [nix.conf](command-ref/conf-file.md)
- [Glossary](glossary.md)
- [Hacking](hacking.md)
- [Contributing](contributing/contributing.md)
- [Hacking](contributing/hacking.md)
- [CLI guideline](contributing/cli-guideline.md)
- [Release Notes](release-notes/release-notes.md)
- [Release 2.3 (2019-09-04)](release-notes/rl-2.3.md)
- [Release 2.2 (2019-01-11)](release-notes/rl-2.2.md)

30
doc/manual/src/command-ref/conf-file-prefix.md
View file

@ -19,19 +19,33 @@ By default Nix reads settings from the following places:
and `XDG_CONFIG_HOME`. If these are unset, it will look in
`$HOME/.config/nix.conf`.
The configuration files consist of `name =
value` pairs, one per line. Other files can be included with a line like
`include
path`, where *path* is interpreted relative to the current conf file and
a missing file is an error unless `!include` is used instead. Comments
- If `NIX_CONFIG` is set, its contents is treated as the contents of
a configuration file.
The configuration files consist of `name = value` pairs, one per
line. Other files can be included with a line like `include path`,
where *path* is interpreted relative to the current conf file and a
missing file is an error unless `!include` is used instead. Comments
start with a `#` character. Here is an example configuration file:
keep-outputs = true # Nice for developers
keep-derivations = true # Idem
You can override settings on the command line using the `--option` flag,
e.g. `--option keep-outputs
false`.
You can override settings on the command line using the `--option`
flag, e.g. `--option keep-outputs false`. Every configuration setting
also has a corresponding command line flag, e.g. `--max-jobs 16`; for
Boolean settings, there are two flags to enable or disable the setting
(e.g. `--keep-failed` and `--no-keep-failed`).
A configuration setting usually overrides any previous value. However,
you can prefix the name of the setting by `extra-` to *append* to the
previous value. For instance,
substituters = a b
extra-substituters = c d
defines the `substituters` setting to be `a b c d`. This is also
available as a command line flag (e.g. `--extra-substituters`).
The following settings are currently available:

5
doc/manual/src/command-ref/env-common.md
View file

@ -81,6 +81,11 @@ Most Nix commands interpret the following environment variables:
Overrides the location of the system Nix configuration directory
(default `prefix/etc/nix`).
- `NIX_CONFIG`
Applies settings from Nix configuration from the environment.
The content is treated as if it was read from a Nix configuration file.
Settings are separated by the newline character.
- `NIX_USER_CONF_FILES`
Overrides the location of the user Nix configuration files to load
from (defaults to the XDG spec locations). The variable is treated

2
doc/manual/src/command-ref/nix-hash.md
View file

@ -45,7 +45,7 @@ md5sum`.
- `--type` *hashAlgo*
Use the specified cryptographic hash algorithm, which can be one of
`md5`, `sha1`, and `sha256`.
`md5`, `sha1`, `sha256`, and `sha512`.
- `--to-base16`
Don’t hash anything, but convert the base-32 hash representation

2
doc/manual/src/command-ref/nix-prefetch-url.md
View file

@ -39,7 +39,7 @@ Nix store is also printed.
- `--type` *hashAlgo*
Use the specified cryptographic hash algorithm, which can be one of
`md5`, `sha1`, and `sha256`.
`md5`, `sha1`, `sha256`, and `sha512`.
- `--print-path`
Print the store path of the downloaded file on standard output.

10
doc/manual/src/command-ref/nix-shell.md
View file

@ -32,7 +32,7 @@ URL of a tarball that will be downloaded and unpacked to a temporary
location. The tarball must include a single top-level directory
containing at least a file named `default.nix`.
If the derivation defines the variable `shellHook`, it will be evaluated
If the derivation defines the variable `shellHook`, it will be run
after `$stdenv/setup` has been sourced. Since this hook is not executed
by regular Nix builds, it allows you to perform initialisation specific
to `nix-shell`. For example, the derivation attribute
@ -41,10 +41,12 @@ to `nix-shell`. For example, the derivation attribute
shellHook =
''
echo "Hello shell"
export SOME_API_TOKEN="$(cat ~/.config/some-app/api-token)"
'';
```
will cause `nix-shell` to print `Hello shell`.
will cause `nix-shell` to print `Hello shell` and set the `SOME_API_TOKEN`
environment variable to a user-configured value.
# Options
@ -76,8 +78,8 @@ All options not listed here are passed to `nix-store
cleared before the interactive shell is started, so you get an
environment that more closely corresponds to the “real” Nix build. A
few variables, in particular `HOME`, `USER` and `DISPLAY`, are
retained. Note that `~/.bashrc` and (depending on your Bash
installation) `/etc/bashrc` are still sourced, so any variables set
retained. Note that (depending on your Bash
installation) `/etc/bashrc` is still sourced, so any variables set
there will affect the interactive shell.
- `--packages` / `-p` *packages*…

589
doc/manual/src/contributing/cli-guideline.md Normal file
View file

@ -0,0 +1,589 @@
# CLI guideline
## Goals
Purpose of this document is to provide a clear direction to **help design
delightful command line** experience. This document contain guidelines to
follow to ensure a consistent and approachable user experience.
## Overview
`nix` command provides a single entry to a number of sub-commands that help
**developers and system administrators** in the life-cycle of a software
project. We particularly need to pay special attention to help and assist new
users of Nix.
# Naming the `COMMANDS`
Words matter. Naming is an important part of the usability. Users will be
interacting with Nix on a regular basis so we should **name things for ease of
understanding**.
We recommend following the [Principle of Least
Astonishment](https://en.wikipedia.org/wiki/Principle_of_least_astonishment).
This means that you should **never use acronyms or abbreviations** unless they
are commonly used in other tools (e.g. `nix init`). And if the command name is
too long (> 10-12 characters) then shortening it makes sense (e.g.
“prioritization” → “priority”).
Commands should **follow a noun-verb dialogue**. Although noun-verb formatting
seems backwards from a speaking perspective (i.e. `nix store copy` vs. `nix
copy store`) it allows us to organize commands the same way users think about
completing an action (the group first, then the command).
## Naming rules
Rules are there to guide you by limiting your options. But not everything can
fit the rules all the time. In those cases document the exceptions in [Appendix
1: Commands naming exceptions](#appendix-1-commands-naming-exceptions) and
provide reason. The rules want to force a Nix developer to look, not just at
the command at hand, but also the command in a full context alongside other
`nix` commands.
```shell
$ nix [<GROUP>] <COMMAND> [<ARGUMENTS>] [<OPTIONS>]
```
- `GROUP`, `COMMAND`, `ARGUMENTS` and `OPTIONS` should be lowercase and in a
singular form.
- `GROUP` should be a **NOUN**.
- `COMMAND` should be a **VERB**.
- `ARGUMENTS` and `OPTIONS` are discussed in [*Input* section](#input).
## Classification
Some commands are more important, some less. While we want all of our commands
to be perfect we can only spend limited amount of time testing and improving
them.
This classification tries to separate commands in 3 categories in terms of
their importance in regards to the new users. Users who are likely to be
impacted the most by bad user experience.
- **Main commands**
Commands used for our main use cases and most likely used by new users. We
expect attention to details, such as:
- Proper use of [colors](#colors), [emojis](#special-unicode-characters)
and [aligning of text](#text-alignment).
- [Autocomplete](#shell-completion) of options.
- Show [next possible steps](#next-steps).
- Showing some [“tips”](#educate-the-user) when running logs running tasks
(eg. building / downloading) in order to teach users interesting bits of
Nix ecosystem.
- [Help pages](#help-is-essential) to be as good as we can write them
pointing to external documentation and tutorials for more.
Examples of such commands: `nix init`, `nix develop`, `nix build`, `nix run`,
...
- **Infrequently used commands**
From infrequently used commands we expect less attention to details, but
still some:
- Proper use of [colors](#colors), [emojis](#special-unicode-characters)
and [aligning of text](#text-alignment).
- [Autocomplete](#shell-completion) of options.
Examples of such commands: `nix doctor`, `nix edit`, `nix eval`, ...
- **Utility and scripting commands**
Commands that expose certain internal functionality of `nix`, mostly used by
other scripts.
- [Autocomplete](#shell-completion) of options.
Examples of such commands: `nix store copy`, `nix hash base16`, `nix store
ping`, ...
# Help is essential
Help should be built into your command line so that new users can gradually
discover new features when they need them.
## Looking for help
Since there is no standard way how user will look for help we rely on ways help
is provided by commonly used tools. As a guide for this we took `git` and
whenever in doubt look at it as a preferred direction.
The rules are:
- Help is shown by using `--help` or `help` command (eg `nix` `--``help` or
`nix help`).
- For non-COMMANDs (eg. `nix` `--``help` and `nix store` `--``help`) we **show
a summary** of most common use cases. Summary is presented on the STDOUT
without any use of PAGER.
- For COMMANDs (eg. `nix init` `--``help` or `nix help init`) we display the
man page of that command. By default the PAGER is used (as in `git`).
- At the end of either summary or man page there should be an URL pointing to
an online version of more detailed documentation.
- The structure of summaries and man pages should be the same as in `git`.
## Anticipate where help is needed
Even better then requiring the user to search for help is to anticipate and
predict when user might need it. Either because the lack of discoverability,
typo in the input or simply taking the opportunity to teach the user of
interesting - but less visible - details.
### Shell completion
This type of help is most common and almost expected by users. We need to
**provide the best shell completion** for `bash`, `zsh` and `fish`.
Completion needs to be **context aware**, this mean when a user types:
```shell
$ nix build n<TAB>
```
we need to display a list of flakes starting with `n`.
### Wrong input
As we all know we humans make mistakes, all the time. When a typo - intentional
or unintentional - is made, we should prompt for closest possible options or
point to the documentation which would educate user to not make the same
errors. Here are few examples:
In first example we prompt the user for typing wrong command name:
```shell
$ nix int
------------------------------------------------------------------------
Error! Command `int` not found.
------------------------------------------------------------------------
Did you mean:
|> nix init
|> nix input
```
Sometimes users will make mistake either because of a typo or simply because of
lack of discoverability. Our handling of this cases needs to be context
sensitive.
```shell
$ nix init --template=template#pyton
------------------------------------------------------------------------
Error! Template `template#pyton` not found.
------------------------------------------------------------------------
Initializing Nix project at `/path/to/here`.
Select a template for you new project:
|> template#pyton
template#python-pip
template#python-poetry
```
### Next steps
It can be invaluable to newcomers to show what a possible next steps and what
is the usual development workflow with Nix. For example:
```shell
$ nix init --template=template#python
Initializing project `template#python`
in `/home/USER/dev/new-project`
Next steps
|> nix develop -- to enter development environment
|> nix build -- to build your project
```
### Educate the user
We should take any opportunity to **educate users**, but at the same time we
must **be very very careful to not annoy users**. There is a thin line between
being helpful and being annoying.
An example of educating users might be to provide *Tips* in places where they
are waiting.
```shell
$ nix build
Started building my-project 1.2.3
Downloaded python3.8-poetry 1.2.3 in 5.3 seconds
Downloaded python3.8-requests 1.2.3 in 5.3 seconds
------------------------------------------------------------------------
Press `v` to increase logs verbosity
|> `?` to see other options
------------------------------------------------------------------------
Learn something new with every build...
|> See last logs of a build with `nix log --last` command.
------------------------------------------------------------------------
Evaluated my-project 1.2.3 in 14.43 seconds
Downloading [12 / 200]
|> firefox 1.2.3 [#########> ] 10Mb/s | 2min left
Building [2 / 20]
|> glibc 1.2.3 -> buildPhase: <last log line>
------------------------------------------------------------------------
```
Now **Learn** part of the output is where you educate users. You should only
show it when you know that a build will take some time and not annoy users of
the builds that take only few seconds.
Every feature like this should go though a intensive review and testing to
collect as much a feedback as possible and to fine tune every little detail. If
done right this can be an awesome features beginners and advance users will
love, but if not done perfectly it will annoy users and leave bad impression.
# Input
Input to a command is provided via `ARGUMENTS` and `OPTIONS`.
`ARGUMENTS` represent a required input for a function. When choosing to use
`ARGUMENT` over function please be aware of the downsides that come with it:
- User will need to remember the order of `ARGUMENTS`. This is not a problem if
there is only one `ARGUMENT`.
- With `OPTIONS` it is possible to provide much better auto completion.
- With `OPTIONS` it is possible to provide much better error message.
- Using `OPTIONS` it will mean there is a little bit more typing.
We don’t discourage the use of `ARGUMENTS`, but simply want to make every
developer consider the downsides and choose wisely.
## Naming the `OPTIONS`
Then only naming convention - apart from the ones mentioned in Naming the
`COMMANDS` section is how flags are named.
Flags are a type of `OPTION` that represent an option that can be turned ON of
OFF. We can say **flags are boolean type of** `**OPTION**`.
Here are few examples of flag `OPTIONS`:
- `--colors` vs. `--no-colors` (showing colors in the output)
- `--emojis` vs. `--no-emojis` (showing emojis in the output)
## Prompt when input not provided
For *main commands* (as [per classification](#classification)) we want command
to improve the discoverability of possible input. A new user will most likely
not know which `ARGUMENTS` and `OPTIONS` are required or which values are
possible for those options.
In cases, the user might not provide the input or they provide wrong input,
rather then show the error, prompt a user with an option to find and select
correct input (see examples).
Prompting is of course not required when TTY is not attached to STDIN. This
would mean that scripts wont need to handle prompt, but rather handle errors.
A place to use prompt and provide user with interactive select
```shell
$ nix init
Initializing Nix project at `/path/to/here`.
Select a template for you new project:
|> py
template#python-pip
template#python-poetry
[ Showing 2 templates from 1345 templates ]
```
Another great place to add prompts are **confirmation dialogues for dangerous
actions**. For example when adding new substitutor via `OPTIONS` or via
`flake.nix` we should prompt - for the first time - and let user review what is
going to happen.
```shell
$ nix build --option substitutors https://cache.example.org
------------------------------------------------------------------------
Warning! A security related question need to be answered.
------------------------------------------------------------------------
The following substitutors will be used to in `my-project`:
- https://cache.example.org
Do you allow `my-project` to use above mentioned substitutors?
[y/N] |> y
```
# Output
Terminal output can be quite limiting in many ways. Which should forces us to
think about the experience even more. As with every design the output is a
compromise between being terse and being verbose, between showing help to
beginners and annoying advance users. For this it is important that we know
what are the priorities.
Nix command line should be first and foremost written with beginners in mind.
But users wont stay beginners for long and what was once useful might quickly
become annoying. There is no golden rule that we can give in this guideline
that would make it easier how to draw a line and find best compromise.
What we would encourage is to **build prototypes**, do some **user testing**
and collect **feedback**. Then repeat the cycle few times.
First design the *happy path* and only after your iron it out, continue to work
on **edge cases** (handling and displaying errors, changes of the output by
certain `OPTIONS`, etc…)
## Follow best practices
Needless to say we Nix must be a good citizen and follow best practices in
command line.
In short: **STDOUT is for output, STDERR is for (human) messaging.**
STDOUT and STDERR provide a way for you to output messages to the user while
also allowing them to redirect content to a file. For example:
```shell
$ nix build > build.txt
------------------------------------------------------------------------
Error! Atrribute `bin` missing at (1:94) from string.
------------------------------------------------------------------------
1| with import <nixpkgs> { }; (pkgs.runCommandCC or pkgs.runCommand) "shell" { buildInputs = [ (surge.bin) ]; } ""
```
Because this warning is on STDERR, it doesn’t end up in the file.
But not everything on STDERR is an error though. For example, you can run `nix
build` and collect logs in a file while still seeing the progress.
```
$ nix build > build.txt
Evaluated 1234 files in 1.2 seconds
Downloaded python3.8-poetry 1.2.3 in 5.3 seconds
Downloaded python3.8-requests 1.2.3 in 5.3 seconds
------------------------------------------------------------------------
Press `v` to increase logs verbosity
|> `?` to see other options
------------------------------------------------------------------------
Learn something new with every build...
|> See last logs of a build with `nix log --last` command.
------------------------------------------------------------------------
Evaluated my-project 1.2.3 in 14.43 seconds
Downloading [12 / 200]
|> firefox 1.2.3 [#########> ] 10Mb/s | 2min left
Building [2 / 20]
|> glibc 1.2.3 -> buildPhase: <last log line>
------------------------------------------------------------------------
```
## Errors (WIP)
**TODO**: Once we have implementation for the *happy path* then we will think
how to present errors.
## Not only for humans
Terse, machine-readable output formats can also be useful but shouldn’t get in
the way of making beautiful CLI output. When needed, commands should offer a
`--json` flag to allow users to easily parse and script the CLI.
When TTY is not detected on STDOUT we should remove all design elements (no
colors, no emojis and using ASCII instead of Unicode symbols). The same should
happen when TTY is not detected on STDERR. We should not display progress /
status section, but only print warnings and errors.
## Dialog with the user
CLIs don't always make it clear when an action has taken place. For every
action a user performs, your CLI should provide an equal and appropriate
reaction, clearly highlighting the what just happened. For example:
```shell
$ nix build
Downloaded python3.8-poetry 1.2.3 in 5.3 seconds
Downloaded python3.8-requests 1.2.3 in 5.3 seconds
...
Success! You have successfully built my-project.
$
```
Above command clearly states that command successfully completed. And in case
of `nix build`, which is a command that might take some time to complete, it is
equally important to also show that a command started.
## Text alignment
Text alignment is the number one design element that will present all of the
Nix commands as a family and not as separate tools glued together.
The format we should follow is:
```shell
$ nix COMMAND
VERB_1 NOUN and other words
VERB__1 NOUN and other words
|> Some details
```
Few rules that we can extract from above example:
- Each line should start at least with one space.
- First word should be a VERB and must be aligned to the right.
- Second word should be a NOUN and must be aligned to the left.
- If you can not find a good VERB / NOUN pair, don’t worry make it as
understandable to the user as possible.
- More details of each line can be provided by `|>` character which is serving
as the first word when aligning the text
Don’t forget you should also test your terminal output with colors and emojis
off (`--no-colors --no-emojis`).
## Dim / Bright
After comparing few terminals with different color schemes we would **recommend
to avoid using dimmed text**. The difference from the rest of the text is very
little in many terminal and color scheme combinations. Sometimes the difference
is not even notable, therefore relying on it wouldn’t make much sense.
**The bright text is much better supported** across terminals and color
schemes. Most of the time the difference is perceived as if the bright text
would be bold.
## Colors
Humans are already conditioned by society to attach certain meaning to certain
colors. While the meaning is not universal, a simple collection of colors is
used to represent basic emotions.
Colors that can be used in output
- Red = error, danger, stop
- Green = success, good
- Yellow/Orange = proceed with caution, warning, in progress
- Blue/Magenta = stability, calm
While colors are nice, when command line is used by machines (in automation
scripts) you want to remove the colors. There should be a global `--no-colors`
option that would remove the colors.
## Special (Unicode) characters
Most of the terminal have good support for Unicode characters and you should
use them in your output by default. But always have a backup solution that is
implemented only with ASCII characters and will be used when `--ascii` option
is going to be passed in. Please make sure that you test your output also
without Unicode characters
More they showing all the different Unicode characters it is important to
**establish common set of characters** that we use for certain situations.
## Emojis
Emojis help channel emotions even better than text, colors and special
characters.
We recommend **keeping the set of emojis to a minimum**. This will enable each
emoji to stand out more.
As not everybody is happy about emojis we should provide an `--no-emojis`
option to disable them. Please make sure that you test your output also without
emojis.
## Tables
All commands that are listing certain data can be implemented in some sort of a
table. It’s important that each row of your output is a single ‘entry’ of data.
Never output table borders. It’s noisy and a huge pain for parsing using other
tools such as `grep`.
Be mindful of the screen width. Only show a few columns by default with the
table header, for more the table can be manipulated by the following options:
- `--no-headers`: Show column headers by default but allow to hide them.
- `--columns`: Comma-separated list of column names to add.
- `--sort`: Allow sorting by column. Allow inverse and multi-column sort as well.
## Interactive output
Interactive output was selected to be able to strike the balance between
beginners and advance users. While the default output will target beginners it
can, with a few key strokes, be changed into and advance introspection tool.
### Progress
For longer running commands we should provide and overview of the progress.
This is shown best in `nix build` example:
```shell
$ nix build
Started building my-project 1.2.3
Downloaded python3.8-poetry 1.2.3 in 5.3 seconds
Downloaded python3.8-requests 1.2.3 in 5.3 seconds
------------------------------------------------------------------------
Press `v` to increase logs verbosity
|> `?` to see other options
------------------------------------------------------------------------
Learn something new with every build...
|> See last logs of a build with `nix log --last` command.
------------------------------------------------------------------------
Evaluated my-project 1.2.3 in 14.43 seconds
Downloading [12 / 200]
|> firefox 1.2.3 [#########> ] 10Mb/s | 2min left
Building [2 / 20]
|> glibc 1.2.3 -> buildPhase: <last log line>
------------------------------------------------------------------------
```
### Search
Use a `fzf` like fuzzy search when there are multiple options to choose from.
```shell
$ nix init
Initializing Nix project at `/path/to/here`.
Select a template for you new project:
|> py
template#python-pip
template#python-poetry
[ Showing 2 templates from 1345 templates ]
```
### Prompt
In some situations we need to prompt the user and inform the user about what is
going to happen.
```shell
$ nix build --option substitutors https://cache.example.org
------------------------------------------------------------------------
Warning! A security related question need to be answered.
------------------------------------------------------------------------
The following substitutors will be used to in `my-project`:
- https://cache.example.org
Do you allow `my-project` to use above mentioned substitutors?
[y/N] |> y
```
## Verbosity
There are many ways that you can control verbosity.
Verbosity levels are:
- `ERROR` (level 0)
- `WARN` (level 1)
- `NOTICE` (level 2)
- `INFO` (level 3)
- `TALKATIVE` (level 4)
- `CHATTY` (level 5)
- `DEBUG` (level 6)
- `VOMIT` (level 7)
The default level that the command starts is `ERROR`. The simplest way to
increase the verbosity by stacking `-v` option (eg: `-vvv == level 3 == INFO`).
There are also two shortcuts, `--debug` to run in `DEBUG` verbosity level and
`--quiet` to run in `ERROR` verbosity level.
----------
# Appendix 1: Commands naming exceptions
`nix init` and `nix repl` are well established

1
doc/manual/src/contributing/contributing.md Normal file
View file

@ -0,0 +1 @@
# Contributing

0
doc/manual/src/hacking.md → doc/manual/src/contributing/hacking.md
View file

14
doc/manual/src/installation/installing-binary.md
View file

@ -195,7 +195,7 @@ If you are comfortable navigating these tradeoffs, you can encrypt the
volume with something along the lines of:
```console
alice$ diskutil apfs enableFileVault /nix -user disk
$ diskutil apfs enableFileVault /nix -user disk
```
## Symlink the Nix store to a custom location
@ -234,13 +234,13 @@ as a helpful reference if you run into trouble.
without a reboot:
```console
alice$ /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B
$ /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B
```
3. Create the new APFS volume with diskutil:
```console
alice$ sudo diskutil apfs addVolume diskX APFS 'Nix Store' -mountpoint /nix
$ sudo diskutil apfs addVolume diskX APFS 'Nix Store' -mountpoint /nix
```
4. Using `vifs`, add the new mount to `/etc/fstab`. If it doesn't
@ -280,10 +280,10 @@ it somewhere (e.g. in `/tmp`), and then run the script named `install`
inside the binary tarball:
```console
alice$ cd /tmp
alice$ tar xfj nix-1.8-x86_64-darwin.tar.bz2
alice$ cd nix-1.8-x86_64-darwin
alice$ ./install
$ cd /tmp
$ tar xfj nix-1.8-x86_64-darwin.tar.bz2
$ cd nix-1.8-x86_64-darwin
$ ./install
```
If you need to edit the multi-user installation script to use different

8
flake.lock
View file

@ -18,16 +18,16 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1591633336,
"narHash": "sha256-oVXv4xAnDJB03LvZGbC72vSVlIbbJr8tpjEW5o/Fdek=",
"lastModified": 1602702596,
"narHash": "sha256-fqJ4UgOb4ZUnCDIapDb4gCrtAah5Rnr2/At3IzMitig=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "70717a337f7ae4e486ba71a500367cad697e5f09",
"rev": "ad0d20345219790533ebe06571f82ed6b034db31",
"type": "github"
},
"original": {
"id": "nixpkgs",
"ref": "nixos-20.03-small",
"ref": "nixos-20.09-small",
"type": "indirect"
}
},

179
flake.nix
View file

@ -1,7 +1,7 @@
{
description = "The purely functional package manager";
inputs.nixpkgs.url = "nixpkgs/nixos-20.03-small";
inputs.nixpkgs.url = "nixpkgs/nixos-20.09-small";
inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; };
outputs = { self, nixpkgs, lowdown-src }:
@ -12,11 +12,13 @@
versionSuffix =
if officialRelease
then ""
else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified)}_${self.shortRev or "dirty"}";
else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}";
officialRelease = false;
systems = [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ];
linux64BitSystems = [ "x86_64-linux" "aarch64-linux" ];
linuxSystems = linux64BitSystems ++ [ "i686-linux" ];
systems = linuxSystems ++ [ "x86_64-darwin" ];
forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system);
@ -61,30 +63,37 @@
"LDFLAGS=-fuse-ld=gold"
];
buildDeps =
[ bison
flex
mdbook
lowdown
autoconf-archive
autoreconfHook
curl
nativeBuildDeps =
[
buildPackages.bison
buildPackages.flex
(lib.getBin buildPackages.lowdown)
buildPackages.mdbook
buildPackages.autoconf-archive
buildPackages.autoreconfHook
buildPackages.pkgconfig
# Tests
buildPackages.git
buildPackages.mercurial
buildPackages.jq
];
buildDeps =
[ curl
bzip2 xz brotli zlib editline
openssl pkgconfig sqlite
openssl sqlite
libarchive
boost
nlohmann_json
# Tests
git
mercurial
jq
lowdown
gmock
]
++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal]
++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
++ lib.optional (stdenv.isLinux || stdenv.isDarwin)
++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium;
awsDeps = lib.optional (stdenv.isLinux || stdenv.isDarwin)
(aws-sdk-cpp.override {
apis = ["s3" "transfer"];
customMemoryManagement = false;
@ -108,6 +117,7 @@
nix = with final; with commonDeps pkgs; (stdenv.mkDerivation {
name = "nix-${version}";
inherit version;
src = self;
@ -115,7 +125,8 @@
outputs = [ "out" "dev" "doc" ];
buildInputs = buildDeps;
nativeBuildInputs = nativeBuildDeps;
buildInputs = buildDeps ++ awsDeps;
propagatedBuildInputs = propagatedDeps;
@ -159,14 +170,17 @@
src = self;
nativeBuildInputs =
[ buildPackages.autoconf-archive
buildPackages.autoreconfHook
buildPackages.pkgconfig
];
buildInputs =
[ autoconf-archive
autoreconfHook
nix
[ nix
curl
bzip2
xz
pkgconfig
pkgs.perl
boost
nlohmann_json
@ -197,15 +211,15 @@
src = lowdown-src;
outputs = [ "out" "dev" ];
outputs = [ "out" "bin" "dev" ];
buildInputs = [ which ];
nativeBuildInputs = [ which ];
configurePhase =
''
./configure \
PREFIX=${placeholder "dev"} \
BINDIR=${placeholder "out"}/bin
BINDIR=${placeholder "bin"}/bin
'';
};
@ -214,10 +228,12 @@
hydraJobs = {
# Binary package for various platforms.
build = nixpkgs.lib.genAttrs systems (system: nixpkgsFor.${system}.nix);
build = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix);
buildStatic = nixpkgs.lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static);
# Perl bindings for various platforms.
perlBindings = nixpkgs.lib.genAttrs systems (system: nixpkgsFor.${system}.nix.perl-bindings);
perlBindings = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix.perl-bindings);
# Binary tarball for various platforms, containing a Nix store
# with the closure of 'nix' package, and the second half of
@ -236,6 +252,7 @@
}
''
cp ${installerClosureInfo}/registration $TMPDIR/reginfo
cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh
substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \
--subst-var-by nix ${nix} \
--subst-var-by cacert ${cacert}
@ -254,6 +271,7 @@
# SC1090: Don't worry about not being able to find
# $nix/etc/profile.d/nix.sh
shellcheck --exclude SC1090 $TMPDIR/install
shellcheck $TMPDIR/create-darwin-volume.sh
shellcheck $TMPDIR/install-darwin-multi-user.sh
shellcheck $TMPDIR/install-systemd-multi-user.sh
@ -269,6 +287,7 @@
fi
chmod +x $TMPDIR/install
chmod +x $TMPDIR/create-darwin-volume.sh
chmod +x $TMPDIR/install-darwin-multi-user.sh
chmod +x $TMPDIR/install-systemd-multi-user.sh
chmod +x $TMPDIR/install-multi-user
@ -281,11 +300,15 @@
--absolute-names \
--hard-dereference \
--transform "s,$TMPDIR/install,$dir/install," \
--transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \
--transform "s,$TMPDIR/reginfo,$dir/.reginfo," \
--transform "s,$NIX_STORE,$dir/store,S" \
$TMPDIR/install $TMPDIR/install-darwin-multi-user.sh \
$TMPDIR/install \
$TMPDIR/create-darwin-volume.sh \
$TMPDIR/install-darwin-multi-user.sh \
$TMPDIR/install-systemd-multi-user.sh \
$TMPDIR/install-multi-user $TMPDIR/reginfo \
$TMPDIR/install-multi-user \
$TMPDIR/reginfo \
$(cat ${installerClosureInfo}/store-paths)
'');
@ -301,9 +324,26 @@
''
mkdir -p $out/nix-support
# Converts /nix/store/50p3qk8kka9dl6wyq40vydq945k0j3kv-nix-2.4pre20201102_550e11f/bin/nix
# To 50p3qk8kka9dl6wyq40vydq945k0j3kv/bin/nix
tarballPath() {
# Remove the store prefix
local path=''${1#${builtins.storeDir}/}
# Get the path relative to the derivation root
local rest=''${path#*/}
# Get the derivation hash
local drvHash=''${path%%-*}
echo "$drvHash/$rest"
}
substitute ${./scripts/install.in} $out/install \
${pkgs.lib.concatMapStrings
(system: "--replace '@binaryTarball_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) ")
(system:
'' \
--replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \
--replace '@tarballPath_${system}@' $(tarballPath ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \
''
)
[ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
} \
--replace '@nixVersion@' ${version}
@ -323,7 +363,8 @@
enableParallelBuilding = true;
buildInputs = buildDeps ++ propagatedDeps;
nativeBuildInputs = nativeBuildDeps;
buildInputs = buildDeps ++ propagatedDeps ++ awsDeps;
dontInstall = false;
@ -364,38 +405,6 @@
inherit (self) overlay;
});
# Test whether the binary tarball works in an Ubuntu system.
tests.binaryTarball =
with nixpkgsFor.x86_64-linux;
vmTools.runInLinuxImage (runCommand "nix-binary-tarball-test"
{ diskImage = vmTools.diskImages.ubuntu1204x86_64;
}
''
set -x
useradd -m alice
su - alice -c 'tar xf ${self.hydraJobs.binaryTarball.x86_64-linux}/*.tar.*'
mkdir /dest-nix
mount -o bind /dest-nix /nix # Provide a writable /nix.
chown alice /nix
su - alice -c '_NIX_INSTALLER_TEST=1 ./nix-*/install'
su - alice -c 'nix-store --verify'
su - alice -c 'PAGER= nix-store -qR ${self.hydraJobs.build.x86_64-linux}'
# Check whether 'nix upgrade-nix' works.
cat > /tmp/paths.nix <<EOF
{
x86_64-linux = "${self.hydraJobs.build.x86_64-linux}";
}
EOF
su - alice -c 'nix --experimental-features nix-command upgrade-nix -vvv --nix-store-paths-url file:///tmp/paths.nix'
(! [ -L /home/alice/.profile-1-link ])
su - alice -c 'PAGER= nix-store -qR ${self.hydraJobs.build.x86_64-linux}'
mkdir -p $out/nix-support
touch $out/nix-support/hydra-build-products
umount /nix
'');
/*
# Check whether we can still evaluate all of Nixpkgs.
tests.evalNixpkgs =
@ -429,6 +438,43 @@
packages = forAllSystems (system: {
inherit (nixpkgsFor.${system}) nix;
} // nixpkgs.lib.optionalAttrs (builtins.elem system linux64BitSystems) {
nix-static = let
nixpkgs = nixpkgsFor.${system}.pkgsStatic;
in with commonDeps nixpkgs; nixpkgs.stdenv.mkDerivation {
name = "nix-${version}";
src = self;
VERSION_SUFFIX = versionSuffix;
outputs = [ "out" "dev" "doc" ];
nativeBuildInputs = nativeBuildDeps;
buildInputs = buildDeps ++ propagatedDeps;
configureFlags = [ "--sysconfdir=/etc" ];
enableParallelBuilding = true;
makeFlags = "profiledir=$(out)/etc/profile.d";
doCheck = true;
installFlags = "sysconfdir=$(out)/etc";
postInstall = ''
mkdir -p $doc/nix-support
echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
mkdir -p $out/nix-support
echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products
'';
doInstallCheck = true;
installCheckFlags = "sysconfdir=$(out)/etc";
stripAllList = ["bin"];
};
});
defaultPackage = forAllSystems (system: self.packages.${system}.nix);
@ -442,7 +488,8 @@
outputs = [ "out" "dev" "doc" ];
buildInputs = buildDeps ++ propagatedDeps ++ perlDeps;
nativeBuildInputs = nativeBuildDeps;
buildInputs = buildDeps ++ propagatedDeps ++ awsDeps ++ perlDeps;
inherit configureFlags;

6
local.mk
View file

@ -1,9 +1,3 @@
ifeq ($(MAKECMDGOALS), dist)
dist-files += $(shell cat .dist-files)
endif
dist-files += configure config.h.in perl/configure
clean-files += Makefile.config
GLOBAL_CXXFLAGS += -Wno-deprecated-declarations

2
misc/systemd/local.mk
View file

@ -2,4 +2,6 @@ ifeq ($(OS), Linux)
$(foreach n, nix-daemon.socket nix-daemon.service, $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/systemd/system, 0644)))
clean-files += $(d)/nix-daemon.socket $(d)/nix-daemon.service
endif

2
misc/upstart/local.mk
View file

@ -2,4 +2,6 @@ ifeq ($(OS), Linux)
$(foreach n, nix-daemon.conf, $(eval $(call install-file-in, $(d)/$(n), $(sysconfdir)/init, 0644)))
clean-files += $(d)/nix-daemon.conf
endif

17
mk/dist.mk
View file

@ -1,17 +0,0 @@
ifdef PACKAGE_NAME
dist-name = $(PACKAGE_NAME)-$(PACKAGE_VERSION)
dist: $(dist-name).tar.bz2 $(dist-name).tar.xz
$(dist-name).tar.bz2: $(dist-files)
$(trace-gen) tar cfj $@ $(sort $(dist-files)) --transform 's,^,$(dist-name)/,'
$(dist-name).tar.xz: $(dist-files)
$(trace-gen) tar cfJ $@ $(sort $(dist-files)) --transform 's,^,$(dist-name)/,'
clean-files += $(dist-name).tar.bz2 $(dist-name).tar.xz
print-top-help += echo " dist: Generate a source distribution";
endif

4
mk/lib.mk
View file

@ -10,7 +10,6 @@ bin-scripts :=
noinst-scripts :=
man-pages :=
install-tests :=
dist-files :=
OS = $(shell uname -s)
@ -112,9 +111,6 @@ $(foreach test, $(install-tests), $(eval $(call run-install-test,$(test))))
$(foreach file, $(man-pages), $(eval $(call install-data-in, $(file), $(mandir)/man$(patsubst .%,%,$(suffix $(file))))))
include mk/dist.mk
.PHONY: default all man help
all: $(programs-list) $(libs-list) $(jars-list) $(man-pages)

1
mk/libraries.mk
View file

@ -159,5 +159,4 @@ define build-library
libs-list += $$($(1)_PATH)
endif
clean-files += $$(_d)/*.a $$(_d)/*.$(SO_EXT) $$(_d)/*.o $$(_d)/.*.dep $$($(1)_DEPS) $$($(1)_OBJS)
dist-files += $$(_srcs)
endef

1
mk/programs.mk
View file

@ -79,7 +79,6 @@ define build-program
programs-list += $$($(1)_PATH)
clean-files += $$($(1)_PATH) $$(_d)/*.o $$(_d)/.*.dep $$($(1)_DEPS) $$($(1)_OBJS)
dist-files += $$(_srcs)
# Phony target to run this program (typically as a dependency of 'check').
.PHONY: $(1)_RUN

2
nix-rust/local.mk
View file

@ -30,8 +30,6 @@ ifeq ($(OS), Darwin)
install_name_tool -id $@ $@
endif
dist-files += $(d)/vendor
clean: clean-rust
clean-rust:

10
perl/lib/Nix/Store.xs
View file

@ -110,10 +110,14 @@ SV * queryPathInfo(char * path, int base32)
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
mXPUSHi(info->registrationTime);
mXPUSHi(info->narSize);
AV * arr = newAV();
AV * refs = newAV();
for (auto & i : info->references)
av_push(arr, newSVpv(store()->printStorePath(i).c_str(), 0));
XPUSHs(sv_2mortal(newRV((SV *) arr)));
av_push(refs, newSVpv(store()->printStorePath(i).c_str(), 0));
XPUSHs(sv_2mortal(newRV((SV *) refs)));
AV * sigs = newAV();
for (auto & i : info->sigs)
av_push(sigs, newSVpv(i.c_str(), 0));
XPUSHs(sv_2mortal(newRV((SV *) sigs)));
} catch (Error & e) {
croak("%s", e.what());
}

62
scripts/create-darwin-volume.sh
View file

@ -5,42 +5,13 @@ root_disk() {
diskutil info -plist /
}
apfs_volumes_for() {
disk=$1
diskutil apfs list -plist "$disk"
}
disk_identifier() {
xpath "/plist/dict/key[text()='ParentWholeDisk']/following-sibling::string[1]/text()" 2>/dev/null
}
volume_list_true() {
key=$1
xpath "/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict/key[text()='$key']/following-sibling::true[1]" 2> /dev/null
}
volume_get_string() {
key=$1 i=$2
xpath "/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict[$i]/key[text()='$key']/following-sibling::string[1]/text()" 2> /dev/null
# i.e., "disk1"
root_disk_identifier() {
diskutil info -plist / | xmllint --xpath "/plist/dict/key[text()='ParentWholeDisk']/following-sibling::string[1]/text()" -
}
find_nix_volume() {
disk=$1
i=1
volumes=$(apfs_volumes_for "$disk")
while true; do
name=$(echo "$volumes" | volume_get_string "Name" "$i")
if [ -z "$name" ]; then
break
fi
case "$name" in
[Nn]ix*)
echo "$name"
break
;;
esac
i=$((i+1))
done
diskutil apfs list -plist "$1" | xmllint --xpath "(/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict/key[text()='Name']/following-sibling::string[starts-with(translate(text(),'N','n'),'nix')]/text())[1]" - 2>/dev/null || true
}
test_fstab() {
@ -55,6 +26,20 @@ test_synthetic_conf() {
grep -q "^nix$" /etc/synthetic.conf 2>/dev/null
}
# Create the paths defined in synthetic.conf, saving us a reboot.
create_synthetic_objects(){
# Big Sur takes away the -B flag we were using and replaces it
# with a -t flag that appears to do the same thing (but they
# don't behave exactly the same way in terms of return values).
# This feels a little dirty, but as far as I can tell the
# simplest way to get the right one is to just throw away stderr
# and call both... :]
{
/System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -t || true # Big Sur
/System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B || true # Catalina
} >/dev/null 2>&1
}
test_nix() {
test -d "/nix"
}
@ -89,9 +74,7 @@ test_t2_chip_present(){
}
test_filevault_in_use() {
disk=$1
# list vols on disk | get value of Filevault key | value is true
apfs_volumes_for "$disk" | volume_list_true FileVault | grep -q true
fdesetup isactive >/dev/null
}
# use after error msg for conditions we don't understand
@ -132,7 +115,7 @@ main() {
if ! test_nix; then
echo "Creating mountpoint for /nix..." >&2
/System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B || true
create_synthetic_objects # the ones we defined in synthetic.conf
if ! test_nix; then
sudo mkdir -p /nix 2>/dev/null || true
fi
@ -143,12 +126,12 @@ main() {
fi
fi
disk=$(root_disk | disk_identifier)
disk="$(root_disk_identifier)"
volume=$(find_nix_volume "$disk")
if [ -z "$volume" ]; then
echo "Creating a Nix Store volume..." >&2
if test_filevault_in_use "$disk"; then
if test_filevault_in_use; then
# TODO: Not sure if it's in-scope now, but `diskutil apfs list`
# shows both filevault and encrypted at rest status, and it
# may be the more semantic way to test for this? It'll show
@ -178,6 +161,7 @@ main() {
if ! test_fstab; then
echo "Configuring /etc/fstab..." >&2
label=$(echo "$volume" | sed 's/ /\\040/g')
# shellcheck disable=SC2209
printf "\$a\nLABEL=%s /nix apfs rw,nobrowse\n.\nwq\n" "$label" | EDITOR=ed sudo vifs
fi
}

7
scripts/install-darwin-multi-user.sh
View file

@ -37,6 +37,13 @@ poly_service_setup_note() {
EOF
}
poly_extra_try_me_commands(){
:
}
poly_extra_setup_instructions(){
:
}
poly_configure_nix_daemon_service() {
_sudo "to set up the nix-daemon as a LaunchDaemon" \
cp -f "/nix/var/nix/profiles/default$PLIST_DEST" "$PLIST_DEST"

52
scripts/install-multi-user.sh
View file

@ -71,11 +71,9 @@ uninstall_directions() {
subheader "Uninstalling nix:"
local step=0
if [ -e /run/systemd/system ] && poly_service_installed_check; then
if poly_service_installed_check; then
step=$((step + 1))
poly_service_uninstall_directions "$step"
else
step=$((step + 1))
fi
for profile_target in "${PROFILE_TARGETS[@]}"; do
@ -255,40 +253,20 @@ function finish_success {
echo "To try again later, run \"sudo -i nix-channel --update nixpkgs\"."
fi
if [ -e /run/systemd/system ]; then
cat <<EOF
Before Nix will work in your existing shells, you'll need to close
them and open them again. Other than that, you should be ready to go.
Try it! Open a new terminal, and type:
$(poly_extra_try_me_commands)
$ nix-shell -p nix-info --run "nix-info -m"
$(poly_extra_setup_instructions)
Thank you for using this installer. If you have any feedback, don't
hesitate:
$(contactme)
EOF
else
cat <<EOF
Before Nix will work in your existing shells, you'll need to close
them and open them again. Other than that, you should be ready to go.
Try it! Open a new terminal, and type:
$ sudo nix-daemon
$ nix-shell -p nix-info --run "nix-info -m"
Additionally, you may want to add nix-daemon to your init-system.
Thank you for using this installer. If you have any feedback, don't
hesitate:
$(contactme)
EOF
fi
}
@ -630,24 +608,20 @@ EOF
}
configure_shell_profile() {
# If there is an /etc/profile.d directory, we want to ensure there
# is a nix.sh within it, so we can use the following loop to add
# the source lines to it. Note that I'm _not_ adding the source
# lines here, because we want to be using the regular machinery.
#
# If we go around that machinery, it becomes more complicated and
# adds complications to the uninstall instruction generator and
# old instruction sniffer as well.
if [ -d /etc/profile.d ]; then
_sudo "create a stub /etc/profile.d/nix.sh which will be updated" \
touch /etc/profile.d/nix.sh
fi
for profile_target in "${PROFILE_TARGETS[@]}"; do
if [ -e "$profile_target" ]; then
_sudo "to back up your current $profile_target to $profile_target$PROFILE_BACKUP_SUFFIX" \
cp "$profile_target" "$profile_target$PROFILE_BACKUP_SUFFIX"
else
# try to create the file if its directory exists
target_dir="$(dirname "$profile_target")"
if [ -d "$target_dir" ]; then
_sudo "to create a stub $profile_target which will be updated" \
touch "$profile_target"
fi
fi
if [ -e "$profile_target" ]; then
shell_source_lines \
| _sudo "extend your $profile_target with nix-daemon settings" \
tee -a "$profile_target"
@ -725,9 +699,7 @@ main() {
setup_default_profile
place_nix_configuration
if [ -e /run/systemd/system ]; then
poly_configure_nix_daemon_service
fi
trap finish_success EXIT
}

26
scripts/install-nix-from-closure.sh
View file

@ -26,9 +26,20 @@ fi
# macOS support for 10.12.6 or higher
if [ "$(uname -s)" = "Darwin" ]; then
macos_major=$(sw_vers -productVersion | cut -d '.' -f 2)
macos_minor=$(sw_vers -productVersion | cut -d '.' -f 3)
if [ "$macos_major" -lt 12 ] || { [ "$macos_major" -eq 12 ] && [ "$macos_minor" -lt 6 ]; }; then
IFS='.' read macos_major macos_minor macos_patch << EOF
$(sw_vers -productVersion)
EOF
# TODO: this is a temporary speed-bump to keep people from naively installing Nix
# on macOS Big Sur (11.0+, 10.16+) until nixpkgs updates are ready for them.
# *Ideally* this is gone before next Nix release. If you're intentionally working on
# Nix + Big Sur, just comment out this block and be on your way :)
if [ "$macos_major" -gt 10 ] || { [ "$macos_major" -eq 10 ] && [ "$macos_minor" -gt 15 ]; }; then
echo "$0: nixpkgs isn't quite ready to support macOS $(sw_vers -productVersion) yet"
exit 1
fi
if [ "$macos_major" -lt 10 ] || { [ "$macos_major" -eq 10 ] && [ "$macos_minor" -lt 12 ]; } || { [ "$macos_minor" -eq 12 ] && [ "$macos_patch" -lt 6 ]; }; then
# patch may not be present; command substitution for simplicity
echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.12.6 or higher"
exit 1
fi
@ -87,10 +98,13 @@ while [ $# -gt 0 ]; do
echo ""
echo " --nix-extra-conf-file: Path to nix.conf to prepend when installing /etc/nix.conf"
echo ""
if [ -n "${INVOKED_FROM_INSTALL_IN:-}" ]; then
echo " --tarball-url-prefix URL: Base URL to download the Nix tarball from."
fi
) >&2
# darwin and Catalina+
if [ "$(uname -s)" = "Darwin" ] && [ "$macos_major" -gt 14 ]; then
if [ "$(uname -s)" = "Darwin" ] && { [ "$macos_major" -gt 10 ] || { [ "$macos_major" -eq 10 ] && [ "$macos_minor" -gt 14 ]; }; }; then
(
echo " --darwin-use-unencrypted-nix-store-volume: Create an APFS volume for the Nix"
echo " store and mount it at /nix. This is the recommended way to create"
@ -110,8 +124,8 @@ if [ "$(uname -s)" = "Darwin" ]; then
"$self/create-darwin-volume.sh"
fi
info=$(diskutil info -plist / | xpath "/plist/dict/key[text()='Writable']/following-sibling::true[1]" 2> /dev/null)
if ! [ -e $dest ] && [ -n "$info" ] && [ "$macos_major" -gt 14 ]; then
writable="$(diskutil info -plist / | xmllint --xpath "name(/plist/dict/key[text()='Writable']/following-sibling::*[1])" -)"
if ! [ -e $dest ] && [ "$writable" = "false" ]; then
(
echo ""
echo "Installing on macOS >=10.15 requires relocating the store to an apfs volume."

23
scripts/install-systemd-multi-user.sh
View file

@ -72,7 +72,28 @@ poly_service_setup_note() {
EOF
}
poly_extra_try_me_commands(){
if [ -e /run/systemd/system ]; then
:
else
cat <<EOF
$ sudo nix-daemon
EOF
fi
}
poly_extra_setup_instructions(){
if [ -e /run/systemd/system ]; then
:
else
cat <<EOF
Additionally, you may want to add nix-daemon to your init-system.
EOF
fi
}
poly_configure_nix_daemon_service() {
if [ -e /run/systemd/system ]; then
_sudo "to set up the nix-daemon service" \
systemctl link "/nix/var/nix/profiles/default$SERVICE_SRC"
@ -89,7 +110,7 @@ poly_configure_nix_daemon_service() {
_sudo "to start the nix-daemon.service" \
systemctl restart nix-daemon.service
fi
}
poly_group_exists() {

49
scripts/install.in Normal file → Executable file
View file

@ -25,16 +25,52 @@ require_util() {
}
case "$(uname -s).$(uname -m)" in
Linux.x86_64) system=x86_64-linux; hash=@binaryTarball_x86_64-linux@;;
Linux.i?86) system=i686-linux; hash=@binaryTarball_i686-linux@;;
Linux.aarch64) system=aarch64-linux; hash=@binaryTarball_aarch64-linux@;;
Darwin.x86_64) system=x86_64-darwin; hash=@binaryTarball_x86_64-darwin@;;
Linux.x86_64)
hash=@tarballHash_x86_64-linux@
path=@tarballPath_x86_64-linux@
system=x86_64-linux
;;
Linux.i?86)
hash=@tarballHash_i686-linux@
path=@tarballPath_i686-linux@
system=i686-linux
;;
Linux.aarch64)
hash=@tarballHash_aarch64-linux@
path=@tarballPath_aarch64-linux@
system=aarch64-linux
;;
Darwin.x86_64)
hash=@tarballHash_x86_64-darwin@
path=@tarballPath_x86_64-darwin@
system=x86_64-darwin
;;
Darwin.arm64|Darwin.aarch64)
# check for Rosetta 2 support
if ! [ -f /Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist ]; then
oops "Rosetta 2 is not installed on this ARM64 macOS machine. Run softwareupdate --install-rosetta then restart installation"
fi
hash=@binaryTarball_x86_64-darwin@
path=@tarballPath_x86_64-darwin@
# eventually maybe: aarch64-darwin
system=x86_64-darwin
;;
*) oops "sorry, there is no binary distribution of Nix for your platform";;
esac
url="https://releases.nixos.org/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.xz"
# Use this command-line option to fetch the tarballs using nar-serve or Cachix
if "${1:---tarball-url-prefix}"; then
if [ -z "${2:-}" ]; then
oops "missing argument for --tarball-url-prefix"
fi
url=${2}/${path}
shift 2
else
url=https://releases.nixos.org/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.xz
fi
tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.xz")"
tarball=$tmpDir/nix-@nixVersion@-$system.tar.xz
require_util curl "download the binary tarball"
require_util tar "unpack the binary tarball"
@ -66,6 +102,7 @@ tar -xJf "$tarball" -C "$unpack" || oops "failed to unpack '$url'"
script=$(echo "$unpack"/*/install)
[ -e "$script" ] || oops "installation script is missing from the binary tarball!"
export INVOKED_FROM_INSTALL_IN=1
"$script" "$@"
} # End of wrapping

10
scripts/nix-profile-daemon.sh.in
View file

@ -17,11 +17,21 @@ elif [ -e /etc/pki/tls/certs/ca-bundle.crt ]; then # Fedora, CentOS
export NIX_SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt
else
# Fall back to what is in the nix profiles, favouring whatever is defined last.
check_nix_profiles() {
if [ "$ZSH_VERSION" ]; then
# Zsh by default doesn't split words in unquoted parameter expansion.
# Set local_options for these options to be reverted at the end of the function
# and shwordsplit to force splitting words in $NIX_PROFILES below.
setopt local_options shwordsplit
fi
for i in $NIX_PROFILES; do
if [ -e $i/etc/ssl/certs/ca-bundle.crt ]; then
export NIX_SSL_CERT_FILE=$i/etc/ssl/certs/ca-bundle.crt
fi
done
}
check_nix_profiles
unset -f check_nix_profiles
fi
export PATH="$HOME/.nix-profile/bin:@localstatedir@/nix/profiles/default/bin:$PATH"

2
src/build-remote/build-remote.cc
View file

@ -201,7 +201,7 @@ static int main_build_remote(int argc, char * * argv)
% concatStringsSep<StringSet>(", ", m.mandatoryFeatures);
}
logErrorInfo(lvlInfo, {
logErrorInfo(canBuildLocally ? lvlChatty : lvlWarn, {
.name = "Remote build",
.description = "Failed to find a machine for remote build!",
.hint = hint

9
src/libexpr/eval-cache.cc
View file

@ -525,8 +525,17 @@ string_t AttrCursor::getStringWithContext()
cachedValue = root->db->getAttr(getKey(), root->state.symbols);
if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
if (auto s = std::get_if<string_t>(&cachedValue->second)) {
bool valid = true;
for (auto & c : s->second) {
if (!root->state.store->isValidPath(root->state.store->parseStorePath(c.first))) {
valid = false;
break;
}
}
if (valid) {
debug("using cached string attribute '%s'", getAttrPathStr());
return *s;
}
} else
throw TypeError("'%s' is not a string", getAttrPathStr());
}

51
src/libexpr/eval.cc
View file

@ -27,6 +27,10 @@
#include <gc/gc.h>
#include <gc/gc_cpp.h>
#include <boost/coroutine2/coroutine.hpp>
#include <boost/coroutine2/protected_fixedsize_stack.hpp>
#include <boost/context/stack_context.hpp>
#endif
namespace nix {
@ -208,7 +212,8 @@ bool Value::isTrivial() const
&& (type != tThunk
|| (dynamic_cast<ExprAttrs *>(thunk.expr)
&& ((ExprAttrs *) thunk.expr)->dynamicAttrs.empty())
|| dynamic_cast<ExprLambda *>(thunk.expr));
|| dynamic_cast<ExprLambda *>(thunk.expr)
|| dynamic_cast<ExprList *>(thunk.expr));
}
@ -219,6 +224,31 @@ static void * oomHandler(size_t requested)
/* Convert this to a proper C++ exception. */
throw std::bad_alloc();
}
class BoehmGCStackAllocator : public StackAllocator {
boost::coroutines2::protected_fixedsize_stack stack {
// We allocate 8 MB, the default max stack size on NixOS.
// A smaller stack might be quicker to allocate but reduces the stack
// depth available for source filter expressions etc.
std::max(boost::context::stack_traits::default_size(), static_cast<std::size_t>(8 * 1024 * 1024))
};
public:
boost::context::stack_context allocate() override {
auto sctx = stack.allocate();
GC_add_roots(static_cast<char *>(sctx.sp) - sctx.size, sctx.sp);
return sctx;
}
void deallocate(boost::context::stack_context sctx) override {
GC_remove_roots(static_cast<char *>(sctx.sp) - sctx.size, sctx.sp);
stack.deallocate(sctx);
}
};
static BoehmGCStackAllocator boehmGCStackAllocator;
#endif
@ -256,6 +286,8 @@ void initGC()
GC_set_oom_fn(oomHandler);
StackAllocator::defaultAllocator = &boehmGCStackAllocator;
/* Set the initial heap size to something fairly big (25% of
physical RAM, up to a maximum of 384 MiB) so that in most cases
we don't need to garbage collect at all. (Collection has a
@ -1404,7 +1436,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v)
if (!state.evalBool(env, cond, pos)) {
std::ostringstream out;
cond->show(out);
throwAssertionError(pos, "assertion '%1%' failed at %2%", out.str());
throwAssertionError(pos, "assertion '%1%' failed", out.str());
}
body->eval(state, env, v);
}
@ -2072,10 +2104,19 @@ EvalSettings::EvalSettings()
Strings EvalSettings::getDefaultNixPath()
{
Strings res;
auto add = [&](const Path & p) { if (pathExists(p)) { res.push_back(p); } };
auto add = [&](const Path & p, const std::string & s = std::string()) {
if (pathExists(p)) {
if (s.empty()) {
res.push_back(p);
} else {
res.push_back(s + "=" + p);
}
}
};
add(getHome() + "/.nix-defexpr/channels");
add("nixpkgs=" + settings.nixStateDir + "/nix/profiles/per-user/root/channels/nixpkgs");
add(settings.nixStateDir + "/nix/profiles/per-user/root/channels");
add(settings.nixStateDir + "/profiles/per-user/root/channels/nixpkgs", "nixpkgs");
add(settings.nixStateDir + "/profiles/per-user/root/channels");
return res;
}

81
src/libexpr/flake/config.cc Normal file
View file

@ -0,0 +1,81 @@
#include "flake.hh"
#include <nlohmann/json.hpp>
namespace nix::flake {
// setting name -> setting value -> allow or ignore.
typedef std::map<std::string, std::map<std::string, bool>> TrustedList;
Path trustedListPath()
{
return getDataDir() + "/nix/trusted-settings.json";
}
static TrustedList readTrustedList()
{
auto path = trustedListPath();
if (!pathExists(path)) return {};
auto json = nlohmann::json::parse(readFile(path));
return json;
}
static void writeTrustedList(const TrustedList & trustedList)
{
writeFile(trustedListPath(), nlohmann::json(trustedList).dump());
}
void ConfigFile::apply()
{
std::set<std::string> whitelist{"bash-prompt", "bash-prompt-suffix"};
for (auto & [name, value] : settings) {
auto baseName = hasPrefix(name, "extra-") ? std::string(name, 6) : name;
// FIXME: Move into libutil/config.cc.
std::string valueS;
if (auto s = std::get_if<std::string>(&value))
valueS = *s;
else if (auto n = std::get_if<int64_t>(&value))
valueS = fmt("%d", n);
else if (auto b = std::get_if<Explicit<bool>>(&value))
valueS = b->t ? "true" : "false";
else if (auto ss = std::get_if<std::vector<std::string>>(&value))
valueS = concatStringsSep(" ", *ss); // FIXME: evil
else
assert(false);
if (!whitelist.count(baseName)) {
auto trustedList = readTrustedList();
bool trusted = false;
if (auto saved = get(get(trustedList, name).value_or(std::map<std::string, bool>()), valueS)) {
trusted = *saved;
} else {
// FIXME: filter ANSI escapes, newlines, \r, etc.
if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)?", name, valueS)).value_or('n')) != 'y') {
if (std::tolower(logger->ask("do you want to permanently mark this value as untrusted (y/N)?").value_or('n')) == 'y') {
trustedList[name][valueS] = false;
writeTrustedList(trustedList);
}
} else {
if (std::tolower(logger->ask("do you want to permanently mark this value as trusted (y/N)?").value_or('n')) == 'y') {
trustedList[name][valueS] = trusted = true;
writeTrustedList(trustedList);
}
}
}
if (!trusted) {
warn("ignoring untrusted flake configuration setting '%s'", name);
continue;
}
}
globalConfig.set(name, valueS);
}
}
}

52
src/libexpr/flake/flake.cc
View file

@ -71,11 +71,17 @@ static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree(
return {std::move(tree), resolvedRef, lockedRef};
}
static void expectType(EvalState & state, ValueType type,
Value & value, const Pos & pos)
static void forceTrivialValue(EvalState & state, Value & value, const Pos & pos)
{
if (value.type == tThunk && value.isTrivial())
state.forceValue(value, pos);
}
static void expectType(EvalState & state, ValueType type,
Value & value, const Pos & pos)
{
forceTrivialValue(state, value, pos);
if (value.type != type)
throw Error("expected %s but got %s at %s",
showType(type), showType(value.type), pos);
@ -114,7 +120,6 @@ static FlakeInput parseFlakeInput(EvalState & state,
expectType(state, tString, *attr.value, *attr.pos);
input.follows = parseInputPath(attr.value->string.s);
} else {
state.forceValue(*attr.value);
if (attr.value->type == tString)
attrs.emplace(attr.name, attr.value->string.s);
else
@ -196,11 +201,6 @@ static Flake getFlake(
expectType(state, tAttrs, vInfo, Pos(foFile, state.symbols.create(flakeFile), 0, 0));
auto sEdition = state.symbols.create("edition"); // FIXME: remove soon
if (vInfo.attrs->get(sEdition))
warn("flake '%s' has deprecated attribute 'edition'", lockedRef);
if (auto description = vInfo.attrs->get(state.sDescription)) {
expectType(state, tString, *description->value, *description->pos);
flake.description = description->value->string.s;
@ -228,11 +228,41 @@ static Flake getFlake(
} else
throw Error("flake '%s' lacks attribute 'outputs'", lockedRef);
auto sNixConfig = state.symbols.create("nixConfig");
if (auto nixConfig = vInfo.attrs->get(sNixConfig)) {
expectType(state, tAttrs, *nixConfig->value, *nixConfig->pos);
for (auto & setting : *nixConfig->value->attrs) {
forceTrivialValue(state, *setting.value, *setting.pos);
if (setting.value->type == tString)
flake.config.settings.insert({setting.name, state.forceStringNoCtx(*setting.value, *setting.pos)});
else if (setting.value->type == tInt)
flake.config.settings.insert({setting.name, state.forceInt(*setting.value, *setting.pos)});
else if (setting.value->type == tBool)
flake.config.settings.insert({setting.name, state.forceBool(*setting.value, *setting.pos)});
else if (setting.value->isList()) {
std::vector<std::string> ss;
for (unsigned int n = 0; n < setting.value->listSize(); ++n) {
auto elem = setting.value->listElems()[n];
if (elem->type != tString)
throw TypeError("list element in flake configuration setting '%s' is %s while a string is expected",
setting.name, showType(*setting.value));
ss.push_back(state.forceStringNoCtx(*elem, *setting.pos));
}
flake.config.settings.insert({setting.name, ss});
}
else
throw TypeError("flake configuration setting '%s' is %s",
setting.name, showType(*setting.value));
}
}
for (auto & attr : *vInfo.attrs) {
if (attr.name != sEdition &&
attr.name != state.sDescription &&
if (attr.name != state.sDescription &&
attr.name != sInputs &&
attr.name != sOutputs)
attr.name != sOutputs &&
attr.name != sNixConfig)
throw Error("flake '%s' has an unsupported attribute '%s', at %s",
lockedRef, attr.name, *attr.pos);
}

40
src/libexpr/flake/flake.hh
View file

@ -17,23 +17,55 @@ struct FlakeInput;
typedef std::map<FlakeId, FlakeInput> FlakeInputs;
/* FlakeInput is the 'Flake'-level parsed form of the "input" entries
* in the flake file.
*
* A FlakeInput is normally constructed by the 'parseFlakeInput'
* function which parses the input specification in the '.flake' file
* to create a 'FlakeRef' (a fetcher, the fetcher-specific
* representation of the input specification, and possibly the fetched
* local store path result) and then creating this FlakeInput to hold
* that FlakeRef, along with anything that might override that
* FlakeRef (like command-line overrides or "follows" specifications).
*
* A FlakeInput is also sometimes constructed directly from a FlakeRef
* instead of starting at the flake-file input specification
* (e.g. overrides, follows, and implicit inputs).
*
* A FlakeInput will usually have one of either "ref" or "follows"
* set. If not otherwise specified, a "ref" will be generated to a
* 'type="indirect"' flake, which is treated as simply the name of a
* flake to be resolved in the registry.
*/
struct FlakeInput
{
std::optional<FlakeRef> ref;
bool isFlake = true;
bool isFlake = true; // true = process flake to get outputs, false = (fetched) static source path
std::optional<InputPath> follows;
bool absolute = false; // whether 'follows' is relative to the flake root
FlakeInputs overrides;
};
struct ConfigFile
{
using ConfigValue = std::variant<std::string, int64_t, Explicit<bool>, std::vector<std::string>>;
std::map<std::string, ConfigValue> settings;
void apply();
};
/* The contents of a flake.nix file. */
struct Flake
{
FlakeRef originalRef;
FlakeRef resolvedRef;
FlakeRef lockedRef;
FlakeRef originalRef; // the original flake specification (by the user)
FlakeRef resolvedRef; // registry references and caching resolved to the specific underlying flake
FlakeRef lockedRef; // the specific local store result of invoking the fetcher
std::optional<std::string> description;
std::shared_ptr<const fetchers::Tree> sourceInfo;
FlakeInputs inputs;
ConfigFile config; // 'nixConfig' attribute
~Flake();
};

23
src/libexpr/flake/flakeref.hh
View file

@ -12,10 +12,33 @@ class Store;
typedef std::string FlakeId;
/* A flake reference specifies how to fetch a flake or raw source
* (e.g. from a Git repository). It is created from a URL-like syntax
* (e.g. 'github:NixOS/patchelf'), an attrset representation (e.g. '{
* type="github"; owner = "NixOS"; repo = "patchelf"; }'), or a local
* path.
*
* Each flake will have a number of FlakeRef objects: one for each
* input to the flake.
*
* The normal method of constructing a FlakeRef is by starting with an
* input description (usually the attrs or a url from the flake file),
* locating a fetcher for that input, and then capturing the Input
* object that fetcher generates (usually via
* FlakeRef::fromAttrs(attrs) or parseFlakeRef(url) calls).
*
* The actual fetch not have been performed yet (i.e. a FlakeRef may
* be lazy), but the fetcher can be invoked at any time via the
* FlakeRef to ensure the store is populated with this input.
*/
struct FlakeRef
{
/* fetcher-specific representation of the input, sufficient to
perform the fetch operation. */
fetchers::Input input;
/* sub-path within the fetched input that represents this input */
Path subdir;
bool operator==(const FlakeRef & other) const;

26
src/libexpr/flake/lockfile.cc
View file

@ -34,7 +34,8 @@ LockedNode::LockedNode(const nlohmann::json & json)
, isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true)
{
if (!lockedRef.input.isImmutable())
throw Error("lockfile contains mutable lock '%s'", attrsToJson(lockedRef.input.toAttrs()));
throw Error("lockfile contains mutable lock '%s'",
fetchers::attrsToJSON(lockedRef.input.toAttrs()));
}
StorePath LockedNode::computeStorePath(Store & store) const
@ -77,7 +78,7 @@ LockFile::LockFile(const nlohmann::json & json, const Path & path)
{
if (jsonNode.find("inputs") == jsonNode.end()) return;
for (auto & i : jsonNode["inputs"].items()) {
if (i.value().is_array()) {
if (i.value().is_array()) { // FIXME: remove, obsolete
InputPath path;
for (auto & j : i.value())
path.push_back(j);
@ -86,10 +87,13 @@ LockFile::LockFile(const nlohmann::json & json, const Path & path)
std::string inputKey = i.value();
auto k = nodeMap.find(inputKey);
if (k == nodeMap.end()) {
auto jsonNode2 = json["nodes"][inputKey];
auto input = std::make_shared<LockedNode>(jsonNode2);
auto nodes = json["nodes"];
auto jsonNode2 = nodes.find(inputKey);
if (jsonNode2 == nodes.end())
throw Error("lock file references missing node '%s'", inputKey);
auto input = std::make_shared<LockedNode>(*jsonNode2);
k = nodeMap.insert_or_assign(inputKey, input).first;
getInputs(*input, jsonNode2);
getInputs(*input, *jsonNode2);
}
if (auto child = std::dynamic_pointer_cast<LockedNode>(k->second))
node.inputs.insert_or_assign(i.key(), child);
@ -110,7 +114,7 @@ LockFile::LockFile(const nlohmann::json & json, const Path & path)
// a bit since we don't need to worry about cycles.
}
nlohmann::json LockFile::toJson() const
nlohmann::json LockFile::toJSON() const
{
nlohmann::json nodes;
std::unordered_map<std::shared_ptr<const Node>, std::string> nodeKeys;
@ -154,8 +158,8 @@ nlohmann::json LockFile::toJson() const
}
if (auto lockedNode = std::dynamic_pointer_cast<const LockedNode>(node)) {
n["original"] = fetchers::attrsToJson(lockedNode->originalRef.toAttrs());
n["locked"] = fetchers::attrsToJson(lockedNode->lockedRef.toAttrs());
n["original"] = fetchers::attrsToJSON(lockedNode->originalRef.toAttrs());
n["locked"] = fetchers::attrsToJSON(lockedNode->lockedRef.toAttrs());
if (!lockedNode->isFlake) n["flake"] = false;
}
@ -174,7 +178,7 @@ nlohmann::json LockFile::toJson() const
std::string LockFile::to_string() const
{
return toJson().dump(2);
return toJSON().dump(2);
}
LockFile LockFile::read(const Path & path)
@ -185,7 +189,7 @@ LockFile LockFile::read(const Path & path)
std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile)
{
stream << lockFile.toJson().dump(2);
stream << lockFile.toJSON().dump(2);
return stream;
}
@ -223,7 +227,7 @@ bool LockFile::isImmutable() const
bool LockFile::operator ==(const LockFile & other) const
{
// FIXME: slow
return toJson() == other.toJson();
return toJSON() == other.toJSON();
}
InputPath parseInputPath(std::string_view s)

2
src/libexpr/flake/lockfile.hh
View file

@ -52,7 +52,7 @@ struct LockFile
LockFile() {};
LockFile(const nlohmann::json & json, const Path & path);
nlohmann::json toJson() const;
nlohmann::json toJSON() const;
std::string to_string() const;

4
src/libexpr/lexer.l
View file

@ -12,6 +12,10 @@
%{
#ifdef __clang__
#pragma clang diagnostic ignored "-Wunneeded-internal-declaration"
#endif
#include <boost/lexical_cast.hpp>
#include "nixexpr.hh"

4
src/libexpr/local.mk
View file

@ -15,7 +15,7 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib
libexpr_LIBS = libutil libstore libfetchers
libexpr_LDFLAGS =
libexpr_LDFLAGS = -lboost_context
ifneq ($(OS), FreeBSD)
libexpr_LDFLAGS += -ldl
endif
@ -35,8 +35,6 @@ $(d)/lexer-tab.cc $(d)/lexer-tab.hh: $(d)/lexer.l
clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh
dist-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh
$(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644))
$(foreach i, $(wildcard src/libexpr/flake/*.hh), \

33
src/libexpr/primops.cc
View file

@ -1089,8 +1089,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
// Regular, non-CA derivation should always return a single hash and not
// hash per output.
Hash h = std::get<0>(hashDerivationModulo(*state.store, Derivation(drv), true));
auto hashModulo = hashDerivationModulo(*state.store, Derivation(drv), true);
std::visit(overloaded {
[&](Hash h) {
for (auto & i : outputs) {
auto outPath = state.store->makeOutputPath(i, h, drvName);
drv.env[i] = state.store->printStorePath(outPath);
@ -1101,6 +1102,22 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
},
});
}
},
[&](CaOutputHashes) {
// Shouldn't happen as the toplevel derivation is not CA.
assert(false);
},
[&](DeferredHash _) {
for (auto & i : outputs) {
drv.outputs.insert_or_assign(i,
DerivationOutput {
.output = DerivationOutputDeferred{},
});
}
},
},
hashModulo);
}
/* Write the resulting term into the Nix store directory. */
@ -1115,9 +1132,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
However, we don't bother doing this for floating CA derivations because
their "hash modulo" is indeterminate until built. */
if (drv.type() != DerivationType::CAFloating)
drvHashes.insert_or_assign(drvPath,
hashDerivationModulo(*state.store, Derivation(drv), false));
if (drv.type() != DerivationType::CAFloating) {
auto h = hashDerivationModulo(*state.store, Derivation(drv), false);
drvHashes.lock()->insert_or_assign(drvPath, h);
}
state.mkAttrs(v, 1 + drv.outputs.size());
mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS});
@ -1603,7 +1621,12 @@ static RegisterPrimOp primop_toJSON({
static void prim_fromJSON(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
string s = state.forceStringNoCtx(*args[0], pos);
try {
parseJSON(state, s, v);
} catch (JSONParseError &e) {
e.addTrace(pos, "while decoding a JSON string");
throw e;
}
}
static RegisterPrimOp primop_fromJSON({

9
src/libexpr/primops/fetchTree.cc
View file

@ -39,11 +39,12 @@ void emitTreeAttrs(
// Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev
auto emptyHash = Hash(htSHA1);
mkString(*state.allocAttr(v, state.symbols.create("rev")), emptyHash.gitRev());
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitRev());
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitShortRev());
}
if (input.getType() == "git")
mkBool(*state.allocAttr(v, state.symbols.create("submodules")), maybeGetBoolAttr(input.attrs, "submodules").value_or(false));
mkBool(*state.allocAttr(v, state.symbols.create("submodules")),
fetchers::maybeGetBoolAttr(input.attrs, "submodules").value_or(false));
if (auto revCount = input.getRevCount())
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
@ -101,7 +102,7 @@ static void fetchTree(
else if (attr.value->type == tString)
addURI(state, attrs, attr.name, attr.value->string.s);
else if (attr.value->type == tBool)
attrs.emplace(attr.name, fetchers::Explicit<bool>{attr.value->boolean});
attrs.emplace(attr.name, Explicit<bool>{attr.value->boolean});
else if (attr.value->type == tInt)
attrs.emplace(attr.name, attr.value->integer);
else
@ -211,7 +212,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
? state.store->queryPathInfo(storePath)->narHash
: hashFile(htSHA256, path);
if (hash != *expectedHash)
throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s",
throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s",
*url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true));
}

4
src/libfetchers/attrs.cc
View file

@ -23,7 +23,7 @@ Attrs jsonToAttrs(const nlohmann::json & json)
return attrs;
}
nlohmann::json attrsToJson(const Attrs & attrs)
nlohmann::json attrsToJSON(const Attrs & attrs)
{
nlohmann::json json;
for (auto & attr : attrs) {
@ -44,7 +44,7 @@ std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::strin
if (i == attrs.end()) return {};
if (auto v = std::get_if<std::string>(&i->second))
return *v;
throw Error("input attribute '%s' is not a string %s", name, attrsToJson(attrs).dump());
throw Error("input attribute '%s' is not a string %s", name, attrsToJSON(attrs).dump());
}
std::string getStrAttr(const Attrs & attrs, const std::string & name)

14
src/libfetchers/attrs.hh
View file

@ -8,24 +8,12 @@
namespace nix::fetchers {
/* Wrap bools to prevent string literals (i.e. 'char *') from being
cast to a bool in Attr. */
template<typename T>
struct Explicit {
T t;
bool operator ==(const Explicit<T> & other) const
{
return t == other.t;
}
};
typedef std::variant<std::string, uint64_t, Explicit<bool>> Attr;
typedef std::map<std::string, Attr> Attrs;
Attrs jsonToAttrs(const nlohmann::json & json);
nlohmann::json attrsToJson(const Attrs & attrs);
nlohmann::json attrsToJSON(const Attrs & attrs);
std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::string & name);

20
src/libfetchers/cache.cc
View file

@ -55,8 +55,8 @@ struct CacheImpl : Cache
bool immutable) override
{
_state.lock()->add.use()
(attrsToJson(inAttrs).dump())
(attrsToJson(infoAttrs).dump())
(attrsToJSON(inAttrs).dump())
(attrsToJSON(infoAttrs).dump())
(store->printStorePath(storePath))
(immutable)
(time(0)).exec();
@ -70,7 +70,7 @@ struct CacheImpl : Cache
if (!res->expired)
return std::make_pair(std::move(res->infoAttrs), std::move(res->storePath));
debug("ignoring expired cache entry '%s'",
attrsToJson(inAttrs).dump());
attrsToJSON(inAttrs).dump());
}
return {};
}
@ -81,15 +81,15 @@ struct CacheImpl : Cache
{
auto state(_state.lock());
auto inAttrsJson = attrsToJson(inAttrs).dump();
auto inAttrsJSON = attrsToJSON(inAttrs).dump();
auto stmt(state->lookup.use()(inAttrsJson));
auto stmt(state->lookup.use()(inAttrsJSON));
if (!stmt.next()) {
debug("did not find cache entry for '%s'", inAttrsJson);
debug("did not find cache entry for '%s'", inAttrsJSON);
return {};
}
auto infoJson = stmt.getStr(0);
auto infoJSON = stmt.getStr(0);
auto storePath = store->parseStorePath(stmt.getStr(1));
auto immutable = stmt.getInt(2) != 0;
auto timestamp = stmt.getInt(3);
@ -97,16 +97,16 @@ struct CacheImpl : Cache
store->addTempRoot(storePath);
if (!store->isValidPath(storePath)) {
// FIXME: we could try to substitute 'storePath'.
debug("ignoring disappeared cache entry '%s'", inAttrsJson);
debug("ignoring disappeared cache entry '%s'", inAttrsJSON);
return {};
}
debug("using cache entry '%s' -> '%s', '%s'",
inAttrsJson, infoJson, store->printStorePath(storePath));
inAttrsJSON, infoJSON, store->printStorePath(storePath));
return Result {
.expired = !immutable && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)),
.infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJson)),
.infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJSON)),
.storePath = std::move(storePath)
};
}

6
src/libfetchers/fetchers.cc
View file

@ -65,7 +65,7 @@ Input Input::fromAttrs(Attrs && attrs)
ParsedURL Input::toURL() const
{
if (!scheme)
throw Error("cannot show unsupported input '%s'", attrsToJson(attrs));
throw Error("cannot show unsupported input '%s'", attrsToJSON(attrs));
return scheme->toURL(*this);
}
@ -110,7 +110,7 @@ bool Input::contains(const Input & other) const
std::pair<Tree, Input> Input::fetch(ref<Store> store) const
{
if (!scheme)
throw Error("cannot fetch unsupported input '%s'", attrsToJson(toAttrs()));
throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs()));
/* The tree may already be in the Nix store, or it could be
substituted (which is often faster than fetching from the
@ -247,7 +247,7 @@ std::optional<time_t> Input::getLastModified() const
ParsedURL InputScheme::toURL(const Input & input)
{
throw Error("don't know how to convert input '%s' to a URL", attrsToJson(input.attrs));
throw Error("don't know how to convert input '%s' to a URL", attrsToJSON(input.attrs));
}
Input InputScheme::applyOverrides(

18
src/libfetchers/fetchers.hh
View file

@ -21,6 +21,14 @@ struct Tree
struct InputScheme;
/* The Input object is generated by a specific fetcher, based on the
* user-supplied input attribute in the flake.nix file, and contains
* the information that the specific fetcher needs to perform the
* actual fetch. The Input object is most commonly created via the
* "fromURL()" or "fromAttrs()" static functions which are provided
* the url or attrset specified in the flake file.
*/
struct Input
{
friend struct InputScheme;
@ -84,6 +92,16 @@ public:
std::optional<time_t> getLastModified() const;
};
/* The InputScheme represents a type of fetcher. Each fetcher
* registers with nix at startup time. When processing an input for a
* flake, each scheme is given an opportunity to "recognize" that
* input from the url or attributes in the flake file's specification
* and return an Input object to represent the input if it is
* recognized. The Input object contains the information the fetcher
* needs to actually perform the "fetch()" when called.
*/
struct InputScheme
{
virtual ~InputScheme()

2
src/libfetchers/git.cc
View file

@ -273,7 +273,7 @@ struct GitInputScheme : InputScheme
haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0);
return {
Tree(store->printStorePath(storePath), std::move(storePath)),
Tree(store->toRealPath(storePath), std::move(storePath)),
input
};
}

56
src/libfetchers/mercurial.cc
View file

@ -11,6 +11,36 @@ using namespace std::string_literals;
namespace nix::fetchers {
namespace {
RunOptions hgOptions(const Strings & args) {
RunOptions opts("hg", args);
opts.searchPath = true;
auto env = getEnv();
// Set HGPLAIN: this means we get consistent output from hg and avoids leakage from a user or system .hgrc.
env["HGPLAIN"] = "";
opts.environment = env;
return opts;
}
// runProgram wrapper that uses hgOptions instead of stock RunOptions.
string runHg(const Strings & args, const std::optional<std::string> & input = {})
{
RunOptions opts = hgOptions(args);
opts.input = input;
auto res = runProgram(opts);
if (!statusOk(res.first))
throw ExecError(res.first, fmt("hg %1%", statusToString(res.first)));
return res.second;
}
}
struct MercurialInputScheme : InputScheme
{
std::optional<Input> inputFromURL(const ParsedURL & url) override
@ -100,11 +130,11 @@ struct MercurialInputScheme : InputScheme
assert(sourcePath);
// FIXME: shut up if file is already tracked.
runProgram("hg", true,
runHg(
{ "add", *sourcePath + "/" + std::string(file) });
if (commitMsg)
runProgram("hg", true,
runHg(
{ "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg });
}
@ -130,7 +160,7 @@ struct MercurialInputScheme : InputScheme
if (!input.getRef() && !input.getRev() && isLocal && pathExists(actualUrl + "/.hg")) {
bool clean = runProgram("hg", true, { "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == "";
bool clean = runHg({ "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == "";
if (!clean) {
@ -143,10 +173,10 @@ struct MercurialInputScheme : InputScheme
if (settings.warnDirty)
warn("Mercurial tree '%s' is unclean", actualUrl);
input.attrs.insert_or_assign("ref", chomp(runProgram("hg", true, { "branch", "-R", actualUrl })));
input.attrs.insert_or_assign("ref", chomp(runHg({ "branch", "-R", actualUrl })));
auto files = tokenizeString<std::set<std::string>>(
runProgram("hg", true, { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
runHg({ "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
PathFilter filter = [&](const Path & p) -> bool {
assert(hasPrefix(p, actualUrl));
@ -166,7 +196,7 @@ struct MercurialInputScheme : InputScheme
auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
return {
Tree(store->printStorePath(storePath), std::move(storePath)),
Tree(store->toRealPath(storePath), std::move(storePath)),
input
};
}
@ -224,33 +254,33 @@ struct MercurialInputScheme : InputScheme
if (!(input.getRev()
&& pathExists(cacheDir)
&& runProgram(
RunOptions("hg", { "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" })
hgOptions({ "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" })
.killStderr(true)).second == "1"))
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl));
if (pathExists(cacheDir)) {
try {
runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl });
runHg({ "pull", "-R", cacheDir, "--", actualUrl });
}
catch (ExecError & e) {
string transJournal = cacheDir + "/.hg/store/journal";
/* hg throws "abandoned transaction" error only if this file exists */
if (pathExists(transJournal)) {
runProgram("hg", true, { "recover", "-R", cacheDir });
runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl });
runHg({ "recover", "-R", cacheDir });
runHg({ "pull", "-R", cacheDir, "--", actualUrl });
} else {
throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status)));
}
}
} else {
createDirs(dirOf(cacheDir));
runProgram("hg", true, { "clone", "--noupdate", "--", actualUrl, cacheDir });
runHg({ "clone", "--noupdate", "--", actualUrl, cacheDir });
}
}
auto tokens = tokenizeString<std::vector<std::string>>(
runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" }));
runHg({ "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" }));
assert(tokens.size() == 3);
input.attrs.insert_or_assign("rev", Hash::parseAny(tokens[0], htSHA1).gitRev());
@ -263,7 +293,7 @@ struct MercurialInputScheme : InputScheme
Path tmpDir = createTempDir();
AutoDelete delTmpDir(tmpDir, true);
runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input.getRev()->gitRev(), tmpDir });
runHg({ "archive", "-R", cacheDir, "-r", input.getRev()->gitRev(), tmpDir });
deletePath(tmpDir + "/.hg_archival.txt");

6
src/libfetchers/registry.cc
View file

@ -60,10 +60,10 @@ void Registry::write(const Path & path)
nlohmann::json arr;
for (auto & entry : entries) {
nlohmann::json obj;
obj["from"] = attrsToJson(entry.from.toAttrs());
obj["to"] = attrsToJson(entry.to.toAttrs());
obj["from"] = attrsToJSON(entry.from.toAttrs());
obj["to"] = attrsToJSON(entry.to.toAttrs());
if (!entry.extraAttrs.empty())
obj["to"].update(attrsToJson(entry.extraAttrs));
obj["to"].update(attrsToJSON(entry.extraAttrs));
if (entry.exact)
obj["exact"] = true;
arr.emplace_back(std::move(obj));

2
src/libmain/common-args.cc
View file

@ -44,7 +44,7 @@ MixCommonArgs::MixCommonArgs(const string & programName)
globalConfig.getSettings(settings);
for (auto & s : settings)
if (hasPrefix(s.first, prefix))
completions->add(s.first, s.second.description);
completions->add(s.first, fmt("Set the `%s` setting.", s.first));
}
}
});

4
src/libmain/loggers.cc
View file

@ -12,7 +12,7 @@ LogFormat parseLogFormat(const std::string & logFormatStr) {
else if (logFormatStr == "raw-with-logs")
return LogFormat::rawWithLogs;
else if (logFormatStr == "internal-json")
return LogFormat::internalJson;
return LogFormat::internalJSON;
else if (logFormatStr == "bar")
return LogFormat::bar;
else if (logFormatStr == "bar-with-logs")
@ -26,7 +26,7 @@ Logger * makeDefaultLogger() {
return makeSimpleLogger(false);
case LogFormat::rawWithLogs:
return makeSimpleLogger(true);
case LogFormat::internalJson:
case LogFormat::internalJSON:
return makeJSONLogger(*makeSimpleLogger(true));
case LogFormat::bar:
return makeProgressBar();

2
src/libmain/loggers.hh
View file

@ -7,7 +7,7 @@ namespace nix {
enum class LogFormat {
raw,
rawWithLogs,
internalJson,
internalJSON,
bar,
barWithLogs,
};

11
src/libmain/progress-bar.cc
View file

@ -466,6 +466,17 @@ public:
Logger::writeToStdout(s);
}
}
std::optional<char> ask(std::string_view msg) override
{
auto state(state_.lock());
if (!state->active || !isatty(STDIN_FILENO)) return {};
std::cerr << fmt("\r\e[K%s ", msg);
auto s = trim(readLine(STDIN_FILENO));
if (s.size() != 1) return {};
draw(*state);
return s[0];
}
};
Logger * makeProgressBar(bool printBuildLogs)

25
src/libstore/binary-cache-store.cc
View file

@ -86,8 +86,7 @@ void BinaryCacheStore::getFile(const std::string & path, Sink & sink)
promise.set_exception(std::current_exception());
}
}});
auto data = promise.get_future().get();
sink((unsigned char *) data->data(), data->size());
sink(*promise.get_future().get());
}
std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
@ -434,7 +433,9 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
if (!repair && isValidPath(path))
return path;
auto source = StringSource { s };
StringSink sink;
dumpString(s, sink);
auto source = StringSource { *sink.s };
return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info { path, nar.first };
info.narSize = nar.second;
@ -444,6 +445,24 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
})->path;
}
std::optional<const Realisation> BinaryCacheStore::queryRealisation(const DrvOutput & id)
{
auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi";
auto rawOutputInfo = getFile(outputInfoFilePath);
if (rawOutputInfo) {
return {Realisation::fromJSON(
nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath)};
} else {
return std::nullopt;
}
}
void BinaryCacheStore::registerDrvOutput(const Realisation& info) {
auto filePath = realisationsPrefix + "/" + info.id.to_string() + ".doi";
upsertFile(filePath, info.toJSON().dump(), "application/json");
}
ref<FSAccessor> BinaryCacheStore::getFSAccessor()
{
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), localNarCache);

7
src/libstore/binary-cache-store.hh
View file

@ -33,6 +33,9 @@ private:
protected:
// The prefix under which realisation infos will be stored
const std::string realisationsPrefix = "/realisations";
BinaryCacheStore(const Params & params);
public:
@ -99,6 +102,10 @@ public:
StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) override;
void registerDrvOutput(const Realisation & info) override;
std::optional<const Realisation> queryRealisation(const DrvOutput &) override;
void narFromPath(const StorePath & path, Sink & sink) override;
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,

2120
src/libstore/build.cc → src/libstore/build/derivation-goal.cc
View file

File diff suppressed because it is too large Load diff

369
src/libstore/build/derivation-goal.hh Normal file
View file

@ -0,0 +1,369 @@
#pragma once
#include "parsed-derivations.hh"
#include "lock.hh"
#include "local-store.hh"
#include "goal.hh"
namespace nix {
using std::map;
struct HookInstance;
typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
/* Unless we are repairing, we don't both to test validity and just assume it,
so the choices are `Absent` or `Valid`. */
enum struct PathStatus {
Corrupt,
Absent,
Valid,
};
struct InitialOutputStatus {
StorePath path;
PathStatus status;
/* Valid in the store, and additionally non-corrupt if we are repairing */
bool isValid() const {
return status == PathStatus::Valid;
}
/* Merely present, allowed to be corrupt */
bool isPresent() const {
return status == PathStatus::Corrupt
|| status == PathStatus::Valid;
}
};
struct InitialOutput {
bool wanted;
std::optional<InitialOutputStatus> known;
};
struct DerivationGoal : public Goal
{
/* Whether to use an on-disk .drv file. */
bool useDerivation;
/* The path of the derivation. */
StorePath drvPath;
/* The specific outputs that we need to build. Empty means all of
them. */
StringSet wantedOutputs;
/* Whether additional wanted outputs have been added. */
bool needRestart = false;
/* Whether to retry substituting the outputs after building the
inputs. */
bool retrySubstitution;
/* The derivation stored at drvPath. */
std::unique_ptr<BasicDerivation> drv;
std::unique_ptr<ParsedDerivation> parsedDrv;
/* The remainder is state held during the build. */
/* Locks on (fixed) output paths. */
PathLocks outputLocks;
/* All input paths (that is, the union of FS closures of the
immediate input paths). */
StorePathSet inputPaths;
std::map<std::string, InitialOutput> initialOutputs;
/* User selected for running the builder. */
std::unique_ptr<UserLock> buildUser;
/* The process ID of the builder. */
Pid pid;
/* The temporary directory. */
Path tmpDir;
/* The path of the temporary directory in the sandbox. */
Path tmpDirInSandbox;
/* File descriptor for the log file. */
AutoCloseFD fdLogFile;
std::shared_ptr<BufferedSink> logFileSink, logSink;
/* Number of bytes received from the builder's stdout/stderr. */
unsigned long logSize;
/* The most recent log lines. */
std::list<std::string> logTail;
std::string currentLogLine;
size_t currentLogLinePos = 0; // to handle carriage return
std::string currentHookLine;
/* Pipe for the builder's standard output/error. */
Pipe builderOut;
/* Pipe for synchronising updates to the builder namespaces. */
Pipe userNamespaceSync;
/* The mount namespace of the builder, used to add additional
paths to the sandbox as a result of recursive Nix calls. */
AutoCloseFD sandboxMountNamespace;
/* On Linux, whether we're doing the build in its own user
namespace. */
bool usingUserNamespace = true;
/* The build hook. */
std::unique_ptr<HookInstance> hook;
/* Whether we're currently doing a chroot build. */
bool useChroot = false;
Path chrootRootDir;
/* RAII object to delete the chroot directory. */
std::shared_ptr<AutoDelete> autoDelChroot;
/* The sort of derivation we are building. */
DerivationType derivationType;
/* Whether to run the build in a private network namespace. */
bool privateNetwork = false;
typedef void (DerivationGoal::*GoalState)();
GoalState state;
/* Stuff we need to pass to initChild(). */
struct ChrootPath {
Path source;
bool optional;
ChrootPath(Path source = "", bool optional = false)
: source(source), optional(optional)
{ }
};
typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
DirsInChroot dirsInChroot;
typedef map<string, string> Environment;
Environment env;
#if __APPLE__
typedef string SandboxProfile;
SandboxProfile additionalSandboxProfile;
#endif
/* Hash rewriting. */
StringMap inputRewrites, outputRewrites;
typedef map<StorePath, StorePath> RedirectedOutputs;
RedirectedOutputs redirectedOutputs;
/* The outputs paths used during the build.
- Input-addressed derivations or fixed content-addressed outputs are
sometimes built when some of their outputs already exist, and can not
be hidden via sandboxing. We use temporary locations instead and
rewrite after the build. Otherwise the regular predetermined paths are
put here.
- Floating content-addressed derivations do not know their final build
output paths until the outputs are hashed, so random locations are
used, and then renamed. The randomness helps guard against hidden
self-references.
*/
OutputPathMap scratchOutputs;
/* The final output paths of the build.
- For input-addressed derivations, always the precomputed paths
- For content-addressed derivations, calcuated from whatever the hash
ends up being. (Note that fixed outputs derivations that produce the
"wrong" output still install that data under its true content-address.)
*/
OutputPathMap finalOutputs;
BuildMode buildMode;
/* If we're repairing without a chroot, there may be outputs that
are valid but corrupt. So we redirect these outputs to
temporary paths. */
StorePathSet redirectedBadOutputs;
BuildResult result;
/* The current round, if we're building multiple times. */
size_t curRound = 1;
size_t nrRounds;
/* Path registration info from the previous round, if we're
building multiple times. Since this contains the hash, it
allows us to compare whether two rounds produced the same
result. */
std::map<Path, ValidPathInfo> prevInfos;
uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); }
gid_t sandboxGid() { return usingUserNamespace ? 100 : buildUser->getGID(); }
const static Path homeDir;
std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
std::unique_ptr<Activity> act;
/* Activity that denotes waiting for a lock. */
std::unique_ptr<Activity> actLock;
std::map<ActivityId, Activity> builderActivities;
/* The remote machine on which we're building. */
std::string machineName;
/* The recursive Nix daemon socket. */
AutoCloseFD daemonSocket;
/* The daemon main thread. */
std::thread daemonThread;
/* The daemon worker threads. */
std::vector<std::thread> daemonWorkerThreads;
/* Paths that were added via recursive Nix calls. */
StorePathSet addedPaths;
/* Recursive Nix calls are only allowed to build or realize paths
in the original input closure or added via a recursive Nix call
(so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
/nix/store/<bla> is some arbitrary path in a binary cache). */
bool isAllowed(const StorePath & path)
{
return inputPaths.count(path) || addedPaths.count(path);
}
friend struct RestrictedStore;
DerivationGoal(const StorePath & drvPath,
const StringSet & wantedOutputs, Worker & worker,
BuildMode buildMode = bmNormal);
DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
const StringSet & wantedOutputs, Worker & worker,
BuildMode buildMode = bmNormal);
~DerivationGoal();
/* Whether we need to perform hash rewriting if there are valid output paths. */
bool needsHashRewrite();
void timedOut(Error && ex) override;
string key() override;
void work() override;
/* Add wanted outputs to an already existing derivation goal. */
void addWantedOutputs(const StringSet & outputs);
BuildResult getResult() { return result; }
/* The states. */
void getDerivation();
void loadDerivation();
void haveDerivation();
void outputsSubstitutionTried();
void gaveUpOnSubstitution();
void closureRepaired();
void inputsRealised();
void tryToBuild();
void tryLocalBuild();
void buildDone();
void resolvedFinished();
/* Is the build hook willing to perform the build? */
HookReply tryBuildHook();
/* Start building a derivation. */
void startBuilder();
/* Fill in the environment for the builder. */
void initEnv();
/* Setup tmp dir location. */
void initTmpDir();
/* Write a JSON file containing the derivation attributes. */
void writeStructuredAttrs();
void startDaemon();
void stopDaemon();
/* Add 'path' to the set of paths that may be referenced by the
outputs, and make it appear in the sandbox. */
void addDependency(const StorePath & path);
/* Make a file owned by the builder. */
void chownToBuilder(const Path & path);
/* Run the builder's process. */
void runChild();
/* Check that the derivation outputs all exist and register them
as valid. */
void registerOutputs();
/* Check that an output meets the requirements specified by the
'outputChecks' attribute (or the legacy
'{allowed,disallowed}{References,Requisites}' attributes). */
void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
/* Open a log file and a pipe to it. */
Path openLogFile();
/* Close the log file. */
void closeLogFile();
/* Delete the temporary directory, if we have one. */
void deleteTmpDir(bool force);
/* Callback used by the worker to write to the log. */
void handleChildOutput(int fd, const string & data) override;
void handleEOF(int fd) override;
void flushLine();
/* Wrappers around the corresponding Store methods that first consult the
derivation. This is currently needed because when there is no drv file
there also is no DB entry. */
std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap();
OutputPathMap queryDerivationOutputMap();
/* Return the set of (in)valid paths. */
void checkPathValidity();
/* Forcibly kill the child process, if any. */
void killChild();
/* Create alternative path calculated from but distinct from the
input, so we can avoid overwriting outputs (or other store paths)
that already exist. */
StorePath makeFallbackPath(const StorePath & path);
/* Make a path to another based on the output name along with the
derivation hash. */
/* FIXME add option to randomize, so we can audit whether our
rewrites caught everything */
StorePath makeFallbackPath(std::string_view outputName);
void repairClosure();
void started();
void done(
BuildResult::Status status,
std::optional<Error> ex = {});
StorePathSet exportReferences(const StorePathSet & storePaths);
};
}

89
src/libstore/build/goal.cc Normal file
View file

@ -0,0 +1,89 @@
#include "goal.hh"
#include "worker.hh"
namespace nix {
bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const {
string s1 = a->key();
string s2 = b->key();
return s1 < s2;
}
void addToWeakGoals(WeakGoals & goals, GoalPtr p)
{
// FIXME: necessary?
// FIXME: O(n)
for (auto & i : goals)
if (i.lock() == p) return;
goals.push_back(p);
}
void Goal::addWaitee(GoalPtr waitee)
{
waitees.insert(waitee);
addToWeakGoals(waitee->waiters, shared_from_this());
}
void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
{
assert(waitees.find(waitee) != waitees.end());
waitees.erase(waitee);
trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size()));
if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed;
if (result == ecNoSubstituters) ++nrNoSubstituters;
if (result == ecIncompleteClosure) ++nrIncompleteClosure;
if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) {
/* If we failed and keepGoing is not set, we remove all
remaining waitees. */
for (auto & goal : waitees) {
WeakGoals waiters2;
for (auto & j : goal->waiters)
if (j.lock() != shared_from_this()) waiters2.push_back(j);
goal->waiters = waiters2;
}
waitees.clear();
worker.wakeUp(shared_from_this());
}
}
void Goal::amDone(ExitCode result, std::optional<Error> ex)
{
trace("done");
assert(exitCode == ecBusy);
assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure);
exitCode = result;
if (ex) {
if (!waiters.empty())
logError(ex->info());
else
this->ex = std::move(*ex);
}
for (auto & i : waiters) {
GoalPtr goal = i.lock();
if (goal) goal->waiteeDone(shared_from_this(), result);
}
waiters.clear();
worker.removeGoal(shared_from_this());
}
void Goal::trace(const FormatOrString & fs)
{
debug("%1%: %2%", name, fs.s);
}
}

107
src/libstore/build/goal.hh Normal file
View file

@ -0,0 +1,107 @@
#pragma once
#include "types.hh"
#include "store-api.hh"
namespace nix {
/* Forward definition. */
struct Goal;
class Worker;
/* A pointer to a goal. */
typedef std::shared_ptr<Goal> GoalPtr;
typedef std::weak_ptr<Goal> WeakGoalPtr;
struct CompareGoalPtrs {
bool operator() (const GoalPtr & a, const GoalPtr & b) const;
};
/* Set of goals. */
typedef set<GoalPtr, CompareGoalPtrs> Goals;
typedef list<WeakGoalPtr> WeakGoals;
/* A map of paths to goals (and the other way around). */
typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
struct Goal : public std::enable_shared_from_this<Goal>
{
typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
/* Backlink to the worker. */
Worker & worker;
/* Goals that this goal is waiting for. */
Goals waitees;
/* Goals waiting for this one to finish. Must use weak pointers
here to prevent cycles. */
WeakGoals waiters;
/* Number of goals we are/were waiting for that have failed. */
unsigned int nrFailed;
/* Number of substitution goals we are/were waiting for that
failed because there are no substituters. */
unsigned int nrNoSubstituters;
/* Number of substitution goals we are/were waiting for that
failed because they had unsubstitutable references. */
unsigned int nrIncompleteClosure;
/* Name of this goal for debugging purposes. */
string name;
/* Whether the goal is finished. */
ExitCode exitCode;
/* Exception containing an error message, if any. */
std::optional<Error> ex;
Goal(Worker & worker) : worker(worker)
{
nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
exitCode = ecBusy;
}
virtual ~Goal()
{
trace("goal destroyed");
}
virtual void work() = 0;
void addWaitee(GoalPtr waitee);
virtual void waiteeDone(GoalPtr waitee, ExitCode result);
virtual void handleChildOutput(int fd, const string & data)
{
abort();
}
virtual void handleEOF(int fd)
{
abort();
}
void trace(const FormatOrString & fs);
string getName()
{
return name;
}
/* Callback in case of a timeout. It should wake up its waiters,
get rid of any running child processes that are being monitored
by the worker (important!), etc. */
virtual void timedOut(Error && ex) = 0;
virtual string key() = 0;
void amDone(ExitCode result, std::optional<Error> ex = {});
};
void addToWeakGoals(WeakGoals & goals, GoalPtr p);
}

72
src/libstore/build/hook-instance.cc Normal file
View file

@ -0,0 +1,72 @@
#include "globals.hh"
#include "hook-instance.hh"
namespace nix {
HookInstance::HookInstance()
{
debug("starting build hook '%s'", settings.buildHook);
/* Create a pipe to get the output of the child. */
fromHook.create();
/* Create the communication pipes. */
toHook.create();
/* Create a pipe to get the output of the builder. */
builderOut.create();
/* Fork the hook. */
pid = startProcess([&]() {
commonChildInit(fromHook);
if (chdir("/") == -1) throw SysError("changing into /");
/* Dup the communication pipes. */
if (dup2(toHook.readSide.get(), STDIN_FILENO) == -1)
throw SysError("dupping to-hook read side");
/* Use fd 4 for the builder's stdout/stderr. */
if (dup2(builderOut.writeSide.get(), 4) == -1)
throw SysError("dupping builder's stdout/stderr");
/* Hack: pass the read side of that fd to allow build-remote
to read SSH error messages. */
if (dup2(builderOut.readSide.get(), 5) == -1)
throw SysError("dupping builder's stdout/stderr");
Strings args = {
std::string(baseNameOf(settings.buildHook.get())),
std::to_string(verbosity),
};
execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data());
throw SysError("executing '%s'", settings.buildHook);
});
pid.setSeparatePG(true);
fromHook.writeSide = -1;
toHook.readSide = -1;
sink = FdSink(toHook.writeSide.get());
std::map<std::string, Config::SettingInfo> settings;
globalConfig.getSettings(settings);
for (auto & setting : settings)
sink << 1 << setting.first << setting.second.value;
sink << 0;
}
HookInstance::~HookInstance()
{
try {
toHook.writeSide = -1;
if (pid != -1) pid.kill();
} catch (...) {
ignoreException();
}
}
}

31
src/libstore/build/hook-instance.hh Normal file
View file

@ -0,0 +1,31 @@
#pragma once
#include "logging.hh"
#include "serialise.hh"
namespace nix {
struct HookInstance
{
/* Pipes for talking to the build hook. */
Pipe toHook;
/* Pipe for the hook's standard output/error. */
Pipe fromHook;
/* Pipe for the builder's standard output/error. */
Pipe builderOut;
/* The process ID of the hook. */
Pid pid;
FdSink sink;
std::map<ActivityId, Activity> activities;
HookInstance();
~HookInstance();
};
}

108
src/libstore/build/local-store-build.cc Normal file
View file

@ -0,0 +1,108 @@
#include "machines.hh"
#include "worker.hh"
#include "substitution-goal.hh"
#include "derivation-goal.hh"
namespace nix {
void LocalStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode)
{
Worker worker(*this);
Goals goals;
for (auto & path : drvPaths) {
if (path.path.isDerivation())
goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode));
else
goals.insert(worker.makeSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair));
}
worker.run(goals);
StorePathSet failed;
std::optional<Error> ex;
for (auto & i : goals) {
if (i->ex) {
if (ex)
logError(i->ex->info());
else
ex = i->ex;
}
if (i->exitCode != Goal::ecSuccess) {
if (auto i2 = dynamic_cast<DerivationGoal *>(i.get())) failed.insert(i2->drvPath);
else if (auto i2 = dynamic_cast<SubstitutionGoal *>(i.get())) failed.insert(i2->storePath);
}
}
if (failed.size() == 1 && ex) {
ex->status = worker.exitStatus();
throw *ex;
} else if (!failed.empty()) {
if (ex) logError(ex->info());
throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed));
}
}
BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode)
{
Worker worker(*this);
auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode);
BuildResult result;
try {
worker.run(Goals{goal});
result = goal->getResult();
} catch (Error & e) {
result.status = BuildResult::MiscFailure;
result.errorMsg = e.msg();
}
return result;
}
void LocalStore::ensurePath(const StorePath & path)
{
/* If the path is already valid, we're done. */
if (isValidPath(path)) return;
Worker worker(*this);
GoalPtr goal = worker.makeSubstitutionGoal(path);
Goals goals = {goal};
worker.run(goals);
if (goal->exitCode != Goal::ecSuccess) {
if (goal->ex) {
goal->ex->status = worker.exitStatus();
throw *goal->ex;
} else
throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path));
}
}
void LocalStore::repairPath(const StorePath & path)
{
Worker worker(*this);
GoalPtr goal = worker.makeSubstitutionGoal(path, Repair);
Goals goals = {goal};
worker.run(goals);
if (goal->exitCode != Goal::ecSuccess) {
/* Since substituting the path didn't work, if we have a valid
deriver, then rebuild the deriver. */
auto info = queryPathInfo(path);
if (info->deriver && isValidPath(*info->deriver)) {
goals.clear();
goals.insert(worker.makeDerivationGoal(*info->deriver, StringSet(), bmRepair));
worker.run(goals);
} else
throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path));
}
}
}

296
src/libstore/build/substitution-goal.cc Normal file
View file

@ -0,0 +1,296 @@
#include "worker.hh"
#include "substitution-goal.hh"
#include "nar-info.hh"
#include "finally.hh"
namespace nix {
SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
: Goal(worker)
, storePath(storePath)
, repair(repair)
, ca(ca)
{
state = &SubstitutionGoal::init;
name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath));
trace("created");
maintainExpectedSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.expectedSubstitutions);
}
SubstitutionGoal::~SubstitutionGoal()
{
try {
if (thr.joinable()) {
// FIXME: signal worker thread to quit.
thr.join();
worker.childTerminated(this);
}
} catch (...) {
ignoreException();
}
}
void SubstitutionGoal::work()
{
(this->*state)();
}
void SubstitutionGoal::init()
{
trace("init");
worker.store.addTempRoot(storePath);
/* If the path already exists we're done. */
if (!repair && worker.store.isValidPath(storePath)) {
amDone(ecSuccess);
return;
}
if (settings.readOnlyMode)
throw Error("cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath));
subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
tryNext();
}
void SubstitutionGoal::tryNext()
{
trace("trying next substituter");
if (subs.size() == 0) {
/* None left. Terminate this goal and let someone else deal
with it. */
debug("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath));
/* Hack: don't indicate failure if there were no substituters.
In that case the calling derivation should just do a
build. */
amDone(substituterFailed ? ecFailed : ecNoSubstituters);
if (substituterFailed) {
worker.failedSubstitutions++;
worker.updateProgress();
}
return;
}
sub = subs.front();
subs.pop_front();
if (ca) {
subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca);
if (sub->storeDir == worker.store.storeDir)
assert(subPath == storePath);
} else if (sub->storeDir != worker.store.storeDir) {
tryNext();
return;
}
try {
// FIXME: make async
info = sub->queryPathInfo(subPath ? *subPath : storePath);
} catch (InvalidPath &) {
tryNext();
return;
} catch (SubstituterDisabled &) {
if (settings.tryFallback) {
tryNext();
return;
}
throw;
} catch (Error & e) {
if (settings.tryFallback) {
logError(e.info());
tryNext();
return;
}
throw;
}
if (info->path != storePath) {
if (info->isContentAddressed(*sub) && info->references.empty()) {
auto info2 = std::make_shared<ValidPathInfo>(*info);
info2->path = storePath;
info = info2;
} else {
printError("asked '%s' for '%s' but got '%s'",
sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path));
tryNext();
return;
}
}
/* Update the total expected download size. */
auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info);
maintainExpectedNar = std::make_unique<MaintainCount<uint64_t>>(worker.expectedNarSize, info->narSize);
maintainExpectedDownload =
narInfo && narInfo->fileSize
? std::make_unique<MaintainCount<uint64_t>>(worker.expectedDownloadSize, narInfo->fileSize)
: nullptr;
worker.updateProgress();
/* Bail out early if this substituter lacks a valid
signature. LocalStore::addToStore() also checks for this, but
only after we've downloaded the path. */
if (worker.store.requireSigs
&& !sub->isTrusted
&& !info->checkSignatures(worker.store, worker.store.getPublicKeys()))
{
logWarning({
.name = "Invalid path signature",
.hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'",
sub->getUri(), worker.store.printStorePath(storePath))
});
tryNext();
return;
}
/* To maintain the closure invariant, we first have to realise the
paths referenced by this one. */
for (auto & i : info->references)
if (i != storePath) /* ignore self-references */
addWaitee(worker.makeSubstitutionGoal(i));
if (waitees.empty()) /* to prevent hang (no wake-up event) */
referencesValid();
else
state = &SubstitutionGoal::referencesValid;
}
void SubstitutionGoal::referencesValid()
{
trace("all references realised");
if (nrFailed > 0) {
debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath));
amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed);
return;
}
for (auto & i : info->references)
if (i != storePath) /* ignore self-references */
assert(worker.store.isValidPath(i));
state = &SubstitutionGoal::tryToRun;
worker.wakeUp(shared_from_this());
}
void SubstitutionGoal::tryToRun()
{
trace("trying to run");
/* Make sure that we are allowed to start a build. Note that even
if maxBuildJobs == 0 (no local builds allowed), we still allow
a substituter to run. This is because substitutions cannot be
distributed to another machine via the build hook. */
if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) {
worker.waitForBuildSlot(shared_from_this());
return;
}
maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions);
worker.updateProgress();
outPipe.create();
promise = std::promise<void>();
thr = std::thread([this]() {
try {
/* Wake up the worker loop when we're done. */
Finally updateStats([this]() { outPipe.writeSide = -1; });
Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()});
PushActivity pact(act.id);
copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()),
subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
promise.set_value();
} catch (...) {
promise.set_exception(std::current_exception());
}
});
worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false);
state = &SubstitutionGoal::finished;
}
void SubstitutionGoal::finished()
{
trace("substitute finished");
thr.join();
worker.childTerminated(this);
try {
promise.get_future().get();
} catch (std::exception & e) {
printError(e.what());
/* Cause the parent build to fail unless --fallback is given,
or the substitute has disappeared. The latter case behaves
the same as the substitute never having existed in the
first place. */
try {
throw;
} catch (SubstituteGone &) {
} catch (...) {
substituterFailed = true;
}
/* Try the next substitute. */
state = &SubstitutionGoal::tryNext;
worker.wakeUp(shared_from_this());
return;
}
worker.markContentsGood(storePath);
printMsg(lvlChatty, "substitution of path '%s' succeeded", worker.store.printStorePath(storePath));
maintainRunningSubstitutions.reset();
maintainExpectedSubstitutions.reset();
worker.doneSubstitutions++;
if (maintainExpectedDownload) {
auto fileSize = maintainExpectedDownload->delta;
maintainExpectedDownload.reset();
worker.doneDownloadSize += fileSize;
}
worker.doneNarSize += maintainExpectedNar->delta;
maintainExpectedNar.reset();
worker.updateProgress();
amDone(ecSuccess);
}
void SubstitutionGoal::handleChildOutput(int fd, const string & data)
{
}
void SubstitutionGoal::handleEOF(int fd)
{
if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this());
}
}

83
src/libstore/build/substitution-goal.hh Normal file
View file

@ -0,0 +1,83 @@
#pragma once
#include "lock.hh"
#include "store-api.hh"
#include "goal.hh"
namespace nix {
class Worker;
struct SubstitutionGoal : public Goal
{
/* The store path that should be realised through a substitute. */
StorePath storePath;
/* The path the substituter refers to the path as. This will be
* different when the stores have different names. */
std::optional<StorePath> subPath;
/* The remaining substituters. */
std::list<ref<Store>> subs;
/* The current substituter. */
std::shared_ptr<Store> sub;
/* Whether a substituter failed. */
bool substituterFailed = false;
/* Path info returned by the substituter's query info operation. */
std::shared_ptr<const ValidPathInfo> info;
/* Pipe for the substituter's standard output. */
Pipe outPipe;
/* The substituter thread. */
std::thread thr;
std::promise<void> promise;
/* Whether to try to repair a valid path. */
RepairFlag repair;
/* Location where we're downloading the substitute. Differs from
storePath when doing a repair. */
Path destPath;
std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions,
maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload;
typedef void (SubstitutionGoal::*GoalState)();
GoalState state;
/* Content address for recomputing store path */
std::optional<ContentAddress> ca;
SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
~SubstitutionGoal();
void timedOut(Error && ex) override { abort(); };
string key() override
{
/* "a$" ensures substitution goals happen before derivation
goals. */
return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath);
}
void work() override;
/* The states. */
void init();
void tryNext();
void gotInfo();
void referencesValid();
void tryToRun();
void finished();
/* Callback used by the worker to write to the log. */
void handleChildOutput(int fd, const string & data) override;
void handleEOF(int fd) override;
};
}

475
src/libstore/build/worker.cc Normal file
View file

@ -0,0 +1,475 @@
#include "machines.hh"
#include "worker.hh"
#include "substitution-goal.hh"
#include "derivation-goal.hh"
#include "hook-instance.hh"
#include <poll.h>
namespace nix {
Worker::Worker(LocalStore & store)
: act(*logger, actRealise)
, actDerivations(*logger, actBuilds)
, actSubstitutions(*logger, actCopyPaths)
, store(store)
{
/* Debugging: prevent recursive workers. */
nrLocalBuilds = 0;
lastWokenUp = steady_time_point::min();
permanentFailure = false;
timedOut = false;
hashMismatch = false;
checkMismatch = false;
}
Worker::~Worker()
{
/* Explicitly get rid of all strong pointers now. After this all
goals that refer to this worker should be gone. (Otherwise we
are in trouble, since goals may call childTerminated() etc. in
their destructors). */
topGoals.clear();
assert(expectedSubstitutions == 0);
assert(expectedDownloadSize == 0);
assert(expectedNarSize == 0);
}
std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
const StorePath & drvPath,
const StringSet & wantedOutputs,
std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal)
{
std::weak_ptr<DerivationGoal> & goal_weak = derivationGoals[drvPath];
std::shared_ptr<DerivationGoal> goal = goal_weak.lock();
if (!goal) {
goal = mkDrvGoal();
goal_weak = goal;
wakeUp(goal);
} else {
goal->addWantedOutputs(wantedOutputs);
}
return goal;
}
std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath,
const StringSet & wantedOutputs, BuildMode buildMode)
{
return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
return std::make_shared<DerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
});
}
std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath,
const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode)
{
return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
return std::make_shared<DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
});
}
std::shared_ptr<SubstitutionGoal> Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
{
std::weak_ptr<SubstitutionGoal> & goal_weak = substitutionGoals[path];
auto goal = goal_weak.lock(); // FIXME
if (!goal) {
goal = std::make_shared<SubstitutionGoal>(path, *this, repair, ca);
goal_weak = goal;
wakeUp(goal);
}
return goal;
}
template<typename G>
static void removeGoal(std::shared_ptr<G> goal, std::map<StorePath, std::weak_ptr<G>> & goalMap)
{
/* !!! inefficient */
for (auto i = goalMap.begin();
i != goalMap.end(); )
if (i->second.lock() == goal) {
auto j = i; ++j;
goalMap.erase(i);
i = j;
}
else ++i;
}
void Worker::removeGoal(GoalPtr goal)
{
if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
nix::removeGoal(drvGoal, derivationGoals);
else if (auto subGoal = std::dynamic_pointer_cast<SubstitutionGoal>(goal))
nix::removeGoal(subGoal, substitutionGoals);
else
assert(false);
if (topGoals.find(goal) != topGoals.end()) {
topGoals.erase(goal);
/* If a top-level goal failed, then kill all other goals
(unless keepGoing was set). */
if (goal->exitCode == Goal::ecFailed && !settings.keepGoing)
topGoals.clear();
}
/* Wake up goals waiting for any goal to finish. */
for (auto & i : waitingForAnyGoal) {
GoalPtr goal = i.lock();
if (goal) wakeUp(goal);
}
waitingForAnyGoal.clear();
}
void Worker::wakeUp(GoalPtr goal)
{
goal->trace("woken up");
addToWeakGoals(awake, goal);
}
unsigned Worker::getNrLocalBuilds()
{
return nrLocalBuilds;
}
void Worker::childStarted(GoalPtr goal, const set<int> & fds,
bool inBuildSlot, bool respectTimeouts)
{
Child child;
child.goal = goal;
child.goal2 = goal.get();
child.fds = fds;
child.timeStarted = child.lastOutput = steady_time_point::clock::now();
child.inBuildSlot = inBuildSlot;
child.respectTimeouts = respectTimeouts;
children.emplace_back(child);
if (inBuildSlot) nrLocalBuilds++;
}
void Worker::childTerminated(Goal * goal, bool wakeSleepers)
{
auto i = std::find_if(children.begin(), children.end(),
[&](const Child & child) { return child.goal2 == goal; });
if (i == children.end()) return;
if (i->inBuildSlot) {
assert(nrLocalBuilds > 0);
nrLocalBuilds--;
}
children.erase(i);
if (wakeSleepers) {
/* Wake up goals waiting for a build slot. */
for (auto & j : wantingToBuild) {
GoalPtr goal = j.lock();
if (goal) wakeUp(goal);
}
wantingToBuild.clear();
}
}
void Worker::waitForBuildSlot(GoalPtr goal)
{
debug("wait for build slot");
if (getNrLocalBuilds() < settings.maxBuildJobs)
wakeUp(goal); /* we can do it right away */
else
addToWeakGoals(wantingToBuild, goal);
}
void Worker::waitForAnyGoal(GoalPtr goal)
{
debug("wait for any goal");
addToWeakGoals(waitingForAnyGoal, goal);
}
void Worker::waitForAWhile(GoalPtr goal)
{
debug("wait for a while");
addToWeakGoals(waitingForAWhile, goal);
}
void Worker::run(const Goals & _topGoals)
{
std::vector<nix::StorePathWithOutputs> topPaths;
for (auto & i : _topGoals) {
topGoals.insert(i);
if (auto goal = dynamic_cast<DerivationGoal *>(i.get())) {
topPaths.push_back({goal->drvPath, goal->wantedOutputs});
} else if (auto goal = dynamic_cast<SubstitutionGoal *>(i.get())) {
topPaths.push_back({goal->storePath});
}
}
/* Call queryMissing() efficiently query substitutes. */
StorePathSet willBuild, willSubstitute, unknown;
uint64_t downloadSize, narSize;
store.queryMissing(topPaths, willBuild, willSubstitute, unknown, downloadSize, narSize);
debug("entered goal loop");
while (1) {
checkInterrupt();
store.autoGC(false);
/* Call every wake goal (in the ordering established by
CompareGoalPtrs). */
while (!awake.empty() && !topGoals.empty()) {
Goals awake2;
for (auto & i : awake) {
GoalPtr goal = i.lock();
if (goal) awake2.insert(goal);
}
awake.clear();
for (auto & goal : awake2) {
checkInterrupt();
goal->work();
if (topGoals.empty()) break; // stuff may have been cancelled
}
}
if (topGoals.empty()) break;
/* Wait for input. */
if (!children.empty() || !waitingForAWhile.empty())
waitForInput();
else {
if (awake.empty() && 0 == settings.maxBuildJobs)
{
if (getMachines().empty())
throw Error("unable to start any build; either increase '--max-jobs' "
"or enable remote builds."
"\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
else
throw Error("unable to start any build; remote machines may not have "
"all required system features."
"\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
}
assert(!awake.empty());
}
}
/* If --keep-going is not set, it's possible that the main goal
exited while some of its subgoals were still active. But if
--keep-going *is* set, then they must all be finished now. */
assert(!settings.keepGoing || awake.empty());
assert(!settings.keepGoing || wantingToBuild.empty());
assert(!settings.keepGoing || children.empty());
}
void Worker::waitForInput()
{
printMsg(lvlVomit, "waiting for children");
/* Process output from the file descriptors attached to the
children, namely log output and output path creation commands.
We also use this to detect child termination: if we get EOF on
the logger pipe of a build, we assume that the builder has
terminated. */
bool useTimeout = false;
long timeout = 0;
auto before = steady_time_point::clock::now();
/* If we're monitoring for silence on stdout/stderr, or if there
is a build timeout, then wait for input until the first
deadline for any child. */
auto nearest = steady_time_point::max(); // nearest deadline
if (settings.minFree.get() != 0)
// Periodicallty wake up to see if we need to run the garbage collector.
nearest = before + std::chrono::seconds(10);
for (auto & i : children) {
if (!i.respectTimeouts) continue;
if (0 != settings.maxSilentTime)
nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime));
if (0 != settings.buildTimeout)
nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout));
}
if (nearest != steady_time_point::max()) {
timeout = std::max(1L, (long) std::chrono::duration_cast<std::chrono::seconds>(nearest - before).count());
useTimeout = true;
}
/* If we are polling goals that are waiting for a lock, then wake
up after a few seconds at most. */
if (!waitingForAWhile.empty()) {
useTimeout = true;
if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before;
timeout = std::max(1L,
(long) std::chrono::duration_cast<std::chrono::seconds>(
lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count());
} else lastWokenUp = steady_time_point::min();
if (useTimeout)
vomit("sleeping %d seconds", timeout);
/* Use select() to wait for the input side of any logger pipe to
become `available'. Note that `available' (i.e., non-blocking)
includes EOF. */
std::vector<struct pollfd> pollStatus;
std::map <int, int> fdToPollStatus;
for (auto & i : children) {
for (auto & j : i.fds) {
pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN });
fdToPollStatus[j] = pollStatus.size() - 1;
}
}
if (poll(pollStatus.data(), pollStatus.size(),
useTimeout ? timeout * 1000 : -1) == -1) {
if (errno == EINTR) return;
throw SysError("waiting for input");
}
auto after = steady_time_point::clock::now();
/* Process all available file descriptors. FIXME: this is
O(children * fds). */
decltype(children)::iterator i;
for (auto j = children.begin(); j != children.end(); j = i) {
i = std::next(j);
checkInterrupt();
GoalPtr goal = j->goal.lock();
assert(goal);
set<int> fds2(j->fds);
std::vector<unsigned char> buffer(4096);
for (auto & k : fds2) {
if (pollStatus.at(fdToPollStatus.at(k)).revents) {
ssize_t rd = ::read(k, buffer.data(), buffer.size());
// FIXME: is there a cleaner way to handle pt close
// than EIO? Is this even standard?
if (rd == 0 || (rd == -1 && errno == EIO)) {
debug("%1%: got EOF", goal->getName());
goal->handleEOF(k);
j->fds.erase(k);
} else if (rd == -1) {
if (errno != EINTR)
throw SysError("%s: read failed", goal->getName());
} else {
printMsg(lvlVomit, "%1%: read %2% bytes",
goal->getName(), rd);
string data((char *) buffer.data(), rd);
j->lastOutput = after;
goal->handleChildOutput(k, data);
}
}
}
if (goal->exitCode == Goal::ecBusy &&
0 != settings.maxSilentTime &&
j->respectTimeouts &&
after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime))
{
goal->timedOut(Error(
"%1% timed out after %2% seconds of silence",
goal->getName(), settings.maxSilentTime));
}
else if (goal->exitCode == Goal::ecBusy &&
0 != settings.buildTimeout &&
j->respectTimeouts &&
after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout))
{
goal->timedOut(Error(
"%1% timed out after %2% seconds",
goal->getName(), settings.buildTimeout));
}
}
if (!waitingForAWhile.empty() && lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) {
lastWokenUp = after;
for (auto & i : waitingForAWhile) {
GoalPtr goal = i.lock();
if (goal) wakeUp(goal);
}
waitingForAWhile.clear();
}
}
unsigned int Worker::exitStatus()
{
/*
* 1100100
* ^^^^
* |||`- timeout
* ||`-- output hash mismatch
* |`--- build failure
* `---- not deterministic
*/
unsigned int mask = 0;
bool buildFailure = permanentFailure || timedOut || hashMismatch;
if (buildFailure)
mask |= 0x04; // 100
if (timedOut)
mask |= 0x01; // 101
if (hashMismatch)
mask |= 0x02; // 102
if (checkMismatch) {
mask |= 0x08; // 104
}
if (mask)
mask |= 0x60;
return mask ? mask : 1;
}
bool Worker::pathContentsGood(const StorePath & path)
{
auto i = pathContentsGoodCache.find(path);
if (i != pathContentsGoodCache.end()) return i->second;
printInfo("checking path '%s'...", store.printStorePath(path));
auto info = store.queryPathInfo(path);
bool res;
if (!pathExists(store.printStorePath(path)))
res = false;
else {
HashResult current = hashPath(info->narHash.type, store.printStorePath(path));
Hash nullHash(htSHA256);
res = info->narHash == nullHash || info->narHash == current.first;
}
pathContentsGoodCache.insert_or_assign(path, res);
if (!res)
logError({
.name = "Corrupted path",
.hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path))
});
return res;
}
void Worker::markContentsGood(const StorePath & path)
{
pathContentsGoodCache.insert_or_assign(path, true);
}
GoalPtr upcast_goal(std::shared_ptr<SubstitutionGoal> subGoal) {
return subGoal;
}
}

207
src/libstore/build/worker.hh Normal file
View file

@ -0,0 +1,207 @@
#pragma once
#include "types.hh"
#include "lock.hh"
#include "local-store.hh"
#include "goal.hh"
namespace nix {
/* Forward definition. */
struct DerivationGoal;
struct SubstitutionGoal;
/* Workaround for not being able to declare a something like
class SubstitutionGoal : public Goal;
even when Goal is a complete type.
This is still a static cast. The purpose of exporting it is to define it in
a place where `SubstitutionGoal` is concrete, and use it in a place where it
is opaque. */
GoalPtr upcast_goal(std::shared_ptr<SubstitutionGoal> subGoal);
typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
/* A mapping used to remember for each child process to what goal it
belongs, and file descriptors for receiving log data and output
path creation commands. */
struct Child
{
WeakGoalPtr goal;
Goal * goal2; // ugly hackery
set<int> fds;
bool respectTimeouts;
bool inBuildSlot;
steady_time_point lastOutput; /* time we last got output on stdout/stderr */
steady_time_point timeStarted;
};
/* Forward definition. */
struct HookInstance;
/* The worker class. */
class Worker
{
private:
/* Note: the worker should only have strong pointers to the
top-level goals. */
/* The top-level goals of the worker. */
Goals topGoals;
/* Goals that are ready to do some work. */
WeakGoals awake;
/* Goals waiting for a build slot. */
WeakGoals wantingToBuild;
/* Child processes currently running. */
std::list<Child> children;
/* Number of build slots occupied. This includes local builds and
substitutions but not remote builds via the build hook. */
unsigned int nrLocalBuilds;
/* Maps used to prevent multiple instantiations of a goal for the
same derivation / path. */
std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals;
std::map<StorePath, std::weak_ptr<SubstitutionGoal>> substitutionGoals;
/* Goals waiting for busy paths to be unlocked. */
WeakGoals waitingForAnyGoal;
/* Goals sleeping for a few seconds (polling a lock). */
WeakGoals waitingForAWhile;
/* Last time the goals in `waitingForAWhile' where woken up. */
steady_time_point lastWokenUp;
/* Cache for pathContentsGood(). */
std::map<StorePath, bool> pathContentsGoodCache;
public:
const Activity act;
const Activity actDerivations;
const Activity actSubstitutions;
/* Set if at least one derivation had a BuildError (i.e. permanent
failure). */
bool permanentFailure;
/* Set if at least one derivation had a timeout. */
bool timedOut;
/* Set if at least one derivation fails with a hash mismatch. */
bool hashMismatch;
/* Set if at least one derivation is not deterministic in check mode. */
bool checkMismatch;
LocalStore & store;
std::unique_ptr<HookInstance> hook;
uint64_t expectedBuilds = 0;
uint64_t doneBuilds = 0;
uint64_t failedBuilds = 0;
uint64_t runningBuilds = 0;
uint64_t expectedSubstitutions = 0;
uint64_t doneSubstitutions = 0;
uint64_t failedSubstitutions = 0;
uint64_t runningSubstitutions = 0;
uint64_t expectedDownloadSize = 0;
uint64_t doneDownloadSize = 0;
uint64_t expectedNarSize = 0;
uint64_t doneNarSize = 0;
/* Whether to ask the build hook if it can build a derivation. If
it answers with "decline-permanently", we don't try again. */
bool tryBuildHook = true;
Worker(LocalStore & store);
~Worker();
/* Make a goal (with caching). */
/* derivation goal */
private:
std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
const StorePath & drvPath, const StringSet & wantedOutputs,
std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);
public:
std::shared_ptr<DerivationGoal> makeDerivationGoal(
const StorePath & drvPath,
const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
const StorePath & drvPath, const BasicDerivation & drv,
const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
/* substitution goal */
std::shared_ptr<SubstitutionGoal> makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
/* Remove a dead goal. */
void removeGoal(GoalPtr goal);
/* Wake up a goal (i.e., there is something for it to do). */
void wakeUp(GoalPtr goal);
/* Return the number of local build and substitution processes
currently running (but not remote builds via the build
hook). */
unsigned int getNrLocalBuilds();
/* Registers a running child process. `inBuildSlot' means that
the process counts towards the jobs limit. */
void childStarted(GoalPtr goal, const set<int> & fds,
bool inBuildSlot, bool respectTimeouts);
/* Unregisters a running child process. `wakeSleepers' should be
false if there is no sense in waking up goals that are sleeping
because they can't run yet (e.g., there is no free build slot,
or the hook would still say `postpone'). */
void childTerminated(Goal * goal, bool wakeSleepers = true);
/* Put `goal' to sleep until a build slot becomes available (which
might be right away). */
void waitForBuildSlot(GoalPtr goal);
/* Wait for any goal to finish. Pretty indiscriminate way to
wait for some resource that some other goal is holding. */
void waitForAnyGoal(GoalPtr goal);
/* Wait for a few seconds and then retry this goal. Used when
waiting for a lock held by another process. This kind of
polling is inefficient, but POSIX doesn't really provide a way
to wait for multiple locks in the main select() loop. */
void waitForAWhile(GoalPtr goal);
/* Loop until the specified top-level goals have finished. */
void run(const Goals & topGoals);
/* Wait for input to become available. */
void waitForInput();
unsigned int exitStatus();
/* Check whether the given valid path exists and has the right
contents. */
bool pathContentsGood(const StorePath & path);
void markContentsGood(const StorePath & path);
void updateProgress()
{
actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds);
actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions);
act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize);
act.setExpected(actCopyPath, expectedNarSize + doneNarSize);
}
};
}

11
src/libstore/ca-specific-schema.sql Normal file
View file

@ -0,0 +1,11 @@
-- Extension of the sql schema for content-addressed derivations.
-- Won't be loaded unless the experimental feature `ca-derivations`
-- is enabled
create table if not exists Realisations (
drvPath text not null,
outputName text not null, -- symbolic output id, usually "out"
outputPath integer not null,
primary key (drvPath, outputName),
foreign key (outputPath) references ValidPaths(id) on delete cascade
);

43
src/libstore/daemon.cc
View file

@ -153,10 +153,10 @@ struct TunnelSink : Sink
{
Sink & to;
TunnelSink(Sink & to) : to(to) { }
virtual void operator () (const unsigned char * data, size_t len)
void operator () (std::string_view data)
{
to << STDERR_WRITE;
writeString(data, len, to);
writeString(data, to);
}
};
@ -165,7 +165,7 @@ struct TunnelSource : BufferedSource
Source & from;
BufferedSink & to;
TunnelSource(Source & from, BufferedSink & to) : from(from), to(to) { }
size_t readUnbuffered(unsigned char * data, size_t len) override
size_t readUnbuffered(char * data, size_t len) override
{
to << STDERR_READ << len;
to.flush();
@ -215,6 +215,8 @@ struct ClientSettings
for (auto & s : ss)
if (trusted.count(s))
subs.push_back(s);
else if (!hasSuffix(s, "/") && trusted.count(s + "/"))
subs.push_back(s + "/");
else
warn("ignoring untrusted substituter '%s'", s);
res = subs;
@ -231,8 +233,6 @@ struct ClientSettings
settings.set(name, value);
else if (setSubstituters(settings.substituters))
;
else if (setSubstituters(settings.extraSubstituters))
;
else
debug("ignoring the client-specified setting '%s', because it is a restricted setting and you are not a trusted user", name);
} catch (UsageError & e) {
@ -276,8 +276,17 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopQueryValidPaths: {
auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {});
SubstituteFlag substitute = NoSubstitute;
if (GET_PROTOCOL_MINOR(clientVersion) >= 27) {
substitute = readInt(from) ? Substitute : NoSubstitute;
}
logger->startWork();
auto res = store->queryValidPaths(paths);
if (substitute) {
store->substitutePaths(paths);
}
auto res = store->queryValidPaths(paths, substitute);
logger->stopWork();
worker_proto::write(*store, to, res);
break;
@ -859,6 +868,28 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
break;
}
case wopRegisterDrvOutput: {
logger->startWork();
auto outputId = DrvOutput::parse(readString(from));
auto outputPath = StorePath(readString(from));
auto resolvedDrv = StorePath(readString(from));
store->registerDrvOutput(Realisation{
.id = outputId, .outPath = outputPath});
logger->stopWork();
break;
}
case wopQueryRealisation: {
logger->startWork();
auto outputId = DrvOutput::parse(readString(from));
auto info = store->queryRealisation(outputId);
logger->stopWork();
std::set<StorePath> outPaths;
if (info) outPaths.insert(info->outPath);
worker_proto::write(*store, to, outPaths);
break;
}
default:
throw Error("invalid operation %1%", op);
}

156
src/libstore/derivations.cc
View file

@ -21,6 +21,9 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string
[](DerivationOutputCAFloating dof) -> std::optional<StorePath> {
return std::nullopt;
},
[](DerivationOutputDeferred) -> std::optional<StorePath> {
return std::nullopt;
},
}, output);
}
@ -37,6 +40,7 @@ bool derivationIsCA(DerivationType dt) {
case DerivationType::InputAddressed: return false;
case DerivationType::CAFixed: return true;
case DerivationType::CAFloating: return true;
case DerivationType::DeferredInputAddressed: return false;
};
// Since enums can have non-variant values, but making a `default:` would
// disable exhaustiveness warnings.
@ -48,6 +52,7 @@ bool derivationIsFixed(DerivationType dt) {
case DerivationType::InputAddressed: return false;
case DerivationType::CAFixed: return true;
case DerivationType::CAFloating: return false;
case DerivationType::DeferredInputAddressed: return false;
};
assert(false);
}
@ -57,6 +62,7 @@ bool derivationIsImpure(DerivationType dt) {
case DerivationType::InputAddressed: return false;
case DerivationType::CAFixed: return true;
case DerivationType::CAFloating: return false;
case DerivationType::DeferredInputAddressed: return false;
};
assert(false);
}
@ -180,6 +186,11 @@ static DerivationOutput parseDerivationOutput(const Store & store,
};
}
} else {
if (pathS == "") {
return DerivationOutput {
.output = DerivationOutputDeferred { }
};
}
validatePath(pathS);
return DerivationOutput {
.output = DerivationOutputInputAddressed {
@ -325,6 +336,11 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
s += ','; printUnquotedString(s, "");
},
[&](DerivationOutputDeferred) {
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, "");
}
}, i.second.output);
s += ')';
}
@ -389,7 +405,7 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName
DerivationType BasicDerivation::type() const
{
std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs;
std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs, deferredIAOutputs;
std::optional<HashType> floatingHashType;
for (auto & i : outputs) {
std::visit(overloaded {
@ -408,29 +424,34 @@ DerivationType BasicDerivation::type() const
throw Error("All floating outputs must use the same hash type");
}
},
[&](DerivationOutputDeferred _) {
deferredIAOutputs.insert(i.first);
},
}, i.second.output);
}
if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) {
if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
throw Error("Must have at least one output");
} else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) {
} else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
return DerivationType::InputAddressed;
} else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty()) {
} else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
if (fixedCAOutputs.size() > 1)
// FIXME: Experimental feature?
throw Error("Only one fixed output is allowed for now");
if (*fixedCAOutputs.begin() != "out")
throw Error("Single fixed output must be named \"out\"");
return DerivationType::CAFixed;
} else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty()) {
} else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
return DerivationType::CAFloating;
} else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && !deferredIAOutputs.empty()) {
return DerivationType::DeferredInputAddressed;
} else {
throw Error("Can't mix derivation output types");
}
}
DrvHashes drvHashes;
Sync<DrvHashes> drvHashes;
/* pathDerivationModulo and hashDerivationModulo are mutually recursive
*/
@ -438,20 +459,22 @@ DrvHashes drvHashes;
/* Look up the derivation by value and memoize the
`hashDerivationModulo` call.
*/
static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath & drvPath)
static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath & drvPath)
{
auto h = drvHashes.find(drvPath);
if (h == drvHashes.end()) {
assert(store.isValidPath(drvPath));
// Cache it
h = drvHashes.insert_or_assign(
drvPath,
hashDerivationModulo(
store,
store.readDerivation(drvPath),
false)).first;
}
{
auto hashes = drvHashes.lock();
auto h = hashes->find(drvPath);
if (h != hashes->end()) {
return h->second;
}
}
auto h = hashDerivationModulo(
store,
store.readInvalidDerivation(drvPath),
false);
// Cache it
drvHashes.lock()->insert_or_assign(drvPath, h);
return h;
}
/* See the header for interface details. These are the implementation details.
@ -473,10 +496,9 @@ static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath
*/
DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
{
bool isDeferred = false;
/* Return a fixed hash for fixed-output derivations. */
switch (drv.type()) {
case DerivationType::CAFloating:
throw Error("Regular input-addressed derivations are not yet allowed to depend on CA derivations");
case DerivationType::CAFixed: {
std::map<std::string, Hash> outputHashes;
for (const auto & i : drv.outputs) {
@ -489,8 +511,13 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
}
return outputHashes;
}
case DerivationType::CAFloating:
isDeferred = true;
break;
case DerivationType::InputAddressed:
break;
case DerivationType::DeferredInputAddressed:
break;
}
/* For other derivations, replace the inputs paths with recursive
@ -503,6 +530,10 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
[&](Hash drvHash) {
inputs2.insert_or_assign(drvHash.to_string(Base16, false), i.second);
},
[&](DeferredHash deferredHash) {
isDeferred = true;
inputs2.insert_or_assign(deferredHash.hash.to_string(Base16, false), i.second);
},
// CA derivation's output hashes
[&](CaOutputHashes outputHashes) {
std::set<std::string> justOut = { "out" };
@ -517,7 +548,34 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
}, res);
}
return hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
auto hash = hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
if (isDeferred)
return DeferredHash { hash };
else
return hash;
}
std::map<std::string, Hash> staticOutputHashes(Store& store, const Derivation& drv)
{
std::map<std::string, Hash> res;
std::visit(overloaded {
[&](Hash drvHash) {
for (auto & outputName : drv.outputNames()) {
res.insert({outputName, drvHash});
}
},
[&](DeferredHash deferredHash) {
for (auto & outputName : drv.outputNames()) {
res.insert({outputName, deferredHash.hash});
}
},
[&](CaOutputHashes outputHashes) {
res = outputHashes;
},
}, hashDerivationModulo(store, drv, true));
return res;
}
@ -620,6 +678,11 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
<< (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType))
<< "";
},
[&](DerivationOutputDeferred) {
out << ""
<< ""
<< "";
},
}, i.second.output);
}
worker_proto::write(store, out, drv.inputSrcs);
@ -645,7 +708,6 @@ std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath
}
// N.B. Outputs are left unchanged
static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) {
debug("Rewriting the derivation");
@ -666,12 +728,24 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
newEnv.emplace(envName, envValue);
}
drv.env = newEnv;
auto hashModulo = hashDerivationModulo(store, Derivation(drv), true);
for (auto & [outputName, output] : drv.outputs) {
if (std::holds_alternative<DerivationOutputDeferred>(output.output)) {
Hash h = std::get<Hash>(hashModulo);
auto outPath = store.makeOutputPath(outputName, h, drv.name);
drv.env[outputName] = store.printStorePath(outPath);
output = DerivationOutput {
.output = DerivationOutputInputAddressed {
.path = std::move(outPath),
},
};
}
}
}
Sync<DrvPathResolutions> drvPathResolutions;
std::optional<BasicDerivation> Derivation::tryResolve(Store & store) {
std::optional<BasicDerivation> Derivation::tryResolveUncached(Store & store) {
BasicDerivation resolved { *this };
// Input paths that we'll want to rewrite in the derivation
@ -697,4 +771,34 @@ std::optional<BasicDerivation> Derivation::tryResolve(Store & store) {
return resolved;
}
std::optional<BasicDerivation> Derivation::tryResolve(Store& store)
{
auto drvPath = writeDerivation(store, *this, NoRepair, false);
return Derivation::tryResolve(store, drvPath);
}
std::optional<BasicDerivation> Derivation::tryResolve(Store& store, const StorePath& drvPath)
{
// This is quite dirty and leaky, but will disappear once #4340 is merged
static Sync<std::map<StorePath, std::optional<Derivation>>> resolutionsCache;
{
auto resolutions = resolutionsCache.lock();
auto resolvedDrvIter = resolutions->find(drvPath);
if (resolvedDrvIter != resolutions->end()) {
auto & [_, resolvedDrv] = *resolvedDrvIter;
return *resolvedDrv;
}
}
/* Try resolve drv and use that path instead. */
auto drv = store.readDerivation(drvPath);
auto attempt = drv.tryResolveUncached(store);
if (!attempt)
return std::nullopt;
/* Store in memo table. */
resolutionsCache.lock()->insert_or_assign(drvPath, *attempt);
return *attempt;
}
}

37
src/libstore/derivations.hh
View file

@ -18,8 +18,6 @@ namespace nix {
/* The traditional non-fixed-output derivation type. */
struct DerivationOutputInputAddressed
{
/* Will need to become `std::optional<StorePath>` once input-addressed
derivations are allowed to depend on cont-addressed derivations */
StorePath path;
};
@ -41,12 +39,18 @@ struct DerivationOutputCAFloating
HashType hashType;
};
/* Input-addressed output which depends on a (CA) derivation whose hash isn't
* known atm
*/
struct DerivationOutputDeferred {};
struct DerivationOutput
{
std::variant<
DerivationOutputInputAddressed,
DerivationOutputCAFixed,
DerivationOutputCAFloating
DerivationOutputCAFloating,
DerivationOutputDeferred
> output;
std::optional<HashType> hashAlgoOpt(const Store & store) const;
/* Note, when you use this function you should make sure that you're passing
@ -72,6 +76,7 @@ typedef std::map<string, string> StringPairs;
enum struct DerivationType : uint8_t {
InputAddressed,
DeferredInputAddressed,
CAFixed,
CAFloating,
};
@ -133,10 +138,14 @@ struct Derivation : BasicDerivation
2. Input placeholders are replaced with realized input store paths. */
std::optional<BasicDerivation> tryResolve(Store & store);
static std::optional<BasicDerivation> tryResolve(Store & store, const StorePath & drvPath);
Derivation() = default;
Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { }
Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { }
private:
std::optional<BasicDerivation> tryResolveUncached(Store & store);
};
@ -167,9 +176,12 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName
// whose output hashes are always known since they are fixed up-front.
typedef std::map<std::string, Hash> CaOutputHashes;
struct DeferredHash { Hash hash; };
typedef std::variant<
Hash, // regular DRV normalized hash
CaOutputHashes
CaOutputHashes, // Fixed-output derivation hashes
DeferredHash // Deferred hashes for floating outputs drvs and their dependencies
> DrvHashModulo;
/* Returns hashes with the details of fixed-output subderivations
@ -197,20 +209,17 @@ typedef std::variant<
*/
DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs);
/*
Return a map associating each output to a hash that uniquely identifies its
derivation (modulo the self-references).
*/
std::map<std::string, Hash> staticOutputHashes(Store& store, const Derivation& drv);
/* Memoisation of hashDerivationModulo(). */
typedef std::map<StorePath, DrvHashModulo> DrvHashes;
extern DrvHashes drvHashes; // FIXME: global, not thread-safe
/* Memoisation of `readDerivation(..).resove()`. */
typedef std::map<
StorePath,
std::optional<StorePath>
> DrvPathResolutions;
// FIXME: global, though at least thread-safe.
// FIXME: arguably overlaps with hashDerivationModulo memo table.
extern Sync<DrvPathResolutions> drvPathResolutions;
extern Sync<DrvHashes> drvHashes;
bool wantOutput(const string & output, const std::set<string> & wanted);

3
src/libstore/dummy-store.cc
View file

@ -60,6 +60,9 @@ struct DummyStore : public Store, public virtual DummyStoreConfig
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode) override
{ unsupported("buildDerivation"); }
std::optional<const Realisation> queryRealisation(const DrvOutput&) override
{ unsupported("queryRealisation"); }
};
static RegisterStoreImplementation<DummyStore, DummyStoreConfig> regDummyStore;

18
src/libstore/filetransfer.cc
View file

@ -95,18 +95,18 @@ struct curlFileTransfer : public FileTransfer
fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri),
{request.uri}, request.parentAct)
, callback(std::move(callback))
, finalSink([this](const unsigned char * data, size_t len) {
, finalSink([this](std::string_view data) {
if (this->request.dataCallback) {
auto httpStatus = getHTTPStatus();
/* Only write data to the sink if this is a
successful response. */
if (successfulStatuses.count(httpStatus)) {
writtenToSink += len;
this->request.dataCallback((char *) data, len);
writtenToSink += data.size();
this->request.dataCallback(data);
}
} else
this->result.data->append((char *) data, len);
this->result.data->append(data);
})
{
if (!request.expectedETag.empty())
@ -171,8 +171,8 @@ struct curlFileTransfer : public FileTransfer
}
if (errorSink)
(*errorSink)((unsigned char *) contents, realSize);
(*decompressionSink)((unsigned char *) contents, realSize);
(*errorSink)({(char *) contents, realSize});
(*decompressionSink)({(char *) contents, realSize});
return realSize;
} catch (...) {
@ -776,7 +776,7 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink)
state->request.notify_one();
});
request.dataCallback = [_state](char * buf, size_t len) {
request.dataCallback = [_state](std::string_view data) {
auto state(_state->lock());
@ -794,7 +794,7 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink)
/* Append data to the buffer and wake up the calling
thread. */
state->data.append(buf, len);
state->data.append(data);
state->avail.notify_one();
};
@ -840,7 +840,7 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink)
if it's blocked on a full buffer. We don't hold the state
lock while doing this to prevent blocking the download
thread if sink() takes a long time. */
sink((unsigned char *) chunk.data(), chunk.size());
sink(chunk);
}
}

2
src/libstore/filetransfer.hh
View file

@ -61,7 +61,7 @@ struct FileTransferRequest
bool decompress = true;
std::shared_ptr<std::string> data;
std::string mimeType;
std::function<void(char *, size_t)> dataCallback;
std::function<void(std::string_view data)> dataCallback;
FileTransferRequest(const std::string & uri)
: uri(uri), parentAct(getCurActivity()) { }

2
src/libstore/gc.cc
View file

@ -683,7 +683,7 @@ void LocalStore::removeUnusedLinks(const GCState & state)
struct stat st;
if (stat(linksDir.c_str(), &st) == -1)
throw SysError("statting '%1%'", linksDir);
auto overhead = st.st_blocks * 512ULL;
int64_t overhead = st.st_blocks * 512ULL;
printInfo("note: currently hard linking saves %.2f MiB",
((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));

15
src/libstore/globals.cc
View file

@ -86,6 +86,12 @@ void loadConfFile()
for (auto file = files.rbegin(); file != files.rend(); file++) {
globalConfig.applyConfigFile(*file);
}
auto nixConfEnv = getEnv("NIX_CONFIG");
if (nixConfEnv.has_value()) {
globalConfig.applyConfig(nixConfEnv.value(), "NIX_CONFIG");
}
}
std::vector<Path> getUserConfigFiles()
@ -154,7 +160,7 @@ NLOHMANN_JSON_SERIALIZE_ENUM(SandboxMode, {
{SandboxMode::smDisabled, false},
});
template<> void BaseSetting<SandboxMode>::set(const std::string & str)
template<> void BaseSetting<SandboxMode>::set(const std::string & str, bool append)
{
if (str == "true") value = smEnabled;
else if (str == "relaxed") value = smRelaxed;
@ -162,6 +168,11 @@ template<> void BaseSetting<SandboxMode>::set(const std::string & str)
else throw UsageError("option '%s' has invalid value '%s'", name, str);
}
template<> bool BaseSetting<SandboxMode>::isAppendable()
{
return false;
}
template<> std::string BaseSetting<SandboxMode>::to_string() const
{
if (value == smEnabled) return "true";
@ -192,7 +203,7 @@ template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::s
});
}
void MaxBuildJobsSetting::set(const std::string & str)
void MaxBuildJobsSetting::set(const std::string & str, bool append)
{
if (str == "auto") value = std::max(1U, std::thread::hardware_concurrency());
else if (!string2Int(str, value))

25
src/libstore/globals.hh
View file

@ -25,7 +25,7 @@ struct MaxBuildJobsSetting : public BaseSetting<unsigned int>
options->addSetting(this);
}
void set(const std::string & str) override;
void set(const std::string & str, bool append = false) override;
};
class Settings : public Config {
@ -413,14 +413,6 @@ public:
Setting<bool> sandboxFallback{this, true, "sandbox-fallback",
"Whether to disable sandboxing when the kernel doesn't allow it."};
Setting<PathSet> extraSandboxPaths{
this, {}, "extra-sandbox-paths",
R"(
A list of additional paths appended to `sandbox-paths`. Useful if
you want to extend its default value.
)",
{"build-extra-chroot-dirs", "build-extra-sandbox-paths"}};
Setting<size_t> buildRepeat{
this, 0, "repeat",
R"(
@ -591,7 +583,7 @@ public:
Setting<Strings> substituters{
this,
nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"} : Strings(),
Strings{"https://cache.nixos.org/"},
"substituters",
R"(
A list of URLs of substituters, separated by whitespace. The default
@ -599,17 +591,6 @@ public:
)",
{"binary-caches"}};
// FIXME: provide a way to add to option values.
Setting<Strings> extraSubstituters{
this, {}, "extra-substituters",
R"(
Additional binary caches appended to those specified in
`substituters`. When used by unprivileged users, untrusted
substituters (i.e. those not listed in `trusted-substituters`) are
silently ignored.
)",
{"extra-binary-caches"}};
Setting<StringSet> trustedSubstituters{
this, {}, "trusted-substituters",
R"(
@ -886,7 +867,7 @@ public:
Example `~/.config/nix/nix.conf`:
```
access-tokens = "github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk"
access-tokens = github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk
```
Example `~/code/flake.nix`:

4
src/libstore/legacy-ssh-store.cc
View file

@ -333,6 +333,10 @@ public:
auto conn(connections->get());
return conn->remoteVersion;
}
std::optional<const Realisation> queryRealisation(const DrvOutput&) override
// TODO: Implement
{ unsupported("queryRealisation"); }
};
static RegisterStoreImplementation<LegacySSHStore, LegacySSHStoreConfig> regLegacySSHStore;

1
src/libstore/local-binary-cache-store.cc
View file

@ -87,6 +87,7 @@ protected:
void LocalBinaryCacheStore::init()
{
createDirs(binaryCacheDir + "/nar");
createDirs(binaryCacheDir + realisationsPrefix);
if (writeDebugInfo)
createDirs(binaryCacheDir + "/debuginfo");
BinaryCacheStore::init();

291
src/libstore/local-store.cc
View file

@ -7,6 +7,7 @@
#include "nar-info.hh"
#include "references.hh"
#include "callback.hh"
#include "topo-sort.hh"
#include <iostream>
#include <algorithm>
@ -41,6 +42,61 @@
namespace nix {
struct LocalStore::State::Stmts {
/* Some precompiled SQLite statements. */
SQLiteStmt RegisterValidPath;
SQLiteStmt UpdatePathInfo;
SQLiteStmt AddReference;
SQLiteStmt QueryPathInfo;
SQLiteStmt QueryReferences;
SQLiteStmt QueryReferrers;
SQLiteStmt InvalidatePath;
SQLiteStmt AddDerivationOutput;
SQLiteStmt RegisterRealisedOutput;
SQLiteStmt QueryValidDerivers;
SQLiteStmt QueryDerivationOutputs;
SQLiteStmt QueryRealisedOutput;
SQLiteStmt QueryAllRealisedOutputs;
SQLiteStmt QueryPathFromHashPart;
SQLiteStmt QueryValidPaths;
};
int getSchema(Path schemaPath)
{
int curSchema = 0;
if (pathExists(schemaPath)) {
string s = readFile(schemaPath);
if (!string2Int(s, curSchema))
throw Error("'%1%' is corrupt", schemaPath);
}
return curSchema;
}
void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd)
{
const int nixCASchemaVersion = 1;
int curCASchema = getSchema(schemaPath);
if (curCASchema != nixCASchemaVersion) {
if (curCASchema > nixCASchemaVersion) {
throw Error("current Nix store ca-schema is version %1%, but I only support %2%",
curCASchema, nixCASchemaVersion);
}
if (!lockFile(lockFd.get(), ltWrite, false)) {
printInfo("waiting for exclusive access to the Nix store for ca drvs...");
lockFile(lockFd.get(), ltWrite, true);
}
if (curCASchema == 0) {
static const char schema[] =
#include "ca-specific-schema.sql.gen.hh"
;
db.exec(schema);
}
writeFile(schemaPath, fmt("%d", nixCASchemaVersion));
lockFile(lockFd.get(), ltRead, true);
}
}
LocalStore::LocalStore(const Params & params)
: StoreConfig(params)
@ -59,6 +115,7 @@ LocalStore::LocalStore(const Params & params)
, locksHeld(tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS").value_or("")))
{
auto state(_state.lock());
state->stmts = std::make_unique<State::Stmts>();
/* Create missing state directories if they don't already exist. */
createDirs(realStoreDir);
@ -221,32 +278,58 @@ LocalStore::LocalStore(const Params & params)
else openDB(*state, false);
if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
migrateCASchema(state->db, dbDir + "/ca-schema", globalLock);
}
/* Prepare SQL statements. */
state->stmtRegisterValidPath.create(state->db,
state->stmts->RegisterValidPath.create(state->db,
"insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs, ca) values (?, ?, ?, ?, ?, ?, ?, ?);");
state->stmtUpdatePathInfo.create(state->db,
state->stmts->UpdatePathInfo.create(state->db,
"update ValidPaths set narSize = ?, hash = ?, ultimate = ?, sigs = ?, ca = ? where path = ?;");
state->stmtAddReference.create(state->db,
state->stmts->AddReference.create(state->db,
"insert or replace into Refs (referrer, reference) values (?, ?);");
state->stmtQueryPathInfo.create(state->db,
state->stmts->QueryPathInfo.create(state->db,
"select id, hash, registrationTime, deriver, narSize, ultimate, sigs, ca from ValidPaths where path = ?;");
state->stmtQueryReferences.create(state->db,
state->stmts->QueryReferences.create(state->db,
"select path from Refs join ValidPaths on reference = id where referrer = ?;");
state->stmtQueryReferrers.create(state->db,
state->stmts->QueryReferrers.create(state->db,
"select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);");
state->stmtInvalidatePath.create(state->db,
state->stmts->InvalidatePath.create(state->db,
"delete from ValidPaths where path = ?;");
state->stmtAddDerivationOutput.create(state->db,
state->stmts->AddDerivationOutput.create(state->db,
"insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);");
state->stmtQueryValidDerivers.create(state->db,
state->stmts->QueryValidDerivers.create(state->db,
"select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;");
state->stmtQueryDerivationOutputs.create(state->db,
state->stmts->QueryDerivationOutputs.create(state->db,
"select id, path from DerivationOutputs where drv = ?;");
// Use "path >= ?" with limit 1 rather than "path like '?%'" to
// ensure efficient lookup.
state->stmtQueryPathFromHashPart.create(state->db,
state->stmts->QueryPathFromHashPart.create(state->db,
"select path from ValidPaths where path >= ? limit 1;");
state->stmtQueryValidPaths.create(state->db, "select path from ValidPaths");
state->stmts->QueryValidPaths.create(state->db, "select path from ValidPaths");
if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
state->stmts->RegisterRealisedOutput.create(state->db,
R"(
insert or replace into Realisations (drvPath, outputName, outputPath)
values (?, ?, (select id from ValidPaths where path = ?))
;
)");
state->stmts->QueryRealisedOutput.create(state->db,
R"(
select Output.path from Realisations
inner join ValidPaths as Output on Output.id = Realisations.outputPath
where drvPath = ? and outputName = ?
;
)");
state->stmts->QueryAllRealisedOutputs.create(state->db,
R"(
select outputName, Output.path from Realisations
inner join ValidPaths as Output on Output.id = Realisations.outputPath
where drvPath = ?
;
)");
}
}
@ -284,16 +367,7 @@ std::string LocalStore::getUri()
int LocalStore::getSchema()
{
int curSchema = 0;
if (pathExists(schemaPath)) {
string s = readFile(schemaPath);
if (!string2Int(s, curSchema))
throw Error("'%1%' is corrupt", schemaPath);
}
return curSchema;
}
{ return nix::getSchema(schemaPath); }
void LocalStore::openDB(State & state, bool create)
{
@ -573,21 +647,29 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
[&](DerivationOutputCAFloating _) {
/* Nothing to check */
},
[&](DerivationOutputDeferred) {
},
}, i.second.output);
}
}
void LocalStore::linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output)
void LocalStore::registerDrvOutput(const Realisation & info)
{
auto state(_state.lock());
return linkDeriverToPath(*state, queryValidPathId(*state, deriver), outputName, output);
retrySQLite<void>([&]() {
state->stmts->RegisterRealisedOutput.use()
(info.id.strHash())
(info.id.outputName)
(printStorePath(info.outPath))
.exec();
});
}
void LocalStore::linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output)
void LocalStore::cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output)
{
retrySQLite<void>([&]() {
state.stmtAddDerivationOutput.use()
state.stmts->AddDerivationOutput.use()
(deriver)
(outputName)
(printStorePath(output))
@ -604,7 +686,7 @@ uint64_t LocalStore::addValidPath(State & state,
throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't",
printStorePath(info.path));
state.stmtRegisterValidPath.use()
state.stmts->RegisterValidPath.use()
(printStorePath(info.path))
(info.narHash.to_string(Base16, true))
(info.registrationTime == 0 ? time(0) : info.registrationTime)
@ -621,7 +703,7 @@ uint64_t LocalStore::addValidPath(State & state,
efficiently query whether a path is an output of some
derivation. */
if (info.path.isDerivation()) {
auto drv = readDerivation(info.path);
auto drv = readInvalidDerivation(info.path);
/* Verify that the output paths in the derivation are correct
(i.e., follow the scheme for computing output paths from
@ -634,7 +716,7 @@ uint64_t LocalStore::addValidPath(State & state,
/* Floating CA derivations have indeterminate output paths until
they are built, so don't register anything in that case */
if (i.second.second)
linkDeriverToPath(state, id, i.first, *i.second.second);
cacheDrvOutputMapping(state, id, i.first, *i.second.second);
}
}
@ -656,7 +738,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
auto state(_state.lock());
/* Get the path info. */
auto useQueryPathInfo(state->stmtQueryPathInfo.use()(printStorePath(path)));
auto useQueryPathInfo(state->stmts->QueryPathInfo.use()(printStorePath(path)));
if (!useQueryPathInfo.next())
return std::shared_ptr<ValidPathInfo>();
@ -676,7 +758,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
info->registrationTime = useQueryPathInfo.getInt(2);
auto s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 3);
auto s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 3);
if (s) info->deriver = parseStorePath(s);
/* Note that narSize = NULL yields 0. */
@ -684,14 +766,14 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
info->ultimate = useQueryPathInfo.getInt(5) == 1;
s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 6);
s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 6);
if (s) info->sigs = tokenizeString<StringSet>(s, " ");
s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 7);
s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 7);
if (s) info->ca = parseContentAddressOpt(s);
/* Get the references. */
auto useQueryReferences(state->stmtQueryReferences.use()(info->id));
auto useQueryReferences(state->stmts->QueryReferences.use()(info->id));
while (useQueryReferences.next())
info->references.insert(parseStorePath(useQueryReferences.getStr(0)));
@ -706,7 +788,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
/* Update path info in the database. */
void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
{
state.stmtUpdatePathInfo.use()
state.stmts->UpdatePathInfo.use()
(info.narSize, info.narSize != 0)
(info.narHash.to_string(Base16, true))
(info.ultimate ? 1 : 0, info.ultimate)
@ -719,7 +801,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
uint64_t LocalStore::queryValidPathId(State & state, const StorePath & path)
{
auto use(state.stmtQueryPathInfo.use()(printStorePath(path)));
auto use(state.stmts->QueryPathInfo.use()(printStorePath(path)));
if (!use.next())
throw InvalidPath("path '%s' is not valid", printStorePath(path));
return use.getInt(0);
@ -728,7 +810,7 @@ uint64_t LocalStore::queryValidPathId(State & state, const StorePath & path)
bool LocalStore::isValidPath_(State & state, const StorePath & path)
{
return state.stmtQueryPathInfo.use()(printStorePath(path)).next();
return state.stmts->QueryPathInfo.use()(printStorePath(path)).next();
}
@ -754,7 +836,7 @@ StorePathSet LocalStore::queryAllValidPaths()
{
return retrySQLite<StorePathSet>([&]() {
auto state(_state.lock());
auto use(state->stmtQueryValidPaths.use());
auto use(state->stmts->QueryValidPaths.use());
StorePathSet res;
while (use.next()) res.insert(parseStorePath(use.getStr(0)));
return res;
@ -764,7 +846,7 @@ StorePathSet LocalStore::queryAllValidPaths()
void LocalStore::queryReferrers(State & state, const StorePath & path, StorePathSet & referrers)
{
auto useQueryReferrers(state.stmtQueryReferrers.use()(printStorePath(path)));
auto useQueryReferrers(state.stmts->QueryReferrers.use()(printStorePath(path)));
while (useQueryReferrers.next())
referrers.insert(parseStorePath(useQueryReferrers.getStr(0)));
@ -785,7 +867,7 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
return retrySQLite<StorePathSet>([&]() {
auto state(_state.lock());
auto useQueryValidDerivers(state->stmtQueryValidDerivers.use()(printStorePath(path)));
auto useQueryValidDerivers(state->stmts->QueryValidDerivers.use()(printStorePath(path)));
StorePathSet derivers;
while (useQueryValidDerivers.next())
@ -796,69 +878,38 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
}
std::map<std::string, std::optional<StorePath>> LocalStore::queryPartialDerivationOutputMap(const StorePath & path_)
std::map<std::string, std::optional<StorePath>>
LocalStore::queryDerivationOutputMapNoResolve(const StorePath& path_)
{
auto path = path_;
std::map<std::string, std::optional<StorePath>> outputs;
Derivation drv = readDerivation(path);
for (auto & [outName, _] : drv.outputs) {
outputs.insert_or_assign(outName, std::nullopt);
}
bool haveCached = false;
{
auto resolutions = drvPathResolutions.lock();
auto resolvedPathOptIter = resolutions->find(path);
if (resolvedPathOptIter != resolutions->end()) {
auto & [_, resolvedPathOpt] = *resolvedPathOptIter;
if (resolvedPathOpt)
path = *resolvedPathOpt;
haveCached = true;
}
}
/* can't just use else-if instead of `!haveCached` because we need to unlock
`drvPathResolutions` before it is locked in `Derivation::resolve`. */
if (!haveCached && drv.type() == DerivationType::CAFloating) {
/* Try resolve drv and use that path instead. */
auto attempt = drv.tryResolve(*this);
if (!attempt)
/* If we cannot resolve the derivation, we cannot have any path
assigned so we return the map of all std::nullopts. */
return outputs;
/* Just compute store path */
auto pathResolved = writeDerivation(*this, *std::move(attempt), NoRepair, true);
/* Store in memo table. */
/* FIXME: memo logic should not be local-store specific, should have
wrapper-method instead. */
drvPathResolutions.lock()->insert_or_assign(path, pathResolved);
path = std::move(pathResolved);
}
return retrySQLite<std::map<std::string, std::optional<StorePath>>>([&]() {
auto outputs = retrySQLite<std::map<std::string, std::optional<StorePath>>>([&]() {
auto state(_state.lock());
std::map<std::string, std::optional<StorePath>> outputs;
uint64_t drvId;
try {
drvId = queryValidPathId(*state, path);
} catch (InvalidPath &) {
/* FIXME? if the derivation doesn't exist, we cannot have a mapping
for it. */
return outputs;
}
auto useQueryDerivationOutputs {
state->stmtQueryDerivationOutputs.use()
(drvId)
};
while (useQueryDerivationOutputs.next())
auto use(state->stmts->QueryDerivationOutputs.use()(drvId));
while (use.next())
outputs.insert_or_assign(
useQueryDerivationOutputs.getStr(0),
parseStorePath(useQueryDerivationOutputs.getStr(1))
);
use.getStr(0), parseStorePath(use.getStr(1)));
return outputs;
});
}
if (!settings.isExperimentalFeatureEnabled("ca-derivations"))
return outputs;
auto drv = readInvalidDerivation(path);
auto drvHashes = staticOutputHashes(*this, drv);
for (auto& [outputName, hash] : drvHashes) {
auto realisation = queryRealisation(DrvOutput{hash, outputName});
if (realisation)
outputs.insert_or_assign(outputName, realisation->outPath);
else
outputs.insert_or_assign(outputName, std::nullopt);
}
return outputs;
}
std::optional<StorePath> LocalStore::queryPathFromHashPart(const std::string & hashPart)
{
@ -869,11 +920,11 @@ std::optional<StorePath> LocalStore::queryPathFromHashPart(const std::string & h
return retrySQLite<std::optional<StorePath>>([&]() -> std::optional<StorePath> {
auto state(_state.lock());
auto useQueryPathFromHashPart(state->stmtQueryPathFromHashPart.use()(prefix));
auto useQueryPathFromHashPart(state->stmts->QueryPathFromHashPart.use()(prefix));
if (!useQueryPathFromHashPart.next()) return {};
const char * s = (const char *) sqlite3_column_text(state->stmtQueryPathFromHashPart, 0);
const char * s = (const char *) sqlite3_column_text(state->stmts->QueryPathFromHashPart, 0);
if (s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0)
return parseStorePath(s);
return {};
@ -957,9 +1008,7 @@ void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, Subst
void LocalStore::registerValidPath(const ValidPathInfo & info)
{
ValidPathInfos infos;
infos.push_back(info);
registerValidPaths(infos);
registerValidPaths({{info.path, info}});
}
@ -977,7 +1026,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
SQLiteTxn txn(state->db);
StorePathSet paths;
for (auto & i : infos) {
for (auto & [_, i] : infos) {
assert(i.narHash.type == htSHA256);
if (isValidPath_(*state, i.path))
updatePathInfo(*state, i);
@ -986,26 +1035,37 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
paths.insert(i.path);
}
for (auto & i : infos) {
for (auto & [_, i] : infos) {
auto referrer = queryValidPathId(*state, i.path);
for (auto & j : i.references)
state->stmtAddReference.use()(referrer)(queryValidPathId(*state, j)).exec();
state->stmts->AddReference.use()(referrer)(queryValidPathId(*state, j)).exec();
}
/* Check that the derivation outputs are correct. We can't do
this in addValidPath() above, because the references might
not be valid yet. */
for (auto & i : infos)
for (auto & [_, i] : infos)
if (i.path.isDerivation()) {
// FIXME: inefficient; we already loaded the derivation in addValidPath().
checkDerivationOutputs(i.path, readDerivation(i.path));
checkDerivationOutputs(i.path,
readInvalidDerivation(i.path));
}
/* Do a topological sort of the paths. This will throw an
error if a cycle is detected and roll back the
transaction. Cycles can only occur when a derivation
has multiple outputs. */
topoSortPaths(paths);
topoSort(paths,
{[&](const StorePath & path) {
auto i = infos.find(path);
return i == infos.end() ? StorePathSet() : i->second.references;
}},
{[&](const StorePath & path, const StorePath & parent) {
return BuildError(
"cycle detected in the references of '%s' from '%s'",
printStorePath(path),
printStorePath(parent));
}});
txn.commit();
});
@ -1018,7 +1078,7 @@ void LocalStore::invalidatePath(State & state, const StorePath & path)
{
debug("invalidating path '%s'", printStorePath(path));
state.stmtInvalidatePath.use()(printStorePath(path)).exec();
state.stmts->InvalidatePath.use()(printStorePath(path)).exec();
/* Note that the foreign key constraints on the Refs table take
care of deleting the references entries for `path'. */
@ -1083,11 +1143,11 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
auto hashResult = hashSink->finish();
if (hashResult.first != info.narHash)
throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s",
throw Error("hash mismatch importing path '%s';\n specified: %s\n got: %s",
printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true));
if (hashResult.second != info.narSize)
throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s",
throw Error("size mismatch importing path '%s';\n specified: %s\n got: %s",
printStorePath(info.path), info.narSize, hashResult.second);
autoGC();
@ -1131,7 +1191,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
dump.resize(oldSize + want);
auto got = 0;
try {
got = source.read((uint8_t *) dump.data() + oldSize, want);
got = source.read(dump.data() + oldSize, want);
} catch (EndOfFile &) {
inMemory = true;
break;
@ -1584,5 +1644,18 @@ void LocalStore::createUser(const std::string & userName, uid_t userId)
}
}
std::optional<const Realisation> LocalStore::queryRealisation(
const DrvOutput& id) {
typedef std::optional<const Realisation> Ret;
return retrySQLite<Ret>([&]() -> Ret {
auto state(_state.lock());
auto use(state->stmts->QueryRealisedOutput.use()(id.strHash())(
id.outputName));
if (!use.next())
return std::nullopt;
auto outputPath = parseStorePath(use.getStr(0));
return Ret{
Realisation{.id = id, .outPath = outputPath}};
});
}
} // namespace nix

35
src/libstore/local-store.hh
View file

@ -55,19 +55,8 @@ private:
/* The SQLite database object. */
SQLite db;
/* Some precompiled SQLite statements. */
SQLiteStmt stmtRegisterValidPath;
SQLiteStmt stmtUpdatePathInfo;
SQLiteStmt stmtAddReference;
SQLiteStmt stmtQueryPathInfo;
SQLiteStmt stmtQueryReferences;
SQLiteStmt stmtQueryReferrers;
SQLiteStmt stmtInvalidatePath;
SQLiteStmt stmtAddDerivationOutput;
SQLiteStmt stmtQueryValidDerivers;
SQLiteStmt stmtQueryDerivationOutputs;
SQLiteStmt stmtQueryPathFromHashPart;
SQLiteStmt stmtQueryValidPaths;
struct Stmts;
std::unique_ptr<Stmts> stmts;
/* The file to which we write our temporary roots. */
AutoCloseFD fdTempRoots;
@ -90,7 +79,7 @@ private:
std::unique_ptr<PublicKeys> publicKeys;
};
Sync<State, std::recursive_mutex> _state;
Sync<State> _state;
public:
@ -138,7 +127,7 @@ public:
StorePathSet queryValidDerivers(const StorePath & path) override;
std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override;
std::map<std::string, std::optional<StorePath>> queryDerivationOutputMapNoResolve(const StorePath & path) override;
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
@ -219,6 +208,13 @@ public:
garbage until it exceeds maxFree. */
void autoGC(bool sync = true);
/* Register the store path 'output' as the output named 'outputName' of
derivation 'deriver'. */
void registerDrvOutput(const Realisation & info) override;
void cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output);
std::optional<const Realisation> queryRealisation(const DrvOutput&) override;
private:
int getSchema();
@ -287,17 +283,12 @@ private:
specified by the ‘secret-key-files’ option. */
void signPathInfo(ValidPathInfo & info);
/* Register the store path 'output' as the output named 'outputName' of
derivation 'deriver'. */
void linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output);
void linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output);
Path getRealStoreDir() override { return realStoreDir; }
void createUser(const std::string & userName, uid_t userId) override;
friend class DerivationGoal;
friend class SubstitutionGoal;
friend struct DerivationGoal;
friend struct SubstitutionGoal;
};

11
src/libstore/local.mk
View file

@ -4,7 +4,7 @@ libstore_NAME = libnixstore
libstore_DIR := $(d)
libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc)
libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/build/*.cc)
libstore_LIBS = libutil
@ -32,7 +32,7 @@ ifeq ($(HAVE_SECCOMP), 1)
endif
libstore_CXXFLAGS += \
-I src/libutil -I src/libstore \
-I src/libutil -I src/libstore -I src/libstore/build \
-DNIX_PREFIX=\"$(prefix)\" \
-DNIX_STORE_DIR=\"$(storedir)\" \
-DNIX_DATA_DIR=\"$(datadir)\" \
@ -48,7 +48,7 @@ ifneq ($(sandbox_shell),)
libstore_CXXFLAGS += -DSANDBOX_SHELL="\"$(sandbox_shell)\""
endif
$(d)/local-store.cc: $(d)/schema.sql.gen.hh
$(d)/local-store.cc: $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh
$(d)/build.cc:
@ -58,9 +58,12 @@ $(d)/build.cc:
@echo ')foo"' >> $@.tmp
@mv $@.tmp $@
clean-files += $(d)/schema.sql.gen.hh
clean-files += $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh
$(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644))
$(foreach i, $(wildcard src/libstore/builtins/*.hh), \
$(eval $(call install-file-in, $(i), $(includedir)/nix/builtins, 0644)))
$(foreach i, $(wildcard src/libstore/build/*.hh), \
$(eval $(call install-file-in, $(i), $(includedir)/nix/build, 0644)))

93
src/libstore/lock.cc Normal file
View file

@ -0,0 +1,93 @@
#include "lock.hh"
#include "globals.hh"
#include "pathlocks.hh"
#include <grp.h>
#include <pwd.h>
#include <fcntl.h>
#include <unistd.h>
namespace nix {
UserLock::UserLock()
{
assert(settings.buildUsersGroup != "");
createDirs(settings.nixStateDir + "/userpool");
}
bool UserLock::findFreeUser() {
if (enabled()) return true;
/* Get the members of the build-users-group. */
struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
if (!gr)
throw Error("the group '%1%' specified in 'build-users-group' does not exist",
settings.buildUsersGroup);
gid = gr->gr_gid;
/* Copy the result of getgrnam. */
Strings users;
for (char * * p = gr->gr_mem; *p; ++p) {
debug("found build user '%1%'", *p);
users.push_back(*p);
}
if (users.empty())
throw Error("the build users group '%1%' has no members",
settings.buildUsersGroup);
/* Find a user account that isn't currently in use for another
build. */
for (auto & i : users) {
debug("trying user '%1%'", i);
struct passwd * pw = getpwnam(i.c_str());
if (!pw)
throw Error("the user '%1%' in the group '%2%' does not exist",
i, settings.buildUsersGroup);
fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str();
AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
if (!fd)
throw SysError("opening user lock '%1%'", fnUserLock);
if (lockFile(fd.get(), ltWrite, false)) {
fdUserLock = std::move(fd);
user = i;
uid = pw->pw_uid;
/* Sanity check... */
if (uid == getuid() || uid == geteuid())
throw Error("the Nix user should not be a member of '%1%'",
settings.buildUsersGroup);
#if __linux__
/* Get the list of supplementary groups of this build user. This
is usually either empty or contains a group such as "kvm". */
supplementaryGIDs.resize(10);
int ngroups = supplementaryGIDs.size();
int err = getgrouplist(pw->pw_name, pw->pw_gid,
supplementaryGIDs.data(), &ngroups);
if (err == -1)
throw Error("failed to get list of supplementary groups for '%1%'", pw->pw_name);
supplementaryGIDs.resize(ngroups);
#endif
isEnabled = true;
return true;
}
}
return false;
}
void UserLock::kill()
{
killUser(uid);
}
}

37
src/libstore/lock.hh Normal file
View file

@ -0,0 +1,37 @@
#pragma once
#include "sync.hh"
#include "types.hh"
#include "util.hh"
namespace nix {
class UserLock
{
private:
Path fnUserLock;
AutoCloseFD fdUserLock;
bool isEnabled = false;
string user;
uid_t uid = 0;
gid_t gid = 0;
std::vector<gid_t> supplementaryGIDs;
public:
UserLock();
void kill();
string getUser() { return user; }
uid_t getUID() { assert(uid); return uid; }
uid_t getGID() { assert(gid); return gid; }
std::vector<gid_t> getSupplementaryGIDs() { return supplementaryGIDs; }
bool findFreeUser();
bool enabled() { return isEnabled; }
};
}

4
src/libstore/nar-accessor.cc
View file

@ -87,7 +87,7 @@ struct NarAccessor : public FSAccessor
parents.top()->start = pos;
}
void receiveContents(unsigned char * data, size_t len) override
void receiveContents(std::string_view data) override
{ }
void createSymlink(const Path & path, const string & target) override
@ -96,7 +96,7 @@ struct NarAccessor : public FSAccessor
NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target});
}
size_t read(unsigned char * data, size_t len) override
size_t read(char * data, size_t len) override
{
auto n = source.read(data, len);
pos += n;

2
src/libstore/path-info.hh
View file

@ -107,6 +107,6 @@ struct ValidPathInfo
virtual ~ValidPathInfo() { }
};
typedef list<ValidPathInfo> ValidPathInfos;
typedef std::map<StorePath, ValidPathInfo> ValidPathInfos;
}

49
src/libstore/realisation.cc Normal file
View file

@ -0,0 +1,49 @@
#include "realisation.hh"
#include "store-api.hh"
#include <nlohmann/json.hpp>
namespace nix {
MakeError(InvalidDerivationOutputId, Error);
DrvOutput DrvOutput::parse(const std::string &strRep) {
size_t n = strRep.find("!");
if (n == strRep.npos)
throw InvalidDerivationOutputId("Invalid derivation output id %s", strRep);
return DrvOutput{
.drvHash = Hash::parseAnyPrefixed(strRep.substr(0, n)),
.outputName = strRep.substr(n+1),
};
}
std::string DrvOutput::to_string() const {
return strHash() + "!" + outputName;
}
nlohmann::json Realisation::toJSON() const {
return nlohmann::json{
{"id", id.to_string()},
{"outPath", outPath.to_string()},
};
}
Realisation Realisation::fromJSON(
const nlohmann::json& json,
const std::string& whence) {
auto getField = [&](std::string fieldName) -> std::string {
auto fieldIterator = json.find(fieldName);
if (fieldIterator == json.end())
throw Error(
"Drv output info file '%1%' is corrupt, missing field %2%",
whence, fieldName);
return *fieldIterator;
};
return Realisation{
.id = DrvOutput::parse(getField("id")),
.outPath = StorePath(getField("outPath")),
};
}
} // namespace nix

39
src/libstore/realisation.hh Normal file
View file

@ -0,0 +1,39 @@
#pragma once
#include "path.hh"
#include <nlohmann/json_fwd.hpp>
namespace nix {
struct DrvOutput {
// The hash modulo of the derivation
Hash drvHash;
std::string outputName;
std::string to_string() const;
std::string strHash() const
{ return drvHash.to_string(Base16, true); }
static DrvOutput parse(const std::string &);
bool operator<(const DrvOutput& other) const { return to_pair() < other.to_pair(); }
bool operator==(const DrvOutput& other) const { return to_pair() == other.to_pair(); }
private:
// Just to make comparison operators easier to write
std::pair<Hash, std::string> to_pair() const
{ return std::make_pair(drvHash, outputName); }
};
struct Realisation {
DrvOutput id;
StorePath outPath;
nlohmann::json toJSON() const;
static Realisation fromJSON(const nlohmann::json& json, const std::string& whence);
};
typedef std::map<DrvOutput, Realisation> DrvOutputs;
}

40
src/libstore/references.cc
View file

@ -55,25 +55,21 @@ struct RefScanSink : Sink
RefScanSink() { }
void operator () (const unsigned char * data, size_t len);
};
void RefScanSink::operator () (const unsigned char * data, size_t len)
{
void operator () (std::string_view data) override
{
/* It's possible that a reference spans the previous and current
fragment, so search in the concatenation of the tail of the
previous fragment and the start of the current fragment. */
string s = tail + string((const char *) data, len > refLength ? refLength : len);
string s = tail + std::string(data, 0, refLength);
search((const unsigned char *) s.data(), s.size(), hashes, seen);
search(data, len, hashes, seen);
search((const unsigned char *) data.data(), data.size(), hashes, seen);
size_t tailLen = len <= refLength ? len : refLength;
tail =
string(tail, tail.size() < refLength - tailLen ? 0 : tail.size() - (refLength - tailLen)) +
string((const char *) data + len - tailLen, tailLen);
}
size_t tailLen = data.size() <= refLength ? data.size() : refLength;
tail = std::string(tail, tail.size() < refLength - tailLen ? 0 : tail.size() - (refLength - tailLen));
tail.append({data.data() + data.size() - tailLen, tailLen});
}
};
std::pair<PathSet, HashResult> scanForReferences(const string & path,
@ -129,10 +125,10 @@ RewritingSink::RewritingSink(const std::string & from, const std::string & to, S
assert(from.size() == to.size());
}
void RewritingSink::operator () (const unsigned char * data, size_t len)
void RewritingSink::operator () (std::string_view data)
{
std::string s(prev);
s.append((const char *) data, len);
s.append(data);
size_t j = 0;
while ((j = s.find(from, j)) != string::npos) {
@ -146,14 +142,14 @@ void RewritingSink::operator () (const unsigned char * data, size_t len)
pos += consumed;
if (consumed) nextSink((unsigned char *) s.data(), consumed);
if (consumed) nextSink(s.substr(0, consumed));
}
void RewritingSink::flush()
{
if (prev.empty()) return;
pos += prev.size();
nextSink((unsigned char *) prev.data(), prev.size());
nextSink(prev);
prev.clear();
}
@ -163,9 +159,9 @@ HashModuloSink::HashModuloSink(HashType ht, const std::string & modulus)
{
}
void HashModuloSink::operator () (const unsigned char * data, size_t len)
void HashModuloSink::operator () (std::string_view data)
{
rewritingSink(data, len);
rewritingSink(data);
}
HashResult HashModuloSink::finish()
@ -176,10 +172,8 @@ HashResult HashModuloSink::finish()
NAR with self-references and a NAR with some of the
self-references already zeroed out do not produce a hash
collision. FIXME: proof. */
for (auto & pos : rewritingSink.matches) {
auto s = fmt("|%d", pos);
hashSink((unsigned char *) s.data(), s.size());
}
for (auto & pos : rewritingSink.matches)
hashSink(fmt("|%d", pos));
auto h = hashSink.finish();
return {h.first, rewritingSink.pos};

4
src/libstore/references.hh
View file

@ -19,7 +19,7 @@ struct RewritingSink : Sink
RewritingSink(const std::string & from, const std::string & to, Sink & nextSink);
void operator () (const unsigned char * data, size_t len) override;
void operator () (std::string_view data) override;
void flush();
};
@ -31,7 +31,7 @@ struct HashModuloSink : AbstractHashSink
HashModuloSink(HashType ht, const std::string & modulus);
void operator () (const unsigned char * data, size_t len) override;
void operator () (std::string_view data) override;
HashResult finish() override;
};

2
src/libstore/remote-fs-accessor.cc
View file

@ -75,7 +75,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
throw SysError("seeking in '%s'", cacheFile);
std::string buf(length, 0);
readFull(fd.get(), (unsigned char *) buf.data(), length);
readFull(fd.get(), buf.data(), length);
return buf;
});

Some files were not shown because too many files have changed in this diff Show more