Add benchmarking scripts
These scripts were originally written by horrors, and have since been
hacked up a lot by jade. We are putting them up as a CL since it is
better to have checked in benchmarking scripts than to not have
benchmarking scripts.
cc: #23
Co-authored-by: eldritch horrors <pennae@lix.systems>
Change-Id: I95c2f9d24753ac468944c5781deec9508fd5cb8c
This commit is contained in:
parent
1e74bffd5c
commit
f4fc2b5d10
3
bench/.gitignore
vendored
Normal file
3
bench/.gitignore
vendored
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
bench-*.json
|
||||
bench-*.md
|
||||
nixpkgs
|
||||
91
bench/README.md
Normal file
91
bench/README.md
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
# Benchmarking scripts for Lix
|
||||
|
||||
These are very much WIP, and have a few clumsy assumptions that we would
|
||||
somewhat rather be fixed, but we have committed them to let others be able to
|
||||
do benchmarking in the mean time.
|
||||
|
||||
## Benchmarking procedure
|
||||
|
||||
Build some Lixes you want to compare, by whichever means you wish.
|
||||
|
||||
Get a computer that is not busy and *strongly preferably* is bare-metal or at
|
||||
least not a cloud VM (e.g. go make coffee when running benchmarks).
|
||||
|
||||
From the root of a Lix checkout, run `./bench/bench.sh resultlink-one
|
||||
resultlink-two`, where `resultlink-one` and `resultlink-two` are the result
|
||||
links from the builds you want to test (they can be any directory with bin/nix
|
||||
in it, however).
|
||||
|
||||
To get the summary again, run `./bench/summarize.jq bench/bench-*.json`.
|
||||
|
||||
## Example results
|
||||
|
||||
(vim tip: `:r !bench/summarize.jq bench/bench-*.json` to dump it directly into
|
||||
your editor)
|
||||
|
||||
```
|
||||
result-asserts/bin/nix --extra-experimental-features 'nix-command flakes' search --no-eval-cache github:nixos/nixpkgs/e1fa12d4f6
|
||||
c6fe19ccb59cac54b5b3f25e160870 hello
|
||||
mean: 15.993s ± 0.081s
|
||||
user: 13.321s | system: 1.865s
|
||||
median: 15.994s
|
||||
range: 15.829s ... 16.096s
|
||||
relative: 1
|
||||
result/bin/nix --extra-experimental-features 'nix-command flakes' search --no-eval-cache github:nixos/nixpkgs/e1fa12d4f6c6fe19cc
|
||||
b59cac54b5b3f25e160870 hello
|
||||
mean: 15.897s ± 0.075s
|
||||
user: 13.248s | system: 1.843s
|
||||
median: 15.88s
|
||||
range: 15.807s ... 16.047s
|
||||
relative: 0.994
|
||||
|
||||
---
|
||||
|
||||
result/bin/nix --extra-experimental-features 'nix-command flakes' eval -f bench/nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix
|
||||
mean: 0.4s ± 0.024s
|
||||
user: 0.335s | system: 0.046s
|
||||
median: 0.386s
|
||||
range: 0.379s ... 0.43s
|
||||
relative: 1
|
||||
|
||||
result-asserts/bin/nix --extra-experimental-features 'nix-command flakes' eval -f bench/nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix
|
||||
mean: 0.404s ± 0.024s
|
||||
user: 0.338s | system: 0.046s
|
||||
median: 0.386s
|
||||
range: 0.384s ... 0.436s
|
||||
relative: 1.008
|
||||
|
||||
---
|
||||
|
||||
result-asserts/bin/nix --extra-experimental-features 'nix-command flakes' eval --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'
|
||||
mean: 5.838s ± 0.023s
|
||||
user: 5.083s | system: 0.464s
|
||||
median: 5.845s
|
||||
range: 5.799s ... 5.867s
|
||||
relative: 1
|
||||
|
||||
result/bin/nix --extra-experimental-features 'nix-command flakes' eval --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'
|
||||
mean: 5.788s ± 0.044s
|
||||
user: 5.056s | system: 0.439s
|
||||
median: 5.79s
|
||||
range: 5.715s ... 5.876s
|
||||
relative: 0.991
|
||||
|
||||
---
|
||||
|
||||
GC_INITIAL_HEAP_SIZE=10g result-asserts/bin/nix eval --extra-experimental-features 'nix-command flakes' --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'
|
||||
mean: 4.147s ± 0.021s
|
||||
user: 3.457s | system: 0.487s
|
||||
median: 4.147s
|
||||
range: 4.123s ... 4.195s
|
||||
relative: 1
|
||||
|
||||
GC_INITIAL_HEAP_SIZE=10g result/bin/nix eval --extra-experimental-features 'nix-command flakes' --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'
|
||||
mean: 4.149s ± 0.027s
|
||||
user: 3.483s | system: 0.456s
|
||||
median: 4.142s
|
||||
range: 4.126s ... 4.215s
|
||||
relative: 1
|
||||
|
||||
---
|
||||
```
|
||||
62
bench/bench.sh
Executable file
62
bench/bench.sh
Executable file
|
|
@ -0,0 +1,62 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
shopt -s inherit_errexit
|
||||
|
||||
scriptdir=$(cd "$(dirname -- "$0")" ; pwd -P)
|
||||
cd "$scriptdir/.."
|
||||
|
||||
if [[ $# -lt 2 ]]; then
|
||||
# FIXME(jade): it is a reasonable use case to want to run a benchmark run
|
||||
# on just one build. However, since we are using hyperfine in comparison
|
||||
# mode, we would have to combine the JSON ourselves to support that, which
|
||||
# would probably be better done by writing a benchmarking script in
|
||||
# not-bash.
|
||||
echo "Fewer than two result dirs given, nothing to compare!" >&2
|
||||
echo "Pass some directories (with names indicating which alternative they are) with bin/nix in them" >&2
|
||||
echo "Usage: ./bench/bench.sh result-1 result-2 [result-3...]" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
_exit=""
|
||||
trap "$_exit" EXIT
|
||||
|
||||
# XXX: yes this is very silly. flakes~!!
|
||||
nix build --impure --expr '(builtins.getFlake "git+file:.").inputs.nixpkgs.outPath' -o bench/nixpkgs
|
||||
|
||||
export NIX_REMOTE="$(mktemp -d)"
|
||||
_exit='rm -rfv "$NIX_REMOTE"; $_exit'
|
||||
export NIX_PATH="nixpkgs=bench/nixpkgs:nixos-config=bench/configuration.nix"
|
||||
|
||||
builds=("$@")
|
||||
|
||||
flake_args="--extra-experimental-features 'nix-command flakes'"
|
||||
|
||||
hyperfineArgs=(
|
||||
--parameter-list BUILD "$(IFS=,; echo "${builds[*]}")"
|
||||
--warmup 2 --runs 10
|
||||
)
|
||||
|
||||
declare -A cases
|
||||
cases=(
|
||||
[search]="{BUILD}/bin/nix $flake_args search --no-eval-cache github:nixos/nixpkgs/e1fa12d4f6c6fe19ccb59cac54b5b3f25e160870 hello"
|
||||
[rebuild]="{BUILD}/bin/nix $flake_args eval --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'"
|
||||
[rebuild-lh]="GC_INITIAL_HEAP_SIZE=10g {BUILD}/bin/nix eval $flake_args --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'"
|
||||
[parse]="{BUILD}/bin/nix $flake_args eval -f bench/nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix"
|
||||
)
|
||||
|
||||
benches=(
|
||||
rebuild
|
||||
rebuild-lh
|
||||
search
|
||||
parse
|
||||
)
|
||||
|
||||
for k in "${benches[@]}"; do
|
||||
taskset -c 2,3 \
|
||||
chrt -f 50 \
|
||||
hyperfine "${hyperfineArgs[@]}" --export-json="bench/bench-${k}.json" --export-markdown="bench/bench-${k}.md" "${cases[$k]}"
|
||||
done
|
||||
|
||||
echo "Benchmarks summary (from ./bench/summarize.jq bench/bench-*.json)"
|
||||
bench/summarize.jq bench/*.json
|
||||
325
bench/configuration.nix
Normal file
325
bench/configuration.nix
Normal file
|
|
@ -0,0 +1,325 @@
|
|||
{
|
||||
config,
|
||||
pkgs,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
|
||||
{
|
||||
boot = {
|
||||
initrd = {
|
||||
availableKernelModules = [
|
||||
"xhci_pci"
|
||||
"ahci"
|
||||
];
|
||||
kernelModules = [ "dm-snapshot" ];
|
||||
luks.devices = {
|
||||
croot = {
|
||||
device = "/dev/sdb";
|
||||
allowDiscards = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
kernelModules = [ "kvm-intel" ];
|
||||
kernelPackages = pkgs.linuxPackages_latest;
|
||||
|
||||
loader = {
|
||||
systemd-boot.enable = true;
|
||||
efi.canTouchEfiVariables = true;
|
||||
};
|
||||
};
|
||||
|
||||
hardware = {
|
||||
enableRedistributableFirmware = true;
|
||||
cpu.intel.updateMicrocode = true;
|
||||
opengl.driSupport32Bit = true;
|
||||
opengl.extraPackages = with pkgs; [
|
||||
vaapiIntel
|
||||
intel-media-driver
|
||||
intel-compute-runtime
|
||||
];
|
||||
};
|
||||
|
||||
fileSystems = {
|
||||
"/" = {
|
||||
device = "/dev/sda2";
|
||||
fsType = "xfs";
|
||||
options = [ "noatime" ];
|
||||
};
|
||||
|
||||
"/boot" = {
|
||||
device = "/dev/sda1";
|
||||
fsType = "vfat";
|
||||
};
|
||||
|
||||
"/nas" = {
|
||||
device = "nas:/";
|
||||
fsType = "nfs4";
|
||||
options = [
|
||||
"ro"
|
||||
"x-systemd.automount"
|
||||
];
|
||||
};
|
||||
};
|
||||
swapDevices = [ { device = "/dev/swap"; } ];
|
||||
|
||||
networking = {
|
||||
useDHCP = false;
|
||||
hostName = "host";
|
||||
wireless = {
|
||||
enable = true;
|
||||
interfaces = [ "eth1" ];
|
||||
};
|
||||
interfaces = {
|
||||
eth0.useDHCP = true;
|
||||
eth1.useDHCP = true;
|
||||
};
|
||||
wg-quick.interfaces = {
|
||||
wg0 = {
|
||||
address = [ "2001:db8::1" ];
|
||||
privateKeyFile = "/etc/secrets/wg0.key";
|
||||
peers = [
|
||||
{
|
||||
publicKey = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=";
|
||||
endpoint = "[2001:db8::2]:61021";
|
||||
allowedIPs = [ "2001::db8:1::/64" ];
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
firewall.allowedUDPPorts = [ 4567 ];
|
||||
};
|
||||
|
||||
i18n = {
|
||||
defaultLocale = "en_US.UTF-8";
|
||||
inputMethod.enabled = "ibus";
|
||||
};
|
||||
|
||||
services = {
|
||||
xserver = {
|
||||
enable = true;
|
||||
layout = "us";
|
||||
xkbVariant = "altgr-intl";
|
||||
xkbOptions = "ctrl:nocaps";
|
||||
libinput.enable = true;
|
||||
wacom.enable = true;
|
||||
videoDrivers = [ "modesetting" ];
|
||||
modules = [ pkgs.xf86_input_wacom ];
|
||||
|
||||
displayManager.sx.enable = true;
|
||||
windowManager.i3.enable = true;
|
||||
};
|
||||
|
||||
udev.extraHwdb = ''
|
||||
# not like this mattered at all
|
||||
# we're not running udev from here
|
||||
'';
|
||||
|
||||
udev.extraRules = ''
|
||||
# ACTION=="add", SUBSYSTEM=="input", ...
|
||||
'';
|
||||
};
|
||||
|
||||
sound.enable = true;
|
||||
hardware.pulseaudio = {
|
||||
enable = true;
|
||||
package = pkgs.pulseaudioFull;
|
||||
daemon.config = {
|
||||
lock-memory = "yes";
|
||||
realtime-scheduling = "yes";
|
||||
rlimit-rtprio = "-1";
|
||||
};
|
||||
};
|
||||
|
||||
programs = {
|
||||
light.enable = true;
|
||||
wireshark = {
|
||||
enable = true;
|
||||
package = pkgs.wireshark-qt;
|
||||
};
|
||||
gnupg.agent = {
|
||||
enable = true;
|
||||
};
|
||||
};
|
||||
|
||||
fonts.packages = with pkgs; [
|
||||
font-awesome
|
||||
noto-fonts
|
||||
noto-fonts-cjk
|
||||
noto-fonts-emoji
|
||||
noto-fonts-extra
|
||||
dejavu_fonts
|
||||
powerline-fonts
|
||||
source-code-pro
|
||||
cantarell-fonts
|
||||
];
|
||||
|
||||
users = {
|
||||
mutableUsers = false;
|
||||
|
||||
users = {
|
||||
user = {
|
||||
isNormalUser = true;
|
||||
group = "user";
|
||||
extraGroups = [
|
||||
"wheel"
|
||||
"video"
|
||||
"audio"
|
||||
"dialout"
|
||||
"users"
|
||||
"kvm"
|
||||
"wireshark"
|
||||
];
|
||||
password = "unimportant";
|
||||
};
|
||||
};
|
||||
|
||||
groups = {
|
||||
user = { };
|
||||
};
|
||||
};
|
||||
|
||||
security = {
|
||||
pam.loginLimits = [
|
||||
{
|
||||
domain = "@audio";
|
||||
item = "memlock";
|
||||
type = "-";
|
||||
value = "unlimited";
|
||||
}
|
||||
{
|
||||
domain = "@audio";
|
||||
item = "rtprio";
|
||||
type = "-";
|
||||
value = "99";
|
||||
}
|
||||
{
|
||||
domain = "@audio";
|
||||
item = "nofile";
|
||||
type = "soft";
|
||||
value = "99999";
|
||||
}
|
||||
{
|
||||
domain = "@audio";
|
||||
item = "nofile";
|
||||
type = "hard";
|
||||
value = "99999";
|
||||
}
|
||||
];
|
||||
|
||||
sudo.extraRules = [
|
||||
{
|
||||
users = [ "user" ];
|
||||
commands = [
|
||||
{
|
||||
command = "${pkgs.linuxPackages.cpupower}/bin/cpupower";
|
||||
options = [ "NOPASSWD" ];
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
};
|
||||
|
||||
environment.systemPackages = with pkgs; [
|
||||
a2jmidid
|
||||
age
|
||||
ardour
|
||||
bemenu
|
||||
blender
|
||||
breeze-icons
|
||||
breeze-qt5
|
||||
bubblewrap
|
||||
calf
|
||||
claws-mail
|
||||
darktable
|
||||
duperemove
|
||||
emacs
|
||||
feh
|
||||
file
|
||||
firefox
|
||||
fluidsynth
|
||||
gnome3.adwaita-icon-theme
|
||||
gnuplot
|
||||
graphviz
|
||||
helm
|
||||
i3status-rust
|
||||
inkscape
|
||||
jack2
|
||||
jq
|
||||
krita
|
||||
ldns
|
||||
libqalculate
|
||||
libreoffice
|
||||
man-pages
|
||||
nheko
|
||||
nix-diff
|
||||
nix-index
|
||||
nix-output-monitor
|
||||
open-music-kontrollers.patchmatrix
|
||||
pamixer
|
||||
pavucontrol
|
||||
pciutils
|
||||
picom
|
||||
pwgen
|
||||
redshift
|
||||
ripgrep
|
||||
rlwrap
|
||||
silver-searcher
|
||||
soundfont-fluid
|
||||
whois
|
||||
wol
|
||||
xclip
|
||||
xdot
|
||||
xdotool
|
||||
xorg.xkbcomp
|
||||
yt-dlp
|
||||
zathura
|
||||
borgbackup
|
||||
linuxPackages.cpupower
|
||||
mtr
|
||||
kitty
|
||||
xf86_input_wacom
|
||||
];
|
||||
|
||||
environment.pathsToLink = [ "/share/soundfonts" ];
|
||||
|
||||
systemd.user.services.run-python = {
|
||||
after = [ "network-online.target" ];
|
||||
script = ''
|
||||
exec ${pkgs.python3}/bin/python
|
||||
'';
|
||||
serviceConfig = {
|
||||
CapabilityBoundingSet = [ "" ];
|
||||
KeyringMode = "private";
|
||||
LockPersonality = true;
|
||||
MemoryDenyWriteExecute = true;
|
||||
NoNewPrivileges = true;
|
||||
PrivateDevices = true;
|
||||
PrivateTmp = true;
|
||||
PrivateUsers = true;
|
||||
ProcSubset = "pid";
|
||||
ProtectClock = true;
|
||||
ProtectControlGroups = true;
|
||||
ProtectHome = true;
|
||||
ProtectHostname = true;
|
||||
ProtectKernelLogs = true;
|
||||
ProtectKernelModules = true;
|
||||
ProtectKernelTunables = true;
|
||||
ProtectProc = "invisible";
|
||||
ProtectSystem = "strict";
|
||||
RestrictAddressFamilies = "AF_INET AF_INET6";
|
||||
RestrictNamespaces = true;
|
||||
RestrictRealtime = true;
|
||||
RestrictSUIDSGID = true;
|
||||
SystemCallArchitectures = "native";
|
||||
SystemCallFilter = [
|
||||
"@system-service"
|
||||
"~ @resources @privileged"
|
||||
];
|
||||
UMask = "077";
|
||||
};
|
||||
};
|
||||
|
||||
system.stateVersion = "23.11";
|
||||
}
|
||||
22
bench/summarize.jq
Executable file
22
bench/summarize.jq
Executable file
|
|
@ -0,0 +1,22 @@
|
|||
#!/usr/bin/env -S jq -Mrf
|
||||
|
||||
def round3:
|
||||
. * 1000 | round | . / 1000
|
||||
;
|
||||
|
||||
def stats($first):
|
||||
[
|
||||
" mean: \(.mean | round3)s ± \(.stddev | round3)s",
|
||||
" user: \(.user | round3)s | system: \(.system | round3)s",
|
||||
" median: \(.median | round3)s",
|
||||
" range: \(.min | round3)s ... \(.max | round3)s",
|
||||
" relative: \(.mean / $first.mean | round3)"
|
||||
]
|
||||
| join("\n")
|
||||
;
|
||||
|
||||
def fmt($first):
|
||||
"\(.command)\n" + (. | stats($first))
|
||||
;
|
||||
|
||||
[.results | .[0] as $first | .[] | fmt($first)] | join("\n\n") | (. + "\n\n---\n")
|
||||
Loading…
Reference in a new issue