From f4fc2b5d10c5bfab6d037e7c9c944d2277f70c58 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Sun, 7 Apr 2024 18:24:03 -0700 Subject: [PATCH] Add benchmarking scripts These scripts were originally written by horrors, and have since been hacked up a lot by jade. We are putting them up as a CL since it is better to have checked in benchmarking scripts than to not have benchmarking scripts. cc: https://git.lix.systems/lix-project/lix/issues/23 Co-authored-by: eldritch horrors Change-Id: I95c2f9d24753ac468944c5781deec9508fd5cb8c --- bench/.gitignore | 3 + bench/README.md | 91 +++++++++++ bench/bench.sh | 62 ++++++++ bench/configuration.nix | 325 ++++++++++++++++++++++++++++++++++++++++ bench/summarize.jq | 22 +++ 5 files changed, 503 insertions(+) create mode 100644 bench/.gitignore create mode 100644 bench/README.md create mode 100755 bench/bench.sh create mode 100644 bench/configuration.nix create mode 100755 bench/summarize.jq diff --git a/bench/.gitignore b/bench/.gitignore new file mode 100644 index 000000000..8115aa6f2 --- /dev/null +++ b/bench/.gitignore @@ -0,0 +1,3 @@ +bench-*.json +bench-*.md +nixpkgs diff --git a/bench/README.md b/bench/README.md new file mode 100644 index 000000000..4aefcd7a8 --- /dev/null +++ b/bench/README.md @@ -0,0 +1,91 @@ +# Benchmarking scripts for Lix + +These are very much WIP, and have a few clumsy assumptions that we would +somewhat rather be fixed, but we have committed them to let others be able to +do benchmarking in the mean time. + +## Benchmarking procedure + +Build some Lixes you want to compare, by whichever means you wish. + +Get a computer that is not busy and *strongly preferably* is bare-metal or at +least not a cloud VM (e.g. go make coffee when running benchmarks). + +From the root of a Lix checkout, run `./bench/bench.sh resultlink-one +resultlink-two`, where `resultlink-one` and `resultlink-two` are the result +links from the builds you want to test (they can be any directory with bin/nix +in it, however). + +To get the summary again, run `./bench/summarize.jq bench/bench-*.json`. + +## Example results + +(vim tip: `:r !bench/summarize.jq bench/bench-*.json` to dump it directly into +your editor) + +``` +result-asserts/bin/nix --extra-experimental-features 'nix-command flakes' search --no-eval-cache github:nixos/nixpkgs/e1fa12d4f6 +c6fe19ccb59cac54b5b3f25e160870 hello + mean: 15.993s ± 0.081s + user: 13.321s | system: 1.865s + median: 15.994s + range: 15.829s ... 16.096s + relative: 1 +result/bin/nix --extra-experimental-features 'nix-command flakes' search --no-eval-cache github:nixos/nixpkgs/e1fa12d4f6c6fe19cc +b59cac54b5b3f25e160870 hello + mean: 15.897s ± 0.075s + user: 13.248s | system: 1.843s + median: 15.88s + range: 15.807s ... 16.047s + relative: 0.994 + +--- + +result/bin/nix --extra-experimental-features 'nix-command flakes' eval -f bench/nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix + mean: 0.4s ± 0.024s + user: 0.335s | system: 0.046s + median: 0.386s + range: 0.379s ... 0.43s + relative: 1 + +result-asserts/bin/nix --extra-experimental-features 'nix-command flakes' eval -f bench/nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix + mean: 0.404s ± 0.024s + user: 0.338s | system: 0.046s + median: 0.386s + range: 0.384s ... 0.436s + relative: 1.008 + +--- + +result-asserts/bin/nix --extra-experimental-features 'nix-command flakes' eval --raw --impure --expr 'with import {}; system' + mean: 5.838s ± 0.023s + user: 5.083s | system: 0.464s + median: 5.845s + range: 5.799s ... 5.867s + relative: 1 + +result/bin/nix --extra-experimental-features 'nix-command flakes' eval --raw --impure --expr 'with import {}; system' + mean: 5.788s ± 0.044s + user: 5.056s | system: 0.439s + median: 5.79s + range: 5.715s ... 5.876s + relative: 0.991 + +--- + +GC_INITIAL_HEAP_SIZE=10g result-asserts/bin/nix eval --extra-experimental-features 'nix-command flakes' --raw --impure --expr 'with import {}; system' + mean: 4.147s ± 0.021s + user: 3.457s | system: 0.487s + median: 4.147s + range: 4.123s ... 4.195s + relative: 1 + +GC_INITIAL_HEAP_SIZE=10g result/bin/nix eval --extra-experimental-features 'nix-command flakes' --raw --impure --expr 'with import {}; system' + mean: 4.149s ± 0.027s + user: 3.483s | system: 0.456s + median: 4.142s + range: 4.126s ... 4.215s + relative: 1 + +--- +``` diff --git a/bench/bench.sh b/bench/bench.sh new file mode 100755 index 000000000..70acd4640 --- /dev/null +++ b/bench/bench.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +set -euo pipefail +shopt -s inherit_errexit + +scriptdir=$(cd "$(dirname -- "$0")" ; pwd -P) +cd "$scriptdir/.." + +if [[ $# -lt 2 ]]; then + # FIXME(jade): it is a reasonable use case to want to run a benchmark run + # on just one build. However, since we are using hyperfine in comparison + # mode, we would have to combine the JSON ourselves to support that, which + # would probably be better done by writing a benchmarking script in + # not-bash. + echo "Fewer than two result dirs given, nothing to compare!" >&2 + echo "Pass some directories (with names indicating which alternative they are) with bin/nix in them" >&2 + echo "Usage: ./bench/bench.sh result-1 result-2 [result-3...]" >&2 + exit 1 +fi + +_exit="" +trap "$_exit" EXIT + +# XXX: yes this is very silly. flakes~!! +nix build --impure --expr '(builtins.getFlake "git+file:.").inputs.nixpkgs.outPath' -o bench/nixpkgs + +export NIX_REMOTE="$(mktemp -d)" +_exit='rm -rfv "$NIX_REMOTE"; $_exit' +export NIX_PATH="nixpkgs=bench/nixpkgs:nixos-config=bench/configuration.nix" + +builds=("$@") + +flake_args="--extra-experimental-features 'nix-command flakes'" + +hyperfineArgs=( + --parameter-list BUILD "$(IFS=,; echo "${builds[*]}")" + --warmup 2 --runs 10 +) + +declare -A cases +cases=( + [search]="{BUILD}/bin/nix $flake_args search --no-eval-cache github:nixos/nixpkgs/e1fa12d4f6c6fe19ccb59cac54b5b3f25e160870 hello" + [rebuild]="{BUILD}/bin/nix $flake_args eval --raw --impure --expr 'with import {}; system'" + [rebuild-lh]="GC_INITIAL_HEAP_SIZE=10g {BUILD}/bin/nix eval $flake_args --raw --impure --expr 'with import {}; system'" + [parse]="{BUILD}/bin/nix $flake_args eval -f bench/nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix" +) + +benches=( + rebuild + rebuild-lh + search + parse +) + +for k in "${benches[@]}"; do + taskset -c 2,3 \ + chrt -f 50 \ + hyperfine "${hyperfineArgs[@]}" --export-json="bench/bench-${k}.json" --export-markdown="bench/bench-${k}.md" "${cases[$k]}" +done + +echo "Benchmarks summary (from ./bench/summarize.jq bench/bench-*.json)" +bench/summarize.jq bench/*.json diff --git a/bench/configuration.nix b/bench/configuration.nix new file mode 100644 index 000000000..54782a1d3 --- /dev/null +++ b/bench/configuration.nix @@ -0,0 +1,325 @@ +{ + config, + pkgs, + lib, + ... +}: + +{ + boot = { + initrd = { + availableKernelModules = [ + "xhci_pci" + "ahci" + ]; + kernelModules = [ "dm-snapshot" ]; + luks.devices = { + croot = { + device = "/dev/sdb"; + allowDiscards = true; + }; + }; + }; + kernelModules = [ "kvm-intel" ]; + kernelPackages = pkgs.linuxPackages_latest; + + loader = { + systemd-boot.enable = true; + efi.canTouchEfiVariables = true; + }; + }; + + hardware = { + enableRedistributableFirmware = true; + cpu.intel.updateMicrocode = true; + opengl.driSupport32Bit = true; + opengl.extraPackages = with pkgs; [ + vaapiIntel + intel-media-driver + intel-compute-runtime + ]; + }; + + fileSystems = { + "/" = { + device = "/dev/sda2"; + fsType = "xfs"; + options = [ "noatime" ]; + }; + + "/boot" = { + device = "/dev/sda1"; + fsType = "vfat"; + }; + + "/nas" = { + device = "nas:/"; + fsType = "nfs4"; + options = [ + "ro" + "x-systemd.automount" + ]; + }; + }; + swapDevices = [ { device = "/dev/swap"; } ]; + + networking = { + useDHCP = false; + hostName = "host"; + wireless = { + enable = true; + interfaces = [ "eth1" ]; + }; + interfaces = { + eth0.useDHCP = true; + eth1.useDHCP = true; + }; + wg-quick.interfaces = { + wg0 = { + address = [ "2001:db8::1" ]; + privateKeyFile = "/etc/secrets/wg0.key"; + peers = [ + { + publicKey = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="; + endpoint = "[2001:db8::2]:61021"; + allowedIPs = [ "2001::db8:1::/64" ]; + } + ]; + }; + }; + + firewall.allowedUDPPorts = [ 4567 ]; + }; + + i18n = { + defaultLocale = "en_US.UTF-8"; + inputMethod.enabled = "ibus"; + }; + + services = { + xserver = { + enable = true; + layout = "us"; + xkbVariant = "altgr-intl"; + xkbOptions = "ctrl:nocaps"; + libinput.enable = true; + wacom.enable = true; + videoDrivers = [ "modesetting" ]; + modules = [ pkgs.xf86_input_wacom ]; + + displayManager.sx.enable = true; + windowManager.i3.enable = true; + }; + + udev.extraHwdb = '' + # not like this mattered at all + # we're not running udev from here + ''; + + udev.extraRules = '' + # ACTION=="add", SUBSYSTEM=="input", ... + ''; + }; + + sound.enable = true; + hardware.pulseaudio = { + enable = true; + package = pkgs.pulseaudioFull; + daemon.config = { + lock-memory = "yes"; + realtime-scheduling = "yes"; + rlimit-rtprio = "-1"; + }; + }; + + programs = { + light.enable = true; + wireshark = { + enable = true; + package = pkgs.wireshark-qt; + }; + gnupg.agent = { + enable = true; + }; + }; + + fonts.packages = with pkgs; [ + font-awesome + noto-fonts + noto-fonts-cjk + noto-fonts-emoji + noto-fonts-extra + dejavu_fonts + powerline-fonts + source-code-pro + cantarell-fonts + ]; + + users = { + mutableUsers = false; + + users = { + user = { + isNormalUser = true; + group = "user"; + extraGroups = [ + "wheel" + "video" + "audio" + "dialout" + "users" + "kvm" + "wireshark" + ]; + password = "unimportant"; + }; + }; + + groups = { + user = { }; + }; + }; + + security = { + pam.loginLimits = [ + { + domain = "@audio"; + item = "memlock"; + type = "-"; + value = "unlimited"; + } + { + domain = "@audio"; + item = "rtprio"; + type = "-"; + value = "99"; + } + { + domain = "@audio"; + item = "nofile"; + type = "soft"; + value = "99999"; + } + { + domain = "@audio"; + item = "nofile"; + type = "hard"; + value = "99999"; + } + ]; + + sudo.extraRules = [ + { + users = [ "user" ]; + commands = [ + { + command = "${pkgs.linuxPackages.cpupower}/bin/cpupower"; + options = [ "NOPASSWD" ]; + } + ]; + } + ]; + }; + + environment.systemPackages = with pkgs; [ + a2jmidid + age + ardour + bemenu + blender + breeze-icons + breeze-qt5 + bubblewrap + calf + claws-mail + darktable + duperemove + emacs + feh + file + firefox + fluidsynth + gnome3.adwaita-icon-theme + gnuplot + graphviz + helm + i3status-rust + inkscape + jack2 + jq + krita + ldns + libqalculate + libreoffice + man-pages + nheko + nix-diff + nix-index + nix-output-monitor + open-music-kontrollers.patchmatrix + pamixer + pavucontrol + pciutils + picom + pwgen + redshift + ripgrep + rlwrap + silver-searcher + soundfont-fluid + whois + wol + xclip + xdot + xdotool + xorg.xkbcomp + yt-dlp + zathura + borgbackup + linuxPackages.cpupower + mtr + kitty + xf86_input_wacom + ]; + + environment.pathsToLink = [ "/share/soundfonts" ]; + + systemd.user.services.run-python = { + after = [ "network-online.target" ]; + script = '' + exec ${pkgs.python3}/bin/python + ''; + serviceConfig = { + CapabilityBoundingSet = [ "" ]; + KeyringMode = "private"; + LockPersonality = true; + MemoryDenyWriteExecute = true; + NoNewPrivileges = true; + PrivateDevices = true; + PrivateTmp = true; + PrivateUsers = true; + ProcSubset = "pid"; + ProtectClock = true; + ProtectControlGroups = true; + ProtectHome = true; + ProtectHostname = true; + ProtectKernelLogs = true; + ProtectKernelModules = true; + ProtectKernelTunables = true; + ProtectProc = "invisible"; + ProtectSystem = "strict"; + RestrictAddressFamilies = "AF_INET AF_INET6"; + RestrictNamespaces = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + SystemCallArchitectures = "native"; + SystemCallFilter = [ + "@system-service" + "~ @resources @privileged" + ]; + UMask = "077"; + }; + }; + + system.stateVersion = "23.11"; +} diff --git a/bench/summarize.jq b/bench/summarize.jq new file mode 100755 index 000000000..5d1449108 --- /dev/null +++ b/bench/summarize.jq @@ -0,0 +1,22 @@ +#!/usr/bin/env -S jq -Mrf + +def round3: + . * 1000 | round | . / 1000 + ; + +def stats($first): + [ + " mean: \(.mean | round3)s ± \(.stddev | round3)s", + " user: \(.user | round3)s | system: \(.system | round3)s", + " median: \(.median | round3)s", + " range: \(.min | round3)s ... \(.max | round3)s", + " relative: \(.mean / $first.mean | round3)" + ] + | join("\n") + ; + +def fmt($first): + "\(.command)\n" + (. | stats($first)) + ; + +[.results | .[0] as $first | .[] | fmt($first)] | join("\n\n") | (. + "\n\n---\n")