shared-public-infra/modules/nix-daemon.nix

{ lib
, config
, pkgs
, inputs
, ...
}:

let
  gcc-system-features = arch: lib.optionals (arch != null) ([ "gccarch-${arch}" ]
    ++ map (x: "gccarch-${x}") lib.systems.architectures.inferiors.${arch});
in
{
  options = {
    simd.arch = lib.mkOption {
      type = with lib.types; nullOr str;
      default = null;
      description = ''
        Microarchitecture string for nixpkgs.hostPlatform.gcc.march and to generate system-features.
        Can be determined with: gcc -march=native -Q --help=target | grep march
      '';
    };
  };

  imports = [ ./builder.nix ];

  config = {
    warnings = lib.optionals (config.simd.arch == null) [ "Please set simd.arch for ${config.networking.hostName}" ];
    # Allow more open files for non-root users to run NixOS VM tests.
    security.pam.loginLimits = [
      { domain = "*"; item = "nofile"; type = "-"; value = "20480"; }
    ];

    # Memory accounting techniques
    systemd.services.nix-daemon.serviceConfig = {
      MemoryAccounting = true;
      MemoryMax = "225G";
      MemoryHigh = "220G";
      MemorySwapMax = "2G";
      ManagedOOMSwap = "kill";
      ManagedOOMMemoryPressure = "kill";
      MemoryPressureWatch = "on";
    };

    nix = {
      # Garbage-collect often
      gc.automatic = true;
      gc.dates = "*:45";
      gc.options = ''--max-freed "$((128 * 1024**3 - 1024 * $(df -P -k /nix/store | tail -n 1 | ${pkgs.gawk}/bin/awk '{ print $4 }')))"'';

      # Randomize GC to avoid thundering herd effects.
      gc.randomizedDelaySec = "1800";

      # Inchallah, it works.
      package = pkgs.nixVersions.nix_2_18;
      # package = lib.mkForce inputs.nixpkgs-unstable.legacyPackages.x86_64-linux.nixVersions.nix_2_17;

      # should be enough?
      nrBuildUsers = 128;

      settings = {
        keep-outputs = false;
        keep-derivations = false;
        use-cgroups = true;
        http-connections = 0;
        auto-allocate-uids = true;
        cores = 64; # 128 is too much, it will explode the RAM for now. Let's keep it serious.
        max-jobs = 2; # Do not build more than 2 derivations at once in the event, both of them are too big, yes this is stupid, fix it in Nix.
        fsync-metadata = true;
        substituters = [
          "https://nix-community.cachix.org"
          "https://tum-dse.cachix.org"
        ];
        system-features = [ "benchmark" "big-parallel" "kvm" "nixos-test" ] ++ gcc-system-features config.simd.arch;
        trusted-public-keys = [
          "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
          "tum-dse.cachix.org-1:v67rK18oLwgO0Z4b69l30SrV1yRtqxKpiHodG4YxhNM="
        ];
        experimental-features = [
          "auto-allocate-uids"
          # "ca-derivations" this feature is really extremely broken.
          "cgroups"
          "fetch-closure"
          "impure-derivations"
        ];
      };
    };

    nixpkgs.config.allowUnfree = true;
  };
}