Add Loki + Promtail setup

More later, Loki also later.

common/sysadmin/default.nix

@@ -17,6 +17,7 @@ in
         pv
         kitty.terminfo
         config.boot.kernelPackages.perf
+        bcc
         tcpdump
         ncdu
       ] ++ lib.optional (lib.hasAttr "pwru" pkgs) pkgs.pwru;

services/gerrit/default.nix

@@ -28,6 +28,8 @@ in
   config = mkIf cfg.enable {
     networking.firewall.allowedTCPPorts = [ 29418 ];
 
+    environment.systemPackages = [ pkgs.openjdk17_headless ];
+
     fileSystems."/var/lib/gerrit" = mkIf (cfg.data != "/var/lib/gerrit") {
       device = cfg.data;
       options = [ "bind" ];
@@ -70,18 +72,49 @@ in
       jvmPackage = pkgs.openjdk17_headless;
 
       settings = {
+        # Performance settings
         sshd.threads = 64;
         sshd.batchThreads = 8;
+
+        gc.aggressive = true;
         gc.interval = "1 day";
-        database.poolLimit = "250";
+
+        database.poolLimit = 250;
         database.poolMaxIdle = 16;
-        http.maxThreads = 100;
-        core.packedGitLimit = "4g";
-        core.packedGitWindowSize = "16k";
-        core.packedGitOpenFiles = "4096";
+
+        httpd.maxThreads = 100;
+
         receive.timeout = "4min";
-        transfer.timeout = "4min";
-        pack.threads = "8";
+        # Default is 0, infinite.
+        transfer.timeout = "30min";
+
+        # We may overshoot but it's OK.
+        core.packedGitWindowSize = "256k";
+        # Sum of all current packfiles is ~1.2G
+        # Largest packfile is 906MB.
+        # Average packfile is ~5-10MB.
+        core.packedGitLimit = "1g";
+        # We have plenty of memory, let's avoid file system cache → Gerrit needless copies.
+        core.packedGitUseStrongRefs = true;
+        core.packedGitOpenFiles = 4096;
+        # Big files in nixpkgs are usually lockfiles or machine-generated expressions
+        # containing a lot of hashes, they would weigh at most ~15MB.
+        core.streamFileThreshold = "20m";
+        # `mmap()` rather than `mmap()+read()` at the risk of running out of virtual address space.
+        core.packedGitMmap = true;
+
+        ## Takes more CPU but the transfer is smaller.
+        pack.deltacompression = false;
+        pack.threads = 8;
+
+        # FIXME(raito):
+        # Are we supposed to have private / hidden references?
+        # For a public server, that seems unlikely.
+        # But, we should be careful with this option.
+        # https://gerrit-documentation.storage.googleapis.com/Documentation/3.9.5/config-gerrit.html#receive.checkReferencedObjectsAreReachable
+        receive.checkReferencedObjectsAreReachable = false;
+
+        # Other settings
         log.jsonLogging = true;
         log.textLogging = false;
         sshd.advertisedAddress = "cl.forkos.org:29418";
@@ -90,11 +123,18 @@ in
         change.enableAttentionSet = true;
         change.enableAssignee = false;
 
+        user = {
+          name = "ForkOS Gerrit";
+          email = "gerrit@forkos.org";
+          anonymousCoward = "ForkOS contributor";
+        };
+
         # Configures gerrit for being reverse-proxied by nginx as per
         # https://gerrit-review.googlesource.com/Documentation/config-reverseproxy.html
         gerrit = {
           canonicalWebUrl = "https://cl.forkos.org";
           docUrl = "/Documentation";
+          defaultBranch = "refs/heads/main";
         };
 
         httpd.listenUrl = "proxy-https://${cfgGerrit.listenAddress}";

services/gerrit/www.nix

@@ -12,21 +12,30 @@ in
           add_header Permissions-Policy "interest-cohort=()";
         '';
         recommendedProxySettings = false;
+        commonHttpConfig = ''
+              log_format upstream_time '$remote_addr - $remote_user [$time_local] '
+                             '"$request" $status $body_bytes_sent '
+                             '"$http_referer" "$http_user_agent"'
+                             'rt=$request_time uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"';
+        '';
       };
       services.nginx.virtualHosts.gerrit = {
         serverName = builtins.head cfg.domains;
         serverAliases = builtins.tail cfg.domains;
         enableACME = true;
         forceSSL = true;
-
         extraConfig = ''
+          access_log /var/log/nginx/gerrit-access.log upstream_time;
+
           location / {
             proxy_pass http://localhost:4778;
             proxy_set_header  X-Forwarded-For $remote_addr;
             # The :443 suffix is a workaround for https://b.tvl.fyi/issues/88.
             proxy_set_header  Host $host:443;
             # Gerrit can throw a lot of data.
-            proxy_buffering off;
+            proxy_buffering on;
+            # NGINX should not give up super fast. Things can take time.
+            proxy_read_timeout 3600;
           }
 
           location = /robots.txt {

services/hydra/default.nix

@@ -53,7 +53,7 @@ in {
 
       buildMachinesFiles = [
         (pkgs.writeText "hydra-builders.conf" ''
-          ssh://bagel-builder@epyc.infra.newtype.fr i686-linux,x86_64-linux ${config.age.secrets.hydra-ssh-key-priv.path} 8 1 big-parallel,kvm,nixos-test - c3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSUJwcFBwKzhsdDFSTDNodW5aaGlXRUUvY1laaHJXYjFzaVhKVWpiU2l6Rzggcm9vdEBlcHljCg==
+          ssh://bagel-builder@epyc.infra.newtype.fr i686-linux,x86_64-linux ${config.age.secrets.hydra-ssh-key-priv.path} 8 1 big-parallel,kvm,nixos-test - c3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSU9YVDlJbml0MU1oS3Q0cmpCQU5McTB0MGJQd3cvV1FaOTZ1QjRBRURybWwgcm9vdEBuaXhvcwo=
         '')
       ];