From 9e7e6d42ab6203d787490cdf888f384e4066d396 Mon Sep 17 00:00:00 2001
From: K900 <me@0upti.me>
Date: Wed, 10 Jul 2024 14:55:28 +0300
Subject: [PATCH] Make nginx/loki/mimir go fast

---
 common/base-server.nix             |  6 ++++++
 services/monitoring/lgtm/loki.nix  | 19 +++++++++++++------
 services/monitoring/lgtm/mimir.nix | 22 ++++++++++++++++------
 3 files changed, 35 insertions(+), 12 deletions(-)

diff --git a/common/base-server.nix b/common/base-server.nix
index d65ee90..82b39d9 100644
--- a/common/base-server.nix
+++ b/common/base-server.nix
@@ -13,6 +13,12 @@
     recommendedTlsSettings = lib.mkDefault true;
     recommendedProxySettings = lib.mkDefault true;
     recommendedGzipSettings = lib.mkDefault true;
+    eventsConfig = ''
+      worker_connections 8192;
+    '';
+    appendConfig = ''
+      worker_rlimit_nofile 16384;
+    '';
   };
 
   nix.gc = {
diff --git a/services/monitoring/lgtm/loki.nix b/services/monitoring/lgtm/loki.nix
index e9ba317..dfe6931 100644
--- a/services/monitoring/lgtm/loki.nix
+++ b/services/monitoring/lgtm/loki.nix
@@ -88,12 +88,19 @@ in
 
     systemd.services.loki.serviceConfig.EnvironmentFile = [ config.age.secrets.loki-environment.path ];
 
-    services.nginx.virtualHosts."loki.forkos.org" = {
-      enableACME = true;
-      forceSSL = true;
-      locations."/loki/api/v1/push" = {
-        proxyPass = "http://localhost:${toString config.services.loki.configuration.server.http_listen_port}";
-        basicAuthFile = config.age.secrets.metrics-push-htpasswd.path;
+    services.nginx = {
+      upstreams.loki = {
+        servers."127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}" = {};
+        extraConfig = "keepalive 16;";
+      };
+
+      virtualHosts."loki.forkos.org" = {
+        enableACME = true;
+        forceSSL = true;
+        locations."/loki/api/v1/push" = {
+          proxyPass = "http://loki";
+          basicAuthFile = config.age.secrets.metrics-push-htpasswd.path;
+        };
       };
     };
 
diff --git a/services/monitoring/lgtm/mimir.nix b/services/monitoring/lgtm/mimir.nix
index cb4f911..bf2d09e 100644
--- a/services/monitoring/lgtm/mimir.nix
+++ b/services/monitoring/lgtm/mimir.nix
@@ -49,6 +49,9 @@ in
 
         ingester.ring.replication_factor = 1;
 
+        distributor.instance_limits.max_ingestion_rate = 0; # unlimited
+        limits.ingestion_rate = 1000000; # can't set to unlimited :(
+
         blocks_storage.backend = "s3";
         ruler_storage = {
           backend = "local";
@@ -80,12 +83,19 @@ in
 
     systemd.services.mimir.serviceConfig.EnvironmentFile = [ config.age.secrets.mimir-environment.path ];
 
-    services.nginx.virtualHosts."mimir.forkos.org" = {
-      enableACME = true;
-      forceSSL = true;
-      locations."/api/v1/push" = {
-        proxyPass = "http://localhost:${toString mimirPort}";
-        basicAuthFile = config.age.secrets.metrics-push-htpasswd.path;
+    services.nginx = {
+      upstreams.mimir = {
+        servers."127.0.0.1:${toString mimirPort}" = {};
+        extraConfig = "keepalive 16;";
+      };
+      
+      virtualHosts."mimir.forkos.org" = {
+        enableACME = true;
+        forceSSL = true;
+        locations."/api/v1/push" = {
+          proxyPass = "http://mimir";
+          basicAuthFile = config.age.secrets.metrics-push-htpasswd.path;
+        };
       };
     };