Set up tempo

hosts/meta01/default.nix

@@ -24,6 +24,7 @@
   bagel.services.grafana.enable = true;
   bagel.services.grapevine.enable = true;
   bagel.services.pyroscope.enable = true;
+  bagel.services.tempo.enable = true;
   bagel.services.hookshot = {
     enable = true;
     admins = [

secrets.nix

@@ -16,6 +16,7 @@ let
     loki-environment = [ machines.meta01 ];
     gerrit-prometheus-bearer-token = [ machines.gerrit01 machines.meta01 ];
     pyroscope-secrets = [ machines.meta01 ];
+    tempo-environment = [ machines.meta01 ];
 
     buildbot-worker-password = [ machines.buildbot ];
     buildbot-oauth-secret = [ machines.buildbot ];

secrets/tempo-environment.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 j2r2qQ kbi4mciOrjd7/X86xfmkDaMZhvZakoSJ6qjqLF3ljkE
+Q2BsgMLJ8AmjhnggRi+wkICj18NCA2HW1t8clemReUw
+-> ssh-ed25519 K3b7BA wNGmX9S9bJgd2JDte9QoNDfyycgmq4JMu2bc5nyYYik
+uUiutxAI3nI0M51W97aPRVE/l4dV2PEjph8eWOMLHIE
+-> ssh-ed25519 +qVung raYJ5vwMP9JopSdfa+ofkLY/gc0zcW4wTNBFTca+MXw
+sa/rWGSYrI4y6rn4JSboldWKUGvx6HbtsYo78AFOkBo
+-> ssh-rsa krWCLQ
+FLq8NwkiGw2gXptVVY393f0p9hFom57xHWPxtAlzOcRT8gvWu/uwgV+0raOcOcJa
+xxr5Sib+2D3UnUhprVPmH5Os9bI2seFAiej1MVVWLqvMtQHLFwnrzZTyZpxsXpQq
+5qQhNEADuQc4uD/ELVjGHKt6nF1Cl/GbgNLIOF/ITZ0pm1O1MjtT6MYJhQJhc6sb
+sno/wQyTXjj7rC06nyLX/rgOWrJSOeaz9eVp0A8k8/I0TXu/vRCW9gqWtv2m8sbh
+1uUHIm0l8f3z+zrL6OlZnpMFw4jpiiGoCYKPzD17I0onDYIjtdVS5iO9BsckxV/a
+wQWbyONUwbGCfeNSVAzZbg
+-> ssh-ed25519 /vwQcQ jwf7fwy4wKz7q761DNu8SyFHGgFlwq4P/Pn44Nido3E
+1q/jvt/vtD4ziY3eCDqk1XwMPpNUd80POTV2VVsumCE
+-> ssh-ed25519 0R97PA XeuziQ+wsoh0KSHXk5Qkl1kQOsAu1Ax1zTg13+XWd3M
+B1KHKm3tx/EsnE6hY+w7ya1ilhYiUs9AbwARHNkJi90
+--- JgQA6gCYZu8xcbXEl9VypccEIBO6uAJIdhBefr4doRQ
+V3ZðõÚ<EFBFBD>ç-·Ý.ê«sòÀ³3 ÎiS‰a5#¿Ð{åÔÈ®Dý˜YêNèãëù«ýoL+ÔÝ#–M<sws P»¢+í¢Ó‰ïBDoÊξÆÏuFí”Ç^Â¥•<C2A5>—ÝG@ÍM×ÛãÐØìq¦ºG^Qb s<
;ÂÒnC+ÖÊxª_­Úì]S<16>Ð

services/monitoring/lgtm/default.nix

@@ -3,5 +3,6 @@
     ./grafana.nix
     ./loki.nix
     ./mimir.nix
+    ./tempo.nix
   ];
 }

services/monitoring/lgtm/grafana.nix

@@ -132,6 +132,14 @@ in
                 access = "proxy";
                 url = "http://127.0.0.1:4040";
               }
+              {
+                name = "Tempo";
+                type = "tempo";
+                uid = "tempo";
+                access = "proxy";
+                url = "http://127.0.0.1:9190";
+                jsonData.streamingEnabled.search = true;
+              }
             ];
           };
         };

services/monitoring/lgtm/tempo.nix

@@ -0,0 +1,79 @@
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.bagel.services.tempo;
+  inherit (lib) mkEnableOption mkIf;
+in
+{
+  options.bagel.services.tempo.enable = mkEnableOption "Tempo trace store";
+
+  config = mkIf cfg.enable {
+    age.secrets = {
+      metrics-push-htpasswd = {
+        file = ../../../secrets/metrics-push-htpasswd.age;
+        owner = "nginx";
+      };
+      tempo-environment.file = ../../../secrets/tempo-environment.age;
+    };
+
+    services.tempo = {
+      enable = true;
+      extraFlags = ["--config.expand-env=true"];
+      settings = {
+        multitenancy_enabled = false;
+        stream_over_http_enabled = true;
+
+        server = {
+          http_listen_port = 9190;
+          grpc_listen_port = 9195;
+        };
+        distributor.receivers.otlp.protocols.http.endpoint = "127.0.0.1:4138";
+
+        storage.trace = {
+          backend = "s3";
+          s3 = {
+            endpoint = "s3.delroth.net";
+            bucket = "bagel-tempo";
+            secret_key = "\${S3_KEY}"; # This is a secret injected via an environment variable
+            access_key = "\${S3_KEY_ID}";
+          };
+          wal.path = "/var/lib/tempo/traces-wal";
+        };
+
+        metrics_generator.storage = {
+          path = "/var/lib/tempo/metrics-wal";
+          remote_write = [
+            {
+              url = "http://127.0.0.1:9009/api/v1/push";
+            }
+          ];
+        };
+
+        overrides.defaults.metrics_generator.processors = [ "span-metrics" ];
+      };
+    };
+
+    systemd.services.tempo.serviceConfig.EnvironmentFile = [ config.age.secrets.tempo-environment.path ];
+
+    services.nginx = {
+      upstreams.tempo = {
+        servers."${config.services.tempo.settings.distributor.receivers.otlp.protocols.http.endpoint}" = {};
+        extraConfig = "keepalive 16;";
+      };
+
+      virtualHosts."tempo.forkos.org" = {
+        enableACME = true;
+        forceSSL = true;
+        locations."/" = {
+          proxyPass = "http://tempo";
+          basicAuthFile = config.age.secrets.metrics-push-htpasswd.path;
+        };
+      };
+    };
+
+    bagel.monitoring.grafana-agent.exporters.tempo.port = 9190;
+  };
+}

terraform/gandi.nix

@@ -81,6 +81,7 @@ in
       (record "loki" 300 "CNAME" ["meta01.infra.p"])
       (record "mimir" 300 "CNAME" ["meta01.infra.p"])
       (record "pyroscope" 300 "CNAME" ["meta01.infra.p"])
+      (record "tempo" 300 "CNAME" ["meta01.infra.p"])
       (record "matrix" 300 "CNAME" ["meta01.infra.p"])
       (record "alerts" 300 "CNAME" ["meta01.infra.p"])
       (record "buildbot" 300 "CNAME" ["buildbot.infra.p"])