From c1712dc1fab1f18b1fea7ececbbef1c734a2d3ea Mon Sep 17 00:00:00 2001 From: K900 Date: Sat, 31 Aug 2024 15:05:30 +0300 Subject: [PATCH] Set up tempo --- hosts/meta01/default.nix | 1 + secrets.nix | 1 + secrets/tempo-environment.age | 20 +++++++ services/monitoring/lgtm/default.nix | 3 +- services/monitoring/lgtm/grafana.nix | 10 +++- services/monitoring/lgtm/tempo.nix | 79 ++++++++++++++++++++++++++++ terraform/gandi.nix | 1 + 7 files changed, 113 insertions(+), 2 deletions(-) create mode 100644 secrets/tempo-environment.age create mode 100644 services/monitoring/lgtm/tempo.nix diff --git a/hosts/meta01/default.nix b/hosts/meta01/default.nix index c50e0e6..865be67 100755 --- a/hosts/meta01/default.nix +++ b/hosts/meta01/default.nix @@ -24,6 +24,7 @@ bagel.services.grafana.enable = true; bagel.services.grapevine.enable = true; bagel.services.pyroscope.enable = true; + bagel.services.tempo.enable = true; bagel.services.hookshot = { enable = true; admins = [ diff --git a/secrets.nix b/secrets.nix index 92818c4..deb69e9 100644 --- a/secrets.nix +++ b/secrets.nix @@ -16,6 +16,7 @@ let loki-environment = [ machines.meta01 ]; gerrit-prometheus-bearer-token = [ machines.gerrit01 machines.meta01 ]; pyroscope-secrets = [ machines.meta01 ]; + tempo-environment = [ machines.meta01 ]; buildbot-worker-password = [ machines.buildbot ]; buildbot-oauth-secret = [ machines.buildbot ]; diff --git a/secrets/tempo-environment.age b/secrets/tempo-environment.age new file mode 100644 index 0000000..de77af1 --- /dev/null +++ b/secrets/tempo-environment.age @@ -0,0 +1,20 @@ +age-encryption.org/v1 +-> ssh-ed25519 j2r2qQ kbi4mciOrjd7/X86xfmkDaMZhvZakoSJ6qjqLF3ljkE +Q2BsgMLJ8AmjhnggRi+wkICj18NCA2HW1t8clemReUw +-> ssh-ed25519 K3b7BA wNGmX9S9bJgd2JDte9QoNDfyycgmq4JMu2bc5nyYYik +uUiutxAI3nI0M51W97aPRVE/l4dV2PEjph8eWOMLHIE +-> ssh-ed25519 +qVung raYJ5vwMP9JopSdfa+ofkLY/gc0zcW4wTNBFTca+MXw +sa/rWGSYrI4y6rn4JSboldWKUGvx6HbtsYo78AFOkBo +-> ssh-rsa krWCLQ +FLq8NwkiGw2gXptVVY393f0p9hFom57xHWPxtAlzOcRT8gvWu/uwgV+0raOcOcJa +xxr5Sib+2D3UnUhprVPmH5Os9bI2seFAiej1MVVWLqvMtQHLFwnrzZTyZpxsXpQq +5qQhNEADuQc4uD/ELVjGHKt6nF1Cl/GbgNLIOF/ITZ0pm1O1MjtT6MYJhQJhc6sb +sno/wQyTXjj7rC06nyLX/rgOWrJSOeaz9eVp0A8k8/I0TXu/vRCW9gqWtv2m8sbh +1uUHIm0l8f3z+zrL6OlZnpMFw4jpiiGoCYKPzD17I0onDYIjtdVS5iO9BsckxV/a +wQWbyONUwbGCfeNSVAzZbg +-> ssh-ed25519 /vwQcQ jwf7fwy4wKz7q761DNu8SyFHGgFlwq4P/Pn44Nido3E +1q/jvt/vtD4ziY3eCDqk1XwMPpNUd80POTV2VVsumCE +-> ssh-ed25519 0R97PA XeuziQ+wsoh0KSHXk5Qkl1kQOsAu1Ax1zTg13+XWd3M +B1KHKm3tx/EsnE6hY+w7ya1ilhYiUs9AbwARHNkJi90 +--- JgQA6gCYZu8xcbXEl9VypccEIBO6uAJIdhBefr4doRQ +V3Zڏ-.s3 iSa5#{ȮDYNoL+#M<swsP+ӉBDouF^¥G@M qG^Qbs<;nC+x_]S \ No newline at end of file diff --git a/services/monitoring/lgtm/default.nix b/services/monitoring/lgtm/default.nix index 700428e..0db72ac 100644 --- a/services/monitoring/lgtm/default.nix +++ b/services/monitoring/lgtm/default.nix @@ -3,5 +3,6 @@ ./grafana.nix ./loki.nix ./mimir.nix + ./tempo.nix ]; -} \ No newline at end of file +} diff --git a/services/monitoring/lgtm/grafana.nix b/services/monitoring/lgtm/grafana.nix index d803304..9688a5e 100644 --- a/services/monitoring/lgtm/grafana.nix +++ b/services/monitoring/lgtm/grafana.nix @@ -8,7 +8,7 @@ let cfg = config.bagel.services.grafana; inherit (lib) mkEnableOption mkIf; - generatedJsonnetDashboards = (pkgs.callPackage ../../../dashboards { + generatedJsonnetDashboards = (pkgs.callPackage ../../../dashboards { inherit (inputs) gerrit-dashboard; }).allDashboards; in @@ -132,6 +132,14 @@ in access = "proxy"; url = "http://127.0.0.1:4040"; } + { + name = "Tempo"; + type = "tempo"; + uid = "tempo"; + access = "proxy"; + url = "http://127.0.0.1:9190"; + jsonData.streamingEnabled.search = true; + } ]; }; }; diff --git a/services/monitoring/lgtm/tempo.nix b/services/monitoring/lgtm/tempo.nix new file mode 100644 index 0000000..3f7da98 --- /dev/null +++ b/services/monitoring/lgtm/tempo.nix @@ -0,0 +1,79 @@ +{ + config, + lib, + ... +}: +let + cfg = config.bagel.services.tempo; + inherit (lib) mkEnableOption mkIf; +in +{ + options.bagel.services.tempo.enable = mkEnableOption "Tempo trace store"; + + config = mkIf cfg.enable { + age.secrets = { + metrics-push-htpasswd = { + file = ../../../secrets/metrics-push-htpasswd.age; + owner = "nginx"; + }; + tempo-environment.file = ../../../secrets/tempo-environment.age; + }; + + services.tempo = { + enable = true; + extraFlags = ["--config.expand-env=true"]; + settings = { + multitenancy_enabled = false; + stream_over_http_enabled = true; + + server = { + http_listen_port = 9190; + grpc_listen_port = 9195; + }; + distributor.receivers.otlp.protocols.http.endpoint = "127.0.0.1:4138"; + + storage.trace = { + backend = "s3"; + s3 = { + endpoint = "s3.delroth.net"; + bucket = "bagel-tempo"; + secret_key = "\${S3_KEY}"; # This is a secret injected via an environment variable + access_key = "\${S3_KEY_ID}"; + }; + wal.path = "/var/lib/tempo/traces-wal"; + }; + + metrics_generator.storage = { + path = "/var/lib/tempo/metrics-wal"; + remote_write = [ + { + url = "http://127.0.0.1:9009/api/v1/push"; + } + ]; + }; + + overrides.defaults.metrics_generator.processors = [ "span-metrics" ]; + }; + }; + + systemd.services.tempo.serviceConfig.EnvironmentFile = [ config.age.secrets.tempo-environment.path ]; + + services.nginx = { + upstreams.tempo = { + servers."${config.services.tempo.settings.distributor.receivers.otlp.protocols.http.endpoint}" = {}; + extraConfig = "keepalive 16;"; + }; + + virtualHosts."tempo.forkos.org" = { + enableACME = true; + forceSSL = true; + locations."/" = { + proxyPass = "http://tempo"; + basicAuthFile = config.age.secrets.metrics-push-htpasswd.path; + }; + }; + }; + + bagel.monitoring.grafana-agent.exporters.tempo.port = 9190; + }; +} diff --git a/terraform/gandi.nix b/terraform/gandi.nix index 9dd3ceb..f40716b 100644 --- a/terraform/gandi.nix +++ b/terraform/gandi.nix @@ -81,6 +81,7 @@ in (record "loki" 300 "CNAME" ["meta01.infra.p"]) (record "mimir" 300 "CNAME" ["meta01.infra.p"]) (record "pyroscope" 300 "CNAME" ["meta01.infra.p"]) + (record "tempo" 300 "CNAME" ["meta01.infra.p"]) (record "matrix" 300 "CNAME" ["meta01.infra.p"]) (record "alerts" 300 "CNAME" ["meta01.infra.p"]) (record "buildbot" 300 "CNAME" ["buildbot.infra.p"])