chore(o11y): filter by tenancy on node_exporter

Signed-off-by: Raito Bezarius <masterancpp@gmail.com>
fix: make all buildbot postgres faster
2024-10-22 16:57:37 +02:00 · 2024-10-22 16:57:24 +02:00 · 2024-10-22 16:57:06 +02:00 · 2024-10-22 16:57:06 +02:00 · 2024-10-22 16:57:06 +02:00 · 2024-10-21 16:31:26 +03:00
17 changed files with 581 additions and 415 deletions
--- a/common/admins.nix
+++ b/common/admins.nix
@ -2,7 +2,7 @@
 let
  inherit (lib) genAttrs;
 in
-# Note: to add somefew in this list. 
+# Note: to add somefew in this list.
 # Ensure their SSH key is already in common/ssh-keys.nix with
 # the same username for here, so that the keys is automatically added.
 {
@ -19,12 +19,14 @@ in
      "thubrecht"
      "winter"
      "yuka"
+      "ckie"
    ];

    lix-infra.members = [
      "raito"
      "hexchen"
      "jade"
+      "pennae"
    ];
  };
  bagel.users = genAttrs [
@ -39,5 +41,7 @@ in
    "thubrecht"
    "winter"
    "yuka"
+    "ckie"
+    "pennae"
  ] (name: {});
 }
--- a/common/ssh-keys.nix
+++ b/common/ssh-keys.nix
@ -63,5 +63,10 @@
    thubrecht = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPM1jpXR7BWQa7Sed7ii3SbvIPRRlKb3G91qC0vOwfJn" ];
    yuka = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIxQ3NYBi8v1f/vhxLKDcA6upmX0pctRDbnK6SER5OUR yureka" ];
    winter = [ "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIH/LDRUG+U+++UmlxvA2kspioTjktQZ8taDcHq8gVlkfAAAABHNzaDo=" ];
+    ckie = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIH3uTwzSSMAPg84fwbNp2cq9+BdLFeA1VzDGth4zCAbz https://mei.puppycat.house" ];
+    pennae = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIC5Wf5/IbyFpdziWfwxkQqxOf3r1L9pYn6xQBEKFwmMY"
+      "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIK8icXjHkb4XzbIVN3djH4CE7RvgGd+3xbG4cgh0Yls5AAAABHNzaDo="
+    ];
  };
 }
--- a/flake.lock
+++ b/flake.lock
@ -87,16 +87,16 @@
        "treefmt-nix": "treefmt-nix"
      },
      "locked": {
-        "lastModified": 1728307353,
-        "narHash": "sha256-eVSDu52qZn48c1HfDlH79JkDIzuE7qyY0ipPMbrpYzE=",
-        "ref": "refs/heads/forkos",
-        "rev": "60860d308404efc14cff66513f9e8e4a002756c3",
-        "revCount": 299,
+        "lastModified": 1728837991,
+        "narHash": "sha256-+jXVHPmX9eUtH2JhMKye0Tm2KMQTmD8FlHHfbcaXMOI=",
+        "ref": "refs/heads/bring-back-old-gerrit-reporting",
+        "rev": "879e9cdcdf2d7e6566ee512d015acc4d23f35517",
+        "revCount": 302,
        "type": "git",
        "url": "https://git.lix.systems/lix-project/buildbot-nix.git"
      },
      "original": {
-        "ref": "refs/heads/forkos",
+        "ref": "refs/heads/bring-back-old-gerrit-reporting",
        "type": "git",
        "url": "https://git.lix.systems/lix-project/buildbot-nix.git"
      }
--- a/flake.nix
+++ b/flake.nix
@ -22,7 +22,7 @@
    gerrit-dashboard.url = "git+https://git.lix.systems/the-distro/gerrit-monitoring.git";
    gerrit-dashboard.flake = false;

-    buildbot-nix.url = "git+https://git.lix.systems/lix-project/buildbot-nix.git?ref=refs/heads/forkos";
+    buildbot-nix.url = "git+https://git.lix.systems/lix-project/buildbot-nix.git?ref=refs/heads/bring-back-old-gerrit-reporting";
    buildbot-nix.inputs.nixpkgs.follows = "nixpkgs";

    channel-scripts.url = "git+https://git.lix.systems/the-distro/channel-scripts.git";
--- a/hosts/buildbot-lix/default.nix
+++ b/hosts/buildbot-lix/default.nix
@ -38,7 +38,9 @@
      port = 2022;
      username = "buildbot";
    };
-    cors.allowedOrigin = "*.lix.systems";
+    cors.allowedOrigins = [
+      "https://*.lix.systems"
+    ];
    projects = [
      "lix"
      "lix-installer"
--- a/hosts/buildbot/default.nix
+++ b/hosts/buildbot/default.nix
@ -36,13 +36,15 @@
        port = cfgGerrit.port;
        username = "buildbot";
      };
-    cors.allowedOrigin = "*.forkos.org";
+    cors.allowedOrigins = [
+      "https://*.forkos.org"
+    ];
    projects = [
      "buildbot-test"
      "nixpkgs"
      "infra"
    ];
-    builders = [ "builder-10" ];
+    builders = [ "builder-9" ];
  };

  i18n.defaultLocale = "en_US.UTF-8";
--- a/hosts/gerrit01/default.nix
+++ b/hosts/gerrit01/default.nix
@ -23,6 +23,9 @@
    };
  };

+  # Block all these crawlers!!
+  bagel.services.nginx.crawler-blocker.enable = true;
+
  fileSystems."/gerrit-data" = {
    device = "/dev/disk/by-uuid/d1062305-0dea-4740-9a27-b6b1691862a4";
    fsType = "ext4";
--- a/services/block-crawlers/blocked-ua.txt
+++ b/services/block-crawlers/blocked-ua.txt
@ -0,0 +1,40 @@
+AI2Bot
+Ai2Bot-Dolma
+Amazonbot
+anthropic-ai
+Applebot
+Applebot-Extended
+Bytespider
+CCBot
+ChatGPT-User
+Claude-Web
+ClaudeBot
+cohere-ai
+Diffbot
+FacebookBot
+facebookexternalhit
+FriendlyCrawler
+Google-Extended
+GoogleOther
+GoogleOther-Image
+GoogleOther-Video
+GPTBot
+iaskspider/2.0
+ICC-Crawler
+ImagesiftBot
+img2dataset
+ISSCyberRiskCrawler
+Kangaroo Bot
+Meta-ExternalAgent
+Meta-ExternalFetcher
+OAI-SearchBot
+omgili
+omgilibot
+PerplexityBot
+PetalBot
+Scrapy
+Sidetrade indexer bot
+Timpibot
+VelenPublicWebCrawler
+Webzio-Extended
+YouBot
--- a/services/block-crawlers/default.nix
+++ b/services/block-crawlers/default.nix
@ -0,0 +1,32 @@
+{ pkgs, config, lib, ... }:
+let
+  inherit (lib) mkEnableOption mkIf mkOption types concatStringsSep mkDefault splitString;
+  cfg = config.bagel.services.nginx.crawler-blocker;
+  mkRobotsFile = blockedUAs: pkgs.writeText "robots.txt" ''
+    ${concatStringsSep "\n" (map (ua: "User-agent: ${ua}") blockedUAs)}
+    Disallow: /
+  '';
+in
+{
+  options = {
+    bagel.services.nginx.crawler-blocker = {
+      enable = mkEnableOption "the crawler blocker";
+
+      userAgents = mkOption {
+        type = types.listOf types.str;
+        default = splitString "\n" (builtins.readFile ./blocked-ua.txt);
+      };
+    };
+
+    services.nginx.virtualHosts = mkOption {
+      type = types.attrsOf (types.submodule {
+        config = {
+          locations."= /robots.txt" = mkIf cfg.enable (mkDefault {
+            alias = mkRobotsFile cfg.userAgents;
+          });
+        };
+      });
+    };
+  };
+}
+
--- a/services/buildbot/default.nix
+++ b/services/buildbot/default.nix
@ -53,9 +53,9 @@ in
      };
    };

-    cors.allowedOrigin = mkOption {
-      type = types.str;
-      example = "*.forkos.org";
+    cors.allowedOrigins = mkOption {
+      type = types.listOf types.str;
+      example = [ "*.forkos.org" ];
      description = "Allowed origin for Buildbot and NGINX for CORS without the protocol";
    };

@ -100,6 +100,7 @@ in
    };

    services.nginx = {
+      recommendedProxySettings = true;
      appendHttpConfig = ''
        # Our session stuff is too big with the TWISTED_COOKIE in addition.
        # Default is usually 4k or 8k.
@ -109,8 +110,8 @@ in
        forceSSL = true;
        enableACME = true;
        extraConfig = ''
+          # This is needed so that logged-in users in Buildbot can include their credentials in their requests.
          add_header Access-Control-Allow-Credentials 'true' always;
-          add_header Access-Control-Allow-Origin 'https://${cfg.cors.allowedOrigin}' always;
        '';
      };
    };
@ -155,9 +156,8 @@ in
      # we can replace all of this with automatic localworker generation on buildbot-nix side.
      workersFile = config.age.secrets.buildbot-workers.path;

-      allowedOrigins = [
-        cfg.cors.allowedOrigin
-      ];
+      # We rely on NGINX to do the CORS dance.
+      allowedOrigins = cfg.cors.allowedOrigins;

      buildMachines = map (n: {
          hostName = nodes.${n}.config.networking.fqdn;
@ -194,6 +194,32 @@ in
      RestartSteps = 10;
    };

+    services.postgresql.settings = {
+      # DB Version: 15
+      # OS Type: linux
+      # DB Type: web
+      # Total Memory (RAM): 64 GB
+      # CPUs num: 16
+      # Connections num: 100
+      # Data Storage: ssd
+      max_connections = 100;
+      shared_buffers = "16GB";
+      effective_cache_size = "48GB";
+      maintenance_work_mem = "2GB";
+      checkpoint_completion_target = 0.9;
+      wal_buffers = "16MB";
+      default_statistics_target = 100;
+      random_page_cost = 1.1;
+      effective_io_concurrency = 200;
+      work_mem = "41943kB";
+      huge_pages = "try";
+      min_wal_size = "1GB";
+      max_wal_size = "4GB";
+      max_worker_processes = 16;
+      max_parallel_workers_per_gather = 4;
+      max_parallel_workers = 16;
+      max_parallel_maintenance_workers = 4;
+    };

    nix.settings.keep-derivations = true;
    nix.gc = {
--- a/services/default.nix
+++ b/services/default.nix
@ -1,5 +1,6 @@
 {
  imports = [
+    ./block-crawlers
    ./gerrit
    ./channel-scripts
    ./hydra
--- a/services/gerrit/default.nix
+++ b/services/gerrit/default.nix
@ -129,7 +129,9 @@ in
      serverId = "9e5216ad-038d-4d74-a4e8-716515834a94";

      builtinPlugins = [
-        "gitiles"
+        # Disable gitiles as it generates too much traffic.
+        # Prefer git.forkos.org.
+        # "gitiles"
        "codemirror-editor"
        "reviewnotes"
        "download-commands"
--- a/services/gerrit/www.nix
+++ b/services/gerrit/www.nix
@ -29,10 +29,6 @@ in
            # NGINX should not give up super fast. Things can take time.
            proxy_read_timeout 3600;
          }
-
-          location = /robots.txt {
-            return 200 'User-agent: *\nAllow: /';
-          }
        '';
      };

--- a/services/matrix/default.nix
+++ b/services/matrix/default.nix
@ -32,6 +32,11 @@ in
          ];
          server_name = "forkos.org";
          database.backend = "rocksdb";
+
+          server_discovery = {
+            server.authority = "matrix.forkos.org:443";
+            client.base_url = "https://matrix.forkos.org";
+          };
        };
      };

@ -48,18 +53,7 @@ in
          "forkos.org" = {
            forceSSL = true;
            enableACME = true;
-            locations = {
-              "= /.well-known/matrix/server".extraConfig = ''
-                add_header Content-Type application/json;
-                add_header Access-Control-Allow-Origin *;
-                return 200 '{"m.server": "matrix.forkos.org:443"}';
-              '';
-              "= /.well-known/matrix/client".extraConfig = ''
-                add_header Content-Type application/json;
-                add_header Access-Control-Allow-Origin *;
-                return 200 '{"m.homeserver": {"base_url": "https://matrix.forkos.org/"}, "m.identity_server": {"base_url": "https://matrix.org/"}, "org.matrix.msc3575.proxy": {"url": "https://matrix.forkos.org"}}';
-              '';
-            };
+            locations."/.well-known/matrix".proxyPass = "http://grapevine";
          };
        };
      };
--- a/services/monitoring/exporters/postgres.nix
+++ b/services/monitoring/exporters/postgres.nix
@ -23,7 +23,10 @@ in
    };

    services.postgresql.settings.shared_preload_libraries = "pg_stat_statements";
+    systemd.services.postgresql.postStart = lib.mkAfter ''
+      ${config.services.postgresql.package}/bin/psql -U postgres -c "CREATE EXTENSION IF NOT EXISTS pg_stat_statements;";
+    '';

    bagel.monitoring.grafana-agent.exporters.postgres.port = 9104;
  };
-}
+}
--- a/services/monitoring/lgtm/dashboards/node_exporter.json
+++ b/services/monitoring/lgtm/dashboards/node_exporter.json
--- a/services/monitoring/lgtm/dashboards/postgres2.json
+++ b/services/monitoring/lgtm/dashboards/postgres2.json
@ -20,7 +20,6 @@
  "fiscalYearStartMonth": 0,
  "gnetId": 9628,
  "graphTooltip": 0,
-  "id": 27,
  "links": [],
  "liveNow": false,
  "panels": [
@ -101,6 +100,7 @@
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "mean"
@ -112,14 +112,14 @@
        "textMode": "name",
        "wideLayout": true
      },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_static{hostname=\"$hostname\"}",
+          "expr": "pg_static{tenant=\"$tenant\",hostname=\"$hostname\"}",
          "format": "time_series",
          "instant": true,
          "intervalFactor": 1,
@ -182,6 +182,7 @@
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "mean"
@ -193,14 +194,14 @@
        "textMode": "auto",
        "wideLayout": true
      },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_postmaster_start_time_seconds{hostname=\"$hostname\"} * 1000",
+          "expr": "pg_postmaster_start_time_seconds{tenant=\"$tenant\",hostname=\"$hostname\"} * 1000",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "",
@ -261,6 +262,7 @@
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
@ -272,14 +274,14 @@
        "textMode": "auto",
        "wideLayout": true
      },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "SUM(pg_stat_database_tup_fetched{datname=~\"$datname\", hostname=~\"$hostname\"})",
+          "expr": "SUM(pg_stat_database_tup_fetched{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"})",
          "format": "time_series",
          "intervalFactor": 2,
          "refId": "A",
@ -340,6 +342,7 @@
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
@ -351,14 +354,14 @@
        "textMode": "auto",
        "wideLayout": true
      },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "SUM(pg_stat_database_tup_inserted{datname=~\"$datname\", hostname=~\"$hostname\"})",
+          "expr": "SUM(pg_stat_database_tup_inserted{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"})",
          "format": "time_series",
          "intervalFactor": 2,
          "refId": "A",
@ -419,6 +422,7 @@
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
@ -430,14 +434,14 @@
        "textMode": "auto",
        "wideLayout": true
      },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "SUM(pg_stat_database_tup_updated{datname=~\"$datname\", hostname=~\"$hostname\"})",
+          "expr": "SUM(pg_stat_database_tup_updated{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"})",
          "format": "time_series",
          "intervalFactor": 2,
          "refId": "A",
@ -498,6 +502,7 @@
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "mean"
@ -509,14 +514,14 @@
        "textMode": "auto",
        "wideLayout": true
      },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_settings_max_connections{hostname=\"$hostname\"}",
+          "expr": "pg_settings_max_connections{tenant=\"$tenant\",hostname=\"$hostname\"}",
          "format": "time_series",
          "intervalFactor": 1,
          "refId": "A"
@ -602,6 +607,7 @@
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
@ -613,14 +619,14 @@
        "textMode": "auto",
        "wideLayout": true
      },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_settings_shared_buffers_bytes{hostname=\"$hostname\"}",
+          "expr": "pg_settings_shared_buffers_bytes{tenant=\"$tenant\",hostname=\"$hostname\"}",
          "format": "time_series",
          "intervalFactor": 1,
          "refId": "A"
@ -680,6 +686,7 @@
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
@ -691,14 +698,14 @@
        "textMode": "auto",
        "wideLayout": true
      },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_settings_effective_cache_size_bytes{hostname=\"$hostname\"}",
+          "expr": "pg_settings_effective_cache_size_bytes{tenant=\"$tenant\",hostname=\"$hostname\"}",
          "format": "time_series",
          "intervalFactor": 1,
          "refId": "A"
@ -758,6 +765,7 @@
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
@ -769,14 +777,14 @@
        "textMode": "auto",
        "wideLayout": true
      },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_settings_maintenance_work_mem_bytes{hostname=\"$hostname\"}",
+          "expr": "pg_settings_maintenance_work_mem_bytes{tenant=\"$tenant\",hostname=\"$hostname\"}",
          "format": "time_series",
          "intervalFactor": 1,
          "refId": "A"
@ -836,6 +844,7 @@
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
@ -847,14 +856,14 @@
        "textMode": "auto",
        "wideLayout": true
      },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_settings_work_mem_bytes{hostname=\"$hostname\"}",
+          "expr": "pg_settings_work_mem_bytes{tenant=\"$tenant\",hostname=\"$hostname\"}",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "",
@ -916,6 +925,7 @@
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
@ -927,14 +937,14 @@
        "textMode": "auto",
        "wideLayout": true
      },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_settings_max_wal_size_bytes{hostname=\"$hostname\"}",
+          "expr": "pg_settings_max_wal_size_bytes{tenant=\"$tenant\",hostname=\"$hostname\"}",
          "format": "time_series",
          "intervalFactor": 1,
          "refId": "A"
@ -994,6 +1004,7 @@
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
@ -1005,14 +1016,14 @@
        "textMode": "auto",
        "wideLayout": true
      },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_settings_random_page_cost{hostname=\"$hostname\"}",
+          "expr": "pg_settings_random_page_cost{tenant=\"$tenant\",hostname=\"$hostname\"}",
          "format": "time_series",
          "intervalFactor": 1,
          "refId": "A"
@ -1072,6 +1083,7 @@
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
@ -1083,7 +1095,7 @@
        "textMode": "auto",
        "wideLayout": true
      },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
      "targets": [
        {
          "datasource": {
@ -1150,6 +1162,7 @@
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "mean"
@ -1161,14 +1174,14 @@
        "textMode": "auto",
        "wideLayout": true
      },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_settings_max_worker_processes{hostname=\"$hostname\"}",
+          "expr": "pg_settings_max_worker_processes{tenant=\"$tenant\",hostname=\"$hostname\"}",
          "format": "time_series",
          "intervalFactor": 1,
          "refId": "A"
@ -1228,6 +1241,7 @@
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
@ -1239,14 +1253,14 @@
        "textMode": "auto",
        "wideLayout": true
      },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_settings_max_parallel_workers{hostname=\"$hostname\"}",
+          "expr": "pg_settings_max_parallel_workers{tenant=\"$tenant\",hostname=\"$hostname\"}",
          "format": "time_series",
          "intervalFactor": 1,
          "refId": "A"
@ -1298,6 +1312,7 @@
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
+            "barWidthFactor": 0.6,
            "drawStyle": "points",
            "fillOpacity": 10,
            "gradientMode": "none",
@ -1372,7 +1387,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_stat_activity_count{datname=~\"$datname\", hostname=~\"$hostname\", state=\"active\"} !=0",
+          "expr": "pg_stat_activity_count{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\", state=\"active\"} !=0",
          "format": "time_series",
          "interval": "",
          "intervalFactor": 2,
@ -1401,6 +1416,7 @@
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
+            "barWidthFactor": 0.6,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
@ -1474,7 +1490,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "irate(pg_stat_database_xact_commit{hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
+          "expr": "irate(pg_stat_database_xact_commit{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "{{datname}} commits",
@ -1485,7 +1501,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "irate(pg_stat_database_xact_rollback{hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
+          "expr": "irate(pg_stat_database_xact_rollback{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "{{datname}} rollbacks",
@ -1512,6 +1528,7 @@
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
+            "barWidthFactor": 0.6,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
@ -1585,7 +1602,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_stat_database_tup_updated{datname=~\"$datname\", hostname=~\"$hostname\"} != 0",
+          "expr": "pg_stat_database_tup_updated{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"} != 0",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "{{datname}}",
@ -1613,6 +1630,7 @@
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
+            "barWidthFactor": 0.6,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
@ -1686,7 +1704,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_stat_database_tup_fetched{datname=~\"$datname\", hostname=~\"$hostname\"} != 0",
+          "expr": "pg_stat_database_tup_fetched{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"} != 0",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "{{datname}}",
@ -1714,6 +1732,7 @@
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
+            "barWidthFactor": 0.6,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
@ -1787,7 +1806,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_stat_database_tup_inserted{datname=~\"$datname\", hostname=~\"$hostname\"} != 0",
+          "expr": "pg_stat_database_tup_inserted{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"} != 0",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "{{datname}}",
@ -1815,6 +1834,7 @@
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
+            "barWidthFactor": 0.6,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
@ -1890,7 +1910,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_locks_count{datname=~\"$datname\", hostname=~\"$hostname\", mode=~\"$mode\"} != 0",
+          "expr": "pg_locks_count{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\", mode=~\"$mode\"} != 0",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "{{datname}},{{mode}}",
@ -1948,8 +1968,7 @@
            "mode": "absolute",
            "steps": [
              {
-                "color": "green",
-                "value": null
+                "color": "green"
              },
              {
                "color": "red",
@ -1991,7 +2010,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_stat_database_tup_returned{datname=~\"$datname\", hostname=~\"$hostname\"} != 0",
+          "expr": "pg_stat_database_tup_returned{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"} != 0",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "{{datname}}",
@ -2050,8 +2069,7 @@
            "mode": "absolute",
            "steps": [
              {
-                "color": "green",
-                "value": null
+                "color": "green"
              },
              {
                "color": "red",
@ -2092,7 +2110,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_stat_activity_count{datname=~\"$datname\", hostname=~\"$hostname\", state=~\"idle|idle in transaction|idle in transaction (aborted)\"}",
+          "expr": "pg_stat_activity_count{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\", state=~\"idle|idle in transaction|idle in transaction (aborted)\"}",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "{{datname}}, s: {{state}}",
@ -2150,8 +2168,7 @@
            "mode": "absolute",
            "steps": [
              {
-                "color": "green",
-                "value": null
+                "color": "green"
              },
              {
                "color": "red",
@ -2193,7 +2210,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_stat_database_tup_deleted{datname=~\"$datname\", hostname=~\"$hostname\"} != 0",
+          "expr": "pg_stat_database_tup_deleted{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"} != 0",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "{{datname}}",
@ -2252,8 +2269,7 @@
            "mode": "absolute",
            "steps": [
              {
-                "color": "green",
-                "value": null
+                "color": "green"
              },
              {
                "color": "red",
@ -2294,7 +2310,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_stat_database_blks_hit{hostname=\"$hostname\", datname=~\"$datname\"} / (pg_stat_database_blks_read{hostname=\"$hostname\", datname=~\"$datname\"} + pg_stat_database_blks_hit{hostname=\"$hostname\", datname=~\"$datname\"})",
+          "expr": "pg_stat_database_blks_hit{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"} / (pg_stat_database_blks_read{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"} + pg_stat_database_blks_hit{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"})",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "{{ datname }}",
@ -2351,8 +2367,7 @@
            "mode": "absolute",
            "steps": [
              {
-                "color": "green",
-                "value": null
+                "color": "green"
              },
              {
                "color": "red",
@ -2395,7 +2410,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "irate(pg_stat_bgwriter_buffers_backend{hostname=\"$hostname\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_buffers_backend{tenant=\"$tenant\",hostname=\"$hostname\"}[5m])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "buffers_backend",
@ -2406,7 +2421,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "irate(pg_stat_bgwriter_buffers_alloc{hostname=\"$hostname\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_buffers_alloc{tenant=\"$tenant\",hostname=\"$hostname\"}[5m])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "buffers_alloc",
@ -2417,7 +2432,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "irate(pg_stat_bgwriter_buffers_backend_fsync{hostname=\"$hostname\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_buffers_backend_fsync{tenant=\"$tenant\",hostname=\"$hostname\"}[5m])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "backend_fsync",
@ -2428,7 +2443,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "irate(pg_stat_bgwriter_buffers_checkpoint{hostname=\"$hostname\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_buffers_checkpoint{tenant=\"$tenant\",hostname=\"$hostname\"}[5m])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "buffers_checkpoint",
@ -2439,7 +2454,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "irate(pg_stat_bgwriter_buffers_clean{hostname=\"$hostname\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_buffers_clean{tenant=\"$tenant\",hostname=\"$hostname\"}[5m])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "buffers_clean",
@ -2497,8 +2512,7 @@
            "mode": "absolute",
            "steps": [
              {
-                "color": "green",
-                "value": null
+                "color": "green"
              },
              {
                "color": "red",
@ -2540,7 +2554,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "irate(pg_stat_database_conflicts{hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
+          "expr": "irate(pg_stat_database_conflicts{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "{{datname}} conflicts",
@ -2551,7 +2565,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "irate(pg_stat_database_deadlocks{hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
+          "expr": "irate(pg_stat_database_deadlocks{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "{{datname}} deadlocks",
@ -2610,8 +2624,7 @@
            "mode": "absolute",
            "steps": [
              {
-                "color": "green",
-                "value": null
+                "color": "green"
              },
              {
                "color": "red",
@ -2653,7 +2666,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "irate(pg_stat_database_temp_bytes{hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
+          "expr": "irate(pg_stat_database_temp_bytes{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "{{datname}}",
@ -2710,8 +2723,7 @@
            "mode": "absolute",
            "steps": [
              {
-                "color": "green",
-                "value": null
+                "color": "green"
              },
              {
                "color": "red",
@ -2754,7 +2766,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "irate(pg_stat_bgwriter_checkpoint_write_time{hostname=\"$hostname\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_checkpoint_write_time{tenant=\"$tenant\",hostname=\"$hostname\"}[5m])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "write_time - Total amount of time that has been spent in the portion of checkpoint processing where files are written to disk.",
@ -2765,7 +2777,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "irate(pg_stat_bgwriter_checkpoint_sync_time{hostname=\"$hostname\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_checkpoint_sync_time{tenant=\"$tenant\",hostname=\"$hostname\"}[5m])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "sync_time - Total amount of time that has been spent in the portion of checkpoint processing where files are synchronized to disk.",
@ -2866,8 +2878,32 @@
      {
        "current": {
          "selected": false,
-          "text": "bagel-box",
-          "value": "bagel-box"
+          "text": "fake",
+          "value": "fake"
+        },
+        "definition": "label_values(tenant)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "Tenant",
+        "multi": true,
+        "name": "tenant",
+        "options": [],
+        "query": {
+          "qryType": 1,
+          "query": "label_values(tenant)",
+          "refId": "PrometheusVariableQueryEditor-VariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      },
+      {
+        "current": {
+          "selected": true,
+          "text": "buildbot",
+          "value": "buildbot"
        },
        "datasource": {
          "type": "prometheus",
@ -2962,7 +2998,6 @@
    "from": "now-6h",
    "to": "now"
  },
-  "timeRangeUpdatedDuringEditOrView": false,
  "timepicker": {
    "refresh_intervals": [
      "5s",
@ -2993,4 +3028,4 @@
  "uid": "000000039",
  "version": 1,
  "weekStart": ""
-}
+}
Author	SHA1	Message	Date
raito	02f8bc7ca4	chore(o11y): filter by tenancy on node_exporter Signed-off-by: Raito Bezarius <masterancpp@gmail.com>	2024-10-22 16:57:37 +02:00
raito	e0c029ba43	fix: make all buildbot postgres faster work_mem was effectively absurdly low. Signed-off-by: Raito Bezarius <masterancpp@gmail.com>	2024-10-22 16:57:24 +02:00
raito	3ed36f74fd	onboarding: add pennae keys on lix infra Signed-off-by: Raito Bezarius <masterancpp@gmail.com>	2024-10-22 16:57:06 +02:00
raito	b1f4674da0	chore: add tenancy in postgres Signed-off-by: Raito Bezarius <masterancpp@gmail.com>	2024-10-22 16:57:06 +02:00
raito	226eacdeec	chore: add tenancy in node_exporter Signed-off-by: Raito Bezarius <masterancpp@gmail.com>	2024-10-22 16:57:06 +02:00
Ilya K	14935c5e92	fix: update grapevine config	2024-10-21 16:31:26 +03:00
raito	bee402fecc	fix: ensure that pg_stat_statements is always created as an ext Otherwise, we will have issues with this exporter. Signed-off-by: Raito Bezarius <masterancpp@gmail.com>	2024-10-21 14:33:18 +02:00
raito	3efdd0f6c9	fix: disable gitiles on gerrit01 It is generating too much traffic and CPU load for no good reason. Signed-off-by: Raito Bezarius <masterancpp@gmail.com>	2024-10-20 11:24:58 +02:00
raito	8c0c7b517f	feat: block automatically crawlers if the blocker is enabled This help us getting rid of useless traffic by crawlers. It is enabled for gerrit01 which is suffering the most from this. Signed-off-by: Raito Bezarius <masterancpp@gmail.com>	2024-10-19 19:12:10 +02:00
raito	d5500d7c4e	fix(buildbot): bring back the old Gerrit reporting Signed-off-by: Raito Bezarius <masterancpp@gmail.com>	2024-10-18 23:22:51 +00:00
raito	eaf48a0cdd	fix(buildbot): use builder-9 as builder-10 is down Signed-off-by: Raito Bezarius <masterancpp@gmail.com>	2024-10-18 23:22:51 +00:00
raito	e3129fec51	fix(buildbot): fix CORS properly wildcards are not allowed in the headers. We need to include credentials as well. Signed-off-by: Raito Bezarius <masterancpp@gmail.com>	2024-10-18 23:22:51 +00:00
raito	437293bdaa	fix(buildbot): remove CORS wildcards for their precise Gerrit hosts wildcards are not supported in CORS headers, so this design was quite wrong actually. We can just use the actual Gerrit hostname for now. Signed-off-by: Raito Bezarius <masterancpp@gmail.com>	2024-10-18 23:22:51 +00:00
mei (ckie)	df8a57f91a	users: add ckie	2024-10-18 14:43:25 +03:00