Support a federated Prometheus setup

Gerrit instances that are loadbalanced cannot easily e scraped by
an external Prometheus, since the request won't end up at a specified
Gerrit instance. A typical setup to solve this issue, is to install a
local Prometheus and scrape the local Prometheus from the central
Prometheus. This is a so called federated setup.

Now such a setup is supported and can be configured.

Change-Id: I0119d3c1d846cd8e975e5732f4d59cf863c6d2b8

README.md

@@ -163,6 +163,18 @@ configuration parameters:
 | `gerritServers.kubernetes.[*].username`      | Username of Gerrit user with 'View Metrics' capabilities |
 | `gerritServers.kubernetes.[*].password`      | Password of Gerrit user with 'View Metrics' capabilities |
 
+* Federated Prometheus \
+  Load balanced Gerrit instances can't be scraped through the load balancer. For
+  this use cases typically a local Prometheus is installed and then scraped by
+  the central Prometheus in a federated setup.
+
+| option                                           | description                                                   |
+|--------------------------------------------------|---------------------------------------------------------------|
+| `gerritServers.federatedPrometheus.[*].host`     | Host running Gerrit and the Prometheus instance being scraped |
+| `gerritServers.federatedPrometheus.[*].port`     | Port used by Prometheus                                       |
+| `gerritServers.federatedPrometheus.[*].username` | Username for authenticating with Prometheus                   |
+| `gerritServers.federatedPrometheus.[*].password` | Password for authenticating with Prometheus                   |
+
 * Other \
   Gerrit installations with just one replica that can run anywhere, where they
   are reachable via HTTP.

charts/prometheus/configuration/prometheus.secret.yaml

@@ -14,6 +14,10 @@ data:
   #@ for gerrit in data.values.gerritServers.other:
   .pwd_(@= gerrit.host @): #@ base64.encode(gerrit.password)
   #@ end
+  #@yaml/text-templated-strings
+  #@ for prometheus in data.values.gerritServers.federatedPrometheus:
+  .pwd_(@= prometheus.host @): #@ base64.encode(prometheus.password)
+  #@ end
 
   #@ if not data.values.tls.skipVerify:
   server.ca.crt: #@ base64.encode(data.values.tls.caCert)

charts/prometheus/prometheus.yaml

@@ -1298,6 +1298,28 @@ serverFiles:
         replacement: $1
         action: replace
     #@ end
+    #@ for prometheus in data.values.gerritServers.federatedPrometheus:
+    - job_name: #@ "gerrit-{}".format(prometheus.host)
+      honor_labels: true
+      metrics_path: '/federate'
+      params:
+        'match[]':
+          - #@ '{{instance=~"{}.*"}}'.format(prometheus.host)
+      scheme: https
+      tls_config:
+        insecure_skip_verify: #@ data.values.tls.skipVerify
+        #@ if not data.values.tls.skipVerify:
+        ca_file: /etc/secrets/server.ca.crt
+        cert_file: /etc/secrets/server.crt
+        key_file: /etc/secrets/server.key
+        #@ end
+      static_configs:
+        - targets:
+          - #@ '{}:{}'.format(prometheus.host, prometheus.port)
+      basic_auth:
+        username: #@ prometheus.username
+        password_file: #@ "/etc/secrets/.pwd_{}".format(prometheus.host)
+    #@ end
     #@ for gerrit in data.values.gerritServers.other:
     - job_name: #@ "gerrit-{}".format(gerrit.host)
       metrics_path: /a/plugins/metrics-reporter-prometheus/metrics

config.yaml

@@ -8,6 +8,11 @@ gerritServers:
     port: 8080
     username: admin
     password: secret
+  federatedPrometheus:
+  - host: prometheus.example.com
+    port: 9090
+    username: admin
+    password: secret
   other:
   - host: gerrit.example.com
     username: admin