From 0bdb1d02e092cafe9466c1e8e9fedb0c98ef26d6 Mon Sep 17 00:00:00 2001 From: Thomas Draebing Date: Thu, 5 Mar 2020 15:28:26 +0100 Subject: [PATCH] Create promtail config per Gerrit host So far the install-script could only create a single promtail config. Since the monitoring setup is able to monitor multiple Gerrit servers, this caused manual work to create a promtail config per Gerrit server. Now ytt will create a configuration for each Gerrit host configured in the config.yaml. Ytt is only able to do that in a single file. Thus, csplit is used to split the files into separate files that can then be used to configure promtail on the respective hosts. The config- files can then be found under $OUTPUT/promtail/promtail-$GERRIT_HOSTNAME.yaml. Change-Id: Ib09fba83d8a8fbd45b42e9e5388a85a37ab1a952 --- README.md | 89 ++++++++++++++++++++---------------------- config.yaml | 6 +-- promtail/promtail.yaml | 16 +++++--- subcommands/install.py | 24 ++++++++++++ 4 files changed, 80 insertions(+), 55 deletions(-) diff --git a/README.md b/README.md index e5b63f0..0fee843 100644 --- a/README.md +++ b/README.md @@ -76,47 +76,47 @@ setup, some configuration is highly dependent on the specific installation. These options have to be configured in the `./config.yaml` before installing and are listed here: -| option | description | -|-----------------------------------------|------------------------------------------------------------------------------------| -| `gerritServers.[0].host` | Hostname (incl. port, if required) of the Gerrit server to monitor | -| `gerritServers.[0].username` | Username of Gerrit user with 'View Metrics' capabilities | -| `gerritServers.[0].password` | Password of Gerrit user with 'View Metrics' capabilities | -| `namespace` | The namespace the charts are installed to | -| `tls.skipVerify` | Whether to skip TLS certificate verification | -| `tls.caCert` | CA certificate used for TLS certificate verification | -| `promtail.storagePath` | Path to directory, where Promtail is allowed to save files (e.g. `positions.yaml`) | -| `promtail.logPath` | Path to directory containing the Gerrit logs (e.g. `/var/gerrit/logs`) | -| `prometheus.server.host` | Prometheus server ingress hostname | -| `prometheus.server.username` | Username for Prometheus | -| `prometheus.server.password` | Password for Prometheus | -| `prometheus.server.tls.cert` | TLS certificate | -| `prometheus.server.tls.key` | TLS key | -| `prometheus.alertmanager.slack.apiUrl` | API URL of the Slack Webhook | -| `prometheus.alertmanager.slack.channel` | Channel to which the alerts should be posted | -| `loki.host` | Loki ingress hostname | -| `loki.username` | Username for Loki | -| `loki.password` | Password for Loki | -| `loki.s3.protocol` | Protocol used for communicating with S3 | -| `loki.s3.host` | Hostname of the S3 object store | -| `loki.s3.accessToken` | The EC2 accessToken used for authentication with S3 | -| `loki.s3.secret` | The secret associated with the accessToken | -| `loki.s3.bucket` | The name of the S3 bucket | -| `loki.s3.region` | The region in which the S3 bucket is hosted | -| `loki.tls.cert` | TLS certificate | -| `loki.tls.key` | TLS key | -| `grafana.host` | Grafana ingress hostname | -| `grafana.tls.cert` | TLS certificate | -| `grafana.tls.key` | TLS key | -| `grafana.admin.username` | Username for the admin user | -| `grafana.admin.password` | Password for the admin user | -| `grafana.ldap.enabled` | Whether to enable LDAP | -| `grafana.ldap.host` | Hostname of LDAP server | -| `grafana.ldap.port` | Port of LDAP server (Has to be `quoted`!) | -| `grafana.ldap.password` | Password of LDAP server | -| `grafana.ldap.bind_dn` | Bind DN (username) of the LDAP server | -| `grafana.ldap.accountBases` | List of base DNs to discover accounts (Has to have the format `"['a', 'b']"`) | -| `grafana.ldap.groupBases` | List of base DNs to discover groups (Has to have the format `"['a', 'b']"`) | -| `grafana.dashboards.editable` | Whether dashboards can be edited manually in the UI | +| option | description | +|------------------------------------------|------------------------------------------------------------------------------------| +| `gerritServers.[0].host` | Hostname (incl. port, if required) of the Gerrit server to monitor | +| `gerritServers.[0].username` | Username of Gerrit user with 'View Metrics' capabilities | +| `gerritServers.[0].password` | Password of Gerrit user with 'View Metrics' capabilities | +| `gerritServers.[0].promtail.storagePath` | Path to directory, where Promtail is allowed to save files (e.g. `positions.yaml`) | +| `gerritServers.[0].promtail.logPath` | Path to directory containing the Gerrit logs (e.g. `/var/gerrit/logs`) | +| `namespace` | The namespace the charts are installed to | +| `tls.skipVerify` | Whether to skip TLS certificate verification | +| `tls.caCert` | CA certificate used for TLS certificate verification | +| `prometheus.server.host` | Prometheus server ingress hostname | +| `prometheus.server.username` | Username for Prometheus | +| `prometheus.server.password` | Password for Prometheus | +| `prometheus.server.tls.cert` | TLS certificate | +| `prometheus.server.tls.key` | TLS key | +| `prometheus.alertmanager.slack.apiUrl` | API URL of the Slack Webhook | +| `prometheus.alertmanager.slack.channel` | Channel to which the alerts should be posted | +| `loki.host` | Loki ingress hostname | +| `loki.username` | Username for Loki | +| `loki.password` | Password for Loki | +| `loki.s3.protocol` | Protocol used for communicating with S3 | +| `loki.s3.host` | Hostname of the S3 object store | +| `loki.s3.accessToken` | The EC2 accessToken used for authentication with S3 | +| `loki.s3.secret` | The secret associated with the accessToken | +| `loki.s3.bucket` | The name of the S3 bucket | +| `loki.s3.region` | The region in which the S3 bucket is hosted | +| `loki.tls.cert` | TLS certificate | +| `loki.tls.key` | TLS key | +| `grafana.host` | Grafana ingress hostname | +| `grafana.tls.cert` | TLS certificate | +| `grafana.tls.key` | TLS key | +| `grafana.admin.username` | Username for the admin user | +| `grafana.admin.password` | Password for the admin user | +| `grafana.ldap.enabled` | Whether to enable LDAP | +| `grafana.ldap.host` | Hostname of LDAP server | +| `grafana.ldap.port` | Port of LDAP server (Has to be `quoted`!) | +| `grafana.ldap.password` | Password of LDAP server | +| `grafana.ldap.bind_dn` | Bind DN (username) of the LDAP server | +| `grafana.ldap.accountBases` | List of base DNs to discover accounts (Has to have the format `"['a', 'b']"`) | +| `grafana.ldap.groupBases` | List of base DNs to discover groups (Has to have the format `"['a', 'b']"`) | +| `grafana.dashboards.editable` | Whether dashboards can be edited manually in the UI | ### Encryption @@ -165,14 +165,9 @@ result in the following command: ```sh $PATH_TO_PROMTAIL/promtail \ - -config.file=./dist/promtail.yaml \ - -client.external-labels=host=$(hostname) + -config.file=./dist/promtail.yaml ``` -The `-client.external-labels=host=$(hostname)` option will add a label to each job -that contains the hostname. This is useful, if multiple host are scraped for logs -and only one Grafana is used to view the logs. - If TLS-verification is activated, the CA-certificate used for verification (usually the one configured for `tls.caCert`) has to be present in the directory configured for `promtail.storagePath` in the `config.yaml` and has to diff --git a/config.yaml b/config.yaml index c7b9840..fffed5a 100644 --- a/config.yaml +++ b/config.yaml @@ -2,13 +2,13 @@ gerritServers: - host: gerrit.example.com username: admin password: secret + promtail: + storagePath: /var/promtail + logPath: /var/gerrit/logs namespace: namespace tls: skipVerify: true caCert: -promtail: - storagePath: /var/promtail - logPath: /var/gerrit/logs prometheus: server: host: prometheus.example.com diff --git a/promtail/promtail.yaml b/promtail/promtail.yaml index 7bd3b3c..d03e026 100644 --- a/promtail/promtail.yaml +++ b/promtail/promtail.yaml @@ -1,18 +1,20 @@ #@ load("@ytt:data", "data") +#@ for i in range(len(data.values.gerritServers)): +--- server: http_listen_port: 9080 grpc_listen_port: 0 positions: - filename: #@ "{}/positions.yaml".format(data.values.promtail.storagePath) + filename: #@ "{}/positions.yaml".format(data.values.gerritServers[i].promtail.storagePath) clients: - url: #@ "https://{}/loki/api/v1/push".format(data.values.loki.host) tls_config: insecure_skip_verify: #@ data.values.tls.skipVerify #@ if not data.values.tls.skipVerify: - ca_file: #@ "{}/promtail.ca.crt".format(data.values.promtail.storagePath) + ca_file: #@ "{}/promtail.ca.crt".format(data.values.gerritServers[i].promtail.storagePath) #@ end basic_auth: username: #@ data.values.loki.username @@ -24,7 +26,8 @@ scrape_configs: - localhost labels: job: gerrit_error - __path__: #@ "{}/error_log.json".format(data.values.promtail.logPath) + __path__: #@ "{}/error_log.json".format(data.values.gerritServers[i].promtail.logPath) + host: #@ data.values.gerritServers[i].host entry_parser: raw pipeline_stages: - json: @@ -54,7 +57,8 @@ scrape_configs: - localhost labels: job: gerrit_httpd - __path__: #@ "{}/httpd_log.json".format(data.values.promtail.logPath) + __path__: #@ "{}/httpd_log.json".format(data.values.gerritServers[i].promtail.logPath) + host: #@ data.values.gerritServers[i].host entry_parser: raw pipeline_stages: - json: @@ -72,7 +76,8 @@ scrape_configs: - localhost labels: job: gerrit_sshd - __path__: #@ "{}/sshd_log.json".format(data.values.promtail.logPath) + __path__: #@ "{}/sshd_log.json".format(data.values.gerritServers[i].promtail.logPath) + host: #@ data.values.gerritServers[i].host entry_parser: raw pipeline_stages: - json: @@ -84,3 +89,4 @@ scrape_configs: - timestamp: source: timestamp format: 2006-01-02 15:04:05.999 -0700 +#@ end diff --git a/subcommands/install.py b/subcommands/install.py index c2db390..a03bdfb 100644 --- a/subcommands/install.py +++ b/subcommands/install.py @@ -71,6 +71,28 @@ def _create_dashboard_configmaps(output_dir, namespace): yaml.dump(dashboard_cm, f) +def _create_promtail_configs(output_dir): + if not os.path.exists(os.path.join(output_dir, "promtail")): + os.mkdir(os.path.join(output_dir, "promtail")) + + with open(os.path.join(output_dir, "promtail.yaml")) as f: + for config in yaml.load_all(f, Loader=yaml.SafeLoader): + with open( + os.path.join( + output_dir, + "promtail", + "promtail-%s" + % config["scrape_configs"][0]["static_configs"][0]["labels"][ + "host" + ], + ), + "w", + ) as f: + yaml.dump(config, f) + + os.remove(os.path.join(output_dir, "promtail.yaml")) + + def _run_ytt(config, output_dir): config_string = "#@data/values\n---\n" config_string += yaml.dump(config) @@ -168,6 +190,8 @@ def install(config_manager, output_dir, dryrun, update_repo): namespace = config_manager.get_config()["namespace"] _create_dashboard_configmaps(output_dir, namespace) + _create_promtail_configs(output_dir) + if not dryrun: if update_repo: _update_helm_repos()