From ce5b8300f129c14951eba24954d186932cd5d28d Mon Sep 17 00:00:00 2001 From: Thomas Draebing Date: Tue, 25 Aug 2020 13:16:42 +0200 Subject: [PATCH] Start using Grafonnet to create Grafana dashboards Versioning the pure JSON files representing the Grafana dashboards had some disadvantages. It was hard to review them, they were very cluttered and a lot was duplicated. There are some tools that deal with that. One of them is Grafonnet, which is a superset of Jsonnet, a tool to create JSON files using a domain specific language. This change implements the Gerrit Process dashboard in Grafonnet. It also extends the installer to be able to install dashboards in the Jsonnet format. Change-Id: I6235fb7d045bd71557678a4e3b0d4ad4515f4615 --- Pipfile | 1 + Pipfile.lock | 107 +-- README.md | 39 +- dashboards/gerrit-process.json | 850 ------------------ dashboards/gerrit/globals/variables.libsonnet | 21 + .../gerrit/process/gerrit-process.jsonnet | 60 ++ .../gerrit/process/panels/cpu.libsonnet | 28 + .../process/panels/file-descriptors.libsonnet | 23 + .../gerrit/process/panels/gc-time.libsonnet | 62 ++ .../process/panels/jgit-block-cache.libsonnet | 25 + .../gerrit/process/panels/memory.libsonnet | 35 + .../process/panels/system-load.libsonnet | 16 + .../gerrit/process/panels/threads.libsonnet | 16 + dashboards/globals/bar-graph.libsonnet | 32 + dashboards/globals/defaults.libsonnet | 14 + dashboards/globals/grid_pos.libsonnet | 13 + dashboards/globals/line-graph.libsonnet | 32 + dashboards/globals/publish.libsonnet | 32 + jsonnetfile.json | 15 + jsonnetfile.lock.json | 16 + subcommands/install.py | 53 +- 21 files changed, 564 insertions(+), 926 deletions(-) delete mode 100644 dashboards/gerrit-process.json create mode 100644 dashboards/gerrit/globals/variables.libsonnet create mode 100644 dashboards/gerrit/process/gerrit-process.jsonnet create mode 100644 dashboards/gerrit/process/panels/cpu.libsonnet create mode 100644 dashboards/gerrit/process/panels/file-descriptors.libsonnet create mode 100644 dashboards/gerrit/process/panels/gc-time.libsonnet create mode 100644 dashboards/gerrit/process/panels/jgit-block-cache.libsonnet create mode 100644 dashboards/gerrit/process/panels/memory.libsonnet create mode 100644 dashboards/gerrit/process/panels/system-load.libsonnet create mode 100644 dashboards/gerrit/process/panels/threads.libsonnet create mode 100644 dashboards/globals/bar-graph.libsonnet create mode 100644 dashboards/globals/defaults.libsonnet create mode 100644 dashboards/globals/grid_pos.libsonnet create mode 100644 dashboards/globals/line-graph.libsonnet create mode 100644 dashboards/globals/publish.libsonnet create mode 100644 jsonnetfile.json create mode 100644 jsonnetfile.lock.json diff --git a/Pipfile b/Pipfile index a313565..b831d26 100644 --- a/Pipfile +++ b/Pipfile @@ -13,6 +13,7 @@ pyyaml = "~=5.3" passlib = "~=1.7.2" python-gnupg = "~=0.4.5" requests = "~=2.23.0" +jsonnet = "~=0.16.0" [requires] python_version = "3.8" diff --git a/Pipfile.lock b/Pipfile.lock index 1fbfb22..ca8e0ee 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "04289147b549a22aa59a430acffa91aec1ce904346f398e8e01093962f562cb7" + "sha256": "567e752ac5b32023d7b9dbcff1c8b5613b4c4e9dc7d64cb69a2b22f3b9a8a521" }, "pipfile-spec": 6, "requires": { @@ -18,10 +18,10 @@ "default": { "certifi": { "hashes": [ - "sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304", - "sha256:51fcb31174be6e6664c5f69e3e1691a2d72a1a12e90f872cbdb1567eb47b6519" + "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3", + "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41" ], - "version": "==2020.4.5.1" + "version": "==2020.6.20" }, "chardet": { "hashes": [ @@ -32,10 +32,17 @@ }, "idna": { "hashes": [ - "sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb", - "sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa" + "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", + "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" ], - "version": "==2.9" + "version": "==2.10" + }, + "jsonnet": { + "hashes": [ + "sha256:4d6eff8c17e146dccd244eda45317577cd5e264ce8d5d0676f1f36afdc01307e" + ], + "index": "pypi", + "version": "==0.16.0" }, "passlib": { "hashes": [ @@ -47,11 +54,11 @@ }, "python-gnupg": { "hashes": [ - "sha256:3353e59949cd2c15efbf1fca45e347d8a22f4bed0d93e9b89b2657bda19cec05", - "sha256:c095a41f310ad7a4fd393406660ac9bd6c175ccaa0f072f9c18f33be8130a27a" + "sha256:3aa0884b3bd414652c2385b9df39e7b87272c2eca1b8fcc3089bc9e58652019a", + "sha256:cba3566e8a8fb7bb417d6897a6e17bfc7f9371052e57eb0057783c07d762a679" ], "index": "pypi", - "version": "==0.4.5" + "version": "==0.4.6" }, "pyyaml": { "hashes": [ @@ -80,19 +87,19 @@ }, "urllib3": { "hashes": [ - "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc", - "sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc" + "sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a", + "sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461" ], - "version": "==1.25.8" + "version": "==1.25.10" } }, "develop": { "appdirs": { "hashes": [ - "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92", - "sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e" + "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41", + "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128" ], - "version": "==1.4.3" + "version": "==1.4.4" }, "astroid": { "hashes": [ @@ -103,10 +110,10 @@ }, "attrs": { "hashes": [ - "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", - "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" + "sha256:0ef97238856430dcf9228e07f316aefc17e8939fc8507e18c6501b761ef1a42a", + "sha256:2867b7b9f8326499ab5b0e2d12801fa5c98842d2cbd22b35112ae04bf85b4dff" ], - "version": "==19.3.0" + "version": "==20.1.0" }, "black": { "hashes": [ @@ -118,10 +125,10 @@ }, "click": { "hashes": [ - "sha256:8a18b4ea89d8820c5d0c7da8a64b2c324b4dabb695804dbfea19b9be9d88c0cc", - "sha256:e345d143d80bf5ee7534056164e5e112ea5e22716bbb1ce727941f4c8b471b9a" + "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a", + "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc" ], - "version": "==7.1.1" + "version": "==7.1.2" }, "isort": { "hashes": [ @@ -180,43 +187,43 @@ }, "regex": { "hashes": [ - "sha256:08119f707f0ebf2da60d2f24c2f39ca616277bb67ef6c92b72cbf90cbe3a556b", - "sha256:0ce9537396d8f556bcfc317c65b6a0705320701e5ce511f05fc04421ba05b8a8", - "sha256:1cbe0fa0b7f673400eb29e9ef41d4f53638f65f9a2143854de6b1ce2899185c3", - "sha256:2294f8b70e058a2553cd009df003a20802ef75b3c629506be20687df0908177e", - "sha256:23069d9c07e115537f37270d1d5faea3e0bdded8279081c4d4d607a2ad393683", - "sha256:24f4f4062eb16c5bbfff6a22312e8eab92c2c99c51a02e39b4eae54ce8255cd1", - "sha256:295badf61a51add2d428a46b8580309c520d8b26e769868b922750cf3ce67142", - "sha256:2a3bf8b48f8e37c3a40bb3f854bf0121c194e69a650b209628d951190b862de3", - "sha256:4385f12aa289d79419fede43f979e372f527892ac44a541b5446617e4406c468", - "sha256:5635cd1ed0a12b4c42cce18a8d2fb53ff13ff537f09de5fd791e97de27b6400e", - "sha256:5bfed051dbff32fd8945eccca70f5e22b55e4148d2a8a45141a3b053d6455ae3", - "sha256:7e1037073b1b7053ee74c3c6c0ada80f3501ec29d5f46e42669378eae6d4405a", - "sha256:90742c6ff121a9c5b261b9b215cb476eea97df98ea82037ec8ac95d1be7a034f", - "sha256:a58dd45cb865be0ce1d5ecc4cfc85cd8c6867bea66733623e54bd95131f473b6", - "sha256:c087bff162158536387c53647411db09b6ee3f9603c334c90943e97b1052a156", - "sha256:c162a21e0da33eb3d31a3ac17a51db5e634fc347f650d271f0305d96601dc15b", - "sha256:c9423a150d3a4fc0f3f2aae897a59919acd293f4cb397429b120a5fcd96ea3db", - "sha256:ccccdd84912875e34c5ad2d06e1989d890d43af6c2242c6fcfa51556997af6cd", - "sha256:e91ba11da11cf770f389e47c3f5c30473e6d85e06d7fd9dcba0017d2867aab4a", - "sha256:ea4adf02d23b437684cd388d557bf76e3afa72f7fed5bbc013482cc00c816948", - "sha256:fb95debbd1a824b2c4376932f2216cc186912e389bdb0e27147778cf6acb3f89" + "sha256:0dc64ee3f33cd7899f79a8d788abfbec168410be356ed9bd30bbd3f0a23a7204", + "sha256:1269fef3167bb52631ad4fa7dd27bf635d5a0790b8e6222065d42e91bede4162", + "sha256:14a53646369157baa0499513f96091eb70382eb50b2c82393d17d7ec81b7b85f", + "sha256:3a3af27a8d23143c49a3420efe5b3f8cf1a48c6fc8bc6856b03f638abc1833bb", + "sha256:46bac5ca10fb748d6c55843a931855e2727a7a22584f302dd9bb1506e69f83f6", + "sha256:4c037fd14c5f4e308b8370b447b469ca10e69427966527edcab07f52d88388f7", + "sha256:51178c738d559a2d1071ce0b0f56e57eb315bcf8f7d4cf127674b533e3101f88", + "sha256:5ea81ea3dbd6767873c611687141ec7b06ed8bab43f68fad5b7be184a920dc99", + "sha256:6961548bba529cac7c07af2fd4d527c5b91bb8fe18995fed6044ac22b3d14644", + "sha256:75aaa27aa521a182824d89e5ab0a1d16ca207318a6b65042b046053cfc8ed07a", + "sha256:7a2dd66d2d4df34fa82c9dc85657c5e019b87932019947faece7983f2089a840", + "sha256:8a51f2c6d1f884e98846a0a9021ff6861bdb98457879f412fdc2b42d14494067", + "sha256:9c568495e35599625f7b999774e29e8d6b01a6fb684d77dee1f56d41b11b40cd", + "sha256:9eddaafb3c48e0900690c1727fba226c4804b8e6127ea409689c3bb492d06de4", + "sha256:bbb332d45b32df41200380fff14712cb6093b61bd142272a10b16778c418e98e", + "sha256:bc3d98f621898b4a9bc7fecc00513eec8f40b5b83913d74ccb445f037d58cd89", + "sha256:c11d6033115dc4887c456565303f540c44197f4fc1a2bfb192224a301534888e", + "sha256:c50a724d136ec10d920661f1442e4a8b010a4fe5aebd65e0c2241ea41dbe93dc", + "sha256:d0a5095d52b90ff38592bbdc2644f17c6d495762edf47d876049cfd2968fbccf", + "sha256:d6cff2276e502b86a25fd10c2a96973fdb45c7a977dca2138d661417f3728341", + "sha256:e46d13f38cfcbb79bfdb2964b0fe12561fe633caf964a77a5f8d4e45fe5d2ef7" ], - "version": "==2020.4.4" + "version": "==2020.7.14" }, "six": { "hashes": [ - "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", - "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c" + "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", + "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" ], - "version": "==1.14.0" + "version": "==1.15.0" }, "toml": { "hashes": [ - "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c", - "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e" + "sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f", + "sha256:bda89d5935c2eac546d648028b9901107a595863cb36bae0c73ac804a9b4ce88" ], - "version": "==0.10.0" + "version": "==0.10.1" }, "typed-ast": { "hashes": [ diff --git a/README.md b/README.md index bd93f0f..465fff0 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,17 @@ based on a lock-file, ensuring a deterministic Python environment. Instruction o how Pipenv can be installed, can be found [here](https://github.com/pypa/pipenv#installation) +- Jsonnet \ +Jsonnet is used to create the JSON-files describing the Grafana dashboards. +Instruction on how Jsonnet can be installed, can be found +[here](https://github.com/google/jsonnet#packages) + +- Grafonnet \ +Grafonnet should be installed using jsonnet-bundler and the `jsonnetfile.json` +provided by this project. Install jsonnet-bundler as described +[here](https://github.com/jsonnet-bundler/jsonnet-bundler#install). Then run +`jb install` from this project's root directory. + ### Infrastructure - Kubernetes Cluster \ @@ -64,11 +75,29 @@ via the S3 API. ## Add dashboards -To have dashboards deployed automatically during installation, export the dashboards -to a JSON-file or create JSON-files describing the dashboards in another way. -Put these dashboards into the `./dashboards`-directory of this repository. During -the installation the dashboards will be added to a configmap and with this -automatically installed to Grafana. +There are two ways to have dashboards deployed automatically during installation: + +### Using JSON + +One way is to export the dashboards to a JSON-file in the UI or create JSON-files +describing the dashboards in another way. Put these dashboards into the +`./dashboards`-directory of this repository. + +### Using Jsonnet + Grafonnet + +The other way is to use Jsonnet/Grafonnet to programmatically create dashboards. +Install Grafonnet into the project as described above and put your dashboard +jsonnet files into the dashboards-directory or one of its subdirectories. The +jsonnet-based dashboards can be transcribed into json manually using the following +command: + +```sh +jsonnet -J grafonnet-lib --ext-code publish=false dashboards/.jsonnet +``` + +The external variable `publish` should be set to `false`, if the dashboard is +imported via API and to `true`, if it is published to the Grafana homepage or +imported via the UI. ## Configuration diff --git a/dashboards/gerrit-process.json b/dashboards/gerrit-process.json deleted file mode 100644 index f35b504..0000000 --- a/dashboards/gerrit-process.json +++ /dev/null @@ -1,850 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "7.1.5" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "links": [], - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 11, - "w": 12, - "x": 0, - "y": 0 - }, - "hiddenSeries": false, - "id": 14, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "proc_cpu_system_load{instance=\"$instance\",replica=\"$replica\"}", - "legendFormat": "system load", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "System load", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 11, - "w": 12, - "x": 12, - "y": 0 - }, - "hiddenSeries": false, - "id": 2, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "rightSide": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "proc_jvm_memory_heap_committed{instance=\"$instance\",replica=\"$replica\"}", - "legendFormat": "committed heap", - "refId": "C" - }, - { - "expr": "proc_jvm_memory_heap_used{instance=\"$instance\",replica=\"$replica\"}", - "legendFormat": "used heap", - "refId": "B" - }, - { - "expr": "jgit_block_cache_cache_used{instance=\"$instance\",replica=\"$replica\"}", - "instant": false, - "legendFormat": "JGit block cache", - "refId": "A" - }, - { - "expr": "proc_jvm_memory_non_heap_used{instance=\"$instance\",replica=\"$replica\"}", - "legendFormat": "used non-heap", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Memory", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "decbytes", - "label": "Memory Consumption", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 11 - }, - "hiddenSeries": false, - "id": 4, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "rightSide": false, - "show": true, - "sideWidth": 100, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "available CPUs", - "color": "#1F60C4", - "fill": 0 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(proc_cpu_usage{instance=\"$instance\",replica=\"$replica\"}[5m])", - "legendFormat": "used CPUs", - "refId": "A" - }, - { - "expr": "proc_cpu_num_cores{instance=\"$instance\",replica=\"$replica\"}", - "legendFormat": "available CPUs", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "CPU", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "CPU cores", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "decimals": 2, - "fill": 6, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 11 - }, - "hiddenSeries": false, - "id": 6, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": true, - "rightSide": false, - "show": true, - "total": false, - "values": true - }, - "lines": false, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "gc time G1 old gen", - "color": "#F2CC0C" - }, - { - "alias": "gc time G1 young gen", - "color": "#3274D9" - }, - { - "alias": "gc time PS Scavange", - "color": "#8AB8FF" - }, - { - "alias": "gc time PS MarkSweep", - "color": "#E02F44" - } - ], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "increase(proc_jvm_gc_time_G1_Young_Generation{instance=\"$instance\",replica=\"$replica\"}[2m])/increase(proc_uptime{instance=\"$instance\",replica=\"$replica\"}[2m])", - "interval": "1m", - "intervalFactor": 1, - "legendFormat": "gc time G1 young gen", - "refId": "B" - }, - { - "expr": "increase(proc_jvm_gc_time_G1_Old_Generation{instance=\"$instance\",replica=\"$replica\"}[2m])/increase(proc_uptime{instance=\"$instance\",replica=\"$replica\"}[2m])", - "interval": "1m", - "intervalFactor": 1, - "legendFormat": "gc time G1 old gen", - "refId": "A" - }, - { - "expr": "increase(proc_jvm_gc_time_PS_MarkSweep{instance=\"$instance\",replica=\"$replica\"}[2m])/increase(proc_uptime{instance=\"$instance\",replica=\"$replica\"}[2m])", - "interval": "1m", - "intervalFactor": 1, - "legendFormat": "gc time PS MarkSweep", - "refId": "C" - }, - { - "expr": "increase(proc_jvm_gc_time_PS_Scavenge{instance=\"$instance\",replica=\"$replica\"}[2m])/increase(proc_uptime{instance=\"$instance\",replica=\"$replica\"}[2m])", - "interval": "1m", - "intervalFactor": 1, - "legendFormat": "gc time PS Scavange", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Java - % of time spent in GC", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": "GC Time", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "description": "", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 21 - }, - "hiddenSeries": false, - "id": 8, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "proc_jvm_thread_num_live{instance=\"$instance\",replica=\"$replica\"}", - "legendFormat": "Java live threads", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Threads", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Live Threads", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 6, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 21 - }, - "hiddenSeries": false, - "id": 10, - "interval": "", - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "jgit_block_cache_open_files{instance=\"$instance\",replica=\"$replica\"}", - "legendFormat": "jgit block cache", - "refId": "B" - }, - { - "expr": "proc_num_open_fds{instance=\"$instance\",replica=\"$replica\"}-jgit_block_cache_open_files{instance=\"$instance\",replica=\"$replica\"}", - "legendFormat": "other", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Java open file descriptors", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Open File Descriptors", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 2, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 31 - }, - "hiddenSeries": false, - "id": 12, - "interval": "", - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "miss ratio", - "yaxis": 1 - }, - { - "alias": "eviction ratio", - "yaxis": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "increase(jgit_block_cache_miss_count{instance=\"$instance\",replica=\"$replica\"}[2m])/(increase(jgit_block_cache_hit_count{instance=\"$instance\",replica=\"$replica\"}[2m])+increase(jgit_block_cache_miss_count{instance=\"$instance\",replica=\"$replica\"}[2m]))", - "format": "time_series", - "instant": false, - "legendFormat": "miss ratio", - "refId": "A" - }, - { - "expr": "increase(jgit_block_cache_eviction_count{instance=\"$instance\",replica=\"$replica\"}[2m])/(increase(jgit_block_cache_hit_count{instance=\"$instance\",replica=\"$replica\"}[2m])+increase(jgit_block_cache_miss_count{instance=\"$instance\",replica=\"$replica\"}[2m]))", - "format": "time_series", - "instant": false, - "legendFormat": "eviction ratio", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "JGit block cache", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "percentunit", - "label": "miss ratio", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "decimals": null, - "format": "percentunit", - "label": "eviction ratio", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": 1 - } - } - ], - "refresh": "1m", - "schemaVersion": 22, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "allValue": null, - "current": { - "text": "", - "value": "" - }, - "datasource": "${DS_PROMETHEUS}", - "definition": "label_values(git_upload_pack_phase_writing_total, instance)", - "hide": 0, - "includeAll": false, - "label": "Gerrit Instance", - "multi": false, - "name": "instance", - "options": [], - "query": "label_values(git_upload_pack_phase_writing_total, instance)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "", - "value": "" - }, - "datasource": "${DS_PROMETHEUS}", - "definition": "label_values(git_upload_pack_phase_writing_total{instance=\"$instance\"}, replica)", - "hide": 0, - "includeAll": false, - "label": "Replica", - "multi": false, - "name": "replica", - "options": [], - "query": "label_values(git_upload_pack_phase_writing_total{instance=\"$instance\"}, replica)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-24h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ] - }, - "timezone": "", - "title": "Process", - "uid": "MeOVgCPWz", - "version": 5 -} diff --git a/dashboards/gerrit/globals/variables.libsonnet b/dashboards/gerrit/globals/variables.libsonnet new file mode 100644 index 0000000..7f142a3 --- /dev/null +++ b/dashboards/gerrit/globals/variables.libsonnet @@ -0,0 +1,21 @@ +local grafana = import '../../../vendor/grafonnet/grafana.libsonnet'; +local template = grafana.template; + +local defaults = import '../../globals/defaults.libsonnet'; + +{ + instance: template.new( + name='instance', + datasource=defaults.datasource, + query='label_values(git_upload_pack_phase_writing_total, instance)', + label='Gerrit Instance', + refresh='load', + ), + replica: template.new( + name='replica', + datasource=defaults.datasource, + query='label_values(git_upload_pack_phase_writing_total{instance="$instance"}, replica)', + label='Replica', + refresh='time', + ), +} diff --git a/dashboards/gerrit/process/gerrit-process.jsonnet b/dashboards/gerrit/process/gerrit-process.jsonnet new file mode 100644 index 0000000..54a9790 --- /dev/null +++ b/dashboards/gerrit/process/gerrit-process.jsonnet @@ -0,0 +1,60 @@ +local grafana = import '../../../vendor/grafonnet/grafana.libsonnet'; +local dashboard = grafana.dashboard; +local template = grafana.template; +local graphPanel = grafana.graphPanel; +local prometheus = grafana.prometheus; + +local defaults = import '../../globals/defaults.libsonnet'; +local gridPos = import '../../globals/grid_pos.libsonnet'; +local publishVariables = import '../../globals/publish.libsonnet'; +local variables = import '../globals/variables.libsonnet'; + +local cpu_panel = import './panels/cpu.libsonnet'; +local file_descr_panel = import './panels/file-descriptors.libsonnet'; +local gc_time_panel = import './panels/gc-time.libsonnet'; +local jgit_block_cache_panel = import './panels/jgit-block-cache.libsonnet'; +local memory_panel = import './panels/memory.libsonnet'; +local system_load_panel = import './panels/system-load.libsonnet'; +local threads_panel = import './panels/threads.libsonnet'; + +dashboard.new( + 'Gerrit - Process', + tags=['gerrit'], + schemaVersion=defaults.dashboards.schemaVersion, + editable=defaults.dashboards.editable, + time_from=defaults.dashboards.timeFrom, + time_to=defaults.dashboards.timeTo, + refresh=defaults.dashboards.refresh, + graphTooltip='shared_tooltip', +) +.addTemplate(variables.instance) +.addTemplate(variables.replica) +.addPanel( + system_load_panel, + gridPos=gridPos.new(0, 0) +) +.addPanel( + memory_panel, + gridPos=gridPos.new(0, 1) +) +.addPanel( + cpu_panel, + gridPos=gridPos.new(1, 0) +) +.addPanel( + gc_time_panel, + gridPos=gridPos.new(1, 1) +) +.addPanel( + threads_panel, + gridPos=gridPos.new(2, 0) +) +.addPanel( + file_descr_panel, + gridPos=gridPos.new(2, 1) +) +.addPanel( + jgit_block_cache_panel, + gridPos=gridPos.new(3, 0) +) ++ if std.extVar('publish') then publishVariables else {} diff --git a/dashboards/gerrit/process/panels/cpu.libsonnet b/dashboards/gerrit/process/panels/cpu.libsonnet new file mode 100644 index 0000000..37cb8aa --- /dev/null +++ b/dashboards/gerrit/process/panels/cpu.libsonnet @@ -0,0 +1,28 @@ +local grafana = import '../../../../vendor/grafonnet/grafana.libsonnet'; +local prometheus = grafana.prometheus; + +local lineGraph = import '../../../globals/line-graph.libsonnet'; + +lineGraph.new( + title='CPU', + labelY1='CPU cores', +) +.addTarget( + prometheus.target( + 'rate(proc_cpu_usage{instance="$instance",replica="$replica"}[5m])', + legendFormat='used CPUs', + ) +) +.addTarget( + prometheus.target( + 'proc_cpu_num_cores{instance="$instance",replica="$replica"}', + legendFormat='available CPUs', + ) +) +.addSeriesOverride( + { + alias: 'available CPUs', + color: '#1F60C4', + fill: 0, + } +) diff --git a/dashboards/gerrit/process/panels/file-descriptors.libsonnet b/dashboards/gerrit/process/panels/file-descriptors.libsonnet new file mode 100644 index 0000000..e222f44 --- /dev/null +++ b/dashboards/gerrit/process/panels/file-descriptors.libsonnet @@ -0,0 +1,23 @@ +local grafana = import '../../../../vendor/grafonnet/grafana.libsonnet'; +local graphPanel = grafana.graphPanel; +local prometheus = grafana.prometheus; + +local lineGraph = import '../../../globals/line-graph.libsonnet'; + +lineGraph.new( + title='Java open file descriptors', + labelY1='Open File Descriptors', + stack=true, +) +.addTarget( + prometheus.target( + 'jgit_block_cache_open_files{instance="$instance",replica="$replica"}', + legendFormat='jgit block cache', + ) +) +.addTarget( + prometheus.target( + 'proc_num_open_fds{instance="$instance",replica="$replica"}-jgit_block_cache_open_files{instance="$instance",replica="$replica"}', + legendFormat='other', + ) +) diff --git a/dashboards/gerrit/process/panels/gc-time.libsonnet b/dashboards/gerrit/process/panels/gc-time.libsonnet new file mode 100644 index 0000000..bac395e --- /dev/null +++ b/dashboards/gerrit/process/panels/gc-time.libsonnet @@ -0,0 +1,62 @@ +local grafana = import '../../../../vendor/grafonnet/grafana.libsonnet'; +local prometheus = grafana.prometheus; + +local barGraph = import '../../../globals/bar-graph.libsonnet'; + +barGraph.new( + title='Java - % of time spent in GC', + formatY1='percentunit', + labelY1='GC Time', +) +.addTarget( + prometheus.target( + 'increase(proc_jvm_gc_time_G1_Young_Generation{instance="$instance",replica="$replica"}[2m])/increase(proc_uptime{instance="$instance",replica="$replica"}[2m])', + legendFormat='gc time G1 young gen', + interval='1m', + ) +) +.addTarget( + prometheus.target( + 'increase(proc_jvm_gc_time_G1_Old_Generation{instance="$instance",replica="$replica"}[2m])/increase(proc_uptime{instance="$instance",replica="$replica"}[2m])', + legendFormat='gc time G1 old gen', + interval='1m', + ) +) +.addTarget( + prometheus.target( + 'increase(proc_jvm_gc_time_PS_MarkSweep{instance="$instance",replica="$replica"}[2m])/increase(proc_uptime{instance="$instance",replica="$replica"}[2m])', + legendFormat='gc time PS MarkSweep', + interval='1m', + ) +) +.addTarget( + prometheus.target( + 'increase(proc_jvm_gc_time_PS_Scavenge{instance="$instance",replica="$replica"}[2m])/increase(proc_uptime{instance="$instance",replica="$replica"}[2m])', + legendFormat='gc time PS Scavange', + interval='1m', + ) +) +.addSeriesOverride( + { + alias: 'gc time G1 young gen', + color: '#F2CC0C', + } +) +.addSeriesOverride( + { + alias: 'gc time G1 young gen', + color: '#3274D9', + } +) +.addSeriesOverride( + { + alias: 'gc time PS Scavange', + color: '#8AB8FF', + } +) +.addSeriesOverride( + { + alias: 'gc time PS MarkSweep', + color: '#E02F44', + } +) diff --git a/dashboards/gerrit/process/panels/jgit-block-cache.libsonnet b/dashboards/gerrit/process/panels/jgit-block-cache.libsonnet new file mode 100644 index 0000000..41e2aec --- /dev/null +++ b/dashboards/gerrit/process/panels/jgit-block-cache.libsonnet @@ -0,0 +1,25 @@ +local grafana = import '../../../../vendor/grafonnet/grafana.libsonnet'; +local graphPanel = grafana.graphPanel; +local prometheus = grafana.prometheus; + +local lineGraph = import '../../../globals/line-graph.libsonnet'; + +lineGraph.new( + title='JGit block cache', + formatY1='percentunit', + labelY1='miss ratio', + formatY2='percentunit', + labelY2='eviction ratio', +) +.addTarget( + prometheus.target( + 'increase(jgit_block_cache_miss_count{instance="$instance",replica="$replica"}[2m])/(increase(jgit_block_cache_hit_count{instance="$instance",replica="$replica"}[2m])+increase(jgit_block_cache_miss_count{instance="$instance",replica="$replica"}[2m]))', + legendFormat='miss ratio', + ) +) +.addTarget( + prometheus.target( + 'increase(jgit_block_cache_eviction_count{instance="$instance",replica="$replica"}[2m])/(increase(jgit_block_cache_hit_count{instance="$instance",replica="$replica"}[2m])+increase(jgit_block_cache_miss_count{instance="$instance",replica="$replica"}[2m]))', + legendFormat='eviction ratio', + ) +) diff --git a/dashboards/gerrit/process/panels/memory.libsonnet b/dashboards/gerrit/process/panels/memory.libsonnet new file mode 100644 index 0000000..2b38c3e --- /dev/null +++ b/dashboards/gerrit/process/panels/memory.libsonnet @@ -0,0 +1,35 @@ +local grafana = import '../../../../vendor/grafonnet/grafana.libsonnet'; +local graphPanel = grafana.graphPanel; +local prometheus = grafana.prometheus; + +local lineGraph = import '../../../globals/line-graph.libsonnet'; + +lineGraph.new( + title='Memory', + formatY1='decbytes', + labelY1='Memory Consumption', +) +.addTarget( + prometheus.target( + 'proc_jvm_memory_heap_committed{instance="$instance",replica="$replica"}', + legendFormat='committed heap' + ) +) +.addTarget( + prometheus.target( + 'proc_jvm_memory_heap_used{instance="$instance",replica="$replica"}', + legendFormat='used heap' + ) +) +.addTarget( + prometheus.target( + 'jgit_block_cache_cache_used{instance="$instance",replica="$replica"}', + legendFormat='JGit block cache' + ) +) +.addTarget( + prometheus.target( + 'proc_jvm_memory_non_heap_used{instance="$instance",replica="$replica"}', + legendFormat='used non-heap' + ) +) diff --git a/dashboards/gerrit/process/panels/system-load.libsonnet b/dashboards/gerrit/process/panels/system-load.libsonnet new file mode 100644 index 0000000..1ba4389 --- /dev/null +++ b/dashboards/gerrit/process/panels/system-load.libsonnet @@ -0,0 +1,16 @@ +local grafana = import '../../../../vendor/grafonnet/grafana.libsonnet'; +local graphPanel = grafana.graphPanel; +local prometheus = grafana.prometheus; + +local lineGraph = import '../../../globals/line-graph.libsonnet'; + +lineGraph.new( + title='System load', + labelY1='System load', +) +.addTarget( + prometheus.target( + 'proc_cpu_system_load{instance="$instance",replica="$replica"}', + legendFormat='system load', + ) +) diff --git a/dashboards/gerrit/process/panels/threads.libsonnet b/dashboards/gerrit/process/panels/threads.libsonnet new file mode 100644 index 0000000..4792a86 --- /dev/null +++ b/dashboards/gerrit/process/panels/threads.libsonnet @@ -0,0 +1,16 @@ +local grafana = import '../../../../vendor/grafonnet/grafana.libsonnet'; +local graphPanel = grafana.graphPanel; +local prometheus = grafana.prometheus; + +local lineGraph = import '../../../globals/line-graph.libsonnet'; + +lineGraph.new( + title='Threads', + labelY1='Live Threads', +) +.addTarget( + prometheus.target( + 'proc_jvm_thread_num_live{instance="$instance",replica="$replica"}', + legendFormat='Java live threads', + ) +) diff --git a/dashboards/globals/bar-graph.libsonnet b/dashboards/globals/bar-graph.libsonnet new file mode 100644 index 0000000..125d21f --- /dev/null +++ b/dashboards/globals/bar-graph.libsonnet @@ -0,0 +1,32 @@ +local grafana = import '../../vendor/grafonnet/grafana.libsonnet'; +local graphPanel = grafana.graphPanel; + +local defaults = import './defaults.libsonnet'; + +{ + new( + title, + labelY1, + labelY2='', + formatY1='short', + formatY2='short', + stack=false, + ):: graphPanel.new( + title=title, + labelY1=labelY1, + labelY2=labelY2, + formatY1=formatY1, + formatY2=formatY2, + stack=stack, + datasource=defaults.datasource, + fill=1, + legend_alignAsTable=true, + legend_avg=true, + legend_current=true, + legend_max=true, + legend_min=true, + legend_values=true, + lines=false, + bars=true + ), +} diff --git a/dashboards/globals/defaults.libsonnet b/dashboards/globals/defaults.libsonnet new file mode 100644 index 0000000..f49f619 --- /dev/null +++ b/dashboards/globals/defaults.libsonnet @@ -0,0 +1,14 @@ +{ + dashboards: { + editable: false, + schemaVersion: 22, + timeFrom: 'now-24h', + timeTo: 'now', + refresh: '1m', + }, + panels: { + width: 12, + height: 11, + }, + datasource: if std.extVar('publish') then '${DS_PROMETHEUS}' else 'Prometheus', +} diff --git a/dashboards/globals/grid_pos.libsonnet b/dashboards/globals/grid_pos.libsonnet new file mode 100644 index 0000000..b421c9c --- /dev/null +++ b/dashboards/globals/grid_pos.libsonnet @@ -0,0 +1,13 @@ +local defaults = import './defaults.libsonnet'; + +{ + new( + row, + column, + ):: { + x: column * defaults.panels.width, + y: row * defaults.panels.height, + w: defaults.panels.width, + h: defaults.panels.height, + }, +} diff --git a/dashboards/globals/line-graph.libsonnet b/dashboards/globals/line-graph.libsonnet new file mode 100644 index 0000000..35f8a92 --- /dev/null +++ b/dashboards/globals/line-graph.libsonnet @@ -0,0 +1,32 @@ +local grafana = import '../../vendor/grafonnet/grafana.libsonnet'; +local graphPanel = grafana.graphPanel; + +local defaults = import './defaults.libsonnet'; + +{ + new( + title, + labelY1, + labelY2='', + formatY1='short', + formatY2='short', + stack=false, + ):: graphPanel.new( + title=title, + labelY1=labelY1, + labelY2=labelY2, + formatY1=formatY1, + formatY2=formatY2, + stack=stack, + datasource=defaults.datasource, + fill=1, + legend_alignAsTable=true, + legend_avg=true, + legend_current=true, + legend_max=true, + legend_min=true, + legend_values=true, + lines=true, + linewidth=1, + ), +} diff --git a/dashboards/globals/publish.libsonnet b/dashboards/globals/publish.libsonnet new file mode 100644 index 0000000..dfd85d5 --- /dev/null +++ b/dashboards/globals/publish.libsonnet @@ -0,0 +1,32 @@ +{ + __inputs: [ + { + name: 'DS_PROMETHEUS', + label: 'Prometheus', + description: '', + type: 'datasource', + pluginId: 'prometheus', + pluginName: 'Prometheus', + }, + ], + __requires: [ + { + type: 'grafana', + id: 'grafana', + name: 'Grafana', + version: '7.1.5', + }, + { + type: 'panel', + id: 'graph', + name: 'Graph', + version: '', + }, + { + type: 'datasource', + id: 'prometheus', + name: 'Prometheus', + version: '1.0.0', + }, + ], +} diff --git a/jsonnetfile.json b/jsonnetfile.json new file mode 100644 index 0000000..93f3316 --- /dev/null +++ b/jsonnetfile.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "dependencies": [ + { + "source": { + "git": { + "remote": "https://github.com/grafana/grafonnet-lib.git", + "subdir": "grafonnet" + } + }, + "version": "master" + } + ], + "legacyImports": true +} diff --git a/jsonnetfile.lock.json b/jsonnetfile.lock.json new file mode 100644 index 0000000..b754ac2 --- /dev/null +++ b/jsonnetfile.lock.json @@ -0,0 +1,16 @@ +{ + "version": 1, + "dependencies": [ + { + "source": { + "git": { + "remote": "https://github.com/grafana/grafonnet-lib.git", + "subdir": "grafonnet" + } + }, + "version": "3336c69715f8f7a4d637582504c9fabd9d9ca081", + "sum": "w6zS28Rjs9EzRN/WoLLIdi028BvumxDTyLefYVoql2k=" + } + ], + "legacyImports": false +} diff --git a/subcommands/install.py b/subcommands/install.py index 424de72..be28517 100644 --- a/subcommands/install.py +++ b/subcommands/install.py @@ -19,6 +19,7 @@ import subprocess import sys import zipfile +import _jsonnet import requests import yaml @@ -55,31 +56,41 @@ def _create_dashboard_configmaps(output_dir, namespace): if not os.path.exists(output_dir): os.mkdir(output_dir) - for dashboard in os.listdir(dashboards_dir): - dashboard_path = os.path.join(dashboards_dir, dashboard) - dashboard_name = os.path.splitext(dashboard)[0] - output_file = f"{output_dir}/{dashboard_name}.dashboard.yaml" - command = ( - f"kubectl create configmap {dashboard_name} -o yaml " - f"--from-file={dashboard_path} --dry-run=client --namespace={namespace} " - f"> {output_file}" - ) + for dir_path, _, files in os.walk(dashboards_dir): + for dashboard in files: + dashboard_path = os.path.join(dir_path, dashboard) + dashboard_name, ext = os.path.splitext(dashboard) + if ext == ".json": + source = f"--from-file={dashboard_path}" + elif ext == ".jsonnet": + json = _jsonnet.evaluate_file(dashboard_path, ext_codes={"publish": "false"}) + source = f"--from-literal={dashboard_name}.json='{json}'" + else: + continue - try: - subprocess.check_output(command, shell=True) - except subprocess.CalledProcessError as err: - print(err.output) + output_file = f"{output_dir}/{dashboard_name}.dashboard.yaml" - with open(output_file, "r") as f: - dashboard_cm = yaml.load(f, Loader=yaml.SafeLoader) - dashboard_cm["metadata"]["labels"] = dict() - dashboard_cm["metadata"]["labels"]["grafana_dashboard"] = dashboard_name - dashboard_cm["data"][dashboard] = dashboard_cm["data"][dashboard].replace( - '"${DS_PROMETHEUS}"', "null" + command = ( + f"kubectl create configmap {dashboard_name} -o yaml " + f"{source} --dry-run=client --namespace={namespace} " + f"> {output_file}" ) - with open(output_file, "w") as f: - yaml.dump(dashboard_cm, f) + try: + subprocess.check_output(command, shell=True) + except subprocess.CalledProcessError as err: + print(err.output) + + with open(output_file, "r") as f: + dashboard_cm = yaml.load(f, Loader=yaml.SafeLoader) + dashboard_cm["metadata"]["labels"] = dict() + dashboard_cm["metadata"]["labels"]["grafana_dashboard"] = dashboard_name + dashboard_cm["data"][f"{dashboard_name}.json"] = dashboard_cm["data"][ + f"{dashboard_name}.json" + ].replace('"${DS_PROMETHEUS}"', "null") + + with open(output_file, "w") as f: + yaml.dump(dashboard_cm, f) def _create_promtail_configs(config, output_dir):