diff --git a/datadog/README.md b/datadog/README.md new file mode 100644 index 00000000..49634fb4 --- /dev/null +++ b/datadog/README.md @@ -0,0 +1,21 @@ +# Hydra status timeboard + +In order to deploy hydra status dashboard you can: + +* create a deployment + +``` +nixops create -d hydra-status /path/to/hydra/datadog/dd-dashboard.nix +``` + +* setup the default hostname and api/app keys + +``` +nixops set-args -d hydra-status --argst appKey --argstr apiKey --argstr host chef +``` + +* deploy + +``` +nixops deploy -d hydra-status +``` diff --git a/datadog/dd-dashboard.nix b/datadog/dd-dashboard.nix new file mode 100644 index 00000000..e5ab9243 --- /dev/null +++ b/datadog/dd-dashboard.nix @@ -0,0 +1,161 @@ +{ + host +, appKey +, apiKey +, ... +}: +{ + resources.datadogTimeboards.hydra-status = { + inherit appKey apiKey; + description = "Hydra build farm status"; + graphs = [ + { + definition = builtins.toJSON { + requests = [ + { q = "avg:hydra.queue.steps.active{$host}"; } + { q = "avg:hydra.queue.steps.building{$host}"; } + { q = "avg:hydra.queue.steps.copying_to{$host}"; } + { q = "avg:hydra.queue.steps.copying_from{$host}"; } + { q = "avg:hydra.queue.steps.waiting{$host}"; } + ]; + viz = "timeseries"; + }; + title = "Active/building steps"; + } + { + definition = builtins.toJSON { + requests = [ + { q = "avg:hydra.queue.steps.avg_build_time{$host}"; } + { q = "avg:hydra.queue.steps.avg_total_time{$host}"; } + ]; + viz = "timeseries"; + }; + title = "Build/total time per step"; + } + { + definition = builtins.toJSON { + requests = [ + { q = "avg:hydra.queue.steps.finished{$host}"; } + { q = "avg:hydra.queue.builds.finished{$host}"; } + ]; + viz = "timeseries"; + }; + title = "Finished builds/steps"; + } + { + definition = builtins.toJSON { + requests = [ + { q = "max:system.io.await{$host} by {device}"; type = "area"; } + ]; + viz = "timeseries"; + }; + title = "Disk latency (ms, by device)"; + } + { + definition = builtins.toJSON { + requests = [ + { q = "avg:hydra.queue.steps.unfinished{$host}"; } + { q = "avg:hydra.queue.builds.unfinished{$host}"; } + { q = "avg:hydra.queue.steps.runnable{$host}"; } + ]; + viz = "timeseries"; + }; + title = "Unfinished builds/steps"; + } + { + definition = builtins.toJSON { + requests = [ + { q = "avg:system.load.1{$host}"; } + { q = "avg:system.load.5{$host}"; } + { q = "avg:system.load.15{$host}"; } + ]; + viz = "timeseries"; + }; + title = "Load Averages 1-5-15"; + } + { + definition = builtins.toJSON { + requests = [ + { q = "per_hour(ewma_20(avg:hydra.queue.steps.finished{$host}))"; } + { + q = "per_hour(ewma_20(avg:hydra.queue.builds.finished{$host}))"; + } + ]; + viz = "timeseries"; + }; + title = "Finished builds/steps / hour"; + } + { + definition = builtins.toJSON { + requests = [ { q = "avg:hydra.mem.dirty{$host}"; } ]; + viz = "timeseries"; + }; + title = "Dirty memory"; + } + { + definition = builtins.toJSON { + requests = [ + { + aggregator = "avg"; + conditional_formats = []; + q = "avg:system.mem.used{$host}"; + type = "line"; + } + { + conditional_formats = []; + q = "avg:system.mem.free{$host}"; + type = "line"; + } + { + conditional_formats = []; + q = "avg:system.mem.usable{$host}"; + type = "line"; + } + ]; + viz = "timeseries"; + }; + title = "Memory usage"; + } + { + definition = builtins.toJSON { + requests = [ + { q = "avg:hydra.queue.bytes_sent{$host}"; type = "line"; } + { q = "avg:hydra.queue.bytes_received{$host}"; type = "line"; } + ]; + viz = "timeseries"; + }; + title = "Stores paths sent/received"; + } + { + definition = builtins.toJSON { + requests = [ + { q = "per_minute(ewma_20(avg:hydra.queue.bytes_sent{$host}))"; } + { + q = "per_minute(ewma_20(avg:hydra.queue.bytes_received{$host}))"; + } + ]; + viz = "timeseries"; + }; + title = "Store paths sent/received (GiB / minute)"; + } + { + definition = builtins.toJSON { + requests = [ + { q = "avg:hydra.queue.machines.total{$host}"; type = "line"; } + { q = "avg:hydra.queue.machines.in_use{$host}"; type = "line"; } + ]; + viz = "timeseries"; + }; + title = "Total and active machines"; + } + ]; + templateVariables = [ + { + default = "host:${host}"; + name = "host"; + prefix = "host"; + } + ]; + title = "Hydra Status (deployed from nixops)"; + }; +}