the-distro/infra
9
0
Fork 5
Code Issues 37 Pull requests 9 Packages Projects 1 Releases Wiki Activity

Compare commits

base: the-distro:b291caac46e710c96e8343b611fce83e4065530c
Branches Tags
the-distro:main
the-distro:revert-to-bash
the-distro:ofborg
the-distro:vault
the-distro:storage-project
the-distro:block-crawlers
the-distro:fix-buildbot
the-distro:lukegb/rule-94
the-distro:cache-lix
the-distro:build02-lix
the-distro:lukegb/uptime-kuma-root
the-distro:hydra-metrics
the-distro:builder
the-distro:email-groups
the-distro:cache-ipv4
the-distro:format
the-distro:hexchen/colmena-checker
the-distro:hexy/credentials
the-distro:buildbot
the-distro:ckie/moarr-v4
the-distro:gerrit-metrics
the-distro:moar-builders
the-distro:unplug-epyc
the-distro:hydra-wire-up
the-distro:alertmanager
the-distro:fodwatch
the-distro:cadvisor-containers
the-distro:gerrit-http-clones
the-distro:meta01
..
compare: the-distro:5582a0a29b584d6806340e77a3fec8a110868624
Branches Tags
the-distro:revert-to-bash
the-distro:ofborg
the-distro:main
the-distro:vault
the-distro:storage-project
the-distro:block-crawlers
the-distro:fix-buildbot
the-distro:lukegb/rule-94
the-distro:cache-lix
the-distro:build02-lix
the-distro:lukegb/uptime-kuma-root
the-distro:hydra-metrics
the-distro:builder
the-distro:email-groups
the-distro:cache-ipv4
the-distro:format
the-distro:hexchen/colmena-checker
the-distro:hexy/credentials
the-distro:buildbot
the-distro:ckie/moarr-v4
the-distro:gerrit-metrics
the-distro:moar-builders
the-distro:unplug-epyc
the-distro:hydra-wire-up
the-distro:alertmanager
the-distro:fodwatch
the-distro:cadvisor-containers
the-distro:gerrit-http-clones
the-distro:meta01

3 commits
b291caac46 ... 5582a0a29b

Author SHA1 Message Date
Ilya K 5582a0a29b Fix Hydra exporter crash loop nonsense 2024-10-01 19:27:13 +03:00
Ilya K 4ddf87fa8e Add new metric to Hydra exporter 2024-10-01 19:27:05 +03:00
Ilya K 98d899fabc Update Hydra 2024-10-01 19:26:58 +03:00
2 changed files with 14 additions and 10 deletions
Show stats Expand all files Collapse all files

16
flake.lock
View file

@ -480,11 +480,11 @@
] ]
}, },
"locked": { "locked": {
"lastModified": 1724616313, "lastModified": 1727799384,
"narHash": "sha256-9syppf9Gm/6F4wQQAbsf7rGY1DooMsprnsEY/0eaewg=", "narHash": "sha256-TRxR6zELQoUDPhLkmBU8QjgskNyVcblKgQIxhuOyNRc=",
"ref": "refs/heads/main", "ref": "refs/heads/main",
"rev": "44b9a7b95d23e7a8587cb963f00382046707f2db", "rev": "d3257e4761370a44ad8ae883de3ff67ddac4b736",
"revCount": 4202, "revCount": 4215,
"type": "git", "type": "git",
"url": "https://git.lix.systems/lix-project/hydra.git" "url": "https://git.lix.systems/lix-project/hydra.git"
}, },
@ -505,11 +505,11 @@
"pre-commit-hooks": "pre-commit-hooks" "pre-commit-hooks": "pre-commit-hooks"
}, },
"locked": { "locked": {
"lastModified": 1723919517, "lastModified": 1725228396,
"narHash": "sha256-D6+zmRXzr85p7riphuIrJQqangoJe70XM5jHhMWwXws=", "narHash": "sha256-QBXwqyPuHUKBiuyzHBxqH/MpjPY9DQiY2M81P2t6b/0=",
"ref": "refs/heads/main", "ref": "refs/heads/main",
"rev": "278fddc317cf0cf4d3602d0ec0f24d1dd281fadb", "rev": "02eb07cfd539c34c080cb1baf042e5e780c1fcc2",
"revCount": 16138, "revCount": 16214,
"type": "git", "type": "git",
"url": "https://git.lix.systems/lix-project/lix" "url": "https://git.lix.systems/lix-project/lix"
}, },

8
services/monitoring/exporters/hydra/hydra-exporter.py
View file

@ -22,6 +22,9 @@ steps_building = Gauge("hydra_steps_building", "Number of steps currently buildi
steps_copying_to = Gauge( steps_copying_to = Gauge(
"hydra_steps_copying_to", "Number of steps copying inputs to a worker" "hydra_steps_copying_to", "Number of steps copying inputs to a worker"
) )
steps_waiting_for_download_slot = Gauge(
"hydra_steps_waiting_for_download_slot", "Number of steps waiting for download slot"
)
steps_copying_from = Gauge( steps_copying_from = Gauge(
"hydra_steps_copying_from", "Number of steps copying outputs from a worker" "hydra_steps_copying_from", "Number of steps copying outputs from a worker"
) )
@ -230,6 +233,7 @@ def update_metrics(status):
steps_active.set({}, status["nrActiveSteps"]) steps_active.set({}, status["nrActiveSteps"])
steps_building.set({}, status["nrStepsBuilding"]) steps_building.set({}, status["nrStepsBuilding"])
steps_copying_to.set({}, status["nrStepsCopyingTo"]) steps_copying_to.set({}, status["nrStepsCopyingTo"])
steps_waiting_for_download_slot.set({}, status["nrStepsWaitingForDownloadSlot"])
steps_copying_from.set({}, status["nrStepsCopyingFrom"]) steps_copying_from.set({}, status["nrStepsCopyingFrom"])
steps_waiting.set({}, status["nrStepsWaiting"]) steps_waiting.set({}, status["nrStepsWaiting"])
steps_unsupported.set({}, status["nrUnsupportedSteps"]) steps_unsupported.set({}, status["nrUnsupportedSteps"])
@ -334,11 +338,11 @@ async def update_metrics_loop(hydra_url, scrape_interval):
) )
update_metrics(response.json()) update_metrics(response.json())
await asyncio.sleep(scrape_interval)
except Exception as ex: except Exception as ex:
logging.exception("Failed to update metrics", exc_info=ex) logging.exception("Failed to update metrics", exc_info=ex)
await asyncio.sleep(scrape_interval)
@click.command() @click.command()
@click.option("--hydra-url", default="https://hydra.forkos.org/") @click.option("--hydra-url", default="https://hydra.forkos.org/")