Fix Hydra exporter crash loop nonsense

flake.lock

@@ -480,11 +480,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1724616313,
-        "narHash": "sha256-9syppf9Gm/6F4wQQAbsf7rGY1DooMsprnsEY/0eaewg=",
+        "lastModified": 1727799384,
+        "narHash": "sha256-TRxR6zELQoUDPhLkmBU8QjgskNyVcblKgQIxhuOyNRc=",
         "ref": "refs/heads/main",
-        "rev": "44b9a7b95d23e7a8587cb963f00382046707f2db",
-        "revCount": 4202,
+        "rev": "d3257e4761370a44ad8ae883de3ff67ddac4b736",
+        "revCount": 4215,
         "type": "git",
         "url": "https://git.lix.systems/lix-project/hydra.git"
       },
@@ -505,11 +505,11 @@
         "pre-commit-hooks": "pre-commit-hooks"
       },
       "locked": {
-        "lastModified": 1723919517,
-        "narHash": "sha256-D6+zmRXzr85p7riphuIrJQqangoJe70XM5jHhMWwXws=",
+        "lastModified": 1725228396,
+        "narHash": "sha256-QBXwqyPuHUKBiuyzHBxqH/MpjPY9DQiY2M81P2t6b/0=",
         "ref": "refs/heads/main",
-        "rev": "278fddc317cf0cf4d3602d0ec0f24d1dd281fadb",
-        "revCount": 16138,
+        "rev": "02eb07cfd539c34c080cb1baf042e5e780c1fcc2",
+        "revCount": 16214,
         "type": "git",
         "url": "https://git.lix.systems/lix-project/lix"
       },

services/monitoring/exporters/hydra/hydra-exporter.py

@@ -22,6 +22,9 @@ steps_building = Gauge("hydra_steps_building", "Number of steps currently buildi
 steps_copying_to = Gauge(
     "hydra_steps_copying_to", "Number of steps copying inputs to a worker"
 )
+steps_waiting_for_download_slot = Gauge(
+    "hydra_steps_waiting_for_download_slot", "Number of steps waiting for download slot"
+)
 steps_copying_from = Gauge(
     "hydra_steps_copying_from", "Number of steps copying outputs from a worker"
 )
@@ -230,6 +233,7 @@ def update_metrics(status):
     steps_active.set({}, status["nrActiveSteps"])
     steps_building.set({}, status["nrStepsBuilding"])
     steps_copying_to.set({}, status["nrStepsCopyingTo"])
+    steps_waiting_for_download_slot.set({}, status["nrStepsWaitingForDownloadSlot"])
     steps_copying_from.set({}, status["nrStepsCopyingFrom"])
     steps_waiting.set({}, status["nrStepsWaiting"])
     steps_unsupported.set({}, status["nrUnsupportedSteps"])
@@ -334,11 +338,11 @@ async def update_metrics_loop(hydra_url, scrape_interval):
                 )
 
                 update_metrics(response.json())
-
-                await asyncio.sleep(scrape_interval)
             except Exception as ex:
                 logging.exception("Failed to update metrics", exc_info=ex)
 
+            await asyncio.sleep(scrape_interval)
+
 
 @click.command()
 @click.option("--hydra-url", default="https://hydra.forkos.org/")