fix(dns): dnsimple expects FQDNs in CNAMEs

DNSimple doesn't appear to follow the typical behaviour of appending the
domain unless the CNAME is terminated with `.`

To avoid further problems, let's just explicilty use the FQDN for all
CNAMEs.

https://support.dnsimple.com/articles/cname-record/

For comparison:
```
;; ANSWER SECTION:
alerts.forkos.org.	300	IN	CNAME	meta01.infra.p.
```

```
;; ANSWER SECTION:
alerts.forkos.org.	181	IN	CNAME	meta01.infra.p.forkos.org.
meta01.infra.p.forkos.org. 181	IN	A	163.172.69.160
```

common/base-server.nix

@@ -55,4 +55,19 @@
     "en_US.UTF-8/UTF-8"
     "fr_FR.UTF-8/UTF-8"
   ];
+
+  time.timeZone = "UTC";
+
+  security.acme.acceptTerms = true;
+  security.acme.defaults.email = "infra@forkos.org";
+
+  # Enable system diffs.
+  system.activationScripts.system-diff = {
+    supportsDryActivation = true; # safe: only outputs to stdout
+    text = ''
+      if [ -e /run/current-system ]; then
+        PATH=$PATH:${pkgs.nix}/bin ${pkgs.nvd}/bin/nvd diff /run/current-system $systemConfig
+      fi
+    '';
+  };
 }

common/channels.nix

@@ -0,0 +1,32 @@
+# Taken from https://github.com/NixOS/infra/blob/master/channels.nix
+{
+  # "Channel name" = {
+  #   # This should be the <value> part of
+  #   # https://hydra.forkos.org/job/<value>/latest-finished
+  #   job = "project/jobset/jobname";
+  #
+  #   # When adding a new version, determine if it needs to be tagged as a
+  #   # variant -- for example:
+  #   # nixos-xx.xx         => primary
+  #   # nixos-xx.xx-small   => small
+  #   # nixos-xx.xx-darwin  => darwin
+  #   # nixos-xx.xx-aarch64 => aarch64
+  #   variant = "primary";
+  #
+  #   # Channel Status:
+  #   # '*-unstable' channels are always "rolling"
+  #   # Otherwise a release generally progresses through the following phases:
+  #   #
+  #   #  - Directly after branch off                   => "beta"
+  #   #  - Once the channel is released                => "stable"
+  #   #  - Once the next channel is released           => "deprecated"
+  #   #  - N months after the next channel is released => "unmaintained"
+  #   #    (check the release notes for when this should happen)
+  #   status = "beta";
+  # };
+  "forkos-unstable" = {
+    job = "forkos/nixos-main/tested";
+    variant = "primary";
+    status = "rolling";
+  };
+}

common/raito-proxy-aware-nginx.nix

@@ -1,9 +1,10 @@
 # This enables an IPv6-only server which is proxied by kurisu.lahfa.xyz to have proper IPv4 logs via PROXY protocol.
 { config, lib, ... }:
 let
-  inherit (lib) mkEnableOption mkIf;
+  inherit (lib) mkEnableOption mkIf concatStringsSep;
   cfg = config.bagel.raito.v6-proxy-awareness;
-  allowedUpstream = "2001:bc8:38ee:99::1/128";
+  #                      outside of raito infra    inside of raito infra
+  allowedUpstreams = [ "2001:bc8:38ee::1/128"     "2001:bc8:38ee:99::1/128" ];
 in
   {
     options.bagel.raito.v6-proxy-awareness.enable = mkEnableOption "the kurisu.lahfa.xyz's sniproxy awareness for NGINX";
@@ -20,8 +21,8 @@ in
         ];
 
         appendHttpConfig = ''
-          # Kurisu node
-          set_real_ip_from ${allowedUpstream};
+          # Kurisu nodes
+          ${concatStringsSep "\n" (map (up: "set_real_ip_from ${up};") allowedUpstreams)}
           real_ip_header proxy_protocol;
         '';
       };
@@ -29,7 +30,7 @@ in
       # Move to nftables if firewall is enabled.
       networking.nftables.enable = true;
       networking.firewall.extraInputRules = ''
-        ip6 saddr ${allowedUpstream} tcp dport 444 accept
+        ${concatStringsSep "\n" (map (up: "ip6 saddr ${up} tcp dport 444 accept") allowedUpstreams)}
       '';
     };
   }

common/raito-vm.nix

@@ -30,8 +30,6 @@ in
   config = mkIf cfg.enable {
     services.qemuGuest.enable = true;
     systemd.network.enable = true;
-    security.acme.defaults.email = "bagel-acme@lahfa.xyz";
-    security.acme.acceptTerms = true;
     networking.useDHCP = lib.mkDefault false;
 
     systemd.network.networks."10-nat-lan" = {

common/ssh-keys.nix

@@ -2,10 +2,12 @@
   machines = {
     bagel-box = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAsO4bNqY04uG13Pg3ubHfRDssTphDLzZ4YUniE5/p+M";
     meta01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM5t9gYorOWgpCFDJgb24pyCKIabGpeI2H/UfdvXODcT";
+    public01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPBy8G8rfLA6E9i+t5kjVafxU1c2NXATXKxoXTH4Kgtm";
     gerrit01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA+eSZu+u9sCynrMlsmFzQHLIELQAuVg0Cs1pBvwb4+A";
     fodwatch = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFRyTNfvKl5FcSyzGzw+h+bNFNOxdhvI67WdUZ2iIJ1L";
     buildbot = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJgIu6ouagYqBeMLfmn1CbaDJMuZcPH9bnUhkht8GfuB";
     git = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEQJcpkCUOx8+5oukMX6lxrYcIX8FyHu8Mc/3+ieKMUn";
+    build-coord = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINpAEJP7F+XtJBpQP1jTzwXwQgJrFxwEJjPf/rnCXkJA";
     builder-0 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBHSNcDGctvlG6BHcJuYIzW9WsBJsts2vpwSketsbXoL";
     builder-1 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIQOGUjERK7Mx8UPM/rbOdMqVyn1sbWqYOG6CbOzH2wm";
     builder-2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMKzXIqCoYElEKIYgjbSpqEcDeOvV+Wo3Agq3jba83cB";
@@ -17,7 +19,6 @@
     builder-8 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKGSWHNeqT0kF/e4yVy2ieW98X5QMyCYIYZh9WTmQDs1";
     builder-9 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOhws9zGgocVY36dMtOL+CXadpvRMffxoWMkfEcTBJm7";
     builder-10 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE7sgIuTSqZiZhp8TvObSbIEhcHHsL5hcmYA22uzwxth";
-    builder-11 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEAqFo1qJY7MSUkfB+zxXB8Lpt/Iqz/RR5A+zwhpRWhr";
     wob-vpn-gw = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINVytPPW8XnXf/rD5TFzsw//CZc2lBjQLmDzlVGPZsjh";
   };
 

dashboards/default.nix

@@ -0,0 +1,43 @@
+{ gerrit-dashboard, stdenv, symlinkJoin, jsonnet, fetchFromGitHub, lib, ... }:
+let
+  inherit (lib) concatMapStringsSep;
+  datasource-id = "mimir";
+in
+rec {
+  grafonnet = fetchFromGitHub {
+    owner = "grafana";
+    repo = "grafonnet-lib";
+    # TODO: figure out how to read the jsonnet lockfile
+    # and propagate this a bit cleverly.
+    rev = "a1d61cce1da59c71409b99b5c7568511fec661ea";
+    hash = "sha256-fs5JZJbcL6sQXBjYhp5eeRtjTFw0J1O/BcwBC8Vm9EM=";
+  };
+  buildJsonnetDashboards = dashboardSrc: targets: stdenv.mkDerivation {
+    name = "jsonnet-grafana-dashboards";
+    src = dashboardSrc;
+    buildInputs = [ jsonnet ];
+    buildPhase = ''
+      runHook preBuild
+      mkdir -p $out
+      ${concatMapStringsSep "\n" (target: "jsonnet -J ${grafonnet} --ext-str datasource=${datasource-id} --ext-code publish=true $src/${target} > $out/${baseNameOf target}.json") targets}
+      runHook postBuild
+    '';
+  };
+
+  allDashboards = symlinkJoin {
+    name = "all-jsonnet-dashboards";
+    paths = [
+      (buildJsonnetDashboards gerrit-dashboard [
+        "dashboards/gerrit/caches/gerrit-caches.jsonnet"
+        "dashboards/gerrit/fetch-clone/gerrit-fetch-clone.jsonnet"
+        "dashboards/gerrit/fetch-clone/gerrit-phases.jsonnet"
+        "dashboards/gerrit/healthcheck/gerrit-healthcheck.jsonnet"
+        "dashboards/gerrit/latency/gerrit-push-latency.jsonnet"
+        "dashboards/gerrit/latency/gerrit-ui-actions-latency.jsonnet"
+        "dashboards/gerrit/overview/gerrit-overview.jsonnet"
+        "dashboards/gerrit/process/gerrit-process.jsonnet"
+        "dashboards/gerrit/queues/gerrit-queues.jsonnet"
+      ])
+    ];
+  };
+}

flake.lock

@@ -10,11 +10,11 @@
         "systems": "systems"
       },
       "locked": {
-        "lastModified": 1722339003,
-        "narHash": "sha256-ZeS51uJI30ehNkcZ4uKqT4ZDARPyqrHADSKAwv5vVCU=",
+        "lastModified": 1723293904,
+        "narHash": "sha256-b+uqzj+Wa6xgMS9aNbX4I+sXeb5biPDi39VgvSFqFvU=",
         "owner": "ryantm",
         "repo": "agenix",
-        "rev": "3f1dae074a12feb7327b4bf43cbac0d124488bb7",
+        "rev": "f6291c5935fdc4e0bef208cfc0dcab7e3f7a1c41",
         "type": "github"
       },
       "original": {
@@ -101,6 +101,26 @@
         "url": "https://git.lix.systems/lix-project/buildbot-nix.git"
       }
     },
+    "channel-scripts": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1725128016,
+        "narHash": "sha256-4TvaXELsl+1OcGNgqB/5HVXVxBvdIQkhJsY4FyiDcNU=",
+        "ref": "refs/heads/main",
+        "rev": "23b6c38ed7e11417bf624f6e4fb6cde0d2be6400",
+        "revCount": 261,
+        "type": "git",
+        "url": "https://git.lix.systems/the-distro/channel-scripts.git"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://git.lix.systems/the-distro/channel-scripts.git"
+      }
+    },
     "colmena": {
       "inputs": {
         "flake-compat": "flake-compat",
@@ -308,11 +328,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1719994518,
-        "narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=",
+        "lastModified": 1722555600,
+        "narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=",
         "owner": "hercules-ci",
         "repo": "flake-parts",
-        "rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7",
+        "rev": "8471fe90ad337a8074e957b69ca4d0089218391d",
         "type": "github"
       },
       "original": {
@@ -385,6 +405,22 @@
         "type": "github"
       }
     },
+    "gerrit-dashboard": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1724509518,
+        "narHash": "sha256-fwYXZVddxfzrlDa3QnFCwHqrbEX+3PrWy0QOlbO+8jk=",
+        "ref": "refs/heads/master",
+        "rev": "e544abac81c581558d68abb2a8dd583049073939",
+        "revCount": 75,
+        "type": "git",
+        "url": "https://git.lix.systems/the-distro/gerrit-monitoring.git"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://git.lix.systems/the-distro/gerrit-monitoring.git"
+      }
+    },
     "grapevine": {
       "inputs": {
         "attic": "attic",
@@ -395,15 +431,16 @@
         "nix-filter": "nix-filter",
         "nixpkgs": [
           "nixpkgs"
-        ]
+        ],
+        "rust-manifest": "rust-manifest"
       },
       "locked": {
         "host": "gitlab.computer.surgery",
-        "lastModified": 1721671623,
-        "narHash": "sha256-ELE+AD83jG3zIbYITbSfo6Ykn+R1gVjMHoS5rhDccuY=",
+        "lastModified": 1723576377,
+        "narHash": "sha256-sTa4XT5xMQkhhLknOfVd433YS1TvkMrE45qAsI1ZB6U=",
         "owner": "matrix",
         "repo": "grapevine-fork",
-        "rev": "dd24a441121b94d389fb46f08c7ec51886d5aa32",
+        "rev": "3b99032456700d06dd937db6a85976a8be9d4fa7",
         "type": "gitlab"
       },
       "original": {
@@ -443,17 +480,17 @@
         ]
       },
       "locked": {
-        "lastModified": 1722688238,
-        "narHash": "sha256-x6BnYtArF6IDs7bS8ExokgAQBOlrxXxD0EOBIlASmfM=",
+        "lastModified": 1724616313,
+        "narHash": "sha256-9syppf9Gm/6F4wQQAbsf7rGY1DooMsprnsEY/0eaewg=",
         "ref": "refs/heads/main",
-        "rev": "9b5ac87de73ea4646dbb2af979db91f096d29960",
-        "revCount": 4191,
+        "rev": "44b9a7b95d23e7a8587cb963f00382046707f2db",
+        "revCount": 4202,
         "type": "git",
-        "url": "https://git.lix.systems/the-distro/hydra.git"
+        "url": "https://git.lix.systems/lix-project/hydra.git"
       },
       "original": {
         "type": "git",
-        "url": "https://git.lix.systems/the-distro/hydra.git"
+        "url": "https://git.lix.systems/lix-project/hydra.git"
       }
     },
     "lix": {
@@ -468,11 +505,11 @@
         "pre-commit-hooks": "pre-commit-hooks"
       },
       "locked": {
-        "lastModified": 1721091462,
-        "narHash": "sha256-0cmEeoOiB91BviTJHzIyxkY+Gxv3O8ZnnExVAoXEFGI=",
+        "lastModified": 1723919517,
+        "narHash": "sha256-D6+zmRXzr85p7riphuIrJQqangoJe70XM5jHhMWwXws=",
         "ref": "refs/heads/main",
-        "rev": "6b4d46e9e0e1dd80e0977684ab20d14bcd1a6bc3",
-        "revCount": 15967,
+        "rev": "278fddc317cf0cf4d3602d0ec0f24d1dd281fadb",
+        "revCount": 16138,
         "type": "git",
         "url": "https://git.lix.systems/lix-project/lix"
       },
@@ -496,11 +533,11 @@
         "treefmt-nix": "treefmt-nix_2"
       },
       "locked": {
-        "lastModified": 1721195872,
-        "narHash": "sha256-TlvRq634MSl22BWLmpTy2vdtKntbZlsUwdMq8Mp9AWs=",
+        "lastModified": 1723579251,
+        "narHash": "sha256-xnHtfw0gRhV+2S9U7hQwvp2klTy1Iv7FlMMO0/WiMVc=",
         "ref": "refs/heads/main",
-        "rev": "c057494450f2d1420726ddb0bab145a5ff4ddfdd",
-        "revCount": 608,
+        "rev": "42a160bce2fd9ffebc3809746bc80cc7208f9b08",
+        "revCount": 609,
         "type": "git",
         "url": "https://git.lix.systems/lix-project/nix-eval-jobs"
       },
@@ -570,11 +607,11 @@
     "nix2container": {
       "flake": false,
       "locked": {
-        "lastModified": 1712990762,
-        "narHash": "sha256-hO9W3w7NcnYeX8u8cleHiSpK2YJo7ecarFTUlbybl7k=",
+        "lastModified": 1720642556,
+        "narHash": "sha256-qsnqk13UmREKmRT7c8hEnz26X3GFFyIQrqx4EaRc1Is=",
         "owner": "nlewo",
         "repo": "nix2container",
-        "rev": "20aad300c925639d5d6cbe30013c8357ce9f2a2e",
+        "rev": "3853e5caf9ad24103b13aa6e0e8bcebb47649fe4",
         "type": "github"
       },
       "original": {
@@ -633,11 +670,11 @@
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1723151389,
-        "narHash": "sha256-9AVY0ReCmSGXHrlx78+1RrqcDgVSRhHUKDVV1LLBy28=",
+        "lastModified": 1723221148,
+        "narHash": "sha256-7pjpeQlZUNQ4eeVntytU3jkw9dFK3k1Htgk2iuXjaD8=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "13fe00cb6c75461901f072ae62b5805baef9f8b2",
+        "rev": "154bcb95ad51bc257c2ce4043a725de6ca700ef6",
         "type": "github"
       },
       "original": {
@@ -647,29 +684,14 @@
         "type": "github"
       }
     },
-    "nixpkgs_3": {
-      "locked": {
-        "lastModified": 1636823747,
-        "narHash": "sha256-oWo1nElRAOZqEf90Yek2ixdHyjD+gqtS/pAgwaQ9UhQ=",
-        "owner": "nixos",
-        "repo": "nixpkgs",
-        "rev": "f6a2ed2082d9a51668c86ba27d0b5496f7a2ea93",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nixos",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
     "pre-commit-hooks": {
       "flake": false,
       "locked": {
-        "lastModified": 1712055707,
-        "narHash": "sha256-4XLvuSIDZJGS17xEwSrNuJLL7UjDYKGJSbK1WWX2AK8=",
+        "lastModified": 1721042469,
+        "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=",
         "owner": "cachix",
         "repo": "git-hooks.nix",
-        "rev": "e35aed5fda3cc79f88ed7f1795021e559582093a",
+        "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd",
         "type": "github"
       },
       "original": {
@@ -682,7 +704,9 @@
       "inputs": {
         "agenix": "agenix",
         "buildbot-nix": "buildbot-nix",
+        "channel-scripts": "channel-scripts",
         "colmena": "colmena",
+        "gerrit-dashboard": "gerrit-dashboard",
         "grapevine": "grapevine",
         "hydra": "hydra",
         "lix": [
@@ -711,6 +735,18 @@
         "type": "github"
       }
     },
+    "rust-manifest": {
+      "flake": false,
+      "locked": {
+        "narHash": "sha256-aZFye4UrtlcvLHrISldx4g9uGt3thDbVlLMK5keBSj0=",
+        "type": "file",
+        "url": "https://static.rust-lang.org/dist/channel-rust-1.78.0.toml"
+      },
+      "original": {
+        "type": "file",
+        "url": "https://static.rust-lang.org/dist/channel-rust-1.78.0.toml"
+      }
+    },
     "stable": {
       "locked": {
         "lastModified": 1696039360,
@@ -762,7 +798,9 @@
         "bats-assert": "bats-assert",
         "bats-support": "bats-support",
         "flake-utils": "flake-utils_4",
-        "nixpkgs": "nixpkgs_3",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
         "terranix-examples": "terranix-examples"
       },
       "locked": {
@@ -824,11 +862,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1721059077,
-        "narHash": "sha256-gCICMMX7VMSKKt99giDDtRLkHJ0cwSgBtDijJAqTlto=",
+        "lastModified": 1723454642,
+        "narHash": "sha256-S0Gvsenh0II7EAaoc9158ZB4vYyuycvMGKGxIbERNAM=",
         "owner": "numtide",
         "repo": "treefmt-nix",
-        "rev": "0fb28f237f83295b4dd05e342f333b447c097398",
+        "rev": "349de7bc435bdff37785c2466f054ed1766173be",
         "type": "github"
       },
       "original": {

flake.nix

@@ -3,7 +3,9 @@
 
   inputs = {
     nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
+
     terranix.url = "github:terranix/terranix";
+    terranix.inputs.nixpkgs.follows = "nixpkgs";
 
     agenix.url = "github:ryantm/agenix";
     agenix.inputs.nixpkgs.follows = "nixpkgs";
@@ -11,15 +13,21 @@
     colmena.url = "github:zhaofengli/colmena";
     colmena.inputs.nixpkgs.follows = "nixpkgs";
 
-    hydra.url = "git+https://git.lix.systems/the-distro/hydra.git";
+    hydra.url = "git+https://git.lix.systems/lix-project/hydra.git";
     hydra.inputs.nixpkgs.follows = "nixpkgs";
 
     nix-gerrit.url = "git+https://git.lix.systems/the-distro/nix-gerrit.git";
     nix-gerrit.inputs.nixpkgs.follows = "nixpkgs";
 
+    gerrit-dashboard.url = "git+https://git.lix.systems/the-distro/gerrit-monitoring.git";
+    gerrit-dashboard.flake = false;
+
     buildbot-nix.url = "git+https://git.lix.systems/lix-project/buildbot-nix.git?ref=refs/heads/non-flakes";
     buildbot-nix.inputs.nixpkgs.follows = "nixpkgs";
 
+    channel-scripts.url = "git+https://git.lix.systems/the-distro/channel-scripts.git";
+    channel-scripts.inputs.nixpkgs.follows = "nixpkgs";
+
     lix.follows = "hydra/lix";
 
     grapevine = {
@@ -46,6 +54,7 @@
           inputs.hydra.overlays.default
           inputs.lix.overlays.default
           inputs.nix-gerrit.overlays.default
+          inputs.channel-scripts.overlays.default
         ];
       };
       terraform = pkgs.opentofu;
@@ -54,6 +63,7 @@
         modules = [
           ./terraform
           {
+            bagel.dnsimple.enable = true;
             bagel.gandi.enable = true;
             bagel.hydra.enable = true;
           }
@@ -107,7 +117,7 @@
         bagel.baremetal.builders = { enable = true; num = i; netboot = i >= 6; };
       };
 
-      builders = lib.listToAttrs (lib.genList makeBuilder 12);
+      builders = lib.listToAttrs (lib.genList makeBuilder 11);
     in {
       meta.nixpkgs = systemBits.x86_64-linux.pkgs;
       meta.specialArgs.inputs = inputs;
@@ -120,6 +130,7 @@
       wob-vpn-gw.imports = commonModules ++ [ ./hosts/wob-vpn-gw ];
       buildbot.imports = commonModules ++ [ ./hosts/buildbot ];
       public01.imports = commonModules ++ [ ./hosts/public01 ];
+      build-coord.imports = commonModules ++ [ ./hosts/build-coord ];
     } // builders;
 
     hydraJobs = builtins.mapAttrs (n: v: v.config.system.build.netbootDir or v.config.system.build.toplevel) self.nixosConfigurations;

hosts/bagel-box/default.nix

@@ -37,20 +37,11 @@
 
   bagel.services = {
     postgres.enable = true;
-
-    hydra.enable = true;
-    hydra.dbi = "dbi:Pg:dbname=hydra;user=hydra";
-    # Takes 10 builders (0 → 9).
-    hydra.builders = lib.genList (i: "builder-${builtins.toString i}") 10;
-
     ofborg.enable = true;
   };
 
   bagel.sysadmin.enable = true;
 
-  security.acme.acceptTerms = true;
-  security.acme.defaults.email = "infra@forkos.org";
-
   services.openssh.enable = true;
 
   system.stateVersion = "24.11";

hosts/build-coord/default.nix

@@ -0,0 +1,21 @@
+{ lib, ... }:
+{
+  imports = [ ./hardware.nix ];
+
+  networking.hostName = "build-coord";
+  networking.domain = "wob01.infra.forkos.org";
+
+  bagel.sysadmin.enable = true;
+
+  bagel.services = {
+    hydra.enable = true;
+    # Takes 10 builders (0 → 9).
+    hydra.builders = lib.genList (i: "builder-${builtins.toString i}") 10;
+  };
+
+  # Hydra is proxied.
+  bagel.raito.v6-proxy-awareness.enable = true;
+
+  system.stateVersion = "24.05";
+  deployment.targetHost = "build-coord.wob01.infra.forkos.org";
+}

hosts/build-coord/hardware.nix

@@ -0,0 +1,87 @@
+{
+  boot.initrd.availableKernelModules = [ "ahci" "ehci_pci" "usb_storage" "usbhid" "sd_mod" ];
+  boot.initrd.kernelModules = [ "dm-snapshot" ];
+
+  nixpkgs.hostPlatform = "x86_64-linux";
+  hardware.cpu.intel.updateMicrocode = true;
+
+  boot.loader.systemd-boot.enable = true;
+  boot.loader.efi.canTouchEfiVariables = true;
+  boot.initrd.systemd.enable = true;
+
+  boot.initrd.services.lvm.enable = true;
+
+  boot.kernelParams = [
+    "console=tty1"
+    "console=ttyS0,115200"
+  ];
+
+  fileSystems = {
+    "/" = {
+      device = "/dev/disk/by-label/root";
+      fsType = "ext4";
+    };
+
+    "/boot" = {
+      device = "/dev/disk/by-label/BOOT";
+      fsType = "vfat";
+      options = [ "fmask=0022" "dmask=0022" ];
+    };
+  };
+
+  swapDevices = [
+    {
+      device = "/swapfile";
+      size = 20 * 1024;  # 50GiB
+    }
+  ];
+
+  zramSwap = {
+    enable = true;
+    memoryPercent = 25;
+  };
+
+  networking.useNetworkd = true;
+
+  systemd.network = {
+    netdevs = {
+      "40-uplink" = {
+        netdevConfig = {
+          Kind = "bond";
+          Name = "uplink";
+        };
+        bondConfig = {
+          Mode = "802.3ad";
+          TransmitHashPolicy = "layer3+4";
+        };
+      };
+    };
+    networks = {
+      "40-eno1" = {
+        name = "eno1";
+        bond = [ "uplink" ];
+      };
+      "40-eno2" = {
+        name = "eno2";
+        bond = [ "uplink" ];
+      };
+    };
+  };
+  networking.interfaces.uplink.ipv6.addresses = [
+    { address = "2a01:584:11::1:11"; prefixLength = 64; }
+  ];
+  networking.defaultGateway6 = { interface = "uplink"; address = "2a01:584:11::1"; };
+
+  services.coredns = {
+    enable = true;
+    config = ''
+      . {
+        bind lo
+        forward . 2001:4860:4860::6464
+        template ANY A { rcode NOERROR }
+      }
+    '';
+  };
+  services.resolved.enable = false;
+  networking.resolvconf.useLocalResolver = true;
+}

hosts/buildbot/default.nix

@@ -9,8 +9,6 @@
   # TODO: make it the default
   networking.domain = "infra.forkos.org";
 
-  time.timeZone = "Europe/Paris";
-
   bagel.sysadmin.enable = true;
   # Buildbot is proxied.
   bagel.raito.v6-proxy-awareness.enable = true;
@@ -28,7 +26,7 @@
   bagel.services.buildbot = {
     enable = true;
     domain = "buildbot.forkos.org";
-    builders = [ "builder-11" ];
+    builders = [ "builder-10" ];
   };
 
   i18n.defaultLocale = "en_US.UTF-8";

hosts/fodwatch/default.nix

@@ -8,8 +8,6 @@
   networking.hostName = "fodwatch";
   networking.domain = "infra.forkos.org";
 
-  time.timeZone = "Europe/Paris";
-
   bagel.sysadmin.enable = true;
   # Fodwatch will be proxied.
   bagel.raito.v6-proxy-awareness.enable = true;

hosts/gerrit01/default.nix

@@ -9,8 +9,6 @@
   # TODO: make it the default
   networking.domain = "infra.forkos.org";
 
-  time.timeZone = "Europe/Paris";
-
   bagel.sysadmin.enable = true;
   # Gerrit is proxied.
   bagel.raito.v6-proxy-awareness.enable = true;
@@ -32,6 +30,7 @@
 
   bagel.services.gerrit = {
     enable = true;
+    pyroscope.enable = true;
     domains = [
       "cl.forkos.org"
     ];
@@ -122,6 +121,26 @@
     };
   };
 
+  age.secrets.s3-channel-staging-keys.file = ../../secrets/s3-channel-staging-keys.age;
+  bagel.nixpkgs.channel-scripts = {
+    enable = true;
+    otlp.enable = true;
+    nixpkgsUrl = "https://cl.forkos.org/nixpkgs.git";
+    hydraUrl = "https://hydra.forkos.org";
+    binaryCacheUrl = "https://cache.forkos.org";
+    baseUriForGitRevisions = "https://cl.forkos.org/plugins/gitiles/nixpkgs/+";
+    s3 = {
+      release = "bagel-channel-scripts-test";
+      channel = "bagel-channel-scripts-test";
+    };
+    releaseBucketCredentialsFile = config.age.secrets.s3-channel-staging-keys.path;
+    deployKeyFile = config.age.secrets.priv-ssh-key.path;
+    extraArgs = [
+      "--bypass-preflight-checks"
+    ];
+    channels = import ../../common/channels.nix;
+  };
+
   i18n.defaultLocale = "fr_FR.UTF-8";
 
   system.stateVersion = "24.05";

hosts/git/default.nix

@@ -8,8 +8,6 @@ in
   networking.hostName = "git";
   networking.domain = "infra.forkos.org";
 
-  time.timeZone = "Europe/Paris";
-
   bagel.sysadmin.enable = true;
   # Forgejo will be proxied.
   bagel.raito.v6-proxy-awareness.enable = true;

hosts/meta01/default.nix

@@ -2,8 +2,6 @@
   networking.hostName = "meta01";
   networking.domain = "infra.forkos.org";
 
-  time.timeZone = "Europe/Paris";
-
   bagel.sysadmin.enable = true;
   # netbox is proxied.
   bagel.raito.v6-proxy-awareness.enable = true;
@@ -25,6 +23,8 @@
   bagel.services.loki.enable = true;
   bagel.services.grafana.enable = true;
   bagel.services.grapevine.enable = true;
+  bagel.services.pyroscope.enable = true;
+  bagel.services.tempo.enable = true;
   bagel.services.hookshot = {
     enable = true;
     admins = [

hosts/public01/default.nix

@@ -9,11 +9,13 @@
   # TODO: make it the default
   networking.domain = "infra.forkos.org";
 
-  time.timeZone = "Europe/Paris";
-
   bagel.sysadmin.enable = true;
-  # Buildbot is proxied.
+  # Newsletter is proxied.
   bagel.raito.v6-proxy-awareness.enable = true;
+  bagel.newsletter = {
+    enable = true;
+    domain = "news.forkos.org";
+  };
   bagel.hardware.raito-vm = {
     enable = true;
     networking = {
@@ -25,6 +27,17 @@
     };
   };
 
+  bagel.services.s3-revproxy = {
+    enable = true;
+    domain = "forkos.org";
+    s3.apiUrl = "s3.delroth.net";
+    targets = {
+      channels = "bagel-channels";
+      releases = "bagel-releases";
+      channel-scripts-test = "bagel-channel-scripts-test";
+    };
+  };
+
   i18n.defaultLocale = "en_US.UTF-8";
 
   system.stateVersion = "24.05";

hosts/wob-vpn-gw/netboot.nix

@@ -42,8 +42,6 @@ in {
   networking.firewall.extraInputRules = ''
     ip6 saddr 2a01:584:11::/64 tcp sport < 1024 tcp dport 443 accept;
   '';
-  security.acme.acceptTerms = true;
-  security.acme.defaults.email = "infra@forkos.org";
   services.nginx = {
     enable = true;
     virtualHosts."vpn-gw.wob01.infra.forkos.org" = {

overlays/default.nix

@@ -2,5 +2,8 @@
   (final: prev: {
     iusb-spoof = final.callPackage ./iusb-spoof.nix {};
     u-root = final.callPackage ./u-root {};
+    pyroscope = final.callPackage ./pyroscope {};
+    s3-revproxy = final.callPackage ./s3-revproxy {};
+    git-gc-preserve = final.callPackage ./git-gc-preserve {};
   })
 ]

overlays/git-gc-preserve/default.nix

@@ -0,0 +1,9 @@
+{ writeShellApplication, git, nettools }:
+
+writeShellApplication {
+  name = "git-gc-preserve";
+
+  runtimeInputs = [ git nettools ];
+
+  text = (builtins.readFile ./script.sh);
+}

overlays/git-gc-preserve/script.sh

@@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+
+set +o errexit
+# Copyright (C) 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+usage() { # exit code
+  cat <<-EOF
+NAME
+    git-gc-preserve - Run git gc and preserve old packs to avoid races for JGit
+SYNOPSIS
+    git gc-preserve
+DESCRIPTION
+    Runs git gc and can preserve old packs to avoid races with concurrently
+    executed commands in JGit.
+    This command uses custom git config options to configure if preserved packs
+    from the last run of git gc should be pruned and if packs should be preserved.
+    This is similar to the implementation in JGit [1] which is used by
+    JGit to avoid errors [2] in such situations.
+    The command prevents concurrent runs of the command on the same repository
+    by acquiring an exclusive file lock on the file
+      "\$repopath/gc-preserve.pid"
+    If it cannot acquire the lock it fails immediately with exit code 3.
+    Failure Exit Codes
+        1: General failure
+        2: Couldn't determine repository path. If the current working directory
+           is outside of the working tree of the git repository use git option
+           --git-dir to pass the root path of the repository.
+           E.g.
+              $ git --git-dir ~/git/foo gc-preserve
+        3: Another process already runs $0 on the same repository
+    [1] https://git.eclipse.org/r/c/jgit/jgit/+/87969
+    [2] https://git.eclipse.org/r/c/jgit/jgit/+/122288
+CONFIGURATION
+    "pack.prunepreserved": if set to "true" preserved packs from the last gc run
+      are pruned before current packs are preserved.
+    "pack.preserveoldpacks": if set to "true" current packs will be hard linked
+      to objects/pack/preserved before git gc is executed. JGit will
+      fallback to the preserved packs in this directory in case it comes
+      across missing objects which might be caused by a concurrent run of
+      git gc.
+EOF
+  exit "$1"
+}
+# acquire file lock, unlock when the script exits
+lock() { # repo
+  readonly LOCKFILE="$1/gc-preserve.pid"
+  test -f "$LOCKFILE" || touch "$LOCKFILE"
+  exec 9> "$LOCKFILE"
+  if flock -nx 9; then
+    echo -n "$$ $USER@$(hostname)" >&9
+    trap unlock EXIT
+  else
+    echo "$0 is already running"
+    exit 3
+  fi
+}
+unlock() {
+  # only delete if the file descriptor 9 is open
+  if { : >&9 ; } &> /dev/null; then
+    rm -f "$LOCKFILE"
+  fi
+  # close the file handle to release file lock
+  exec 9>&-
+}
+# prune preserved packs if pack.prunepreserved == true
+prune_preserved() { # repo
+  configured=$(git --git-dir="$1" config --get pack.prunepreserved)
+  if [ "$configured" != "true" ]; then
+    return 0
+  fi
+  local preserved=$1/objects/pack/preserved
+  if [ -d "$preserved" ]; then
+    printf "Pruning old preserved packs: "
+    count=$(find "$preserved" -name "*.old-pack" | wc -l)
+    rm -rf "$preserved"
+    echo "$count, done."
+  fi
+}
+# preserve packs if pack.preserveoldpacks == true
+preserve_packs() { # repo
+  configured=$(git --git-dir="$1" config --get pack.preserveoldpacks)
+  if [ "$configured" != "true" ]; then
+    return 0
+  fi
+  local packdir=$1/objects/pack
+  pushd "$packdir" >/dev/null || exit 1
+  mkdir -p preserved
+  printf "Preserving packs: "
+  count=0
+  for file in pack-*{.pack,.idx} ; do
+    ln -f "$file" preserved/"$(get_preserved_packfile_name "$file")"
+    if [[ "$file" == pack-*.pack ]]; then
+      ((count++))
+    fi
+  done
+  echo "$count, done."
+  popd >/dev/null || exit 1
+}
+# pack-0...2.pack to pack-0...2.old-pack
+# pack-0...2.idx to pack-0...2.old-idx
+get_preserved_packfile_name() { # packfile > preserved_packfile
+  local old=${1/%\.pack/.old-pack}
+  old=${old/%\.idx/.old-idx}
+  echo "$old"
+}
+# main
+while [ $# -gt 0 ] ; do
+    case "$1" in
+        -u|-h)  usage 0 ;;
+    esac
+    shift
+done
+args=$(git rev-parse --sq-quote "$@")
+repopath=$(git rev-parse --git-dir)
+if [ -z "$repopath" ]; then
+  usage 2
+fi
+lock "$repopath"
+prune_preserved "$repopath"
+preserve_packs "$repopath"
+git gc ${args:+"$args"} || { EXIT_CODE="$?"; echo "git gc failed"; exit "$EXIT_CODE"; }

overlays/pyroscope/default.nix

@@ -0,0 +1,42 @@
+{ lib
+, buildGoModule
+, fetchFromGitHub
+}:
+
+buildGoModule rec {
+  pname = "pyroscope";
+  version = "1.7.1";
+
+  src = fetchFromGitHub {
+    owner = "grafana";
+    repo = "pyroscope";
+    rev = "v${version}";
+    hash = "sha256-iMP67J0Q8Cgo52iImMzAM3PEkk6uLF7r6v9TyXZVaIE=";
+  };
+
+  env.GOWORK = "off";
+
+  vendorHash = "sha256-ggntpnU9s2rpkv6S0LnZNexrdkBsdsUrGPc93SVrK4M=";
+
+  subPackages = [ "cmd/profilecli" "cmd/pyroscope" ];
+
+  ldflags = [
+    "-extldflags"
+    "-static"
+    "-s"
+    "-w"
+    "-X=github.com/grafana/pyroscope/pkg/util/build.Branch=${src.rev}"
+    "-X=github.com/grafana/pyroscope/pkg/util/build.Version=${version}"
+    "-X=github.com/grafana/pyroscope/pkg/util/build.Revision=${src.rev}"
+    "-X=github.com/grafana/pyroscope/pkg/util/build.BuildDate=1970-01-01T00:00:00Z"
+  ];
+
+  meta = with lib; {
+    description = "Continuous profiling platform";
+    homepage = "https://github.com/grafana/pyroscope";
+    changelog = "https://github.com/grafana/pyroscope/blob/${src.rev}/CHANGELOG.md";
+    license = licenses.agpl3Only;
+    maintainers = with maintainers; [ raitobezarius ];
+    mainProgram = "pyroscope";
+  };
+}

overlays/s3-revproxy/default.nix

@@ -0,0 +1,39 @@
+# Originally written by Jade Lovelace for Lix.
+{ lib, buildGoModule, fetchFromGitHub }:
+buildGoModule rec {
+  pname = "s3-revproxy";
+  version = "4.15.0";
+
+  src = fetchFromGitHub {
+    owner = "oxyno-zeta";
+    repo = "s3-proxy";
+    rev = "v${version}";
+    hash = "sha256-q0cfAo8Uz7wtKljmSDaJ320bjg2yXydvvxubAsMKzbc=";
+  };
+
+  vendorHash = "sha256-dOwNQtTfOCQcjgNBV/FeWdwbW9xi1OK5YD7PBPPDKOQ=";
+
+  ldflags = [
+    "-X github.com/oxyno-zeta/s3-proxy/pkg/s3-proxy/version.Version=${version}"
+    "-X github.com/oxyno-zeta/s3-proxy/pkg/s3-proxy/version.Metadata="
+  ];
+
+  postPatch = ''
+    # Refer to the included templates in the package instead of cwd-relative
+    sed -i "s#Path = \"templates/#Path = \"$out/share/s3-revproxy/templates/#" pkg/s3-proxy/config/config.go
+  '';
+
+  postInstall = ''
+    mkdir -p $out/share/s3-revproxy
+    cp -r templates/ $out/share/s3-revproxy/templates
+  '';
+
+  meta = {
+    description = "S3 Reverse Proxy with GET, PUT and DELETE methods and authentication (OpenID Connect and Basic Auth)";
+    homepage = "https://oxyno-zeta.github.io/s3-proxy";
+    # hm, not having a maintainers entry is kind of inconvenient
+    maintainers = [ ];
+    licenses = lib.licenses.asl20;
+    mainProgram = "s3-proxy";
+  };
+}

pkgs/forgejo/default.nix

@@ -1,6 +1,6 @@
-{ forgejo }:
+{ forgejo-lts }:
 
-forgejo.overrideAttrs (prev: {
+forgejo-lts.overrideAttrs (prev: {
   patches = [
     # Branch divergence calculations for a single branch may take 100-200ms on something as big
     # as nixpkgs. The branch view defaults to 20 branches for each page, taking roughtly 3s to

secrets.nix

@@ -4,15 +4,19 @@ let
   commonKeys = keys.users.delroth ++ keys.users.raito;
 
   secrets = with keys; {
-    hydra-s3-credentials = [ machines.bagel-box ];
-    hydra-signing-priv = [ machines.bagel-box ];
-    hydra-ssh-key-priv = [ machines.bagel-box ];
+    hydra-postgres-key = [ machines.build-coord ];
+    hydra-s3-credentials = [ machines.build-coord ];
+    hydra-signing-priv = [ machines.build-coord ];
+    hydra-ssh-key-priv = [ machines.build-coord ];
+
     netbox-environment = [ machines.meta01 ];
     mimir-environment = [ machines.meta01 ];
     mimir-webhook-url = [ machines.meta01 ];
     grafana-oauth-secret = [ machines.meta01 ];
     loki-environment = [ machines.meta01 ];
     gerrit-prometheus-bearer-token = [ machines.gerrit01 machines.meta01 ];
+    pyroscope-secrets = [ machines.meta01 ];
+    tempo-environment = [ machines.meta01 ];
 
     buildbot-worker-password = [ machines.buildbot ];
     buildbot-oauth-secret = [ machines.buildbot ];
@@ -29,6 +33,14 @@ let
     metrics-push-password = builtins.attrValues machines;
 
     ows-deploy-key = [ machines.gerrit01 ];
+    s3-channel-staging-keys = [ machines.gerrit01 ];
+    s3-channel-keys = [ machines.gerrit01 ];
+
+    postgres-ca-priv = [ machines.bagel-box ];
+    postgres-tls-priv = [ machines.bagel-box ];
+
+    newsletter-secrets = [ machines.public01 ];
+    s3-revproxy-api-keys = [ machines.public01 ];
   };
 in
   builtins.listToAttrs (

secrets/buildbot-oauth-secret.age

@@ -1,20 +1,20 @@
 age-encryption.org/v1
--> ssh-ed25519 87T2Ig g15A5EWi9IhaxPFS6SD6YYm/aFnC0Dum7zK8/ZUtW0s
-791D6C8mAy2dhDAlqRQ+q41FlQTJX2WfZQPjuwetP2A
--> ssh-ed25519 K3b7BA cJY9qIFVmucmMJLTFffkRCNYeudZl+8Yrm5SkxQ4eSI
-97nXyKffZGoGJ6252UKUEJHiFgdk8XUkAAkXy2PLepM
--> ssh-ed25519 +qVung HMBSUjfmaFLVx64epj0djkqNMe3CdKN1fxAVuu+Dtmg
-AxT62n2p/pP9WZmmuHClSKKgXhr4FjEQpEs0HfdNGfw
+-> ssh-ed25519 87T2Ig tzPD1x6XKuDfgJ8jkQnwW/ALp2pkANCeNoO8xdUqq30
+QSsuO6Dwc8QJuY92gXRnWB5aJ2SU9X2uFh01GmLVaQE
+-> ssh-ed25519 K3b7BA 9G9Uw1xY8hq//xphNWrPn5y7vG2o8/kwkC8cJGuf/mI
+Ip0019OUaFq2ZDFI3i77hdsp9IqFV2qqYIB/TnDSXgo
+-> ssh-ed25519 +qVung dx22ef+x9X5mr73L8NUzxYQa640M2XViELjJcpgF3go
+CXyit7pk8SPNHBgULlMQUAasGAn4C36zcwOBDI46nU4
 -> ssh-rsa krWCLQ
-N0Duz2bONcCUZ76QhPsCJ4BHHWqzFdZLqFdl+6GeW+tgIp2Nb4la8eNfgzYGSwTy
-53bRePNMIBTkChXFYt/4fUdqaiiVYg25swMeVLQBJnjJkcAks0Gf44FXLIaoPr1M
-56rtixpSX31WDKwHbUF/40G6Xut8KNlI8BdwiOl9ibgnuEf4mYQbwFbRQbLMK5IK
-Rf/7SEmAqqfY/HG1RqqgCs4kEpvFTKqEEDpgjOoyS2tyKN2351jya91YzotLja4I
-sLoMg/G3UNtxfdaCgK7TP4IxV9blkVMDPAbyR622VbS0sEa7uJGzb86jDDsZXaKX
-9iWK9n4hMKZDv9gBbhTIWg
--> ssh-ed25519 /vwQcQ hMkCrUcLGxdZMYgi1D1Kr5qUdGNfza2UTvRJKiHObgM
-7Lz70zSMPk/tsU1CZGOk/BPA7NSSnSJgFbG5TjyOXvA
--> ssh-ed25519 0R97PA OQjDTknVmrYVclcqlT31YjZx+3a/0GxfjuVQFmPJ7UQ
-KMGTMfO/mO5EAYacyz1hmHnQgzunRqkDeglhbGVNWe4
---- ScDZvSiVSjNXm8TSoLSAM+KpcFORnCXiemYbCBcz2jQ
-™ŸÄhÜ}E¹ÊœËíUÌùᢌƒÿ…<C3BF>é™k¢ág[<5B>ñCƒ"<22>–NÛj•u5«<0C>ÄCXÕöÈGt¡TOmñ
+NlGh0hM10NOuek7MbrFo0iul0kQQtDFmZIhgpyqaATMdCDRBXJOyhASHU5N0zDDJ
+MLaJUV0l2o1ghBF9RhSKdoUPVEn8Cce/nfQepYzMlfc4UG3qWXwabwR6EtqqCZCJ
+jAEWZ8taTKDmzoXwuygCW+bRBuoMMrcfzu7V90N+mQpZWtOScatb6E7d5VRqjlar
+st1ZQu5ccghufyQSUmOC7GpojOyutX5EvbMGn84X4ouZRHRX/8fTgaqicV+aeAIb
+QyXisOrO6C+Jle5qfxzMSe8c/TCyF2574kD6F1BQ9Kpkinn8v7OWcIXtkNmZ5hzK
+vs0Bej8yZVsoBkj1vWAM0A
+-> ssh-ed25519 /vwQcQ n+hr1cV1zRs1S86YnA+0oRB8SCaPKtkoMNe15ZsVVwM
+fdFtUqno07ik6FpW5zMImIjd8wM8dMgwU+RqjeT2PiI
+-> ssh-ed25519 0R97PA ddPILw57gkuKvAqlmpa+MnV/LSEdyQzQaAarCUqQ1xE
+ozK5a6uXZDc17OrX0OZun9hmZwP3H3rYQiNuKnukqsg
+--- f7yGgKQpCPj64Ps0HfMcToYircGH5SPqMzVZrUMB8ZI
+føv[iY\ÅšMP,¯Ùh°Èxb—Ðÿ«J<C2AB>*ºË"”+¬ÒA0T˜?KmˆPÈË2¹'2±‚µ³Ø=¯êÚÏŸj”

secrets/buildbot-worker-password.age

@@ -1,20 +1,20 @@
 age-encryption.org/v1
--> ssh-ed25519 87T2Ig df+IMqWM/HNjaY74zibFQIdUdC3K7uQlm3U9R9NUtFY
-hPSbCuWvqy/7FEj7YScYztyt5GVx4Y7tgGuKKkSKoRg
--> ssh-ed25519 K3b7BA xN8wzUKHqjOb/tqA+EI+0H0MSQRihRfydchwVqYWAVU
-maLMpZe8orvTT6Av+YkhT8FcG4dc7bzDgOW339nSw1g
--> ssh-ed25519 +qVung oM1uphTbjI54t4U9jNd1zORqpjBG17MwDf2eNDmOlkg
-oUHVuQt2SHIwtV82pgnKJ7g2jcVBAHWOzPK46otoh34
+-> ssh-ed25519 87T2Ig p8lEB5da4fIfLH/HKBsghzq5mvQLB69UB4+uAi3DGCw
+NeZ3jPTUKa7MiqjrFPrYuP4VneytQPdBNqf+omPZJYM
+-> ssh-ed25519 K3b7BA uP2K1hU7uLmiHXmmoUdsB7CHQq61ZkEAjG/aK863RDw
+0chTczEMXASdYiwqNxDQ+vMXXhjOf64oIQ2ULZmQI8Y
+-> ssh-ed25519 +qVung jUgEqz3+ypL7mwJ1R7lfeOMhkon/aRrNSJUJT3X7vmU
+pgOiwrp9JiA20yw9bsxi8eiQ9/23CYXKRBGF1pea9eI
 -> ssh-rsa krWCLQ
-eYspf5hUKdFQl1RxPaNTj0viAPd+kzp8Xbwn+q6fSITMacmyTY5J8FckLx2YXDxy
-Qm/OsEK0ZOvxnHMrL0oAJjKSy/MamE+9heT3QO+LUN30QxbOIOqHMrl3waadWZdx
-ZGOWK+r+dKGYNsxFv+t1Y/4DBKKzlXFWhJ0aL7nMOqq9+Ca+UZuE41j7eWGGPPLy
-fuW/iOVVxQ+EEeCDpatQSrFPKaeWCCVP9oIDFtE4dsKxubMa4EpUoag0UvEIW182
-UGS8BvMqYgx+obqJDkhXXBK9apmJS2ojcfdtCbNOCV9Ett72Nm/iY5NjLprFMLde
-8wWGA6s3hBOP39lq0eiSxw
--> ssh-ed25519 /vwQcQ 3zLcLDaDVhIn2knezexYM5Fqu/O9wwORnJIhsXHqgj0
-HchGikQMgkDj0qQgtDdsdKokV+nMjdv6t0uVISeU7Q8
--> ssh-ed25519 0R97PA 6lm6B6B3dzSdhdcf5rjyTu+7cCtWRxVpWeapJX3nbQo
-x/w4dEfFyxPi4lbNEqgjEblPVfQyj+q1JjeQHiVFhDw
---- oo5BK1pG+43amUg803Uv511RNtdQ/PDwlXUrV/AbOAA
-…ÙUqÆçïµ[f7ƒêŒë¼¨FìˆY<13>™Ùm¶ØLS?Úℶ‡÷ƒöæ<Kø©F¤z¥V^³U¨N»¯ôƒ)zÔ<7A>¥ž@<40>SÀF€Y‡ËG2^žƒ˜à„»N|
+snCHrLHzkjimwIxKO90IjnHwOArlozO9kd/aCdZZnYNgh/QG3rUSceSn9yTHbtMV
+izv0SU51LrRU+JyE+a524AxKhyPBvGDig20j7hMy5fVxZqeunztqtlha5gaYYaQg
+Tbfs9tDP+pCIgzMVNqYf6EJ4MK7qjNf9DE5I490Eta5YZxAi/3To3BmZmIYtCz6l
+1kNRiSmWCbZqE25keFgPCgRMFXAFK9W6NmL+HamqCUhjPoJg/Gd4sf39EONT0PYg
+7BpCOAnwwfECHPxpM3qv0h2kJXTb4DZ715cFReSVyQe5fvKv8hoWhl/S+++pEYT8
+u/LKBx/o7e3Kd7cm2RGnBw
+-> ssh-ed25519 /vwQcQ 4+IQPRsMMHmuSGL7T7IbRkTTuL+TTqgdQp5FSbyt8Dw
+KOI0LKQ0oA5XtxaW7wftlEJB0BGVnx41HUJMG92SRUA
+-> ssh-ed25519 0R97PA l1aWUEv8nLEtYnpY1gjTJqk5UYm51NDqOjYmL83rZ10
+B7qDZwCpolkIajqCXeOepwmF6ciJfKvr+AN7VouMUvA
+--- lz/IMMPxBpD3Bzuv9Wl23+swBQHlblhlAO/ZXAgN0hU
+µoÍüÌ<EFBFBD>²-‚Īr °eó|Í?ït
èìÎZ<C38E>¬sÒì!ŸƒÁ<>@Ï'–ìèz6UöÎg
JøÑOµ–s13<31>š‹8<î’%-·Ô‡Eÿ}–Šdm9¿å¢Óoæ

secrets/grafana-oauth-secret.age

@@ -1,20 +1,20 @@
 age-encryption.org/v1
--> ssh-ed25519 j2r2qQ JSveX4zYEjb4jJH4eg4oXA6r3oc0jBx8NgjhN9JrjlQ
-1ZIr/XFClbwJHn0ppJnolpb4QlgZOA8JX5OjjY4x6pU
--> ssh-ed25519 K3b7BA sXUjuZFK0PL/KndxRCJCM5Kg8OmVseRZNWG8mL1alRc
-U9MMgDtqtmsS1W5i04Pa/b4JBTSjK6FffZxgYI3phtg
--> ssh-ed25519 +qVung FNSElbiw0frYcsO0xoyPQgRGqAe/aVX21dTB6yk+GQg
-zHT/xU+yfXYSBO2HLwoHrGf5ns6BDVb8MlhVVQCBlOc
+-> ssh-ed25519 j2r2qQ vwcaLpvGJ9swXnV8idDwi9jdRPSj38As9p2QFkIJ1Xc
+FLnZeblHDQQcWjFm1iaghbvuFgOG3miwtkRE5sz1+X0
+-> ssh-ed25519 K3b7BA 9VRe2rBwg3G9lxxfxL/yLob2NZmLJTBMxzx0Ew8VwmY
+/I2W80UykNvll5o98OPeMpIsddOel9B7uQlio0X3gcs
+-> ssh-ed25519 +qVung VsqKzMD85aps4PIx2zqae2Dj7YWibiaKYb5z7ws8ggM
+Y9dRd/hOz8h4avlutBQ1YZgHIAf/AuTr5WaByKlFbLE
 -> ssh-rsa krWCLQ
-ye0mLiYeyvlp4EZX7mZ3F7B9V9JSeoiCodzccS+5qIEd6gr+RTHSnKYqwf/nwf8F
-qKLwbxWjpmkIzBWeswy8AJ8159aucGEmB+3/tTSwd+QlRkru4Z/7jtfU64KQttgt
-vaRfc9J/85AJJ2V6Sw/xG8SgxyLBbp/XIN2+tmb0g3kAWiuLcrLk3H/MsfmxDVXg
-RQjugP5K2+fEZc77dHQTrMI58K9TrSw1zYA1ee8J/fl9IJ7J77qi5UgizY+YfX8T
-SmR9DeYUe+hKgCB2k/KgAxp4WOQNgUOFBTsE5FW+kQQpfGx5aqR6vCYU+CPsA3Zb
-FwV0l+g4FUVy+xAtqaGSAQ
--> ssh-ed25519 /vwQcQ fbnK1jYiUwUsgD8sSTboJCBfcuwJXKNCaJaWYuIfmVk
-Uj2+uBABMTxq1MBsiHXgkdFMOpIN7gfxoJVKOQff1Pw
--> ssh-ed25519 0R97PA yYOb6AYAFWvm7W2KYT5v9zznkF4Di/vatH48Xgx0x2E
-yUm+MKj9496BkdX2FpLyhML7budUyqT1hL9hpghxSnI
---- ogCPBrmdbeDorj3t5BL05ge6VngXBpUEDW4qaaKIa0U
-%¨šÚlD]Ϫ?©ßŠÑ(ÿ†E/Wu穉T¶îç[}ž$ÁÍS„Šˆ^[:¸]
he0XUœp¸äq<C3A4>`0A
+gjyaUFrIIbZnFTGVw4XEZzkTIP/+qXV6/q0W8Wb4EtqQXDRISFT+bwxQU/S2p5hf
+7+JGcn4BZg6puOJ5BBABWtpn6gcX5OFfga5azIdioF/R19XByT+0SK5njw8g1VPS
+R7o8kQt2yvKWayoq9Cis5XRg+4KANkwOQaNTO8AdiCwgq9nc0Cd9avk8QhaFoR74
+D5cf8jPsufp744rQqwhWDoG533LS1WUUuYZqRmtp2Vz+r583RhSscaNyA7ddr7o6
+e9ZQJyL5bKiN8qe3Xm76lLypf/wg7+aGn8HHnO6GA65g+VYfjLMODEqCN/+uDJtB
+g8v2wzKIGYlZiV1hEjH8nw
+-> ssh-ed25519 /vwQcQ 4pU5JGK5vpZbFgq01a9YY8VmSJvPSHPSZD50TLJwKHc
+L46UA/p+bNSR8cLmL8G7VpmAcZ+sy5AROc4yj2ABOWg
+-> ssh-ed25519 0R97PA Tk00kYLhsEy1HJcmKLgaLWTdNP8XV/cdKHMLzyK6glk
+kwyQZr/h6MutROJmjVfPWGcf9xN5Uc5w5mVyuKcK64g
+--- E0vVtBqbjNkZY0/1dFJ53uVAR7IGPO+OMmXkpJcKmlw
+{ÐQê%è‹õY•B,isr¥1‘<31>|¼yLÕ'7?¶iŠM…¶MU]×ê/d2¸I1u¶2hZjHåh&¥

secrets/hydra-signing-priv.age

@@ -1,20 +1,20 @@
 age-encryption.org/v1
--> ssh-ed25519 +HUDfA NMyTM3c++HKU2klLjAbUUFS81k21LUwEoqR1OUBuLjU
-OrKxpksxoay93URtmN9HhnK43QrM/Gs0qRuENZvHWJI
--> ssh-ed25519 K3b7BA LloEGN8cbVvGraHs5cPIZRJJyTPFrmmeGwZyqov9m2U
-XPvNpQT3aFVoidOhmePGgiTyytIWtd4rs59Qq9xl/I4
--> ssh-ed25519 +qVung 3hat0gKIl1WjXnkP6p+/8RyTxZkaVnLgV9B8plICPlY
-jmRKWCUCDpDExmq4SEq8WpqQheBSRD4uqrTgxy2u6PM
+-> ssh-ed25519 +uvEmw TNYFQxSUv5yMmlTWoIxCOlv6UR+RA50cb5aJbo0yEE0
+Yw3sTPqYf7A33RI87CqoPWe2gh0FuvdBGGKqHV55Atc
+-> ssh-ed25519 K3b7BA vMlHenY6jSIfnxQD6xh09cwwV+YVBkLuSMHcyKD+dCk
+heXkAEqRawBlHqcr6ldmhWmk7qPtGLMDFC3QT79vdMM
+-> ssh-ed25519 +qVung fgimLW5X0z4Eh2u3fIr5bgR5/c1SKam9CKW/2mqtTik
+8VKJJr+FRE0j5YvjfdMXugNA4UwUebKrkeAe+9LYBnQ
 -> ssh-rsa krWCLQ
-QPOl96dmoxY5YtMmL68+6MQpGwZc68ajaRkcEKmYYu4/XB+mffRKsNtyiKJQwEi7
-szvAced8C4RMNrCf3xyF77Sm1UV8YCyaHyplb2/yjv5YDvCDwTp2GnadDoAaLrXU
-jf6ocI8409XWQHEEEofHZRjmfmIBUx1lTwbGFMt48V7MZdadFjXmSmUMvxsu/Rj5
-NLjoPNRBzqPIw6U7nTSmkG2HOeHlA9Z5a33MsXYs8NPH22Spjjy+VvxrLv8VAjnf
-7kGjviW4ZcdEQ7Aox+9V+6qArrIy7lJ9lOIZA2LueCtKhQAmKnInFxRyyN0Nk4ls
-tjlBFJQEG2v14iaHENwRAg
--> ssh-ed25519 /vwQcQ o7qseMFb4ViV7ylSl2ug7xFZn7GZGqCapWRCq2vyVVs
-hKqzk9BcK5l0VhLfPONKKv6SRnDCw2n+RoaeQbOnT8Y
--> ssh-ed25519 0R97PA DTEowwoCXTMGxfQIXOnwn5fjlih0UmQJCKs2II4gCVY
-8BpVhUV8qg3zcCJe7OwHoJrfYIpBtOquqhFyfZx7mRQ
---- r2NmEyV0/Goas5lXMHeFoafcrbMHvs4ob0Zg4cVil3w
-=‚TÏ`Ô}%xÖ¸œêFtÆêlãµ)•ò1Ð]Œ<DvwFøè:Qf@nÇÀU=~FžË-Ò!©ò÷þÌ«i6wXtü'k®ç`)A›äûÌtI1Tî£U+Ú~×’aÒÉПONƒÄèLþI£p@œT)¤'_ÞoDö0»,JfGVó­k’/
+sa3fA4GglovY8H6jimpTvQPW/axun8WADPlIXzpX/Zeshkzem+pQoQqptzDlnmH8
+8AngqXgFYrmHgNNAylavgcrxbjNrtlJU24ldF1YIubz7VsU1678F27LCd9B0c2dn
+X+0CccH19lM8Q+zVI2Wrq9R83MEP/5uOOc+eXXnvNSGqfKgZ2OplG/HUllFS13j6
+uiQy5zwJJKkII7KUThcGteux7NONoLeUqRE8CW2uSeY9fXBWKgxeENKgiT7PEAAo
+nvwWa+GatEYf6eUz8Lph8lETorgP+7JS2VQRAkmhDbjQLTYzfFmiJGE/mzyobslf
+ZEq6Oj5UNgnzdWmK5ZYKPg
+-> ssh-ed25519 /vwQcQ 9EG/cydlzlLd6cFed7DzmwzubzJUXvD9mX3WKDyFD1s
+3Emj+tVZmnsC/YZdChvyaxeObbBsri347vZl0ff9kH4
+-> ssh-ed25519 0R97PA kcIYyWKxpJmjcrel+YodZQiR2zGPqfjzMyJXsz2XOzM
+SUlgGGs2BVRzTHT/ULNo1AiN5SY1BETFtJRY6LDr4JI
+--- l87sO6IuwSeCeQ8ktvYFI0xr4Utcl8KfpAV7WePc1y4
+÷ÚÖ
~÷J3¦Œ§Í‘1íÇè²ù<º?%×ý<0E>›Á‡ÉÒ\—\Ï7\»Ú¨åU-&W'd”{իɼ½u	"Û#Ž}¯õ…x–ìšz¥®Nj„!éfUqDG‘<é<7F>Ñca<63>‚'´+	¸Ï±]Pó»ö€D
Õ¸’´þ<C2B4>£¼­ŽÿÂçÌ\¦<>

secrets/loki-environment.age

@@ -1,21 +1,20 @@
 age-encryption.org/v1
--> ssh-ed25519 j2r2qQ 6qyr94uky6B36UOY0jd5NXgF2rJ3RWBUzZ32c5iOTmY
-fjlI3fjYjwyNQBs4K4pq/5c7oBkf5XUXoGlBOBpmPu4
--> ssh-ed25519 K3b7BA N9VYT/ZslG07KldzO8sPE5TiYYwxJqpYU87ED4PuBXw
-P1s9L57prPqM4fjcYHv+g0rgP/NvFr13CgCxthVHZ4c
--> ssh-ed25519 +qVung Ry8uUFsmYmP+Urw46lhAsCc3S+QiWu1mn8J3rIy+KFQ
-iB7xAfdpHwOzAnLvosJb+F50QKsOYWr7CHC3srsS6ME
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
 -> ssh-rsa krWCLQ
-w0xIVFtUghdAO7SxZD10rBMtdQESEvYUEKxnWzLh0cjcRhaVT/BXSZQsKV2Rupoo
-nDL5uy0k+tPXm0HroZ6VkZ0fH/lOpeUR69ZvJmClKql3Fnf1385+5BvT719cbbaq
-yll49gx0+ms/oB9jS3SPwbOg+UJgnkZCeu9138h3MG7yWNtVuA9l5hsJioVvOVlS
-Z5EXbjdQR9xYjSwR+b8MYZ97ej5fXpuULEopbx2wXt84u1e67vTETqflitR7lrzy
-A6F65g35aagPJZGHzfrKVToy3pfXm9ky/30DolWLD0DpG7G6o/8afy8O4yBAGlv3
-ZLTaUbrdILSz2ff1Njx4Nw
--> ssh-ed25519 /vwQcQ YqqmX/f4whOk97kCgSPo6oj/274eYlBWtS+OahAAQ34
-hoCbhupzSTx+wNIorzYGHyGvU/L8unKEyD7Bqq23YP0
--> ssh-ed25519 0R97PA 17SDtfT9GzAsIsQB24AmYXpW8v4+LEakup+tdFroHTk
-HIvBhAGA2GMVWFBP3OTFEn+XpPFBJDOJDK3SQ94mNKM
---- CD1QrxYGAhhy+l7U5kOXn1shCwz8pYJNuGRugPxmzJw
-ñY¾Æ‹N	Ï<>x
™êÿrR^z[¤ã¸è…•ªa”z
-óæÔÉ¿Ïžu0c¯c;y<>Ÿ¢›&{ñèxA]‚þ†¨Q¨¼_:̱ í€öUoiDl (‹Å
ëwÝKi,j.oFyÌ°$}•Y§@1”È™„Y£²è¶u Ò*¡ÏþÅ<C3BE>¥™0…
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>
+<EFBFBD>

secrets/newsletter-secrets.age

@@ -0,0 +1,21 @@
+age-encryption.org/v1
+-> ssh-ed25519 CyxfgQ fWH/o2+Uf0i/JFIuVjCnkhDIfYndtL8EeDcxSxhKVH8
+ShSPmdwnxzDuUe/kCx8e61JJAoHMwguNydn+5OIGuAg
+-> ssh-ed25519 K3b7BA p+wXAGvPqTX63dlZNCTIq3F4QFMWEJH1R+Ex4SJ5UTk
+1sFqFqnUM8YvZy7BEBArg3eLxCCsLXq2jNI7XLKq/Ww
+-> ssh-ed25519 +qVung rcpgzVQ1PmoNF2i0K0nAknzZwPXICBggzqhIZwO+8xY
+9rjsTwLm5u1GOJmnJYriXXAY1unG7y+WJ4G2ltxX34U
+-> ssh-rsa krWCLQ
+seXsQjs62kxn/agyKda2l19PI4xzDl1gM7rEnaEBV8UNLOPNxh41HTnP2etgDXSc
+4eyS3ntHXIOHmN4+JBn+Q/wuhzMGQmAcoFWbjqVVPOrpPYjgCG7q/iUD8kULxLB9
+UpF0gLsg1TnvrkTwlpxr8rP/PM+ZgyQAA84S96j9TW0coyTUoH/ZX1wWGtS4aalm
+aTrOMZGScZu7onTg+tYvR+aBKlFL28h08I5nqbA39srnCNuU68+OUhLgLUfiTscl
+umwNh/C4BP2Tmc6gxQiY8o3tGqGBssGH5+WqKzbK151vJjq80RKAS1HCaSSfmxkP
+vWkXWN3NQkJyqCBpuPYilg
+-> ssh-ed25519 /vwQcQ eUH0B+cCoUubIKbG+bA25kRj0TnZabB6t8jVK04NrFs
+ovkI0C4W5CJXMZIZdpaTtQNc+TGkQ3Yq87Dei3BMUsA
+-> ssh-ed25519 0R97PA u/I45pxH3Bnja/Jw/6IukINRuC0e1IKu8UVygVgIomc
+xyHuiHf1/nJirnhXbGHJnextGQa95tDo/RPRRnDCkIg
+--- LGqO4Bsa8bofD1W5YrQp75SlGLNg1XaFZ0rPUuvLPTo
+Êçã ‹ÜmlW£{@I3…*¹ŒÇ™@ÞªL7Wª
¤ÝŒY
+n
õö~Tb\V‘•ÜvPÙpPô
ïoÌS"ôm/Ûµ/bÝp’Äžêq¸£¦šeDj6–ÆþTì)

secrets/tempo-environment.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 j2r2qQ kbi4mciOrjd7/X86xfmkDaMZhvZakoSJ6qjqLF3ljkE
+Q2BsgMLJ8AmjhnggRi+wkICj18NCA2HW1t8clemReUw
+-> ssh-ed25519 K3b7BA wNGmX9S9bJgd2JDte9QoNDfyycgmq4JMu2bc5nyYYik
+uUiutxAI3nI0M51W97aPRVE/l4dV2PEjph8eWOMLHIE
+-> ssh-ed25519 +qVung raYJ5vwMP9JopSdfa+ofkLY/gc0zcW4wTNBFTca+MXw
+sa/rWGSYrI4y6rn4JSboldWKUGvx6HbtsYo78AFOkBo
+-> ssh-rsa krWCLQ
+FLq8NwkiGw2gXptVVY393f0p9hFom57xHWPxtAlzOcRT8gvWu/uwgV+0raOcOcJa
+xxr5Sib+2D3UnUhprVPmH5Os9bI2seFAiej1MVVWLqvMtQHLFwnrzZTyZpxsXpQq
+5qQhNEADuQc4uD/ELVjGHKt6nF1Cl/GbgNLIOF/ITZ0pm1O1MjtT6MYJhQJhc6sb
+sno/wQyTXjj7rC06nyLX/rgOWrJSOeaz9eVp0A8k8/I0TXu/vRCW9gqWtv2m8sbh
+1uUHIm0l8f3z+zrL6OlZnpMFw4jpiiGoCYKPzD17I0onDYIjtdVS5iO9BsckxV/a
+wQWbyONUwbGCfeNSVAzZbg
+-> ssh-ed25519 /vwQcQ jwf7fwy4wKz7q761DNu8SyFHGgFlwq4P/Pn44Nido3E
+1q/jvt/vtD4ziY3eCDqk1XwMPpNUd80POTV2VVsumCE
+-> ssh-ed25519 0R97PA XeuziQ+wsoh0KSHXk5Qkl1kQOsAu1Ax1zTg13+XWd3M
+B1KHKm3tx/EsnE6hY+w7ya1ilhYiUs9AbwARHNkJi90
+--- JgQA6gCYZu8xcbXEl9VypccEIBO6uAJIdhBefr4doRQ
+V3ZðõÚ<EFBFBD>ç-·Ý.ê«sòÀ³3 ÎiS‰a5#¿Ð{åÔÈ®Dý˜YêNèãëù«ýoL+ÔÝ#–M<sws P»¢+í¢Ó‰ïBDoÊξÆÏuFí”Ç^Â¥•<C2A5>—ÝG@ÍM×ÛãÐØìq¦ºG^Qb s<
;ÂÒnC+ÖÊxª_­Úì]S<16>Ð

services/baremetal-builder/assignments.nix

@@ -20,7 +20,7 @@ in builtins.listToAttrs (
   }; }
   ++
   # These are not currently used for hydra
-  genBuilders { offset = 10; count = 2; f = name: {
+  genBuilders { offset = 10; count = 1; f = name: {
     cores = 8;
     max-jobs = 8;
     supported-features = [ "kvm" "nixos-test" "big-parallel" ];

services/baremetal-builder/default.nix

@@ -164,8 +164,8 @@ in
       script = ''
         while : ; do
           percent_filled=$(($(stat -f --format="100-(100*%a/%b)" /mnt)))
-          if [ "$percent_filled" -gt "54" ]; then
-            ${config.nix.package.out}/bin/nix-store --gc --max-freed 50G --store /mnt
+          if [ "$percent_filled" -gt "85" ]; then
+            ${config.nix.package.out}/bin/nix-store --gc --max-freed 80G --store /mnt
           else
             break
           fi

services/channel-scripts/default.nix

@@ -0,0 +1,192 @@
+{ lib, config, pkgs, ... }:
+# FIXME(raito): I'm really really really not happy with this design of NixOS module, clean up all of this someday.
+let
+  inherit (lib) mkEnableOption mkOption types mkIf mapAttrsToList mkPackageOption concatStringsSep mkMerge;
+  cfg = config.bagel.nixpkgs.channel-scripts;
+  toml = pkgs.formats.toml { };
+  configFile = toml.generate "forkos.toml" cfg.settings;
+  orderLib = import ./service-order.nix { inherit lib; };
+  makeUpdateJob = channelName: mainJob: {
+    name = "update-${channelName}";
+    value = {
+      description = "Update channel ${channelName}";
+      path = with pkgs; [ git ];
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = false;
+        User = "channel-scripts";
+        DynamicUser = true;
+        StateDirectory = "channel-scripts";
+        MemoryHigh = "80%";
+        EnvironmentFile = [
+          cfg.releaseBucketCredentialsFile
+        ];
+        Environment = cfg.extraEnvironment;
+        # TODO: we should have our own secret for this.
+        LoadCredential = [ "password:${config.age.secrets.alloy-push-password.path}" ];
+      };
+      unitConfig.After = [ "networking.target" ];
+      script =
+      ''
+        # A stateful copy of nixpkgs
+        dir=/var/lib/channel-scripts/nixpkgs
+        if ! [[ -e $dir ]]; then
+          git clone --bare ${cfg.nixpkgsUrl} $dir
+        fi
+        GIT_DIR=$dir git config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'
+
+        CREDENTIAL=$(echo -en "promtail:$(cat $CREDENTIALS_DIRECTORY/password)" | base64)
+        export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Basic $CREDENTIAL"
+        # TODO: use escapeShellArgs
+        exec ${cfg.package}/bin/mirror-forkos -c ${configFile} ${concatStringsSep " " cfg.extraArgs} apply ${channelName} ${mainJob}
+      '';
+    };
+  };
+  updateJobs = orderLib.mkOrderedChain (mapAttrsToList (n: { job, ... }: makeUpdateJob n job) cfg.channels);
+  channelOpts = { ... }: {
+    options = {
+      job = mkOption {
+        type = types.str;
+        example = "nixos/trunk-combined/tested";
+      };
+
+      variant = mkOption {
+        type = types.enum [ "primary" "small" "darwin" "aarch64" ];
+        example = "primary";
+      };
+
+      status = mkOption {
+        type = types.enum [ "beta" "stable" "deprecated" "unmaintained" "rolling" ];
+        example = "rolling";
+      };
+    };
+  };
+in
+{
+  options.bagel.nixpkgs.channel-scripts = {
+    enable = mkEnableOption ''the channel scripts.
+      Fast forwarding channel branches which are read-only except for this privileged bot
+      based on our Hydra acceptance tests.
+    '';
+
+    otlp.enable = mkEnableOption "the OTLP export process";
+
+    s3 = {
+      release = mkOption {
+        type = types.str;
+      };
+
+      channel = mkOption {
+        type = types.str;
+      };
+    };
+
+    package = mkPackageOption pkgs "mirror-forkos" { };
+
+    settings = mkOption {
+      type = types.attrsOf types.anything;
+    };
+
+    nixpkgsUrl = mkOption {
+      type = types.str;
+      default = "https://cl.forkos.org/nixpkgs.git";
+      description = "URL to the nixpkgs repository to clone and to push to";
+    };
+
+    binaryCacheUrl = mkOption {
+      type = types.str;
+      default = "https://cache.forkos.org";
+      description = "URL to the binary cache";
+    };
+
+    baseUriForGitRevisions = mkOption {
+      type = types.str;
+      description = "Base URI to generate link to a certain revision";
+    };
+
+    extraArgs = mkOption {
+      type = types.listOf types.str;
+      default = [ ];
+      description = "Extra arguments passed to the mirroring program";
+    };
+
+    releaseBucketCredentialsFile = mkOption {
+      type = types.path;
+      description = ''Path to the release bucket credentials file exporting S3-style environment variables.
+        For example, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` for the S3 operations to work.
+      '';
+    };
+
+    deployKeyFile = mkOption {
+      type = types.path;
+      description = ''Path to the private SSH key which is allowed to deploy things to the protected channel references on the Git repository.
+      '';
+    };
+
+    hydraUrl = mkOption {
+      type = types.str;
+      default = "https://hydra.forkos.org";
+      description = "URL to the Hydra instance";
+    };
+
+    channels = mkOption {
+      type = types.attrsOf (types.submodule channelOpts);
+      description = "List of channels to mirror";
+    };
+
+    extraEnvironment = mkOption {
+      type = types.listOf types.str;
+    };
+  };
+
+  config = mkIf cfg.enable {
+    bagel.nixpkgs.channel-scripts.extraEnvironment = mkMerge [
+      ([
+        "RUST_LOG=info"
+      ])
+      (mkIf cfg.otlp.enable [
+        ''OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="https://tempo.forkos.org/v1/traces"''
+      ])
+    ];
+    bagel.nixpkgs.channel-scripts.settings = {
+      hydra_uri = cfg.hydraUrl;
+      binary_cache_uri = cfg.binaryCacheUrl;
+      base_git_uri_for_revision = cfg.baseUriForGitRevisions;
+      nixpkgs_dir = "/var/lib/channel-scripts/nixpkgs";
+      s3_release_bucket_name = cfg.s3.release;
+      s3_channel_bucket_name = cfg.s3.channel;
+    };
+
+    users.users.channel-scripts = {
+      description = "Channel scripts user";
+      isSystemUser = true;
+      group = "channel-scripts";
+    };
+    users.groups.channel-scripts = {};
+
+    systemd.services = (lib.listToAttrs updateJobs) // {
+      "update-all-channels" = {
+        description = "Start all channel updates.";
+        unitConfig = {
+          After = map
+            (service: "${service.name}.service")
+            updateJobs;
+          Wants = map
+            (service: "${service.name}.service")
+            updateJobs;
+        };
+        script = "true";
+      };
+    };
+
+    systemd.timers."update-all-channels" = {
+      description = "Start all channel updates.";
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnUnitInactiveSec = 600;
+        OnBootSec = 900;
+        AccuracySec = 300;
+      };
+    };
+  };
+}

services/channel-scripts/service-order.nix

@@ -0,0 +1,63 @@
+# Vendored from https://raw.githubusercontent.com/NixOS/infra/master/lib/service-order.nix
+# TODO: get rid of me?
+# Ordering Services
+#
+# Given a set of services, make them run one at a time in a specific
+# order, on a timer.
+{ lib }:
+{
+  # Given a list of systemd service, give each one an After
+  # attribute, so they start in a specific order. The returned
+  # list can be converted in to a systemd.services attrset with
+  # `lib.listToAttrs`.
+  #
+  # Example:
+  #
+  #  mkOrderedChain [
+  #    { name = "foo"; value = { script = "true"; }; }
+  #    { name = "bar"; value = { script = "true"; }; }
+  #  ]
+  #
+  # => [
+  #  {
+  #    name = "foo";
+  #    value = {
+  #      script = "true";
+  #      unitConfig = { After = []; };
+  #    };
+  #  }
+  #  {
+  #    name = "bar";
+  #    value = {
+  #      script = "true";
+  #      unitConfig = { After = [ "bar" ]; };
+  #    };
+  #  }
+  #
+  mkOrderedChain = jobs: let
+    unitConfigFrom = job: job.unitConfig or {};
+    afterFrom = job: (unitConfigFrom job).After or [];
+    previousFrom = collector:
+      if collector ? previous
+      then [collector.previous]
+      else [];
+
+      ordered = builtins.foldl'
+        (collector: item: {
+          services = collector.services
+          ++ [{
+            inherit (item) name;
+            value = item.value // {
+              unitConfig = (unitConfigFrom item.value) //
+              {
+                After = (afterFrom item.value) ++
+                  (previousFrom collector);
+              };
+            };
+          }];
+          previous = "${item.name}.service";
+        })
+        { services = []; }
+        jobs;
+  in ordered.services;
+}

services/default.nix

@@ -1,6 +1,7 @@
 {
   imports = [
     ./gerrit
+    ./channel-scripts
     ./hydra
     ./matrix
     ./monitoring
@@ -10,5 +11,7 @@
     ./forgejo
     ./baremetal-builder
     ./buildbot
+    ./newsletter
+    ./s3-revproxy
   ];
 }

services/gerrit/default.nix

@@ -12,6 +12,10 @@ in
 {
   options.bagel.services.gerrit = {
     enable = mkEnableOption "Gerrit";
+    pyroscope.enable = mkEnableOption ''Pyroscope client,
+      this will send profiling of all Java processes on the current host
+      to our Pyroscope instance.
+    '';
     domains = mkOption {
       type = types.listOf types.str;
       description = "List of domains that Gerrit will answer to";
@@ -41,8 +45,13 @@ in
 
   config = mkIf cfg.enable {
     networking.firewall.allowedTCPPorts = [ cfg.port ];
+    age.secrets.alloy-push-password.file = ../../secrets/metrics-push-password.age;
 
-    environment.systemPackages = [ jdk ];
+    environment.systemPackages = [ jdk
+      pkgs.git
+      # https://gerrit.googlesource.com/gerrit/+/refs/heads/master/contrib/git-gc-preserve
+      pkgs.git-gc-preserve
+    ];
 
     fileSystems."/var/lib/gerrit" = mkIf (cfg.data != "/var/lib/gerrit") {
       device = cfg.data;
@@ -55,6 +64,64 @@ in
     };
     users.groups.git = {};
 
+    services.alloy = {
+      enable = cfg.pyroscope.enable;
+      extraFlags = [
+        # Debugging interface.
+        "--server.http.listen-addr=127.0.0.1:15555"
+      ];
+    };
+    systemd.services.alloy.serviceConfig = {
+      User = lib.mkForce "root";
+      Group = lib.mkForce "root";
+      DynamicUser = lib.mkForce false;
+    };
+    systemd.services.alloy.serviceConfig.LoadCredential = [ "password:${config.age.secrets.alloy-push-password.path}" ];
+    environment.etc."alloy/config.alloy".text = ''
+      pyroscope.write "production" {
+        endpoint {
+          url = "https://pyroscope.forkos.org"
+          basic_auth {
+            username = "promtail"
+            password_file = "/run/credentials/alloy.service/password"
+          }
+        }
+      }
+
+      discovery.process "all" {
+        refresh_interval = "60s"
+        discover_config {
+          cwd = true
+          exe = true
+          commandline = true
+          username = true
+          uid = true
+          container_id = true
+        }
+      }
+
+      discovery.relabel "java" {
+        targets = discovery.process.all.targets
+        rule {
+          action = "keep"
+          regex = ".*/java$"
+          source_labels = ["__meta_process_exe"]
+        }
+      }
+
+      pyroscope.java "java" {
+        targets = discovery.relabel.java.output
+        forward_to = [pyroscope.write.production.receiver]
+        profiling_config {
+          interval = "60s"
+          alloc = "512k"
+          cpu = true
+          sample_rate = 100
+          lock = "1ms"
+        }
+      }
+    '';
+
     services.gerrit = {
       enable = true;
       listenAddress = "[::]:4778"; # 4778 - grrt
@@ -111,14 +178,16 @@ in
 
         receive.timeout = "4min";
         # Default is 0, infinite.
-        transfer.timeout = "30min";
+        # When system is under pressure and there's too many packfiles
+        # the search for reuse can take a stupid amount of time.
+        transfer.timeout = "60min";
 
         # We may overshoot but it's OK.
         core.packedGitWindowSize = "256k";
         # Sum of all current packfiles is ~1.2G
         # Largest packfile is 906MB.
         # Average packfile is ~5-10MB.
-        core.packedGitLimit = "1g";
+        core.packedGitLimit = "2g";
         # We have plenty of memory, let's avoid file system cache → Gerrit needless copies.
         core.packedGitUseStrongRefs = true;
         core.packedGitOpenFiles = 4096;

services/hydra/default.nix

@@ -9,7 +9,11 @@ let
 
   mkCacheSettings = settings: builtins.concatStringsSep "&" (
     lib.mapAttrsToList (k: v: "${k}=${v}") settings
-    );
+  );
+
+  mkPgConnString = options: builtins.concatStringsSep ";" (
+    lib.mapAttrsToList (k: v: "${k}=${v}") options
+  );
 
   # XXX: to support Nix's dumb public host key syntax (base64'd), this outputs
   # a string with shell-style command interpolations: $(...).
@@ -50,11 +54,6 @@ in {
   options.bagel.services.hydra = with lib; {
     enable = mkEnableOption "Hydra coordinator";
 
-    dbi = mkOption {
-      type = types.str;
-      description = "DBI connection string for the Hydra postgres database";
-    };
-
     builders = mkOption {
       type = types.listOf types.str;
       description = "List of builders to configure for Hydra";
@@ -69,6 +68,10 @@ in {
 
     age.secrets.hydra-s3-credentials.file = ../../secrets/hydra-s3-credentials.age;
 
+    age.secrets.hydra-postgres-key.group = "hydra";
+    age.secrets.hydra-postgres-key.mode = "0440";
+    age.secrets.hydra-postgres-key.file = ../../secrets/hydra-postgres-key.age;
+
     age.secrets.hydra-signing-priv.owner = "hydra-queue-runner";
     age.secrets.hydra-signing-priv.file = ../../secrets/hydra-signing-priv.age;
 
@@ -97,9 +100,18 @@ in {
     services.hydra-dev = {
       enable = true;
 
-      listenHost = "localhost";
+      listenHost = "127.0.0.1";
       port = port;
-      dbi = cfg.dbi;
+
+      dbi = "dbi:Pg:${mkPgConnString {
+        host = "postgres.forkos.org";
+        dbname = "hydra";
+        user = "hydra";
+        sslmode = "verify-full";
+        sslcert = "${./postgres.crt}";
+        sslkey = config.age.secrets.hydra-postgres-key.path;
+        sslrootcert = "${../postgres/ca.crt}";
+      }}";
 
       hydraURL = "https://hydra.forkos.org";
       useSubstitutes = false;
@@ -134,7 +146,7 @@ in {
 
         server_store_uri = https://bagel-cache.s3-web.delroth.net?local-nar-cache=${narCacheDir}
         binary_cache_public_url = https://bagel-cache.s3-web.delroth.net
-        log_prefix = https://bagel-cache.s3-web.delroth.net
+        log_prefix = https://bagel-cache.s3-web.delroth.net/
 
         upload_logs_to_binary_cache = true
 
@@ -146,6 +158,10 @@ in {
 
         max_output_size = ${builtins.toString (3 * 1024 * 1024 * 1024)}
         max_db_connections = 100
+
+        <git-input>
+          timeout = 1800
+        </git-input>
       '';
     };
 

services/hydra/postgres.crt

@@ -0,0 +1,12 @@
+-----BEGIN CERTIFICATE-----
+MIIBtDCCAVugAwIBAgIQTU55o4gtG8EFZArwsuj4NjAKBggqhkjOPQQDAjAiMSAw
+HgYDVQQDExdGb3JrT1MgUG9zdGdyZXMgUm9vdCBDQTAeFw0yNDA4MTYwNTU0MTda
+Fw0zNDA4MTYxNzU0MTdaMBAxDjAMBgNVBAMTBWh5ZHJhMFkwEwYHKoZIzj0CAQYI
+KoZIzj0DAQcDQgAEnTgiFZOXBrcPlWDxJPXUFgxIi7/T7LmwLtpGPK/G6R8KA9cS
+4UXF5Ifz2dCgozTlhqLROKb81yhNsSy1tOcFyKOBhDCBgTAOBgNVHQ8BAf8EBAMC
+B4AwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMB0GA1UdDgQWBBQGh6HM
+jl41qw/F0vpBdmOWQ2IfGzAfBgNVHSMEGDAWgBQy4WX/hUExQ/i1h7MvF6Ow2irN
+izAQBgNVHREECTAHggVoeWRyYTAKBggqhkjOPQQDAgNHADBEAiAEypqfyMOGbEJv
+dKI1tyj890uq5Osr5+9wxGBvJDMJNwIgefyOdFcvJTzbfHgLmORpBOVtnpbkwj5y
+rMnjT8gYjEA=
+-----END CERTIFICATE-----

services/monitoring/default.nix

@@ -4,5 +4,6 @@
     ./lgtm
     ./agent.nix
     ./hookshot-adapter
+    ./pyroscope
   ];
-}
+}

services/monitoring/lgtm/default.nix

@@ -3,5 +3,6 @@
     ./grafana.nix
     ./loki.nix
     ./mimir.nix
+    ./tempo.nix
   ];
-}
+}

services/monitoring/lgtm/grafana.nix

@@ -1,11 +1,16 @@
 {
   config,
   lib,
+  pkgs,
+  inputs,
   ...
 }:
 let
   cfg = config.bagel.services.grafana;
   inherit (lib) mkEnableOption mkIf;
+  generatedJsonnetDashboards = (pkgs.callPackage ../../../dashboards {
+    inherit (inputs) gerrit-dashboard;
+  }).allDashboards;
 in
 {
   options.bagel.services.grafana.enable = mkEnableOption "Grafana frontend";
@@ -16,8 +21,6 @@ in
       owner = "grafana";
     };
 
-    bagel.services.postgres.enable = true;
-
     services = {
       grafana = {
         enable = true;
@@ -86,6 +89,10 @@ in
                 name = "default";
                 options.path = ./dashboards;
               }
+              {
+                name = "jsonnet";
+                options.path = generatedJsonnetDashboards;
+              }
             ];
           };
 
@@ -118,6 +125,21 @@ in
                   implementation = "mimir";
                 };
               }
+              {
+                name = "Pyroscope";
+                type = "grafana-pyroscope-datasource";
+                uid = "pyroscope";
+                access = "proxy";
+                url = "http://127.0.0.1:4040";
+              }
+              {
+                name = "Tempo";
+                type = "tempo";
+                uid = "tempo";
+                access = "proxy";
+                url = "http://127.0.0.1:9190";
+                jsonData.streamingEnabled.search = true;
+              }
             ];
           };
         };
@@ -147,7 +169,7 @@ in
         };
       };
     };
-    
+
     bagel.monitoring.grafana-agent.exporters.grafana.port = 2342;
   };
 }

services/monitoring/lgtm/tempo.nix

@@ -0,0 +1,79 @@
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.bagel.services.tempo;
+  inherit (lib) mkEnableOption mkIf;
+in
+{
+  options.bagel.services.tempo.enable = mkEnableOption "Tempo trace store";
+
+  config = mkIf cfg.enable {
+    age.secrets = {
+      metrics-push-htpasswd = {
+        file = ../../../secrets/metrics-push-htpasswd.age;
+        owner = "nginx";
+      };
+      tempo-environment.file = ../../../secrets/tempo-environment.age;
+    };
+
+    services.tempo = {
+      enable = true;
+      extraFlags = ["--config.expand-env=true"];
+      settings = {
+        multitenancy_enabled = false;
+        stream_over_http_enabled = true;
+
+        server = {
+          http_listen_port = 9190;
+          grpc_listen_port = 9195;
+        };
+        distributor.receivers.otlp.protocols.http.endpoint = "127.0.0.1:4138";
+
+        storage.trace = {
+          backend = "s3";
+          s3 = {
+            endpoint = "s3.delroth.net";
+            bucket = "bagel-tempo";
+            secret_key = "\${S3_KEY}"; # This is a secret injected via an environment variable
+            access_key = "\${S3_KEY_ID}";
+          };
+          wal.path = "/var/lib/tempo/traces-wal";
+        };
+
+        metrics_generator.storage = {
+          path = "/var/lib/tempo/metrics-wal";
+          remote_write = [
+            {
+              url = "http://127.0.0.1:9009/api/v1/push";
+            }
+          ];
+        };
+
+        overrides.defaults.metrics_generator.processors = [ "span-metrics" ];
+      };
+    };
+
+    systemd.services.tempo.serviceConfig.EnvironmentFile = [ config.age.secrets.tempo-environment.path ];
+
+    services.nginx = {
+      upstreams.tempo = {
+        servers."${config.services.tempo.settings.distributor.receivers.otlp.protocols.http.endpoint}" = {};
+        extraConfig = "keepalive 16;";
+      };
+
+      virtualHosts."tempo.forkos.org" = {
+        enableACME = true;
+        forceSSL = true;
+        locations."/" = {
+          proxyPass = "http://tempo";
+          basicAuthFile = config.age.secrets.metrics-push-htpasswd.path;
+        };
+      };
+    };
+
+    bagel.monitoring.grafana-agent.exporters.tempo.port = 9190;
+  };
+}

services/monitoring/pyroscope/default.nix

@@ -0,0 +1,74 @@
+{ lib, config, ... }:
+let
+  inherit (lib) mkEnableOption mkIf;
+  cfg = config.bagel.services.pyroscope;
+  pyroscopePort = config.services.pyroscope.settings.server.http_listen_port;
+in
+{
+  options.bagel.services.pyroscope = {
+    enable = mkEnableOption "pyroscope server";
+  };
+
+  # TODO: send me to nixpkgs
+  imports = [
+    ./module.nix
+  ];
+
+  config = mkIf cfg.enable {
+    age.secrets.pyroscope-secrets.file = ../../../secrets/pyroscope-secrets.age;
+    services.nginx = {
+      upstreams.pyroscope = {
+        servers."127.0.0.1:${toString pyroscopePort}" = {};
+        extraConfig = "keepalive 16;";
+      };
+
+      virtualHosts."pyroscope.forkos.org" = {
+        enableACME = true;
+        forceSSL = true;
+        locations."/ingest" = {
+          proxyPass = "http://pyroscope";
+          basicAuthFile = config.age.secrets.metrics-push-htpasswd.path;
+        };
+        locations."/push.v1.PusherService/Push" = {
+          proxyPass = "http://pyroscope";
+          basicAuthFile = config.age.secrets.metrics-push-htpasswd.path;
+        };
+      };
+    };
+    services.pyroscope = {
+      enable = true;
+      secretFile = config.age.secrets.pyroscope-secrets.path;
+      settings = {
+        target = "all";
+        multitenancy_enabled = false;
+
+        api.base-url = "https://pyroscope.forkos.org";
+        analytics.reporting_enabled = false;
+
+        storage = {
+          backend = "s3";
+          s3 = {
+            endpoint = "s3.delroth.net";
+            region = "garage";
+            bucket_name = "bagel-pyroscope";
+            access_key_id = "\${S3_KEY_ID}";
+            secret_access_key = "\${S3_KEY}";
+            force_path_style = true;
+          };
+        };
+        server = {
+          http_listen_port = 4040;
+          grpc_listen_port = 9097;
+          grpc_server_max_recv_msg_size = 104857600;
+          grpc_server_max_send_msg_size = 104857600;
+          grpc_server_max_concurrent_streams = 1000;
+        };
+
+        memberlist = {
+          advertise_port = 7948;
+          bind_port = 7948;
+        };
+      };
+    };
+  };
+}

services/monitoring/pyroscope/module.nix

@@ -0,0 +1,42 @@
+{ pkgs, lib, config, ... }:
+let
+  inherit (lib) mkEnableOption mkPackageOption mkOption types mkIf;
+  settingsFormatYaml = pkgs.formats.yaml { };
+  cfg = config.services.pyroscope;
+  configFile = settingsFormatYaml.generate "settings.yaml" cfg.settings;
+in
+{
+  options.services.pyroscope = {
+    enable = mkEnableOption "pyroscope, a continuous profiling platform";
+    package = mkPackageOption pkgs "pyroscope" { };
+    secretFile = mkOption {
+      type = types.path;
+    };
+    settings = mkOption {
+      description = "Pyroscope settings. See <>";
+
+      type = types.submodule {
+        freeformType = settingsFormatYaml.type;
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    systemd.services.pyroscope = {
+      description = "Pyroscope server - a continuous profiling platform";
+      wantedBy = [ "multi-user.target" ];
+      wants = [ "network-online.target" ];
+      after = [ "network-online.target" ];
+      serviceConfig = {
+        ExecStart = "${cfg.package}/bin/pyroscope -config.file ${configFile} -config.expand-env";
+        WorkingDirectory = "/var/lib/pyroscope";
+        User = "pyroscope";
+        DynamicUser = true;
+        Restart = "on-failure";
+        RuntimeDirectory = "pyroscope";
+        StateDirectory = "pyroscope";
+        EnvironmentFile = [ cfg.secretFile ];
+      };
+    };
+  };
+}

services/newsletter/default.nix

@@ -0,0 +1,43 @@
+{ config, lib, ... }:
+let
+  cfg = config.bagel.newsletter;
+  inherit (lib) mkIf mkOption mkEnableOption types;
+  port = 18999;
+  address = "127.0.0.1:${toString port}";
+in
+{
+  options.bagel.newsletter = {
+    enable = mkEnableOption "the newsletter web service (listmonk)";
+    domain = mkOption {
+      type = types.str;
+    };
+  };
+
+  config = mkIf cfg.enable {
+    age.secrets.newsletter-secrets.file = ../../secrets/newsletter-secrets.age;
+    services.listmonk = {
+      enable = true;
+      secretFile = config.age.secrets.newsletter-secrets.path;
+      settings."app" = {
+        inherit address;
+        admin_username = "admin";
+      };
+      database.createLocally = true;
+    };
+
+    services.nginx.enable = true;
+    services.nginx.virtualHosts."${cfg.domain}" = {
+      enableACME = true;
+      forceSSL = true;
+      locations."/".proxyPass = "http://${address}";
+    };
+
+    users.users.listmonk = {
+      isSystemUser = true;
+      group = "listmonk";
+    };
+    users.groups.listmonk = {};
+
+    networking.firewall.allowedTCPPorts = [ 80 443 ];
+  };
+}

services/postgres/ca.crt

@@ -0,0 +1,11 @@
+-----BEGIN CERTIFICATE-----
+MIIBiDCCAS6gAwIBAgIRAIX2H5zZS7SY7/uXfA59emswCgYIKoZIzj0EAwIwIjEg
+MB4GA1UEAxMXRm9ya09TIFBvc3RncmVzIFJvb3QgQ0EwHhcNMjQwODE2MDUyNzIx
+WhcNMzQwODE0MDUyNzIxWjAiMSAwHgYDVQQDExdGb3JrT1MgUG9zdGdyZXMgUm9v
+dCBDQTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABP7WPT1ooa4Irv1V58Bgqg7t
+S4hXymbps7CyFMwy1gILOazDqh1YmgacofWST1gf0qm9Uo4YgKtWyCdZndWjLqmj
+RTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAGAQH/AgEBMB0GA1UdDgQW
+BBQy4WX/hUExQ/i1h7MvF6Ow2irNizAKBggqhkjOPQQDAgNIADBFAiBnpvX9a4N9
+f0pnG58f8GG/Yu91N2s0eESiPcMjzRB3vwIhAPo6YMFzNrB6IWxiUGtlOni1eY06
+iCsMoQ7B0zTfwBGW
+-----END CERTIFICATE-----

services/postgres/default.nix

@@ -10,6 +10,9 @@ in {
   };
 
   config = lib.mkIf cfg.enable {
+    age.secrets.postgresql-tls-priv.owner = "postgres";
+    age.secrets.postgresql-tls-priv.file = ../../secrets/postgres-tls-priv.age;
+
     systemd.tmpfiles.rules = [
       "d /var/db 0755 root root - -"
       "d /var/db/postgresql 0750 postgres postgres - -"
@@ -21,6 +24,8 @@ in {
       package = pkgs.postgresql_16;
       dataDir = dataDir;
 
+      enableTCPIP = true;
+
       # TODO: Where to put this to properly couple things? It doesn't belong
       # here, but using it in services/hydra would require running on
       # localhost. Probably needs to be replaced with some different way of
@@ -41,12 +46,60 @@ in {
         hydra-users postgres postgres
       '';
       authentication = ''
-        local hydra all ident map=hydra-users
+        local hydra all peer map=hydra-users
+
+        # Allow any connection over TLS with a valid client certificate: signed
+        # by our CA, and with username = cert CN.
+        hostssl all all all cert clientcert=verify-full
       '';
 
       settings = {
         max_connections = 500;
+
+        ssl = true;
+        ssl_ca_file = "${./ca.crt}";
+        ssl_cert_file = "${./server.crt}";
+        ssl_key_file = config.age.secrets.postgresql-tls-priv.path;
       };
     };
+
+    networking.firewall.allowedTCPPorts = [ config.services.postgresql.settings.port ];
+
+    # Provisioned on the server so that CA operations can be done there.
+    age.secrets.postgresql-ca-priv.owner = "postgres";
+    age.secrets.postgresql-ca-priv.file = ../../secrets/postgres-ca-priv.age;
+
+    users.users.postgres.packages = [
+      (pkgs.writeShellScriptBin "postgres-mint-new-client" ''
+        #! ${pkgs.runtimeShell}
+
+        set -eo pipefail
+
+        if [ $# -eq 0 ]; then
+          echo "usage: $0 <pg-username>" >&2
+          exit 1
+        fi
+
+        user=$1
+        step=${pkgs.lib.getExe pkgs.step-cli}
+
+        tmpdir=$(mktemp -d)
+        trap "rm -rf $tmpdir" EXIT
+
+        $step certificate create "$user" "$tmpdir/client.crt" "$tmpdir/client.key" \
+          --profile leaf \
+          --ca ${./ca.crt} \
+          --ca-key ${config.age.secrets.postgresql-ca-priv.path} \
+          --not-after 87660h \
+          --no-password --insecure
+
+        echo "Client certificate:"
+        cat "$tmpdir/client.crt"
+        echo
+
+        echo "Client private key:"
+        cat "$tmpdir/client.key"
+      '')
+    ];
   };
 }

services/postgres/server.crt

@@ -0,0 +1,12 @@
+-----BEGIN CERTIFICATE-----
+MIIB0DCCAXegAwIBAgIQTayv82V9dp0ptyMbDcN70zAKBggqhkjOPQQDAjAiMSAw
+HgYDVQQDExdGb3JrT1MgUG9zdGdyZXMgUm9vdCBDQTAeFw0yNDA4MTYwNTMwMjBa
+Fw0zNDA4MTYxNzMwMjBaMB4xHDAaBgNVBAMTE3Bvc3RncmVzLmZvcmtvcy5vcmcw
+WTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAR7St4dklWkvYbCi+3xPY5YFfBwgErz
+8TLtT5F2l5aKN0+I0sBo+ktiTNl8BzaVzXJmLa2xzRt2jQgB2R0IZgyko4GSMIGP
+MA4GA1UdDwEB/wQEAwIHgDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIw
+HQYDVR0OBBYEFCELKmGcbzVZf6Qx1cWCeXVC3XXJMB8GA1UdIwQYMBaAFDLhZf+F
+QTFD+LWHsy8Xo7DaKs2LMB4GA1UdEQQXMBWCE3Bvc3RncmVzLmZvcmtvcy5vcmcw
+CgYIKoZIzj0EAwIDRwAwRAIgJTM0nO2UaJnlGNIOJ1oT7HClNBxmH5oQu2DwVMuS
+MB8CIDbg/nrYjnRmFGGtbWvLbdvHIZ7t0A4kjkLwJ2QCr6Ni
+-----END CERTIFICATE-----

services/s3-revproxy/default.nix

@@ -0,0 +1,141 @@
+# Originally written by Jade Lovelace for Lix.
+# Adapted for ForkOS.
+{ lib, config, ... }:
+let
+  inherit (lib) mkEnableOption mkIf types mkOption mapAttrs mapAttrs' nameValuePair;
+  cfg = config.bagel.services.s3-revproxy;
+  s3RevproxyPort = 10652;
+  mkTarget = { name, bucket ? name }: {
+    mount = {
+      host = "${name}.${cfg.domain}";
+      path = [ "/" ];
+    };
+    actions.GET = {
+      enabled = true;
+      config = {
+        # e.g. /2.90 will 404, so it will redirect to /2.90/ if it is a directory
+        redirectWithTrailingSlashForNotFoundFile = true;
+        indexDocument = "index.html";
+      };
+    };
+
+    bucket = {
+      name = bucket;
+      region = "garage";
+      s3Endpoint = "https://${cfg.s3.apiUrl}";
+      credentials = {
+        accessKey.env = "AWS_ACCESS_KEY_ID";
+        secretKey.env = "AWS_SECRET_KEY";
+      };
+    };
+  };
+  # Makes a subdomain that gets proxied through s3-proxy to provide directory
+  # listings and reasonable 404 pages.
+  # This is not used on cache, since there a directory listing for cache is a
+  # liability at best.
+  mkProxiedSubdomain = subdomain: {
+    enableACME = true;
+    forceSSL = true;
+    locations."/" = {
+      recommendedProxySettings = true;
+      proxyPass = "http://127.0.0.1:${toString s3RevproxyPort}/";
+    };
+  };
+in
+{
+  options.bagel.services.s3-revproxy = {
+    enable = mkEnableOption "a S3 reverse proxy";
+
+    domain = mkOption {
+      type = types.str;
+    };
+
+    s3 = {
+      apiUrl = mkOption {
+        type = types.str;
+        description = "An URL to the S3 API";
+      };
+    };
+
+    targets = mkOption {
+      type = types.attrsOf types.str;
+      description = ''
+        A mapping between a nice name and a bucket name.
+      '';
+    };
+  };
+
+  imports = [
+    ./module.nix
+  ];
+
+  config = mkIf cfg.enable {
+    age.secrets.s3-revproxy-api-keys.file = ../../secrets/s3-revproxy-api-keys.age;
+    # For each target, generate an entry that passes it to the s3-revproxy.
+    services.nginx.virtualHosts = mapAttrs' (subdomain: _: nameValuePair "${subdomain}.${cfg.domain}" (mkProxiedSubdomain subdomain)) cfg.targets;
+    # this solves garage supporting neither anonymous access nor automatic
+    # directory indexing by simply ignoring garage's web server and replacing it
+    # with overengineered golang instead.
+    services.s3-revproxy = {
+      enable = true;
+      settings = {
+        templates = {
+          helpers = [ ./s3-revproxy-templates/_helpers.tpl ];
+          notFoundError = {
+            headers = {
+              "Content-Type" = "{{ template \"main.headers.contentType\" . }}";
+            };
+            status = "404";
+          };
+          folderList = {
+            path = ./s3-revproxy-templates/folder-list.tpl;
+            headers = {
+              "Content-Type" = "{{ template \"main.headers.contentType\" . }}";
+            };
+            # empty s3 directories are not real and cannot hurt you.
+            # due to redirectWithTrailingSlashForNotFoundFile, garbage file names
+            # get redirected as folders, which then appear as empty, yielding
+            # poor UX.
+            status = ''
+              {{- if eq (len .Entries) 0 -}}
+                404
+              {{- else -}}
+                200
+              {{- end -}}
+            '';
+          };
+        };
+        /* For metrics and debugging (e.g. pulling the config)
+        internalServer = {
+          listenAddr = "127.0.0.1";
+          port = 1337;
+        };
+        */
+        server = {
+          listenAddr = "127.0.0.1";
+          port = s3RevproxyPort;
+
+          # it's going right into nginx, so no point
+          compress.enabled = false;
+          cors = {
+            enabled = true;
+            allowMethods = [ "GET" ];
+            allowOrigins = [ "*" ];
+          };
+
+          cache = {
+            noCacheEnabled = false;
+            # Taken from the original Perl script.
+            # maxage=600: Serve from cache for 5 minutes.
+            # stale-while-revaliadate=1800: Serve from cache while updating in the background for 30 minutes.
+            # https://web.dev/stale-while-revalidate/
+            # https://developer.fastly.com/learning/concepts/cache-freshness/
+            cacheControl = "maxage=600,stale-while-revalidate=1800,public";
+          };
+        };
+        targets = mapAttrs (name: bucket: mkTarget { inherit name bucket; }) cfg.targets;
+      };
+      environmentFile = config.age.secrets.s3-revproxy-api-keys.path;
+    };
+  };
+}

services/s3-revproxy/module.nix

@@ -0,0 +1,69 @@
+# Originally, written by Jade Lovelace for Lix.
+{ config, pkgs, lib, ... }:
+let cfg = config.services.s3-revproxy;
+  settingsGenerator = pkgs.formats.yaml {};
+  # Needs to be in a directory, so we might as well implement autoreload, why not!
+  configFile = settingsGenerator.generate "config.yaml" cfg.settings;
+
+  inherit (lib) types;
+in
+{
+  options.services.s3-revproxy = {
+    enable = lib.mkEnableOption "s3 reverse proxy";
+    package = lib.mkPackageOption pkgs "s3-revproxy" {};
+    settings = lib.mkOption {
+      default = { };
+      type = settingsGenerator.type;
+      description = ''
+        Settings to use for the service. See the documentation at https://oxyno-zeta.github.io/s3-proxy/configuration/structure/
+      '';
+    };
+
+    environmentFile = lib.mkOption {
+      type = types.nullOr types.path;
+      default = null;
+      description = ''
+        Environment file to use for s3-revproxy.
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    environment.etc."s3-revproxy/config.yaml".source = configFile;
+    systemd.services.s3-revproxy = {
+      wantedBy = [ "multi-user.target" ];
+
+      serviceConfig = {
+        ExecStart = "${lib.getExe cfg.package} --config /etc/s3-revproxy";
+
+        DynamicUser = true;
+        CapabilityBoundingSet = "";
+        NoNewPrivileges = true;
+        PrivateTmp = true;
+        PrivateUsers = true;
+        PrivateDevices = true;
+        ProtectHome = true;
+        ProtectClock = true;
+        ProtectProc = "noaccess";
+        ProcSubset = "pid";
+        UMask = "0077";
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        ProtectControlGroups = true;
+        ProtectHostname = true;
+        RestrictSUIDSGID = true;
+        RestrictRealtime = true;
+        RestrictNamespaces = true;
+        LockPersonality = true;
+        RemoveIPC = true;
+        SystemCallFilter = [ "@system-service" "~@privileged" ];
+        RestrictAddressFamilies = [ "AF_INET" "AF_INET6" ];
+        MemoryDenyWriteExecute = true;
+        SystemCallArchitectures = "native";
+
+        EnvironmentFile = lib.optionals (cfg.environmentFile != null) [ cfg.environmentFile ];
+      };
+    };
+  };
+}

services/s3-revproxy/s3-revproxy-templates/_helpers.tpl

@@ -0,0 +1,37 @@
+{{/* SPDX-License-Identifier: Apache-2.0 */}}
+{{/* SPDX-FileCopyrightText: 2024 s3-proxy contributors */}}
+
+{{- /* This function will allow to get user identifier. */ -}}
+{{- define "main.userIdentifier" -}}
+{{- if .User -}}
+{{- .User.GetIdentifier -}}
+{{- end -}}
+{{- end -}}
+
+
+{{- /* This function will allow to get the content type header from "Accept" header */ -}}
+{{- define "main.headers.contentType" -}}
+{{- if contains "application/json" (.Request.Header.Get "Accept") -}}
+application/json; charset=utf-8
+{{- else -}}
+text/html; charset=utf-8
+{{- end -}}
+{{- end -}}
+
+{{- /* This will forge the json output of an error */ -}}
+{{- define "main.body.errorJsonBody" -}}
+{"error": {{ .Error.Error | toJson }}}
+{{- end -}}
+
+{{- define "notFoundErrorBody" -}}
+  {{- if contains "application/json" (.Request.Header.Get "Accept") -}}
+  {{ template "main.body.errorJsonBody" . }}
+  {{- else -}}
+  <!DOCTYPE html>
+  <html>
+    <body>
+      <h1>Not Found {{ .Request.URL.Path }}</h1>
+    </body>
+  </html>
+  {{- end -}}
+{{- end -}}

services/s3-revproxy/s3-revproxy-templates/folder-list.tpl

@@ -0,0 +1,52 @@
+{{/* SPDX-License-Identifier: Apache-2.0 */}}
+{{/* SPDX-FileCopyrightText: 2024 s3-proxy contributors */}}
+{{- $root := . -}}
+{{- if eq (len .Entries) 0 -}}
+  {{- include "notFoundErrorBody" . -}}
+{{- else -}}
+
+{{- if contains "application/json" (.Request.Header.Get "Accept") -}}
+[
+  {{- $maxLen := len $root.Entries -}}
+  {{- range $index, $entry := $root.Entries -}}
+  {"name": {{ $entry.Name | toJson -}}
+    ,"etag": {{ $entry.ETag | toJson -}}
+    ,"type": {{ $entry.Type | toJson -}}
+    ,"size": {{ $entry.Size | toJson -}}
+    ,"path": {{ $entry.Path | toJson -}}
+    ,"lastModified": {{ $entry.LastModified | date "2006-01-02T15:04:05Z07:00" | toJson -}}
+  }{{- if ne $index (sub $maxLen 1) -}},{{- end -}}
+  {{- end -}}
+]
+{{- else -}}
+  <!DOCTYPE html>
+  <html>
+    <body>
+      <h1>Index of {{ .Request.URL.Path }}</h1>
+      <table style="width:100%">
+          <thead>
+              <tr>
+                  <th style="border-right:1px solid black;text-align:start">Entry</th>
+                  <th style="border-right:1px solid black;text-align:start">Size</th>
+                  <th style="border-right:1px solid black;text-align:start">Last modified</th>
+              </tr>
+          </thead>
+          <tbody style="border-top:1px solid black">
+            <tr>
+              <td style="border-right:1px solid black;padding: 0 5px"><a href="..">..</a></td>
+              <td style="border-right:1px solid black;padding: 0 5px"> - </td>
+              <td style="padding: 0 5px"> - </td>
+            </tr>
+          {{- range .Entries }}
+            <tr>
+                <td style="border-right:1px solid black;padding: 0 5px"><a href="{{ .Path }}">{{ .Name }}</a></td>
+                <td style="border-right:1px solid black;padding: 0 5px">{{- if eq .Type "FOLDER" -}} - {{- else -}}{{ .Size | humanSize }}{{- end -}}</td>
+                <td style="padding: 0 5px">{{ .LastModified }}</td>
+            </tr>
+          {{- end }}
+          </tbody>
+      </table>
+    </body>
+  </html>
+  {{- end -}}
+{{- end -}}

services/s3-revproxy/s3-revproxy-templates/not-found-error.tpl

@@ -0,0 +1,3 @@
+{{/* SPDX-License-Identifier: Apache-2.0 */}}
+{{/* SPDX-FileCopyrightText: 2024 s3-proxy contributors */}}
+{{- include "notFoundErrorBody" . -}}

terraform/default.nix

@@ -2,6 +2,7 @@
   imports = [
     ./common.nix
     ./gandi.nix
+    ./dnsimple.nix
     ./hydra.nix
     ./state.nix
   ];

terraform/dnsimple.nix

@@ -0,0 +1,147 @@
+{ lib, config, ... }:
+let
+  inherit (lib) mkEnableOption mkIf tf genList;
+  cfg = config.bagel.dnsimple;
+in
+{
+  options.bagel.dnsimple = {
+    enable = mkEnableOption "the DNSimple configuration";
+  };
+
+  config = mkIf cfg.enable {
+    terraform.required_providers.dnsimple = {
+      version = "~> 1.7.0";
+      source = "dnsimple/dnsimple";
+    };
+
+    resource.secret_resource.dnsimple_token.lifecycle.prevent_destroy = true;
+    resource.secret_resource.dnsimple_account.lifecycle.prevent_destroy = true;
+
+    provider.dnsimple = {
+      token = tf.ref "resource.secret_resource.dnsimple_token.value";
+      account = tf.ref "resource.secret_resource.dnsimple_account.value";
+    };
+
+    resource.dnsimple_zone.forkos_org = {
+      name = "forkos.org";
+    };
+
+    resource.dnsimple_zone.fleurixos_org = {
+      name = "fleurixos.org";
+    };
+
+    resource.dnsimple_zone.floral_systems = {
+      name = "floral.systems";
+    };
+
+    resource.dnsimple_zone.flowery_systems = {
+      name = "flowery.systems";
+    };
+
+    resource.dnsimple_zone.petalpkgs_org = {
+      name = "petalpkgs.org";
+    };
+
+    resource.dnsimple_zone.vzfdfp_de = {
+      name = "vzfdfp.de";
+    };
+
+    resource.dnsimple_zone_record = let
+      # https://registry.terraform.io/providers/dnsimple/dnsimple/latest/docs/resources/zone_record
+      canonicalName = zoneName: record: let
+        # TODO: make less fragile and have actual unique and stable names
+        normalize = builtins.replaceStrings ["." "@"] ["_" "_root_"];
+        zone = normalize zoneName;
+        name = normalize record.name;
+      in "${zone}_${record.type}_${name}";
+
+      record = name: ttl: type: value: {
+        inherit name ttl type value;
+      };
+
+      proxyRecords = name: ttl: type: value: [
+        # kurisu.lahfa.xyz running a sniproxy:
+        (record name ttl "A" "163.172.69.160")
+        (record name ttl type value)
+      ];
+
+      # Creates a extra *.p record pointing to the sniproxy
+      dualProxyRecords = name: ttl: type: value: lib.flatten [
+        (record name ttl type value)
+        (proxyRecords "${name}.p" ttl type value)
+      ];
+
+      domain = zoneName: records:
+        builtins.listToAttrs (map (record: {
+          name = canonicalName zoneName record;
+          value = record // {
+            zone_name = zoneName;
+          };
+        }
+      ) (lib.flatten records));
+      zones = domains: lib.zipAttrs (lib.mapAttrsToList (zoneName: records: domain zoneName records) domains);
+    in zones {
+        "forkos.org" = ([
+          # (record "@" 300 "A" "163.172.69.160")
+          (record "@" 300 "AAAA" "2001:bc8:38ee:100:1000::20")
+
+          (dualProxyRecords "bagel-box.infra" 300 "AAAA" "2001:bc8:38ee:100:100::1")
+          (dualProxyRecords "gerrit01.infra" 300 "AAAA" "2001:bc8:38ee:100:1000::10")
+          (dualProxyRecords "meta01.infra" 300 "AAAA" "2001:bc8:38ee:100:1000::20")
+          (dualProxyRecords "fodwatch.infra" 300 "AAAA" "2001:bc8:38ee:100:1000::30")
+          # git.infra.forkos.org exposes opensshd
+          (dualProxyRecords "git.infra" 300 "AAAA" "2001:bc8:38ee:100:1000::41")
+          # git.p.forkos.org exposes forgejo ssh server.
+          (proxyRecords "git.p" 300 "AAAA" "2001:bc8:38ee:100:1000::40")
+          (dualProxyRecords "buildbot.infra" 300 "AAAA" "2001:bc8:38ee:100:1000::50")
+          (dualProxyRecords "public01.infra" 300 "AAAA" "2001:bc8:38ee:100:1000::60")
+
+          (record "cl" 300 "CNAME" "gerrit01.infra.p.forkos.org")
+          (record "fodwatch" 300 "CNAME" "fodwatch.infra.p.forkos.org")
+          # git.p.forkos.org is the proxy variant of the Forgejo server.
+          (record "git" 300 "CNAME" "git.p.forkos.org")
+          (record "netbox" 300 "CNAME" "meta01.infra.p.forkos.org")
+          (record "amqp" 300 "CNAME" "bagel-box.infra.p.forkos.org")
+          (record "grafana" 300 "CNAME" "meta01.infra.p.forkos.org")
+          (record "hydra" 300 "CNAME" "build-coord.wob01.infra.p.forkos.org")
+          (record "loki" 300 "CNAME" "meta01.infra.p.forkos.org")
+          (record "mimir" 300 "CNAME" "meta01.infra.p.forkos.org")
+          (record "pyroscope" 300 "CNAME" "meta01.infra.p.forkos.org")
+          (record "tempo" 300 "CNAME" "meta01.infra.p.forkos.org")
+          (record "matrix" 300 "CNAME" "meta01.infra.p.forkos.org")
+          (record "alerts" 300 "CNAME" "meta01.infra.p.forkos.org")
+          (record "buildbot" 300 "CNAME" "buildbot.infra.p.forkos.org")
+          (record "b" 300 "CNAME" "public01.infra.p.forkos.org")
+          (record "postgres" 300 "CNAME" "bagel-box.infra.p.forkos.org")
+          (record "news" 3600 "CNAME" "public01.infra.p.forkos.org")
+
+          # S3 in delroth's basement
+          (record "cache" 300 "AAAA" "2a02:168:6426::12") # smol.delroth.net
+          (record "cache" 300 "A" "195.39.247.161") # sni proxy
+
+          (record "vpn-gw.wob01.infra" 300 "AAAA" "2a01:584:11::2")
+
+          (dualProxyRecords "build-coord.wob01.infra" 300 "AAAA" "2a01:584:11::1:11")
+          # TODO: do not hardcode, just reuse the Colmena hive module outputs to generate all the required details.
+        ]
+        ++ (map (index: record "builder-${toString index}.wob01.infra" 300 "AAAA" "2a01:584:11::1:${toString index}") (genList lib.id 11))
+        ++ (
+          let
+            # FIXME: figure out a way to poke `config.services.s3-revproxy` and
+            # automate the DNS part away?
+            buckets = [
+              "channels"
+              "releases"
+              "channel-scripts-test"
+            ];
+          in
+          map (bucket: record "${bucket}" 300 "CNAME" "public01.infra.p.forkos.org") buckets
+        ));
+        "flowery.systems" = [
+          (record "" 300 "ALIAS" "news.forkos.org")
+        ];
+        "vzfdfp.de" = [
+        ];
+    };
+  };
+}

terraform/gandi.nix

@@ -56,40 +56,60 @@ in
         }) (lib.flatten records));
 
     in forkosRecords ([
-      # (record "@" 3600 "A" ["163.172.69.160"])
-      (record "@" 3600 "AAAA" ["2001:bc8:38ee:100:1000::20"])
+      # (record "@" 300 "A" ["163.172.69.160"])
+      (record "@" 300 "AAAA" ["2001:bc8:38ee:100:1000::20"])
 
-      (dualProxyRecords "bagel-box.infra" 3600 "AAAA" ["2001:bc8:38ee:100:100::1"])
-      (dualProxyRecords "gerrit01.infra" 3600 "AAAA" ["2001:bc8:38ee:100:1000::10"])
-      (dualProxyRecords "meta01.infra" 3600 "AAAA" ["2001:bc8:38ee:100:1000::20"])
-      (dualProxyRecords "fodwatch.infra" 3600 "AAAA" ["2001:bc8:38ee:100:1000::30"])
+      (dualProxyRecords "bagel-box.infra" 300 "AAAA" ["2001:bc8:38ee:100:100::1"])
+      (dualProxyRecords "gerrit01.infra" 300 "AAAA" ["2001:bc8:38ee:100:1000::10"])
+      (dualProxyRecords "meta01.infra" 300 "AAAA" ["2001:bc8:38ee:100:1000::20"])
+      (dualProxyRecords "fodwatch.infra" 300 "AAAA" ["2001:bc8:38ee:100:1000::30"])
       # git.infra.forkos.org exposes opensshd
-      (dualProxyRecords "git.infra" 3600 "AAAA" ["2001:bc8:38ee:100:1000::41"])
+      (dualProxyRecords "git.infra" 300 "AAAA" ["2001:bc8:38ee:100:1000::41"])
       # git.p.forkos.org exposes forgejo ssh server.
-      (proxyRecords "git.p" 3600 "AAAA" ["2001:bc8:38ee:100:1000::40"])
-      (dualProxyRecords "buildbot.infra" 3600 "AAAA" ["2001:bc8:38ee:100:1000::50"])
-      (dualProxyRecords "public01.infra" 3600 "AAAA" ["2001:bc8:38ee:100:1000::60"])
+      (proxyRecords "git.p" 300 "AAAA" ["2001:bc8:38ee:100:1000::40"])
+      (dualProxyRecords "buildbot.infra" 300 "AAAA" ["2001:bc8:38ee:100:1000::50"])
+      (dualProxyRecords "public01.infra" 300 "AAAA" ["2001:bc8:38ee:100:1000::60"])
 
-      (record "cl" 3600 "CNAME" ["gerrit01.infra.p"])
-      (record "fodwatch" 3600 "CNAME" ["fodwatch.infra.p"])
+      (record "cl" 300 "CNAME" ["gerrit01.infra.p"])
+      (record "fodwatch" 300 "CNAME" ["fodwatch.infra.p"])
       # git.p.forkos.org is the proxy variant of the Forgejo server.
-      (record "git" 3600 "CNAME" ["git.p"])
-      (record "netbox" 3600 "CNAME" ["meta01.infra.p"])
-      (record "amqp" 3600 "CNAME" ["bagel-box.infra.p"])
-      (record "grafana" 3600 "CNAME" ["meta01.infra.p"])
-      (record "hydra" 3600 "CNAME" ["bagel-box.infra.p"])
-      (record "loki" 3600 "CNAME" ["meta01.infra.p"])
-      (record "mimir" 3600 "CNAME" ["meta01.infra.p"])
-      (record "matrix" 3600 "CNAME" ["meta01.infra.p"])
-      (record "alerts" 3600 "CNAME" ["meta01.infra.p"])
-      (record "buildbot" 3600 "CNAME" ["buildbot.infra.p"])
-      (record "b" 3600 "CNAME" ["public01.infra.p"])
+      (record "git" 300 "CNAME" ["git.p"])
+      (record "netbox" 300 "CNAME" ["meta01.infra.p"])
+      (record "amqp" 300 "CNAME" ["bagel-box.infra.p"])
+      (record "grafana" 300 "CNAME" ["meta01.infra.p"])
+      (record "hydra" 300 "CNAME" ["build-coord.wob01.infra.p"])
+      (record "loki" 300 "CNAME" ["meta01.infra.p"])
+      (record "mimir" 300 "CNAME" ["meta01.infra.p"])
+      (record "pyroscope" 300 "CNAME" ["meta01.infra.p"])
+      (record "tempo" 300 "CNAME" ["meta01.infra.p"])
+      (record "matrix" 300 "CNAME" ["meta01.infra.p"])
+      (record "alerts" 300 "CNAME" ["meta01.infra.p"])
+      (record "buildbot" 300 "CNAME" ["buildbot.infra.p"])
+      (record "b" 300 "CNAME" ["public01.infra.p"])
+      (record "postgres" 300 "CNAME" ["bagel-box.infra.p"])
+      (record "news" 3600 "CNAME" ["public01.infra.p"])
 
       # S3 in delroth's basement
-      (record "cache" 3600 "CNAME" ["smol.delroth.net."])
+      (record "cache" 300 "AAAA" ["2a02:168:6426::12"]) # smol.delroth.net
+      (record "cache" 300 "A" ["195.39.247.161"]) # sni proxy
 
-      (record "vpn-gw.wob01.infra" 3600 "AAAA" [ "2a01:584:11::2" ])
+      (record "vpn-gw.wob01.infra" 300 "AAAA" [ "2a01:584:11::2" ])
+
+      (dualProxyRecords "build-coord.wob01.infra" 300 "AAAA" [ "2a01:584:11::1:11" ])
       # TODO: do not hardcode, just reuse the Colmena hive module outputs to generate all the required details.
-    ] ++ map (index: record "builder-${toString index}.wob01.infra" 3600 "AAAA" [ "2a01:584:11::1:${toString index}" ]) (genList lib.id 12));
+    ]
+    ++ (map (index: record "builder-${toString index}.wob01.infra" 300 "AAAA" [ "2a01:584:11::1:${toString index}" ]) (genList lib.id 11))
+    ++ (
+      let
+        # FIXME: figure out a way to poke `config.services.s3-revproxy` and
+        # automate the DNS part away?
+        buckets = [
+          "channels"
+          "releases"
+          "channel-scripts-test"
+        ];
+      in
+      map (bucket: record "${bucket}" 300 "CNAME" [ "public01.infra.p" ]) buckets
+    ));
   };
 }

terraform/hydra.nix

@@ -33,7 +33,7 @@ in
     };
 
     resource.hydra_jobset.k900-experiments = {
-      project = config.resource.hydra_project.forkos.name;
+      project = tf.ref "resource.hydra_project.forkos.name";
       state = "enabled";
       visible = true;
       name = "nixpkgs-experiments";
@@ -55,7 +55,7 @@ in
         {
           name = "nixpkgs";
           type = "git";
-          value = "https://github.com/nixos/nixpkgs 03ff49192b044786362c8c94d8501eac5c6eada4";
+          value = "https://github.com/K900/nixpkgs 467efb1055fa3c0f49d6b6a4505a4c662a836f68";
           notify_committers = false;
         }
         {
@@ -72,7 +72,7 @@ in
     };
 
     resource.hydra_jobset.raito-nixos-rolling-small = {
-      project = config.resource.hydra_project.forkos.name;
+      project = tf.ref "resource.hydra_project.forkos.name";
       state = "enabled";
       visible = true;
       name = "raito-nixos-rolling-small";
@@ -112,7 +112,7 @@ in
     };
 
     resource.hydra_jobset.delroth-nixpkgs-staging-small = {
-      project = config.resource.hydra_project.forkos.name;
+      project = tf.ref "resource.hydra_project.forkos.name";
       state = "enabled";
       visible = true;
       name = "delroth-nixpkgs-staging-small";
@@ -162,7 +162,7 @@ in
     };
 
     resource.hydra_jobset.infra_main = {
-      project = config.resource.hydra_project.infra.name;
+      project = tf.ref "resource.hydra_project.infra.name";
       state = "enabled";
       visible = true;
       name = "main";
@@ -189,7 +189,7 @@ in
     };
 
     resource.hydra_jobset.hydra_main = {
-      project = config.resource.hydra_project.hydra.name;
+      project = tf.ref "resource.hydra_project.hydra.name";
       state = "enabled";
       visible = true;
       name = "main";
@@ -206,7 +206,7 @@ in
     };
 
     resource.hydra_jobset.nixos-staging-next-small = {
-      project = config.resource.hydra_project.forkos.name;
+      project = tf.ref "resource.hydra_project.forkos.name";
       state = "enabled";
       visible = true;
       name = "nixos-staging-next-small";
@@ -246,7 +246,7 @@ in
     };
 
     resource.hydra_jobset.nixpkgs-staging-next = {
-      project = config.resource.hydra_project.forkos.name;
+      project = tf.ref "resource.hydra_project.forkos.name";
       state = "enabled";
       visible = true;
       name = "nixpkgs-staging-next";
@@ -286,7 +286,7 @@ in
     };
 
     resource.hydra_jobset.nixos-main = {
-      project = config.resource.hydra_project.forkos.name;
+      project = tf.ref "resource.hydra_project.forkos.name";
       state = "enabled";
       visible = true;
       name = "nixos-main";