flake: support aarch64-linux

...I don't know how to remove the mention of x86_64-linux for colmena,
or if it actually matters, so I'm just leaving that there for now.

common/admins.nix

@@ -3,12 +3,14 @@ let
 in {
   users.users.root.openssh.authorizedKeys.keys = 
     keys.users.delroth ++
-    keys.users.k900 ++ 
+    keys.users.emilylange ++
-    keys.users.raito ++
+    keys.users.hexchen ++
-    keys.users.maxine ++ 
     keys.users.jade ++
     keys.users.janik ++
+    keys.users.k900 ++
     keys.users.lukegb ++
-    keys.users.emilylange ++
+    keys.users.maxine ++
+    keys.users.raito ++
+    keys.users.thubrecht ++
     keys.users.yuka;
 }

common/ssh-keys.nix

@@ -23,14 +23,12 @@
 
   users = {
     delroth = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII3tjB4KYDok3KlWxdBp/yEmqhhmybd+w0VO4xUwLKKV" ];
-    raito = [
+    emilylange = [ "no-touch-required sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIL7jgq3i+N3gVJhs4shm7Kmw6dIocs2OuR0GBMG1RxfKAAAABHNzaDo=" ];
-      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICaw9ihTG7ucB8P38XdalEWev8+q96e2yNm4B+/I9IJp"
+    hexchen = [
-      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcEkYM1r8QVNM/G5CxJInEdoBCWjEHHDdHlzDYNSUIdHHsn04QY+XI67AdMCm8w30GZnLUIj5RiJEWXREUApby0GrfxGGcy8otforygfgtmuUKAUEHdU2MMwrQI7RtTZ8oQ0USRGuqvmegxz3l5caVU7qGvBllJ4NUHXrkZSja2/51vq80RF4MKkDGiz7xUTixI2UcBwQBCA/kQedKV9G28EH+1XfvePqmMivZjl+7VyHsgUVj9eRGA1XWFw59UPZG8a7VkxO/Eb3K9NF297HUAcFMcbY6cPFi9AaBgu3VC4eetDnoN/+xT1owiHi7BReQhGAy/6cdf7C/my5ehZwD"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINJ0tCxsEilAzV6LaNpUpcjzyEn4ptw8kFz3R+Z3YjEF hexchen@backup"
-      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE0xMwWedkKosax9+7D2OlnMxFL/eV4CvFZLsbLptpXr"
+      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDI3T1eFS77URHZ/HVWkMOqx7W1U54zJtn9C7QWsHOtyH72i/4EVj8SxYqLllElh1kuKUXSUipPeEzVsipFVvfH0wEuTDgFffiSQ3a8lfUgdEBuoySwceEoPgc5deapkOmiDIDeeWlrRe3nqspLRrSWU1DirMxoFPbwqJXRvpl6qJPxRg+2IolDcXlZ6yxB4Vv48vzRfVzZNUz7Pjmy2ebU8PbDoFWL/S3m7yOzQpv3L7KYBz7+rkjuF3AU2vy6CAfIySkVpspZZLtkTGCIJF228ev0e8NvhuN6ZnjzXxVTQOy32HCdPdbBbicu0uHfZ5O7JX9DjGd8kk1r2dnZwwy/ hexchen@yubi5"
-      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU"
+      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC4CLJ+mFfq5XiBXROKewmN9WYmj+79bj/AoaR6Iud2pirulot3tkrrLe2cMjiNWFX8CGVqrsAELKUA8EyUTJfStlcTE0/QNESTRmdDaC+lZL41pWUO9KOiD6/0axAhHXrSJ0ScvbqtD0CtpnCKKxtuOflVPoUGZsH9cLKJNRKfEka0H0GgeKb5Tp618R/WNAQOwaCcXzg/nG4Bgv3gJW4Nm9IKy/MwRZqtILi8Mtd+2diTqpMwyNRmbenmRHCQ1vRw46joYkledVqrmSlfSMFgIHI1zRSBXb/JkG2IvIyB5TGbTkC4N2fqJNpH8wnCKuOvs46xmgdiRA26P48C2em3 hexchen@yubi5c"
     ];
-    k900 = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOi9vgVGs+S5kEsUqHPvyMMh1Q9gqL4TcbHoe5d73tun" ];
-    maxine = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILpWQfhNFdrxMTP/1DwBVuk49f3df9iH7Tbdu8ltIKjr" ];
     jade = [
       "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDNldAg4t13/i69TD786The+U3wbiNUdW2Kc9KNWvEhgpf4y4x4Sft0oYfkPw5cjX4H3APqfD+b7ItAG0GCbwHw6KMYPoVMNK08zBMJUqt1XExbqGeFLqBaeqDsmEAYXJRbjMTAorpOCtgQdoCKK/DvZ51zUWXxT8UBNHSl19Ryv5Ry5VVdbAE35rqs57DQ9+ma6htXnsBEmmnC+1Zv1FE956m/OpBTId50mor7nS2FguAtPZnDPpTd5zl9kZmJEuWCrmy6iinw5V4Uy1mLeZkQv+/FtozbyifCRCvps9nHpv4mBSU5ABLgnRRvXs+D41Jx7xloNADr1nNgpsNrYaTh hed-bot-ssh-tpm-rsa"
       "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIKYljH8iPMrH00lOb3ETxRrZimdKzPPEdsJQ5D5ovtOwAAAACnNzaDpzc2hrZXk= ssh:sshkey"
@@ -41,8 +39,16 @@
       "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIOYg513QZsVzoyVycXZjg4F3T3+OwtcY3WAhrlfyLgLTAAAABHNzaDo="
       "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBLZxVITpJ8xbiCa/u2gjSSIupeiqOnRh+8tFIoVhCON"
     ];
+    k900 = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOi9vgVGs+S5kEsUqHPvyMMh1Q9gqL4TcbHoe5d73tun" ];
     lukegb = [ ''cert-authority,principals="lukegb" ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEqNOwlR7Qa8cbGpDfSCOweDPbAGQOZIcoRgh6s/J8DR'' ];
-    emilylange = [ "no-touch-required sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIL7jgq3i+N3gVJhs4shm7Kmw6dIocs2OuR0GBMG1RxfKAAAABHNzaDo=" ];
+    maxine = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILpWQfhNFdrxMTP/1DwBVuk49f3df9iH7Tbdu8ltIKjr" ];
+    raito = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICaw9ihTG7ucB8P38XdalEWev8+q96e2yNm4B+/I9IJp"
+      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcEkYM1r8QVNM/G5CxJInEdoBCWjEHHDdHlzDYNSUIdHHsn04QY+XI67AdMCm8w30GZnLUIj5RiJEWXREUApby0GrfxGGcy8otforygfgtmuUKAUEHdU2MMwrQI7RtTZ8oQ0USRGuqvmegxz3l5caVU7qGvBllJ4NUHXrkZSja2/51vq80RF4MKkDGiz7xUTixI2UcBwQBCA/kQedKV9G28EH+1XfvePqmMivZjl+7VyHsgUVj9eRGA1XWFw59UPZG8a7VkxO/Eb3K9NF297HUAcFMcbY6cPFi9AaBgu3VC4eetDnoN/+xT1owiHi7BReQhGAy/6cdf7C/my5ehZwD"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE0xMwWedkKosax9+7D2OlnMxFL/eV4CvFZLsbLptpXr"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU"
+    ];
+    thubrecht = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPM1jpXR7BWQa7Sed7ii3SbvIPRRlKb3G91qC0vOwfJn" ];
     yuka = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKath4/fDnlv/4fzxkPrQN1ttmoPRNu/m9bEtdPJBDfY cardno:16_933_242" ];
   };
 }

flake.lock

@@ -64,16 +64,16 @@
         "treefmt-nix": "treefmt-nix"
       },
       "locked": {
-        "lastModified": 1721409873,
+        "lastModified": 1721685540,
-        "narHash": "sha256-h0njWQRvtkjK0NJ/Kgj76sXBhWwq5HGJm7OMcigmNw4=",
+        "narHash": "sha256-sIFaurUhoxZBahwfXpHRfMk41FexvULOe03qRBe7uiA=",
-        "ref": "refs/heads/refactor",
+        "ref": "refs/heads/non-flakes",
-        "rev": "54bba654d4279dfd112345b6470547851feb1457",
+        "rev": "3c903f14c25d87f4fb0b3a0ee7e860b6fa5b2d96",
-        "revCount": 267,
+        "revCount": 290,
         "type": "git",
         "url": "https://git.lix.systems/lix-project/buildbot-nix.git"
       },
       "original": {
-        "ref": "refs/heads/refactor",
+        "ref": "refs/heads/non-flakes",
         "type": "git",
         "url": "https://git.lix.systems/lix-project/buildbot-nix.git"
       }
@@ -258,11 +258,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1721484547,
+        "lastModified": 1721682989,
-        "narHash": "sha256-RXkwCO2V9CcoNKwXdfRQc8dLCZUtDae9LFZ9LsgADWo=",
+        "narHash": "sha256-kjJiZ7m4HKqbZ2mxNQiB32/goKFb8BRi8OqC4wIU0OI=",
         "ref": "refs/heads/main",
-        "rev": "abc9f11417e2de515006e0fe8dd345f815dc92a7",
+        "rev": "4b107e6ff36bd89958fba36e0fe0340903e7cd13",
-        "revCount": 4186,
+        "revCount": 4190,
         "type": "git",
         "url": "https://git.lix.systems/lix-project/hydra.git"
       },

flake.nix

@@ -17,7 +17,7 @@
     nix-gerrit.url = "git+https://git.lix.systems/the-distro/nix-gerrit.git";
     nix-gerrit.inputs.nixpkgs.follows = "nixpkgs";
 
-    buildbot-nix.url = "git+https://git.lix.systems/lix-project/buildbot-nix.git?ref=refs/heads/refactor";
+    buildbot-nix.url = "git+https://git.lix.systems/lix-project/buildbot-nix.git?ref=refs/heads/non-flakes";
     buildbot-nix.inputs.nixpkgs.follows = "nixpkgs";
 
     lix.follows = "hydra/lix";
@@ -74,8 +74,9 @@
         packages = [
           inputs.agenix.packages.${system}.agenix
 
-          pkgs.colmena
           pkgs.opentofu
+
+          (pkgs.callPackage ./lib/colmena-wrapper.nix { })
         ];
       };
     });

hosts/gerrit01/default.nix

@@ -51,7 +51,7 @@
       name = "nixpkgs-${branchName}";
       fromUri = "https://github.com/NixOS/nixpkgs";
       fromRefspec = branchName;
-      localRefspec = "refs/remotes/origin/${branchName}";
+      localRefspec = branchName;
       inherit timer;
     };
     in
@@ -61,9 +61,14 @@
     pushUrl = "ssh://ows_bot@cl.forkos.org:29418/nixpkgs";
     deployKeyPath = config.age.secrets.ows-deploy-key.path;
 
-    branches."refs/heads/master" = mkNixpkgsJob {
+    branches."refs/heads/main" = mkNixpkgsJob {
       timer = "hourly";
-      branchName = "master";
+      branchName = "main";
+    };
+
+    branches."refs/heads/staging" = mkNixpkgsJob {
+      timer = "hourly";
+      branchName = "staging";
     };
 
     branches."refs/heads/release-24.05" = mkNixpkgsJob {
@@ -71,17 +76,27 @@
       branchName = "release-24.05";
     };
 
+    branches."refs/heads/staging-24.05" = mkNixpkgsJob {
+      timer = "hourly";
+      branchName = "staging-24.05";
+    };
+
     branches."refs/heads/release-23.11" = mkNixpkgsJob {
       timer = "hourly";
       branchName = "release-23.11";
     };
 
+    branches."refs/heads/staging-23.11" = mkNixpkgsJob {
+      timer = "hourly";
+      branchName = "staging-23.11";
+    };
+
     # Testing jobs for personal sandbox branches
     branches."refs/heads/sandbox/raito/raito-unstable-small" = {
       name = "raito-unstable-sync";
       fromUri = "https://github.com/NixOS/nixpkgs";
       fromRefspec = "nixos-unstable-small";
-      localRefspec = "refs/remotes/origin/sandbox/raito/raito-unstable-small";
+      localRefspec = "sandbox/raito/raito-unstable-small";
       timer = "*-*-* 12:00:00";
     };
 
@@ -89,7 +104,7 @@
       name = "raito-release-sync";
       fromUri = "https://github.com/NixOS/nixpkgs";
       fromRefspec = "nixos-24.05";
-      localRefspec = "refs/remotes/origin/sandbox/raito/raito-nixos-24.05";
+      localRefspec = "sandbox/raito/raito-nixos-24.05";
       timer = "daily";
     };
   };

lib/colmena-wrapper.nix

@@ -0,0 +1,14 @@
+# A wrapper for colmena that prevents accidentally deploying changes without
+# having pulled.
+{ colmena, runCommandNoCC }:
+runCommandNoCC "colmena-wrapper"
+{
+  env.colmena = "${colmena}/bin/colmena";
+} ''
+  mkdir -p $out
+  ln -s ${colmena}/share $out/share
+  mkdir $out/bin
+
+  substituteAll ${./colmena-wrapper.sh.in} $out/bin/colmena
+  chmod +x $out/bin/colmena
+''

lib/colmena-wrapper.sh.in

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+doChecks() {
+    # creates refs in the refs/prefetch/remotes/origin namespace
+    echo "Prefetching repo changes..." >&2
+    git fetch --quiet --prefetch --no-write-fetch-head origin
+
+    diffs=$(git rev-list --left-right --count HEAD...refs/prefetch/remotes/origin/main)
+    only_in_local=$(echo "$diffs" | cut -f1)
+    only_in_main=$(echo "$diffs" | cut -f2)
+
+    if [[ $only_in_main -gt 0 && ! -v $FOOTGUN_ME_UWU ]]; then
+        echo >&2
+        echo "Attempting to deploy when main has $only_in_main commits not in your branch!" >&2
+        echo "This will probably revert someone's changes. Consider merging them." >&2
+        echo "If you really mean it, set the environment variable FOOTGUN_ME_UWU" >&2
+        exit 1
+    fi
+
+    if [[ $only_in_local -gt 0 ]]; then
+        echo "You have $only_in_local commits not yet pushed to main. Reminder to push them after :)" >&2
+    fi
+}
+
+if [[ $1 == 'apply' ]]; then
+    doChecks
+fi
+
+exec @colmena@ "$@"

services/baremetal-builder/default.nix

@@ -150,7 +150,11 @@ in
       script = "exec ${config.nix.package.out}/bin/nix-store --gc --store /mnt";
       serviceConfig.Type = "oneshot";
       serviceConfig.User = "builder";
-      startAt = "hourly";
+    };
+    systemd.timers.hydra-gc = {
+      timerConfig.OnStartupSec = "4h";
+      timerConfig.OnUnitActiveSec = "20h";
+      wantedBy = [ "timers.target" ];
     };
     systemd.timers.hydra-gc.timerConfig.Persistent = true;
 

services/buildbot/default.nix

@@ -32,7 +32,11 @@ in
     age.secrets.buildbot-workers.file = ../../secrets/buildbot-workers.age;
     age.secrets.buildbot-service-key.file = ../../secrets/buildbot-service-key.age;
     age.secrets.buildbot-signing-key.file = ../../secrets/buildbot-signing-key.age;
-    age.secrets.buildbot-remote-builder-key.file = ../../secrets/buildbot-remote-builder-key.age;
+    age.secrets.buildbot-remote-builder-key = {
+      file = ../../secrets/buildbot-remote-builder-key.age;
+      owner = "buildbot-worker";
+      group = "buildbot-worker";
+    };
 
     services.nginx.virtualHosts.${cfg.domain} = {
       forceSSL = true;
@@ -58,7 +62,7 @@ in
             (_: lib.foldl' lib.add 0)
             (lib.concatMap
               (m: map (s: { ${s} = m.maxJobs; }) m.systems)
-              config.nix.buildMachines))
+              config.services.buildbot-nix.coordinator.buildMachines))
         );
     };
 
@@ -67,6 +71,8 @@ in
 
       inherit (cfg) domain;
 
+      debugging.enable = true;
+
       oauth2 = {
         name = "Lix";
         clientId = "forkos-buildbot";

services/gerrit/one-way-sync.nix

@@ -16,7 +16,8 @@ let
     path = [ pkgs.gitFull pkgs.openssh pkgs.lix ];
     script = ''
       set -xe
-      trap "git worktree prune && git worktree remove -f ${name}" EXIT
+      RUNTIME_DIRECTORY="/run/onewaysync-${name}"
+      trap "git worktree remove -f "$RUNTIME_DIRECTORY"/${name}" EXIT
 
       if [ ! -d "/var/lib/onewaysync/nixpkgs" ]; then
         echo "First run, synchronizing nixpkgs..."
@@ -26,19 +27,19 @@ let
       cd /var/lib/onewaysync/nixpkgs
       echo "Syncing ${fromUri}:${fromRefspec} to /var/lib/onewaysync/nixpkgs:${targetRef}"
       echo "Current ref: $EXPECTED_REF"
-      git worktree add -f ${cfg.workingDir}/${name} ${localRefspec}
+      git worktree add -f "$RUNTIME_DIRECTORY"/${name} refs/remotes/origin/${localRefspec}
-      cd ${cfg.workingDir}/${name}
+      cd "$RUNTIME_DIRECTORY"/${name}
-      git pull origin ${fromRefspec}
+      git pull origin ${localRefspec}
-      EXPECTED_REF=$(git rev-list ${localRefspec} | head -1)
+      EXPECTED_REF=$(git rev-list refs/remotes/origin/${localRefspec} | head -1)
       git config user.name Fork-o-Tron
       git config user.email noreply@forkos.org
       git fetch ${fromUri} ${fromRefspec}
     '' + lib.optionalString (!(lib.hasInfix "staging" localRefspec)) ''
-      OLD_STDENV=$(nix eval -f . stdenv.outPath --store /run/onewaysync)
+      OLD_STDENV=$(nix eval -f . stdenv.outPath --store "$RUNTIME_DIRECTORY")
     '' + ''
       git merge FETCH_HEAD
     '' + lib.optionalString (!(lib.hasInfix "staging" localRefspec)) ''
-      NEW_STDENV=$(nix eval -f . stdenv.outPath --store /run/onewaysync)
+      NEW_STDENV=$(nix eval -f . stdenv.outPath --store "$RUNTIME_DIRECTORY")
       # Do not allow auto-merging a staging iteration
       test "$OLD_STDENV" = "$NEW_STDENV"
     '' + ''
@@ -48,8 +49,8 @@ let
       User = "git";
       Group = "git";
       Type = "oneshot";
-      RuntimeDirectory = "onewaysync";
+      RuntimeDirectory = "onewaysync-${name}";
-      WorkingDirectory = cfg.workingDir;
+      WorkingDirectory = "/run/onewaysync-${name}";
       StateDirectory = "onewaysync";
     };
   };

services/hydra/default.nix

@@ -117,7 +117,7 @@ in {
 
         upload_logs_to_binary_cache = true
 
-        evaluator_workers = 4
+        evaluator_workers = 16
         evaluator_max_memory_size = 4096
         max_concurrent_evals = 1
 

services/monitoring/lgtm/grafana.nix

@@ -36,6 +36,12 @@ in
             host = "/run/postgresql";
           };
 
+          "auth.anonymous" = {
+            enabled = true;
+            org_name = "Main Org.";
+            org_role = "Viewer";
+          };
+
           "auth.generic_oauth" = {
             enabled = true;
 

terraform/gandi.nix

@@ -82,6 +82,9 @@ in
       (record "matrix" 3600 "CNAME" ["meta01.infra.p"])
       (record "buildbot" 3600 "CNAME" ["buildbot.infra.p"])
 
+      # S3 in delroth's basement
+      (record "cache" 3600 "CNAME" ["smol.delroth.net."])
+
       (record "vpn-gw.wob01.infra" 3600 "AAAA" [ "2a01:584:11::2" ])
       # TODO: do not hardcode, just reuse the Colmena hive module outputs to generate all the required details.
     ] ++ map (index: record "builder-${toString index}.wob01.infra" 3600 "AAAA" [ "2a01:584:11::1:${toString index}" ]) (genList lib.id 12));

terraform/hydra.nix

@@ -32,6 +32,45 @@ in
       visible = true;
     };
 
+    resource.hydra_jobset.k900-experiments = {
+      project = config.resource.hydra_project.forkos.name;
+      state = "enabled";
+      visible = true;
+      name = "nixpkgs-experiments";
+      type = "legacy";
+      description = "experiments branch to test things for K900";
+
+      nix_expression = {
+        file = "nixos/release.nix";
+        input = "nixpkgs";
+      };
+
+      check_interval = 0;
+      scheduling_shares = 3000;
+      keep_evaluations = 3;
+
+      email_notifications = false;
+
+      input = [
+        {
+          name = "nixpkgs";
+          type = "git";
+          value = "https://github.com/nixos/nixpkgs 03ff49192b044786362c8c94d8501eac5c6eada4";
+          notify_committers = false;
+        }
+        {
+          name = "officialRelease";
+          type = "boolean";
+          value = false;
+        }
+        {
+          name = "supportedSystems";
+          type = "nix";
+          value = ''[ "x86_64-linux" ]'';
+        }
+      ];
+    };
+
     resource.hydra_jobset.raito-nixos-rolling-small = {
       project = config.resource.hydra_project.forkos.name;
       state = "enabled";