bagel-container: provision a user with Nix store perms for remote builds

Reviewed-on: raito/shared-public-infra#2

.gitignore

@@ -1 +1,3 @@
 .direnv
+result
+.gcroots

configurations.nix

@@ -9,7 +9,6 @@ let
     colmena
     flake-registry
     nixos-hardware
-    nixpkgs-unstable
     srvos
     disko
     ;

flake.lock

@@ -10,11 +10,11 @@
         "systems": "systems"
       },
       "locked": {
-        "lastModified": 1716561646,
-        "narHash": "sha256-UIGtLO89RxKt7RF2iEgPikSdU53r6v/6WYB0RW3k89I=",
+        "lastModified": 1718371084,
+        "narHash": "sha256-abpBi61mg0g+lFFU0zY4C6oP6fBwPzbHPKBGw676xsA=",
         "owner": "ryantm",
         "repo": "agenix",
-        "rev": "c2fc0762bbe8feb06a2e59a364fa81b3a57671c9",
+        "rev": "3a56735779db467538fb2e577eda28a9daacaca6",
         "type": "github"
       },
       "original": {
@@ -32,11 +32,11 @@
         "nixpkgs-stable": "nixpkgs-stable"
       },
       "locked": {
-        "lastModified": 1711742460,
-        "narHash": "sha256-0O4v6e4a1toxXZ2gf5INhg4WPE5C5T+SVvsBt+45Mcc=",
+        "lastModified": 1717279440,
+        "narHash": "sha256-kH04ReTjxOpQumgWnqy40vvQLSnLGxWP6RF3nq5Esrk=",
         "owner": "zhaofengli",
         "repo": "attic",
-        "rev": "4dbdbee45728d8ce5788db6461aaaa89d98081f0",
+        "rev": "717cc95983cdc357bc347d70be20ced21f935843",
         "type": "github"
       },
       "original": {
@@ -76,11 +76,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1702918879,
-        "narHash": "sha256-tWJqzajIvYcaRWxn+cLUB9L9Pv4dQ3Bfit/YjU5ze3g=",
+        "lastModified": 1717025063,
+        "narHash": "sha256-dIubLa56W9sNNz0e8jGxrX3CAkPXsq7snuFA/Ie6dn8=",
         "owner": "ipetkov",
         "repo": "crane",
-        "rev": "7195c00c272fdd92fc74e7d5a0a2844b9fadb2fb",
+        "rev": "480dff0be03dac0e51a8dfc26e882b0d123a450e",
         "type": "github"
       },
       "original": {
@@ -118,11 +118,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1716431128,
-        "narHash": "sha256-t3T8HlX3udO6f4ilLcN+j5eC3m2gqsouzSGiriKK6vk=",
+        "lastModified": 1718846788,
+        "narHash": "sha256-9dtXYtEkmXoUJV+PGLqscqF7qTn4AIhAKpFWRFU2NYs=",
         "owner": "nix-community",
         "repo": "disko",
-        "rev": "7ffc4354dfeb37c8c725ae1465f04a9b45ec8606",
+        "rev": "e1174d991944a01eaaa04bc59c6281edca4c0e6e",
         "type": "github"
       },
       "original": {
@@ -170,11 +170,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1715865404,
-        "narHash": "sha256-/GJvTdTpuDjNn84j82cU6bXztE0MSkdnTWClUCRub78=",
+        "lastModified": 1717285511,
+        "narHash": "sha256-iKzJcpdXih14qYVcZ9QC9XuZYnPc6T8YImb6dX166kw=",
         "owner": "hercules-ci",
         "repo": "flake-parts",
-        "rev": "8dc45382d5206bd292f9c2768b8058a8fd8311d9",
+        "rev": "2a55567fcf15b1b1c7ed712a2c6fadaec7412ea8",
         "type": "github"
       },
       "original": {
@@ -186,11 +186,11 @@
     "flake-registry": {
       "flake": false,
       "locked": {
-        "lastModified": 1705308826,
-        "narHash": "sha256-Z3xTYZ9EcRIqZAufZbci912MUKB0sD+qxi/KTGMFVwY=",
+        "lastModified": 1717415742,
+        "narHash": "sha256-HKvoLGZUsBpjkxWkdtctGYj6RH0bl6vcw0OjTOqyzJk=",
         "owner": "NixOS",
         "repo": "flake-registry",
-        "rev": "9c69f7bd2363e71fe5cd7f608113290c7614dcdd",
+        "rev": "895a65f8d5acf848136ee8fe8e8f736f0d27df96",
         "type": "github"
       },
       "original": {
@@ -257,11 +257,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1717527182,
-        "narHash": "sha256-vWSkg6AMok1UUQiSYVdGMOXKD2cDFnajITiSi0Zjd1A=",
+        "lastModified": 1718530513,
+        "narHash": "sha256-BmO8d0r+BVlwWtMLQEYnwmngqdXIuyFzMwvmTcLMee8=",
         "owner": "rycee",
         "repo": "home-manager",
-        "rev": "845a5c4c073f74105022533907703441e0464bc3",
+        "rev": "a1fddf0967c33754271761d91a3d921772b30d0e",
         "type": "github"
       },
       "original": {
@@ -273,11 +273,11 @@
     },
     "nixos-hardware": {
       "locked": {
-        "lastModified": 1716715385,
-        "narHash": "sha256-fe6Z33pbfqu4TI5ijmcaNc5vRBs633tyxJ12HTghy3w=",
+        "lastModified": 1719069430,
+        "narHash": "sha256-d9KzCJv3UG6nX9Aur5OSEf4Uj+ywuxojhiCiRKYVzXA=",
         "owner": "NixOS",
         "repo": "nixos-hardware",
-        "rev": "2e7d6c568063c83355fe066b8a8917ee758de1b8",
+        "rev": "e8232c132a95ddc62df9d404120ad4ff53862910",
         "type": "github"
       },
       "original": {
@@ -302,6 +302,22 @@
         "type": "github"
       }
     },
+    "nixpkgs-for-kernel": {
+      "locked": {
+        "lastModified": 1709742294,
+        "narHash": "sha256-8iPomMqw7grXVsugMJhsnHdbre8LnXOQUtHtMXRaWqc=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "56051fbe049bf39adc1f08eb51740c226a4c3b90",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "56051fbe049bf39adc1f08eb51740c226a4c3b90",
+        "type": "github"
+      }
+    },
     "nixpkgs-stable": {
       "locked": {
         "lastModified": 1711460390,
@@ -318,13 +334,13 @@
         "type": "github"
       }
     },
-    "nixpkgs-unstable": {
+    "nixpkgs_2": {
       "locked": {
-        "lastModified": 1716715802,
-        "narHash": "sha256-usk0vE7VlxPX8jOavrtpOqphdfqEQpf9lgedlY/r66c=",
+        "lastModified": 1718983919,
+        "narHash": "sha256-+1xgeIow4gJeiwo4ETvMRvWoircnvb0JOt7NS9kUhoM=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "e2dd4e18cc1c7314e24154331bae07df76eb582f",
+        "rev": "90338afd6177fc683a04d934199d693708c85a3b",
         "type": "github"
       },
       "original": {
@@ -334,29 +350,13 @@
         "type": "github"
       }
     },
-    "nixpkgs_2": {
-      "locked": {
-        "lastModified": 1709742294,
-        "narHash": "sha256-8iPomMqw7grXVsugMJhsnHdbre8LnXOQUtHtMXRaWqc=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "56051fbe049bf39adc1f08eb51740c226a4c3b90",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "56051fbe049bf39adc1f08eb51740c226a4c3b90",
-        "type": "github"
-      }
-    },
     "nur": {
       "locked": {
-        "lastModified": 1716741358,
-        "narHash": "sha256-4bxptwbmplGKq3W4tl6Zem/bOHsdLP4DSPcm/FfCaFE=",
+        "lastModified": 1719099906,
+        "narHash": "sha256-xo1cNkVBW7NxTU5zMu0B7ZkismtkHfTRWfhBXbNnp9g=",
         "owner": "nix-community",
         "repo": "NUR",
-        "rev": "c65a3bde6793b437a705edfe5ff8435cbb8307a2",
+        "rev": "315cf1f8c5f5e92150d81ccafba7525c54327094",
         "type": "github"
       },
       "original": {
@@ -376,7 +376,7 @@
         "home-manager": "home-manager_2",
         "nixos-hardware": "nixos-hardware",
         "nixpkgs": "nixpkgs_2",
-        "nixpkgs-unstable": "nixpkgs-unstable",
+        "nixpkgs-for-kernel": "nixpkgs-for-kernel",
         "nur": "nur",
         "srvos": "srvos"
       }
@@ -388,11 +388,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1716425501,
-        "narHash": "sha256-BSLhmGYY1khyyBAjraR+N0Pa9Nha/et5yQQlEZxcfkU=",
+        "lastModified": 1718844164,
+        "narHash": "sha256-QUXWv6llKIQ5To2N24d9dRI78Hqfm9iFyhvmvlOICNo=",
         "owner": "numtide",
         "repo": "srvos",
-        "rev": "1122cd50a23647e09c3e7a679d37ec02113bc412",
+        "rev": "557ff94aa1b48a723f8fa16eb9e7a2e6de991682",
         "type": "github"
       },
       "original": {

flake.nix

@@ -10,9 +10,9 @@
     flake-parts.url = "github:hercules-ci/flake-parts";
     flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs";
 
-    nixpkgs-unstable.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
+    nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
     # contains kernel 6.7.7, do not update
-    nixpkgs.url = "github:NixOS/nixpkgs/56051fbe049bf39adc1f08eb51740c226a4c3b90";
+    nixpkgs-for-kernel.url = "github:NixOS/nixpkgs/56051fbe049bf39adc1f08eb51740c226a4c3b90";
 
     nixos-hardware.url = "github:NixOS/nixos-hardware";
     nur.url = "github:nix-community/NUR";

hosts/epyc.nix

@@ -1,4 +1,4 @@
-{ lib, pkgs, ... }:
+{ inputs, lib, pkgs, ... }:
 let
   gcc-system-features = arch: lib.optionals (arch != null) ([ "gccarch-${arch}" ]
     ++ map (x: "gccarch-${x}") lib.systems.architectures.inferiors.${arch});
@@ -13,6 +13,7 @@ in
     ../modules/android-cache.nix
     ../modules/garage.nix
     ../modules/users/friends.nix
+    ../modules/bagel-container.nix
   ];
 
   networking.hostName = "epyc";
@@ -27,7 +28,11 @@ in
 
   # TODO: there's a critical bug on 6.8+ where btrfs won't mount the rootfs at all.
   # Do not upgrade until it is fixed. Ping Raito when needed.
-  boot.kernelPackages = pkgs.linuxPackages_6_7;
+  boot.kernelPackages = let
+    pkgsForKernel = import inputs.nixpkgs-for-kernel {
+      system = "x86_64-linux";
+    };
+  in pkgsForKernel.linuxPackages_6_7;
 
   # Open public access to our PostgreSQL.
   services.postgresql.enable = true;

modules/bagel-container.nix

@@ -0,0 +1,46 @@
+# Stateful/mutable container used for Bagel (tm) related infra (mostly
+# rebuilding nixpkgs a lot).
+#
+# System image is stored at /var/lib/machines/bagel.
+{
+  systemd.nspawn.bagel = {
+    execConfig = {
+      Boot = true;
+      Ephemeral = false;
+      PrivateUsers = true;
+      NotifyReady = true;
+      LinkJournal = "try-guest";
+    };
+
+    networkConfig = {
+      Bridge = "wan-br";
+      VirtualEthernetExtra = "vb-bagel-v4:host1";
+    };
+  };
+
+  systemd.services."systemd-nspawn@bagel" = {
+    wantedBy = [ "machines.target" ];
+    wants = [ "network.target" ];
+    after = [ "network.target" ];
+    overrideStrategy = "asDropin";
+  };
+
+  systemd.network.networks."20-vb-bagel-v4" = {
+    matchConfig.Name = "vb-bagel-v4";
+    networkConfig.Address = [ "172.16.100.1/24" ];
+    networkConfig.IPMasquerade = true;
+  };
+
+  # Configure a local Nix builder account, since getting sandboxing and KVM
+  # working inside the container will be tricky.
+  users.users.bagel-builder = {
+    isSystemUser = true;
+    group = "nogroup";
+    home = "/var/empty";
+    shell = "/bin/sh";
+    openssh.authorizedKeys.keys = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAvUT9YBig9LQPHgypIBHQuC32XqDKxlFZ2CfgDi0ZKx"
+    ];
+  };
+  nix.settings.trusted-users = [ "bagel-builder" ];
+}

modules/network.nix

@@ -14,8 +14,8 @@
     '')
     config.networking.newtype.hosts);
 
-  # leave container interfaces alone
-  systemd.network.networks."05-veth".extraConfig = ''
+  # leave container interfaces alone unless otherwise specified
+  systemd.network.networks."95-veth".extraConfig = ''
     [Match]
     Driver = veth
 
@@ -34,12 +34,29 @@
     linkConfig.Name = "nat-lan";
   };
 
-  systemd.network.networks."10-wan" = {
-    matchConfig.Name = "wan";
+  systemd.network.netdevs."10-wan-br" = {
+    netdevConfig.Name = "wan-br";
+    netdevConfig.Kind = "bridge";
+    netdevConfig.MACAddress = "none";
+    bridgeConfig.MulticastSnooping = false;
+  };
+
+  systemd.network.links."10-wan-br" = {
+    matchConfig.Name = "wan-br";
+    linkConfig.MACAddressPolicy = "none";
+  };
+
+  systemd.network.networks."10-wan-br" = {
+    matchConfig.Name = "wan-br";
     linkConfig.RequiredForOnline = true;
     networkConfig.Address = [ config.networking.newtype.currentHost.ipv6 ];
   };
 
+  systemd.network.networks."10-wan" = {
+    matchConfig.Name = "wan";
+    networkConfig.Bridge = "wan-br";
+  };
+
   systemd.network.links."10-wan" = {
     matchConfig.MACAddress = "3c:ec:ef:7e:bd:c9";
     linkConfig.Name = "wan";

modules/packages.nix

@@ -36,8 +36,6 @@
     usbutils
 
     ipmitool
-
-    nix-top
     # tries to default to soft-float due to out-dated cc-rs
   ] ++ lib.optional (!stdenv.hostPlatform.isRiscV) bandwhich;
 }