bagel-container: provision a user with Nix store perms for remote builds

feat: cleanup post btrfs epic crash
BTRFS had a bug and we know how it goes. Signed-off-by: Raito Bezarius <masterancpp@gmail.com>
2024-07-04 19:45:48 +00:00 · 2024-07-04 21:25:14 +02:00 · 2024-06-30 14:15:30 +02:00 · 2024-06-23 18:26:57 +00:00 · 2024-06-23 20:24:46 +02:00 · 2024-06-23 19:33:00 +02:00
10 changed files with 137 additions and 90 deletions
--- a/.gitignore
+++ b/.gitignore
@ -1 +1,3 @@
 .direnv
+result
+.gcroots
--- a/configurations.nix
+++ b/configurations.nix
@ -9,7 +9,6 @@ let
    colmena
    flake-registry
    nixos-hardware
-    nixpkgs-unstable
    srvos
    disko
    ;
--- a/flake.lock
+++ b/flake.lock
@ -10,11 +10,11 @@
        "systems": "systems"
      },
      "locked": {
-        "lastModified": 1716561646,
-        "narHash": "sha256-UIGtLO89RxKt7RF2iEgPikSdU53r6v/6WYB0RW3k89I=",
+        "lastModified": 1718371084,
+        "narHash": "sha256-abpBi61mg0g+lFFU0zY4C6oP6fBwPzbHPKBGw676xsA=",
        "owner": "ryantm",
        "repo": "agenix",
-        "rev": "c2fc0762bbe8feb06a2e59a364fa81b3a57671c9",
+        "rev": "3a56735779db467538fb2e577eda28a9daacaca6",
        "type": "github"
      },
      "original": {
@ -32,11 +32,11 @@
        "nixpkgs-stable": "nixpkgs-stable"
      },
      "locked": {
-        "lastModified": 1711742460,
-        "narHash": "sha256-0O4v6e4a1toxXZ2gf5INhg4WPE5C5T+SVvsBt+45Mcc=",
+        "lastModified": 1717279440,
+        "narHash": "sha256-kH04ReTjxOpQumgWnqy40vvQLSnLGxWP6RF3nq5Esrk=",
        "owner": "zhaofengli",
        "repo": "attic",
-        "rev": "4dbdbee45728d8ce5788db6461aaaa89d98081f0",
+        "rev": "717cc95983cdc357bc347d70be20ced21f935843",
        "type": "github"
      },
      "original": {
@ -76,11 +76,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1702918879,
-        "narHash": "sha256-tWJqzajIvYcaRWxn+cLUB9L9Pv4dQ3Bfit/YjU5ze3g=",
+        "lastModified": 1717025063,
+        "narHash": "sha256-dIubLa56W9sNNz0e8jGxrX3CAkPXsq7snuFA/Ie6dn8=",
        "owner": "ipetkov",
        "repo": "crane",
-        "rev": "7195c00c272fdd92fc74e7d5a0a2844b9fadb2fb",
+        "rev": "480dff0be03dac0e51a8dfc26e882b0d123a450e",
        "type": "github"
      },
      "original": {
@ -118,11 +118,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1716431128,
-        "narHash": "sha256-t3T8HlX3udO6f4ilLcN+j5eC3m2gqsouzSGiriKK6vk=",
+        "lastModified": 1718846788,
+        "narHash": "sha256-9dtXYtEkmXoUJV+PGLqscqF7qTn4AIhAKpFWRFU2NYs=",
        "owner": "nix-community",
        "repo": "disko",
-        "rev": "7ffc4354dfeb37c8c725ae1465f04a9b45ec8606",
+        "rev": "e1174d991944a01eaaa04bc59c6281edca4c0e6e",
        "type": "github"
      },
      "original": {
@ -170,11 +170,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1715865404,
-        "narHash": "sha256-/GJvTdTpuDjNn84j82cU6bXztE0MSkdnTWClUCRub78=",
+        "lastModified": 1717285511,
+        "narHash": "sha256-iKzJcpdXih14qYVcZ9QC9XuZYnPc6T8YImb6dX166kw=",
        "owner": "hercules-ci",
        "repo": "flake-parts",
-        "rev": "8dc45382d5206bd292f9c2768b8058a8fd8311d9",
+        "rev": "2a55567fcf15b1b1c7ed712a2c6fadaec7412ea8",
        "type": "github"
      },
      "original": {
@ -186,11 +186,11 @@
    "flake-registry": {
      "flake": false,
      "locked": {
-        "lastModified": 1705308826,
-        "narHash": "sha256-Z3xTYZ9EcRIqZAufZbci912MUKB0sD+qxi/KTGMFVwY=",
+        "lastModified": 1717415742,
+        "narHash": "sha256-HKvoLGZUsBpjkxWkdtctGYj6RH0bl6vcw0OjTOqyzJk=",
        "owner": "NixOS",
        "repo": "flake-registry",
-        "rev": "9c69f7bd2363e71fe5cd7f608113290c7614dcdd",
+        "rev": "895a65f8d5acf848136ee8fe8e8f736f0d27df96",
        "type": "github"
      },
      "original": {
@ -257,11 +257,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1717527182,
-        "narHash": "sha256-vWSkg6AMok1UUQiSYVdGMOXKD2cDFnajITiSi0Zjd1A=",
+        "lastModified": 1718530513,
+        "narHash": "sha256-BmO8d0r+BVlwWtMLQEYnwmngqdXIuyFzMwvmTcLMee8=",
        "owner": "rycee",
        "repo": "home-manager",
-        "rev": "845a5c4c073f74105022533907703441e0464bc3",
+        "rev": "a1fddf0967c33754271761d91a3d921772b30d0e",
        "type": "github"
      },
      "original": {
@ -273,11 +273,11 @@
    },
    "nixos-hardware": {
      "locked": {
-        "lastModified": 1716715385,
-        "narHash": "sha256-fe6Z33pbfqu4TI5ijmcaNc5vRBs633tyxJ12HTghy3w=",
+        "lastModified": 1719069430,
+        "narHash": "sha256-d9KzCJv3UG6nX9Aur5OSEf4Uj+ywuxojhiCiRKYVzXA=",
        "owner": "NixOS",
        "repo": "nixos-hardware",
-        "rev": "2e7d6c568063c83355fe066b8a8917ee758de1b8",
+        "rev": "e8232c132a95ddc62df9d404120ad4ff53862910",
        "type": "github"
      },
      "original": {
@ -318,13 +318,13 @@
        "type": "github"
      }
    },
-    "nixpkgs-unstable": {
+    "nixpkgs_2": {
      "locked": {
-        "lastModified": 1716715802,
-        "narHash": "sha256-usk0vE7VlxPX8jOavrtpOqphdfqEQpf9lgedlY/r66c=",
+        "lastModified": 1718983919,
+        "narHash": "sha256-+1xgeIow4gJeiwo4ETvMRvWoircnvb0JOt7NS9kUhoM=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "e2dd4e18cc1c7314e24154331bae07df76eb582f",
+        "rev": "90338afd6177fc683a04d934199d693708c85a3b",
        "type": "github"
      },
      "original": {
@ -334,29 +334,13 @@
        "type": "github"
      }
    },
-    "nixpkgs_2": {
-      "locked": {
-        "lastModified": 1709742294,
-        "narHash": "sha256-8iPomMqw7grXVsugMJhsnHdbre8LnXOQUtHtMXRaWqc=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "56051fbe049bf39adc1f08eb51740c226a4c3b90",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "56051fbe049bf39adc1f08eb51740c226a4c3b90",
-        "type": "github"
-      }
-    },
    "nur": {
      "locked": {
-        "lastModified": 1716741358,
-        "narHash": "sha256-4bxptwbmplGKq3W4tl6Zem/bOHsdLP4DSPcm/FfCaFE=",
+        "lastModified": 1719099906,
+        "narHash": "sha256-xo1cNkVBW7NxTU5zMu0B7ZkismtkHfTRWfhBXbNnp9g=",
        "owner": "nix-community",
        "repo": "NUR",
-        "rev": "c65a3bde6793b437a705edfe5ff8435cbb8307a2",
+        "rev": "315cf1f8c5f5e92150d81ccafba7525c54327094",
        "type": "github"
      },
      "original": {
@ -376,7 +360,6 @@
        "home-manager": "home-manager_2",
        "nixos-hardware": "nixos-hardware",
        "nixpkgs": "nixpkgs_2",
-        "nixpkgs-unstable": "nixpkgs-unstable",
        "nur": "nur",
        "srvos": "srvos"
      }
@ -388,11 +371,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1716425501,
-        "narHash": "sha256-BSLhmGYY1khyyBAjraR+N0Pa9Nha/et5yQQlEZxcfkU=",
+        "lastModified": 1718844164,
+        "narHash": "sha256-QUXWv6llKIQ5To2N24d9dRI78Hqfm9iFyhvmvlOICNo=",
        "owner": "numtide",
        "repo": "srvos",
-        "rev": "1122cd50a23647e09c3e7a679d37ec02113bc412",
+        "rev": "557ff94aa1b48a723f8fa16eb9e7a2e6de991682",
        "type": "github"
      },
      "original": {
--- a/flake.nix
+++ b/flake.nix
@ -10,9 +10,7 @@
    flake-parts.url = "github:hercules-ci/flake-parts";
    flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs";

-    nixpkgs-unstable.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
-    # contains kernel 6.7.7, do not update
-    nixpkgs.url = "github:NixOS/nixpkgs/56051fbe049bf39adc1f08eb51740c226a4c3b90";
+    nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";

    nixos-hardware.url = "github:NixOS/nixos-hardware";
    nur.url = "github:nix-community/NUR";
--- a/hosts/epyc.nix
+++ b/hosts/epyc.nix
@ -1,7 +1,7 @@
-{ lib, pkgs, ... }:
+{ inputs, lib, pkgs, ... }:
 let
  gcc-system-features = arch: lib.optionals (arch != null) ([ "gccarch-${arch}" ]
-    ++ map (x: "gccarch-${x}") lib.systems.architectures.inferiors.${arch});
+  ++ map (x: "gccarch-${x}") lib.systems.architectures.inferiors.${arch});
 in
 {
  imports = [
@ -9,25 +9,21 @@ in
    ../modules/hardware/supermicro-H12SSL-i.nix
    ../modules/iperf-server.nix
    ../modules/hypervisor.nix
-    ../modules/hydra/coordinator.nix
    ../modules/android-cache.nix
    ../modules/garage.nix
    ../modules/users/friends.nix
+    ../modules/bagel-container.nix
  ];

  networking.hostName = "epyc";

+  security.acme.acceptTerms = true;
+  security.acme.defaults.email = "epyc@lahfa.xyz";
+
  boot.loader.systemd-boot.enable = true;
  boot.loader.efi.canTouchEfiVariables = true;

-  virtualisation.docker = {
-    enable = true;
-    rootless.enable = true;
-  };
-
-  # TODO: there's a critical bug on 6.8+ where btrfs won't mount the rootfs at all.
-  # Do not upgrade until it is fixed. Ping Raito when needed.
-  boot.kernelPackages = pkgs.linuxPackages_6_7;
+  boot.kernelPackages = pkgs.linuxPackages_latest;

  # Open public access to our PostgreSQL.
  services.postgresql.enable = true;
--- a/modules/bagel-container.nix
+++ b/modules/bagel-container.nix
@ -0,0 +1,46 @@
+# Stateful/mutable container used for Bagel (tm) related infra (mostly
+# rebuilding nixpkgs a lot).
+#
+# System image is stored at /var/lib/machines/bagel.
+{
+  systemd.nspawn.bagel = {
+    execConfig = {
+      Boot = true;
+      Ephemeral = false;
+      PrivateUsers = true;
+      NotifyReady = true;
+      LinkJournal = "try-guest";
+    };
+
+    networkConfig = {
+      Bridge = "wan-br";
+      VirtualEthernetExtra = "vb-bagel-v4:host1";
+    };
+  };
+
+  systemd.services."systemd-nspawn@bagel" = {
+    wantedBy = [ "machines.target" ];
+    wants = [ "network.target" ];
+    after = [ "network.target" ];
+    overrideStrategy = "asDropin";
+  };
+
+  systemd.network.networks."20-vb-bagel-v4" = {
+    matchConfig.Name = "vb-bagel-v4";
+    networkConfig.Address = [ "172.16.100.1/24" ];
+    networkConfig.IPMasquerade = true;
+  };
+
+  # Configure a local Nix builder account, since getting sandboxing and KVM
+  # working inside the container will be tricky.
+  users.users.bagel-builder = {
+    isSystemUser = true;
+    group = "nogroup";
+    home = "/var/empty";
+    shell = "/bin/sh";
+    openssh.authorizedKeys.keys = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAvUT9YBig9LQPHgypIBHQuC32XqDKxlFZ2CfgDi0ZKx"
+    ];
+  };
+  nix.settings.trusted-users = [ "bagel-builder" ];
+}
--- a/modules/buildbot/default.nix
+++ b/modules/buildbot/default.nix
@ -34,7 +34,6 @@ in
      pkgs.gh
      pkgs.nix
      pkgs.nix-output-monitor
-      inputs.attic.packages.x86_64-linux.attic
    ];
    environment.PYTHONPATH = "${python.withPackages (_: [package])}/${python.sitePackages}";
    environment.MASTER_URL = ''TCP:2a01\\:e34\\:ec2a\\:8e60\\:8ec7\\:b5d2\\:f663\\:a67a:9989'';
--- a/modules/hardware/supermicro-H12SSL-i.nix
+++ b/modules/hardware/supermicro-H12SSL-i.nix
@ -14,33 +14,43 @@
  boot.kernelModules = [ "kvm-amd" ];
  boot.extraModulePackages = [ ];

-  boot.initrd.extraUtilsCommands = ''
-    copy_bin_and_libs ${pkgs.nvme-cli}/bin/nvme
-    copy_bin_and_libs ${pkgs.util-linux}/bin/blkzone
-    copy_bin_and_libs ${pkgs.util-linux}/bin/lsblk
-  '';
+  boot.initrd.services.lvm.enable = true;
+  boot.initrd.systemd.enable = true;

-  boot.initrd.systemd.enable = lib.mkForce false;
-
-  fileSystems."/" =
-    { device = "/dev/disk/by-uuid/3a81ba8f-f5bb-446c-89a3-ad77e354dae0";
-      fsType = "btrfs";
+  fileSystems."/experiments" =
+    { device = "/dev/disk/by-uuid/40ef7d25-91c5-41e4-a40f-b0fb93658ffe";
+      fsType = "ext4";
    };

-  boot.initrd.luks.devices."nixroot" = {
-   device = "/dev/disk/by-uuid/c10d2822-cb83-4666-98f8-0aa04be259bc";
-   keyFile = "/dev/zero";
-   keyFileSize = 1;
-  };
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/53cc33a3-1488-44c4-8f5d-a2bc67914274";
+      fsType = "xfs";
+    };
+
+  fileSystems."/nix" =
+    { device = "/dev/disk/by-uuid/cee7b903-53f6-4967-b95d-654d34ccd460";
+      fsType = "xfs";
+    };
+
+  fileSystems."/home" =
+    { device = "/dev/disk/by-uuid/5625935d-579b-41e4-be35-03df8437bc2c";
+      fsType = "xfs";
+    };
+
+  fileSystems."/var" =
+    { device = "/dev/disk/by-uuid/33bf7f4e-37f5-4121-84ac-70d06964ea21";
+      fsType = "xfs";
+    };

  fileSystems."/boot" =
    { device = "/dev/disk/by-uuid/AFF2-3149";
      fsType = "vfat";
+      options = [ "fmask=0022" "dmask=0022" ];
    };

  swapDevices =
    [ { device = "/dev/disk/by-uuid/93e251e1-1bfc-4bd4-8585-ea2eae7795bf"; }
-  ];
+    ];

  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
--- a/modules/network.nix
+++ b/modules/network.nix
@ -14,8 +14,8 @@
    '')
    config.networking.newtype.hosts);

-  # leave container interfaces alone
-  systemd.network.networks."05-veth".extraConfig = ''
+  # leave container interfaces alone unless otherwise specified
+  systemd.network.networks."95-veth".extraConfig = ''
    [Match]
    Driver = veth

@ -34,12 +34,29 @@
    linkConfig.Name = "nat-lan";
  };

-  systemd.network.networks."10-wan" = {
-    matchConfig.Name = "wan";
+  systemd.network.netdevs."10-wan-br" = {
+    netdevConfig.Name = "wan-br";
+    netdevConfig.Kind = "bridge";
+    netdevConfig.MACAddress = "none";
+    bridgeConfig.MulticastSnooping = false;
+  };
+
+  systemd.network.links."10-wan-br" = {
+    matchConfig.Name = "wan-br";
+    linkConfig.MACAddressPolicy = "none";
+  };
+
+  systemd.network.networks."10-wan-br" = {
+    matchConfig.Name = "wan-br";
    linkConfig.RequiredForOnline = true;
    networkConfig.Address = [ config.networking.newtype.currentHost.ipv6 ];
  };

+  systemd.network.networks."10-wan" = {
+    matchConfig.Name = "wan";
+    networkConfig.Bridge = "wan-br";
+  };
+
  systemd.network.links."10-wan" = {
    matchConfig.MACAddress = "3c:ec:ef:7e:bd:c9";
    linkConfig.Name = "wan";
--- a/modules/packages.nix
+++ b/modules/packages.nix
@ -6,7 +6,6 @@
    whois

    nix-output-monitor
-    inputs.attic.packages.x86_64-linux.attic
    jq
    psmisc
    libarchive
@ -36,8 +35,6 @@
    usbutils

    ipmitool
-
-    nix-top
    # tries to default to soft-float due to out-dated cc-rs
  ] ++ lib.optional (!stdenv.hostPlatform.isRiscV) bandwhich;
 }
Author	SHA1	Message	Date
Pierre Bourdon	d5cf620e0a	bagel-container: provision a user with Nix store perms for remote builds	2024-07-04 19:45:48 +00:00
Raito Bezarius	1b1c6b2849	feat: cleanup post btrfs epic crash BTRFS had a bug and we know how it goes. Signed-off-by: Raito Bezarius <masterancpp@gmail.com>	2024-07-04 21:25:14 +02:00
Raito Bezarius	c96bce2975	fix: pin kernel to 6.7.5 6.7.6, 6.7.7 are not working for unknown reasons even though a previous NixOS stable 23.11 revision with 6.7.7 worked… Let's keep it that way for now, pending bisection between 6.7.5..6.7.6. Signed-off-by: Raito Bezarius <masterancpp@gmail.com>	2024-06-30 14:15:30 +02:00
raito	edf11d7650	Merge pull request 'bagel factory: init' (#2 ) from delroth/raito-shared-public-infra:container-prep into main Reviewed-on: raito/shared-public-infra#2	2024-06-23 18:26:57 +00:00
Pierre Bourdon	a1c645a1e6	bagel-container: provide IPv4 NAT for outbound access	2024-06-23 20:24:46 +02:00
Pierre Bourdon	7d9d2a93df	modules: add bagel-container	2024-06-23 19:33:00 +02:00
Pierre Bourdon	9e609128af	network: add a wan bridge for VMs/containers	2024-06-23 19:33:00 +02:00
Pierre Bourdon	6fe7c98069	gitignore: also ignore Nix result/ and colmena .gcroots/	2024-06-23 03:13:50 +02:00
Pierre Bourdon	f7902ae1c7	Bump nixpkgs to unstable and only pin for kernelPackages.	2024-06-23 03:13:50 +02:00