Compare commits

..

1 commit
main ... acls

Author SHA1 Message Date
130faa2836 feat: finer-grained ACLs for server accesses
In the process of adding multi-tenant infrastructure, it seems relevant
to add finer-grained ACLs.

Signed-off-by: Raito Bezarius <masterancpp@gmail.com>
2024-08-01 23:45:37 +02:00
13 changed files with 184 additions and 313 deletions

View file

@ -1,16 +1,21 @@
{ lib, ... }:
let
keys = import ./ssh-keys.nix;
in {
users.users.root.openssh.authorizedKeys.keys =
keys.users.delroth ++
keys.users.emilylange ++
keys.users.hexchen ++
keys.users.jade ++
keys.users.janik ++
keys.users.k900 ++
keys.users.lukegb ++
keys.users.maxine ++
keys.users.raito ++
keys.users.thubrecht ++
keys.users.yuka;
}
inherit (lib) genAttrs;
buildInfraMembers = [
"delroth"
"emilylange"
"hexchen"
"jade"
"janik"
"k900"
"maxine"
"raito"
"thubrecht"
"yuka"
];
in
{
bagel.admins.users = genAttrs buildInfraMembers (username: {
groups = [ "build-infra" ];
});
}

View file

@ -1,6 +1,7 @@
{
imports = [
./admins.nix
./server-acl.nix
./base-server.nix
./hardening.nix
./nix.nix

View file

@ -14,7 +14,7 @@
# Use our cache and trust its signing key. Still use cache.nixos.org as
# fallback.
nix.settings.substituters = [ "https://cache.forkos.org/" ];
nix.settings.substituters = [ "https://bagel-cache.s3-web.delroth.net/" ];
nix.settings.trusted-public-keys = [
"cache.forkos.org:xfXIUJO1yiEITJmYsVmNDa9BFSlgTh/YqZ+4ei1EhQg="
];

51
common/server-acl.nix Normal file
View file

@ -0,0 +1,51 @@
{ lib, config, ... }:
let
keys = import ./ssh-keys.nix;
inherit (lib) mkOption types length filterAttrs any catAttrs concatLists attrValues;
cfg = config.bagel.admins;
userOpts = { name, ... }: {
options = {
groups = mkOption {
type = types.listOf types.str;
description = "List of groups this user is part of";
example = [ "build-infra" ];
default = [ ];
};
sshKeys = mkOption {
type = types.listOf types.str;
description = "List of SSH keys associated to this user, defaults to `ssh-keys.nix` entries.";
default = keys.users.${name} or [ ];
};
};
};
isAllowedGroup = group: any (allowedGroup: group == allowedGroup) cfg.allowedGroups;
rootKeys = concatLists (catAttrs "sshKeys" (attrValues (filterAttrs (username: { groups, ... }: any isAllowedGroup groups) cfg.users)));
in
{
options.bagel.admins = {
allowedGroups = mkOption {
type = types.listOf types.str;
default = [ "catch-all" ];
description = "List of groups which are allowed to admin this machine.";
example = [ "lix" "build-infra" ];
};
users = mkOption {
type = types.attrsOf (types.submodule userOpts);
description = "Attribute set of admins with their groups and credentials, the username is the key of the attrset";
};
};
config = {
assertions = [
{ assertion = length config.users.users.root.openssh.authorizedKeys.keys > 0;
# TODO: you can add printing of `concatStringsSep ", " cfg.allowedGroups` to diagnose
# which are the allowed groups and existing admins.
message = "root@${config.networking.fqdnOrHostName} has no SSH key attached, this machine will lose its access if you deploy it successfully! Set a valid `bagel.admins.allowedGroups` or ensure you have at least one administrator of the relevant group registered";
}
];
users.users.root.openssh.authorizedKeys.keys = rootKeys;
};
}

View file

@ -64,11 +64,11 @@
"treefmt-nix": "treefmt-nix"
},
"locked": {
"lastModified": 1722939563,
"narHash": "sha256-lMe8aXgF550iQLRaoU+yn8yYQ4x2qiyqANgsFyjfWwA=",
"lastModified": 1722339507,
"narHash": "sha256-CdkQx6l0CuPtU9tMw3u73PAMhj+8YUAlTUVyrTCZas8=",
"ref": "refs/heads/non-flakes",
"rev": "4a162a8aa5dad6cecdb33bd8534e67e0bdaeb13f",
"revCount": 295,
"rev": "15963fa0e687cb39660aadf147402c955d671029",
"revCount": 292,
"type": "git",
"url": "https://git.lix.systems/lix-project/buildbot-nix.git"
},
@ -258,17 +258,17 @@
]
},
"locked": {
"lastModified": 1722688238,
"narHash": "sha256-x6BnYtArF6IDs7bS8ExokgAQBOlrxXxD0EOBIlASmfM=",
"lastModified": 1721682989,
"narHash": "sha256-kjJiZ7m4HKqbZ2mxNQiB32/goKFb8BRi8OqC4wIU0OI=",
"ref": "refs/heads/main",
"rev": "9b5ac87de73ea4646dbb2af979db91f096d29960",
"revCount": 4191,
"rev": "4b107e6ff36bd89958fba36e0fe0340903e7cd13",
"revCount": 4190,
"type": "git",
"url": "https://git.lix.systems/the-distro/hydra.git"
"url": "https://git.lix.systems/lix-project/hydra.git"
},
"original": {
"type": "git",
"url": "https://git.lix.systems/the-distro/hydra.git"
"url": "https://git.lix.systems/lix-project/hydra.git"
}
},
"lix": {

View file

@ -11,7 +11,7 @@
colmena.url = "github:zhaofengli/colmena";
colmena.inputs.nixpkgs.follows = "nixpkgs";
hydra.url = "git+https://git.lix.systems/the-distro/hydra.git";
hydra.url = "git+https://git.lix.systems/lix-project/hydra.git";
hydra.inputs.nixpkgs.follows = "nixpkgs";
nix-gerrit.url = "git+https://git.lix.systems/the-distro/nix-gerrit.git";
@ -92,6 +92,13 @@
./services
./common
{
# This means that anyone with @build-infra permissions
# can ssh on root of every machines handled here.
bagel.admins.allowedGroups = [
"build-infra"
];
}
];
makeBuilder = i: lib.nameValuePair "builder-${toString i}" {

View file

@ -1,63 +1,56 @@
{ config, lib, pkgs, nodes, modulesPath, ... }:
{ lib, pkgs, nodes, config, ... }:
# The way the connection is established is specific to the wob01 site and the Intel S2600KPR blades.
# Proper netboot is not possible, because while the blades and the APU board (which is the netboot
# server here) are in the same L2 network, the uplink connection of each blade is an LACP LAG,
# meaning that the switch on the other side will only enable the port if it sees valid LACP packets.
# IPXE sends out these LACP packets while it is probing the ports, however the NICs of the blades
# do not have a flash which IPXE could be written to.
# We work around this by presenting a virtual floppy drive using the "IUSB" protocol of the BMC.
# This virtual floppy drive contains an per-blade customized initramfs which will initialize the
# network connection including IP configuration and load the actual image off hydra.
# This virtual floppy drive contains an per-blade customized IPXE script which will initialize the
# network connection including IP configuration and chainload the actual script off the netboot
# server.
let
netboot-server-ip = "2a01:584:11::2";
netbootNodes = lib.filterAttrs (_: node: node.config.bagel.baremetal.builders.enable && node.config.bagel.baremetal.builders.netboot) nodes;
in {
assertions = [
{
assertion = !(lib.elem 443 config.networking.firewall.allowedTCPPorts);
message = ''
Port 443 is in networking.firewalls.allowedTCPPorts, but should be only manually
allowed for specific IPs and source ports in ${builtins.toJSON __curPos}
'';
}
];
networking.firewall.allowedTCPPorts = [ 80 ];
systemd.services = lib.mapAttrs' (nodename: node: let
ip = "2a01:584:11::1:${toString node.config.bagel.baremetal.builders.num}";
bmcIp = "192.168.1.${toString (node.config.bagel.baremetal.builders.num * 4 + 2)}";
notipxe = node.config.system.build.notipxe.config.system.build.usbImage;
gw = "2a01:584:11::1";
dns = "2a01:580:6000::ff01";
ipxe = node.pkgs.ipxe.override {
embedScript = builtins.toFile "bootstrap-${node.config.networking.hostName}.ipxe" ''
#!ipxe
ifopen net0
echo ip ${ip}/64
set net0/ip6:ipv6 ${ip}
set net0/len6:int8 64
echo gw ${gw}
set net0/gateway6:ipv6 ${gw}
echo dns ${dns}
set net0/dns6:ipv6 ${dns}
# wait for the lacp link to come up
ping --count 20 ${gw}
chain https://hydra.forkos.org/job/infra/main/${node.config.networking.hostName}/latest/download-by-type/file/ipxe
# if it fails, show a shell
shell
'';
};
in lib.nameValuePair "iusb-spoof-${nodename}" {
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Restart = "always";
Restart = "on-failure";
};
script = ''
AUTH_TOKEN=$(${pkgs.iusb-spoof}/bin/make-token ${bmcIp})
exec ${pkgs.iusb-spoof}/bin/iusb-spoof -r ${bmcIp} 5123 $AUTH_TOKEN ${notipxe}
exec ${pkgs.iusb-spoof}/bin/iusb-spoof -r ${bmcIp} 5123 $AUTH_TOKEN ${ipxe}/ipxe-efi.usb
'';
}) netbootNodes;
# Since the builders are stateless, they can not store their ssh hostkeys
networking.firewall.allowedTCPPorts = [ 80 ]; # for ACME
networking.firewall.extraInputRules = ''
ip6 saddr 2a01:584:11::/64 tcp sport < 1024 tcp dport 443 accept;
'';
security.acme.acceptTerms = true;
security.acme.defaults.email = "infra@forkos.org";
services.nginx = {
enable = true;
virtualHosts."vpn-gw.wob01.infra.forkos.org" = {
enableACME = true;
forceSSL = true;
locations = lib.mapAttrs' (nodename: node: let
ip = "2a01:584:11::1:${toString node.config.bagel.baremetal.builders.num}";
in lib.nameValuePair "/${nodename}/" {
root = "/var/www";
extraConfig = ''
allow ${ip};
deny all;
'';
}) netbootNodes;
};
};
}) (lib.filterAttrs (_: node: node.config.bagel.baremetal.builders.enable && node.config.bagel.baremetal.builders.netboot) nodes);
}

View file

@ -1,6 +1,5 @@
[
(final: prev: {
iusb-spoof = final.callPackage ./iusb-spoof.nix {};
u-root = final.callPackage ./u-root {};
})
]

View file

@ -9,7 +9,7 @@ rustPlatform.buildRustPackage rec {
src = builtins.fetchGit {
url = "https://git.lix.systems/the-distro/iusb-spoof/";
rev = "fafd47986239cc2f4dfbbae74b17555608806581";
rev = "a1ec0384e724f609bb8e391512a8fa76d9894e55";
};
cargoLock.lockFile = src + "/Cargo.lock";

View file

@ -1,20 +0,0 @@
{ buildGoModule, fetchFromGitHub }:
buildGoModule rec {
pname = "u-root";
version = "0.14.0";
src = fetchFromGitHub {
owner = "u-root";
repo = "u-root";
rev = "v${version}";
hash = "sha256-8zA3pHf45MdUcq/MA/mf0KCTxB1viHieU/oigYwIPgo=";
};
patches = [
./u-root-allow-https.patch
];
vendorHash = null;
doCheck = false;
}

View file

@ -1,12 +0,0 @@
diff --git a/pkg/curl/schemes.go b/pkg/curl/schemes.go
index 8bac3bc0..cd396cbc 100644
--- a/pkg/curl/schemes.go
+++ b/pkg/curl/schemes.go
@@ -81,6 +81,7 @@ var (
DefaultSchemes = Schemes{
"tftp": DefaultTFTPClient,
"http": DefaultHTTPClient,
+ "https": DefaultHTTPClient,
"file": &LocalFileClient{},
}
)

View file

@ -1,10 +1,8 @@
{ pkgs, lib, config, ... }:
{ pkgs, lib, config, extendModules, ... }:
let
cfg = config.bagel.baremetal.builders;
in
{
imports = [ ./netboot.nix ];
options = {
bagel.baremetal.builders = {
@ -44,20 +42,10 @@ in
];
};
nix.settings = {
trusted-users = [ "builder" "buildbot" ];
inherit ((import ./assignments.nix).${config.networking.hostName}) max-jobs cores;
};
services.openssh.extraConfig = ''
Match User buildbot
AllowAgentForwarding no
AllowTcpForwarding no
PermitTTY no
PermitTunnel no
X11Forwarding no
ForceCommand ${config.nix.package.out}/bin/nix-daemon --store /mnt --stdio
Match All
'';
nixpkgs.hostPlatform = "x86_64-linux";
hardware.cpu.intel.updateMicrocode = true;
@ -67,36 +55,31 @@ in
boot.initrd.services.lvm.enable = true;
fileSystems."/" = {
device = "/dev/disk/by-label/root";
fsType = "xfs";
};
fileSystems."/mnt" = {
device = "/dev/disk/by-label/hydra";
fsType = "xfs";
options = ["logbsize=256k"];
};
boot.kernel.sysctl."fs.xfs.xfssyncd_centisecs" = "12000";
fileSystems = lib.mkMerge [
(lib.mkIf (!cfg.netboot) {
"/" = {
device = "/dev/disk/by-label/root";
fsType = "xfs";
};
"/boot" = {
device = "/dev/disk/by-label/BOOT";
fsType = "vfat";
options = [ "fmask=0022" "dmask=0022" ];
};
})
{
"/mnt" = {
device = "/dev/disk/by-label/hydra";
fsType = "xfs";
options = ["logbsize=256k"];
};
# We want the tmp filesystem on the same filesystem as the hydra store, so that builds can use reflinks
fileSystems."/tmp" = {
device = "/mnt/tmp";
options = [ "bind" ];
};
# We want the tmp filesystem on the same filesystem as the hydra store, so that builds can use reflinks
"/tmp" = {
device = "/mnt/tmp";
options = [ "bind" ];
};
}
];
fileSystems."/boot" = {
device = "/dev/disk/by-label/BOOT";
fsType = "vfat";
options = [ "fmask=0022" "dmask=0022" ];
};
swapDevices = lib.optionals (!cfg.netboot) [
swapDevices = [
{
device = "/swapfile";
size = 50 * 1024; # 50GiB
@ -109,8 +92,8 @@ in
};
boot.kernelParams = [
"console=tty1"
"console=ttyS0,115200"
"console=tty1"
];
networking.useNetworkd = true;
@ -194,6 +177,39 @@ in
environment.systemPackages = [ pkgs.ipmitool ];
system.build = lib.mkIf cfg.netboot {
netbootVariant = extendModules {
modules = [
(
{ modulesPath, ... }:
{
imports = [ (modulesPath + "/installer/netboot/netboot.nix") ];
}
)
];
};
netbootDir = let
kernelTarget = pkgs.stdenv.hostPlatform.linux-kernel.target;
build = config.system.build.netbootVariant.config.system.build;
in
pkgs.symlinkJoin {
name = "netboot";
paths = [
build.netbootRamdisk
build.kernel
build.netbootIpxeScript
];
postBuild = ''
mkdir -p $out/nix-support
echo "file ${kernelTarget} ${build.kernel}/${kernelTarget}" >> $out/nix-support/hydra-build-products
echo "file initrd ${build.netbootRamdisk}/initrd" >> $out/nix-support/hydra-build-products
echo "file ipxe ${build.netbootIpxeScript}/netboot.ipxe" >> $out/nix-support/hydra-build-products
'';
preferLocalBuild = true;
};
};
system.stateVersion = "24.05";
};
}

View file

@ -1,169 +0,0 @@
{ modulesPath, pkgs, lib, config, extendModules, ... }@node:
let
cfg = config.bagel.baremetal.builders;
in
{
config = lib.mkIf (cfg.enable && cfg.netboot) {
systemd.services.sshd.after = [ "provision-ssh-hostkey.service" ];
systemd.services.provision-ssh-hostkey = {
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
};
script = ''
mkdir -p /etc/ssh
umask 0077
until ${pkgs.iputils}/bin/ping -c 1 vpn-gw.wob01.infra.forkos.org; do sleep 1; done
${pkgs.curl}/bin/curl --local-port 25-1024 https://vpn-gw.wob01.infra.forkos.org/${config.networking.hostName}/ssh_host_ed25519_key > /etc/ssh/ssh_host_ed25519_key
# Run the activation script again to trigger agenix decryption
/run/current-system/activate
'';
};
system.build = {
# Build a kernel and initramfs which will download the IPXE script from hydra using
# u-root pxeboot tool and kexec into the final netbooted system.
notipxe = import (modulesPath + "/..") {
system = "x86_64-linux";
configuration =
{ pkgs, config, ... }:
{
system.stateVersion = "24.11";
boot.initrd.availableKernelModules = [ "ahci" "ehci_pci" "usb_storage" "usbhid" "sd_mod" "igb" "bonding" ];
boot.kernelParams = [ "console=ttyS0,115200" "panic=1" "boot.panic_on_fail" ];
#boot.initrd.systemd.emergencyAccess = true;
networking.hostName = "${node.config.networking.hostName}-boot";
nixpkgs.overlays = import ../../overlays;
boot.loader.grub.enable = false;
fileSystems."/".device = "bogus"; # this config will never be booted
boot.initrd.systemd.enable = true;
boot.initrd.systemd.network = {
enable = true;
networks = node.config.systemd.network.networks;
netdevs = node.config.systemd.network.netdevs;
};
boot.initrd.systemd.storePaths = [
"${pkgs.u-root}/bin/pxeboot"
"${pkgs.iputils}/bin/ping"
];
boot.initrd.systemd.services.kexec = {
serviceConfig.Restart = "on-failure";
serviceConfig.Type = "oneshot";
wantedBy = [ "initrd-root-fs.target" ];
before = [ "sysroot.mount" ];
script = ''
ln -sf /dev/console /dev/tty
until ${pkgs.iputils}/bin/ping -c 1 hydra.forkos.org; do sleep 1; done
${pkgs.u-root}/bin/pxeboot -v -ipv4=false -file https://hydra.forkos.org/job/infra/main/${node.config.networking.hostName}/latest/download-by-type/file/ipxe
'';
};
boot.initrd.systemd.contents."/etc/ssl/certs/ca-certificates.crt".source = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
boot.initrd.services.resolved.enable = false;
boot.initrd.systemd.contents."/etc/resolv.conf".text = ''
nameserver 2001:4860:4860::6464
'';
boot.initrd.systemd.contents."/etc/systemd/journald.conf".text = ''
[Journal]
ForwardToConsole=yes
MaxLevelConsole=debug
'';
# Provide a bootable USB drive image
system.build.usbImage = pkgs.callPackage ({ stdenv, runCommand, dosfstools, e2fsprogs, mtools, libfaketime, util-linux, nukeReferences }:
runCommand "boot-img-${node.config.networking.hostName}" {
nativeBuildInputs = [ dosfstools e2fsprogs libfaketime mtools util-linux ];
outputs = [ "out" "firmware_part" ];
} ''
export img=$out
truncate -s 40M $img
sfdisk $img <<EOF
label: gpt
label-id: F222513B-DED1-49FA-B591-20CE86A2FE7F
type=C12A7328-F81F-11D2-BA4B-00A0C93EC93B, bootable
EOF
# Create a FAT32 /boot/firmware partition of suitable size into firmware_part.img
eval $(partx $img -o START,SECTORS --nr 1 --pairs)
truncate -s $((2081 * 512 + SECTORS * 512)) firmware_part.img
mkfs.vfat --invariant -i 2e24ec82 -n BOOT firmware_part.img
# Populate the files intended for /boot/firmware
mkdir -p firmware/EFI/BOOT firmware/loader/entries
cp ${pkgs.systemd}/lib/systemd/boot/efi/systemd-boot*.efi firmware/EFI/BOOT/BOOT${lib.toUpper stdenv.hostPlatform.efiArch}.EFI
cat > firmware/loader/loader.conf << EOF
default foo
EOF
cat > firmware/loader/entries/default.conf << EOF
title Default
linux /EFI/${pkgs.stdenv.hostPlatform.linux-kernel.target}
initrd /EFI/initrd
options init=${config.system.build.toplevel}/init ${toString config.boot.kernelParams}
EOF
cp ${config.system.build.kernel}/${pkgs.stdenv.hostPlatform.linux-kernel.target} firmware/EFI/${pkgs.stdenv.hostPlatform.linux-kernel.target}
cp ${config.system.build.initialRamdisk}/${config.system.boot.loader.initrdFile} firmware/EFI/initrd
find firmware -exec touch --date=2000-01-01 {} +
# Copy the populated /boot/firmware into the SD image
cd firmware
# Force a fixed order in mcopy for better determinism, and avoid file globbing
for d in $(find . -type d -mindepth 1 | sort); do
faketime "2000-01-01 00:00:00" mmd -i ../firmware_part.img "::/$d"
done
for f in $(find . -type f | sort); do
mcopy -pvm -i ../firmware_part.img "$f" "::/$f"
done
cd ..
# Verify the FAT partition before copying it.
fsck.vfat -vn firmware_part.img
dd conv=notrunc if=firmware_part.img of=$img seek=$START count=$SECTORS
cp firmware_part.img $firmware_part
''
) {};
}
;
};
# This is the config which will actually be booted
netbootVariant = extendModules {
modules = [
(
{ modulesPath, ... }:
{
imports = [ (modulesPath + "/installer/netboot/netboot.nix") ];
}
)
];
};
# A derivation combining all the artifacts required for netbooting for the hydra job
netbootDir = let
kernelTarget = pkgs.stdenv.hostPlatform.linux-kernel.target;
build = config.system.build.netbootVariant.config.system.build;
in
pkgs.symlinkJoin {
name = "netboot";
paths = [
build.netbootRamdisk
build.kernel
build.netbootIpxeScript
];
postBuild = ''
mkdir -p $out/nix-support
echo "file ${kernelTarget} $out/${kernelTarget}" >> $out/nix-support/hydra-build-products
echo "file initrd $out/initrd" >> $out/nix-support/hydra-build-products
echo "file ipxe $out/netboot.ipxe" >> $out/nix-support/hydra-build-products
'';
preferLocalBuild = true;
};
};
};
}