Compare commits

..

No commits in common. "main" and "hydra-metrics" have entirely different histories.

96 changed files with 956 additions and 2215 deletions

11
.envrc
View file

@ -1,11 +1,2 @@
#!/usr/bin/env bash
# the shebang is ignored, but nice for editors
# shellcheck shell=bash
if type -P lorri &>/dev/null; then
eval "$(lorri direnv --flake .)"
else
echo 'while direnv evaluated .envrc, could not find the command "lorri" [https://github.com/nix-community/lorri]'
use flake
fi
use flake

View file

@ -1,32 +1 @@
# Infrastructure for the donut shaped thing that is absolutely not a donut.
## Quick start
### Build the infrastructure
```
$ colmena build --on @localboot
```
Notice that `@localboot` is load-bearing as we have some machines that _cannot be_ deployed with vanilla Colmena. Fixing this is welcome.
### Recommended deploy process
```
$ colmena apply dry-activate $machine # Verify that the nvd log is reasonable.
$ colmena apply $machine
```
### Recommended upgrade process
```
$ nix flake update
$ colmena apply dry-activate --on @localboot # Verify that the nvd log is reasonable. Run it twice to get only NVD logs shown.
$ colmena apply --on @localboot
```
## Troubleshooting
### I failed to deploy `gerrit01`
Our Gerrit source build is known to have some hiccups sometimes, we are always interested in build logs, feel free to attach information in a new issue so we can make it more reliable.
Infrastructure for the donut shaped thing that is absolutely not a donut.

View file

@ -1,47 +1,17 @@
{ lib, ... }:
let
inherit (lib) genAttrs;
in
# Note: to add somefew in this list.
# Ensure their SSH key is already in common/ssh-keys.nix with
# the same username for here, so that the keys is automatically added.
{
bagel.groups = {
floral-infra.members = [
"delroth"
"emilylange"
"hexchen"
"jade"
"janik"
"k900"
"maxine"
"raito"
"thubrecht"
"winter"
"yuka"
"ckie"
];
lix-infra.members = [
"raito"
"hexchen"
"jade"
"pennae"
];
};
bagel.users = genAttrs [
"delroth"
"emilylange"
"hexchen"
"jade"
"janik"
"k900"
"maxine"
"raito"
"thubrecht"
"winter"
"yuka"
"ckie"
"pennae"
] (name: {});
keys = import ./ssh-keys.nix;
in {
users.users.root.openssh.authorizedKeys.keys =
keys.users.delroth ++
keys.users.emilylange ++
keys.users.hexchen ++
keys.users.jade ++
keys.users.janik ++
keys.users.k900 ++
keys.users.lukegb ++
keys.users.maxine ++
keys.users.raito ++
keys.users.thubrecht ++
keys.users.yuka ++
keys.users.winter;
}

View file

@ -1,13 +1,11 @@
{ lib, pkgs, ... }: {
imports = [
./known-ssh-keys.nix
./cgroups.nix
];
nixpkgs.overlays = import ../overlays;
nix.package = lib.mkDefault pkgs.lix;
system.tools.nixos-option.enable = false;
services.openssh.enable = lib.mkForce true;
networking.nftables.enable = true;
@ -32,7 +30,7 @@
automatic = true;
persistent = true;
dates = lib.mkDefault "daily";
options = lib.mkDefault "--delete-older-than 30d";
options = "--delete-older-than 30d";
};
services.journald.extraConfig = "SystemMaxUse=512M";

View file

@ -1,83 +0,0 @@
# Relatively inspired by fbtax2:
# https://facebookmicrosites.github.io/cgroup2/docs/fbtax-results.html
#
# See also the Chris Down talk at LISA'21:
# https://www.usenix.org/conference/lisa21/presentation/down
{ ... }:
let
systemCriticalSliceConfig = {
ManagedOOMMemoryPressure = "kill";
# guarantee availability of memory
MemoryMin = "192M";
# default 100
IOWeight = 1000;
# default 100
CPUWeight = 1000;
};
in
{
systemd.oomd = {
enable = true;
# why not, we have cgroups at user level now so it'll just kill the
# terminal
enableRootSlice = true;
enableSystemSlice = true;
enableUserSlices = true;
};
systemd.enableCgroupAccounting = true;
systemd.services.nix-daemon = {
serviceConfig = {
# FIXME: how do i deprioritize this for memory
CPUWeight = 10;
IOWeight = 10;
};
};
systemd.slices.hostcritical = {
description = "Ensures that services to keep the system alive remain alive";
unitConfig = {
# required to avoid a dependency cycle on systemd-oomd. systemd will
# actually guess this right but we should fix it anyway.
DefaultDependencies = false;
};
sliceConfig = systemCriticalSliceConfig;
};
# make root logins higher priority for resources
systemd.slices."user-0" = {
sliceConfig = systemCriticalSliceConfig;
};
systemd.slices.system = {
sliceConfig = {
ManagedOOMMemoryPressure = "kill";
ManagedOOMMemoryPressureLimit = "50%";
IOWeight = 100;
};
};
systemd.services.sshd = {
serviceConfig = {
Slice = "hostcritical.slice";
};
};
systemd.services.systemd-oomd = {
serviceConfig = {
Slice = "hostcritical.slice";
};
};
systemd.services.systemd-journald = {
serviceConfig = {
Slice = "hostcritical.slice";
};
};
}

View file

@ -1,14 +1,12 @@
{
imports = [
./admins.nix
./server-acl.nix
./base-server.nix
./hardening.nix
./nix.nix
./raito-proxy-aware-nginx.nix
./raito-vm.nix
./sysadmin
./hardware
./zsh.nix
./secrets.nix
];
}

View file

@ -1,7 +0,0 @@
{ ... }: {
imports = [
./raito-vm.nix
./oracle-vm.nix
./hetzner.nix
];
}

View file

@ -1,76 +0,0 @@
{ lib, config, ... }:
let
cfg = config.bagel.hardware.hetzner;
inherit (lib) mkEnableOption mkIf mkOption types;
in
{
options.bagel.hardware.hetzner = {
enable = mkEnableOption "Hetzner's hardware defaults";
platformType = mkOption {
# Only VMs are supported.
type = types.enum [ "virtual-machine" ];
};
system = mkOption {
# Only the aarch64-linux VM Hetzner is supported.
type = types.enum [ "aarch64-linux" ];
};
networking.wan = {
mac = mkOption {
type = types.str;
description = "MAC address of the WAN interface in the Hetzner machine";
};
address = mkOption {
type = types.listOf types.str;
description = "List of static addresses attached to the WAN interface";
};
};
};
config = mkIf cfg.enable {
# A bunch of stuff is virtio.
boot.initrd.availableKernelModules = [
"xhci_pci"
"usbhid"
"sr_mod"
"virtio_gpu"
"virtio_scsi"
"virtio_rng"
"virtio_pci"
];
boot.loader.systemd-boot.enable = true;
boot.loader.efi.canTouchEfiVariables = true;
networking.useDHCP = lib.mkDefault false;
# Stolen from the netplan provided by aarch64 Ubuntu images.
systemd.network.enable = true;
systemd.network.links."10-wan" = {
linkConfig.Name = "wan";
matchConfig.MACAddress = cfg.networking.mac;
};
systemd.network.networks."10-wan" = {
matchConfig.Name = "wan";
networkingConfig.Address = cfg.networking.address;
linkConfig.RequiredForOnline = true;
DHCP = "ipv4";
routes = [
{
routeConfig = {
Destination = "::/0";
GatewayOnLink = true;
Gateway = "fe80::1";
};
}
];
dhcpV4Config = {
RouteMetric = 100;
UseMTU = true;
};
};
};
}

View file

@ -1,52 +0,0 @@
{ lib, config, modulesPath, ... }:
let
cfg = config.bagel.hardware.oracle-vm;
inherit (lib) mkEnableOption mkIf mkOption types;
in
{
options.bagel.hardware.oracle-vm = {
enable = mkEnableOption "Oracle's VM hardware defaults";
system = mkOption {
# Only the free Oracle VMs are supported.
type = types.enum [ "aarch64-linux" ];
};
};
# Imports a bunch of virtio modules.
imports = [
"${modulesPath}/profiles/qemu-guest.nix"
];
config = mkIf cfg.enable {
boot.loader.systemd-boot.enable = true;
boot.loader.efi.canTouchEfiVariables = true;
boot.initrd.systemd.enable = true;
boot.initrd.availableKernelModules = [
"xhci_pci" "virtio_pci" "usbhid" "sr_mod"
];
boot.initrd.kernelModules = [ ];
boot.kernelModules = [ ];
boot.extraModulePackages = [ ];
nixpkgs.hostPlatform = cfg.system;
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
# (the default) this is the recommended approach. When using systemd-networkd it's
# still possible to use this option, but it's recommended to use it in conjunction
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
networking.useDHCP = lib.mkDefault false;
# Examples:
# 2: enp0s3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
# link/ether 02:00:17:00:91:6e brd ff:ff:ff:ff:ff:ff
# inet 10.0.0.94/24 brd 10.0.0.255 scope global dynamic noprefixroute enp0s3
# valid_lft 44162sec preferred_lft 33362sec
# inet6 fe80::17ff:fe00:916e/64 scope link
# valid_lft forever preferred_lft forever
# [root@build02-aarch64-lahfa:~]# ip r
# default via 10.0.0.1 dev enp0s3 proto dhcp src 10.0.0.94 metric 1002 mtu 9000
networking.interfaces.enp0s3.useDHCP = lib.mkDefault true;
};
}

View file

@ -2,6 +2,5 @@
{
programs.ssh.knownHosts = {
"[cl.forkos.org]:29418".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM82mJ259C8Nc+BHHNBeRWXWhL3dfirQhmFbDAwHMle3";
"[gerrit.lix.systems]:2022".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICC/S6Z56uhv7zBMutkV0nU8eDuRcl3trykGWBch4L/l";
};
}

View file

@ -1,22 +0,0 @@
## This is a simple secret abstraction with multi-tenancy awareness.
{ config, lib, ... }:
let
cfg = config.bagel.secrets;
inherit (lib) mkOption types genAttrs;
in
{
options.bagel.secrets = {
tenant = mkOption {
type = types.enum [ "lix" "floral" ];
};
files = mkOption {
type = types.listOf types.str;
default = [ ];
};
};
config.age.secrets = genAttrs cfg.files (secretFile: {
file = ../secrets/${cfg.tenant}/${secretFile}.age;
});
}

View file

@ -1,69 +0,0 @@
{ lib, config, ... }:
let
keys = import ./ssh-keys.nix;
inherit (lib) mkOption types length concatMap listToAttrs catAttrs attrValues;
cfgAdmins = config.bagel.admins;
cfgGroups = config.bagel.groups;
cfgUsers = config.bagel.users;
userOpts = { name, ... }: {
options = {
sshKeys = mkOption {
type = types.listOf types.str;
description = "List of SSH keys associated to this user, defaults to `ssh-keys.nix` entries.";
default = keys.users.${name} or [ ];
};
};
};
groupOpts = { name, ... }: {
options = {
members = mkOption {
type = types.listOf types.str;
description = "List of users member of this group";
example = [ "raito" ];
default = [ ];
};
};
};
# There might be duplicate in that list. We will turn it into an attribute set.
allowedMembers = listToAttrs (
map (member: {
name = member;
value = cfgUsers.${member};
}) (concatMap (allowedGroup: cfgGroups.${allowedGroup}.members) cfgAdmins.allowedGroups));
rootKeys = concatMap ({ sshKeys, ... }: sshKeys) (attrValues allowedMembers);
in
{
options.bagel.users = mkOption {
type = types.attrsOf (types.submodule userOpts);
description = "User configuration for server ACLs";
};
options.bagel.groups = mkOption {
type = types.attrsOf (types.submodule groupOpts);
description = "Group configuration for server ACLs";
};
options.bagel.admins = {
allowedGroups = mkOption {
type = types.listOf types.str;
default = [ "catch-all" ];
description = "List of groups which are allowed to admin this machine.";
example = [ "lix" "build-infra" ];
};
};
config = {
assertions = [
{ assertion = length config.users.users.root.openssh.authorizedKeys.keys > 0;
# TODO: you can add printing of `concatStringsSep ", " cfg.allowedGroups` to diagnose
# which are the allowed groups and existing admins.
message = "root@${config.networking.fqdnOrHostName} has no SSH key attached, this machine will lose its access if you deploy it successfully! Set a valid `bagel.admins.allowedGroups` or ensure you have at least one administrator of the relevant group registered";
}
];
users.users.root.openssh.authorizedKeys.keys = rootKeys;
};
}

View file

@ -1,6 +1,5 @@
{
machines = {
# Floral
bagel-box = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAsO4bNqY04uG13Pg3ubHfRDssTphDLzZ4YUniE5/p+M";
meta01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM5t9gYorOWgpCFDJgb24pyCKIabGpeI2H/UfdvXODcT";
public01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPBy8G8rfLA6E9i+t5kjVafxU1c2NXATXKxoXTH4Kgtm";
@ -21,16 +20,6 @@
builder-9 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOhws9zGgocVY36dMtOL+CXadpvRMffxoWMkfEcTBJm7";
builder-10 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE7sgIuTSqZiZhp8TvObSbIEhcHHsL5hcmYA22uzwxth";
wob-vpn-gw = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINVytPPW8XnXf/rD5TFzsw//CZc2lBjQLmDzlVGPZsjh";
# Lix
build01-aarch64-lix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICC69NZD/zhIB/wUb5odg46bss5g8hH2fDl22bk4qeSW";
build02-aarch64-lix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGdJE375pe58RJbhKwXRp3D//+SJ3ssiVZrLsM9CLHn0";
build01-aarch64-darwin-lix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMVf1uO0lv5UBti/naW/+amqLxvWZg+StXk9aM+lJ7e4";
buildbot-lix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFoVSh35UqNQZ6ZZ1c6CzqERC40ovQ/KDXz8pC7nNlkR";
# Raito infrastructure
epyc-newtype-fr = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOXT9Init1MhKt4rjBANLq0t0bPww/WQZ96uB4AEDrml";
};
users = {
@ -61,12 +50,7 @@
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU"
];
thubrecht = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPM1jpXR7BWQa7Sed7ii3SbvIPRRlKb3G91qC0vOwfJn" ];
yuka = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIxQ3NYBi8v1f/vhxLKDcA6upmX0pctRDbnK6SER5OUR yureka" ];
yuka = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKath4/fDnlv/4fzxkPrQN1ttmoPRNu/m9bEtdPJBDfY cardno:16_933_242" ];
winter = [ "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIH/LDRUG+U+++UmlxvA2kspioTjktQZ8taDcHq8gVlkfAAAABHNzaDo=" ];
ckie = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIH3uTwzSSMAPg84fwbNp2cq9+BdLFeA1VzDGth4zCAbz https://mei.puppycat.house" ];
pennae = [
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIC5Wf5/IbyFpdziWfwxkQqxOf3r1L9pYn6xQBEKFwmMY"
"sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIK8icXjHkb4XzbIVN3djH4CE7RvgGd+3xbG4cgh0Yls5AAAABHNzaDo="
];
};
}

View file

@ -13,11 +13,7 @@ in
tmux
rsync
fd
eza
grc
ripgrep
delta
tshark
pv
kitty.terminfo
config.boot.kernelPackages.perf

View file

@ -1,4 +1,5 @@
{ lib, pkgs, config, ... }: {
users.defaultUserShell = pkgs.zsh;
programs.zsh = {
enable = true;
enableCompletion = true;

View file

@ -27,17 +27,16 @@
"inputs": {
"crane": "crane",
"flake-compat": "flake-compat_2",
"flake-parts": "flake-parts_2",
"nix-github-actions": "nix-github-actions_2",
"flake-utils": "flake-utils_2",
"nixpkgs": "nixpkgs",
"nixpkgs-stable": "nixpkgs-stable"
},
"locked": {
"lastModified": 1730257295,
"narHash": "sha256-OQl+aAsKiyygvpzck1u0sZf/R4T9zM903CgNDFmmzA8=",
"lastModified": 1711742460,
"narHash": "sha256-0O4v6e4a1toxXZ2gf5INhg4WPE5C5T+SVvsBt+45Mcc=",
"owner": "zhaofengli",
"repo": "attic",
"rev": "48c8b395bfbc6b76c7eae74df6c74351255a095c",
"rev": "4dbdbee45728d8ce5788db6461aaaa89d98081f0",
"type": "github"
},
"original": {
@ -50,11 +49,11 @@
"bats-assert": {
"flake": false,
"locked": {
"lastModified": 1692829535,
"narHash": "sha256-oDqhUQ6Xg7a3xx537SWLGRzqP3oKKeyY4UYGCdz9z/Y=",
"lastModified": 1636059754,
"narHash": "sha256-ewME0l27ZqfmAwJO4h5biTALc9bDLv7Bl3ftBzBuZwk=",
"owner": "bats-core",
"repo": "bats-assert",
"rev": "e2d855bc78619ee15b0c702b5c30fb074101159f",
"rev": "34551b1d7f8c7b677c1a66fc0ac140d6223409e5",
"type": "github"
},
"original": {
@ -66,11 +65,11 @@
"bats-support": {
"flake": false,
"locked": {
"lastModified": 1693050811,
"narHash": "sha256-PxJaH16+QrsfZqtkWVt5K6TwJB5gjIXnbGo+MB84WIU=",
"lastModified": 1548869839,
"narHash": "sha256-Gr4ntadr42F2Ks8Pte2D4wNDbijhujuoJi4OPZnTAZU=",
"owner": "bats-core",
"repo": "bats-support",
"rev": "9bf10e876dd6b624fe44423f0b35e064225f7556",
"rev": "d140a65044b2d6810381935ae7f0c94c7023c8c3",
"type": "github"
},
"original": {
@ -88,16 +87,16 @@
"treefmt-nix": "treefmt-nix"
},
"locked": {
"lastModified": 1730064416,
"narHash": "sha256-Opbtu9hKijGkEx+GYbSu3MJms3lFxZmAGTFyckguWMM=",
"ref": "refs/heads/forkos",
"rev": "79137b14f3cb376204f739f44b05aebfc288ca89",
"revCount": 310,
"lastModified": 1722939563,
"narHash": "sha256-lMe8aXgF550iQLRaoU+yn8yYQ4x2qiyqANgsFyjfWwA=",
"ref": "refs/heads/non-flakes",
"rev": "4a162a8aa5dad6cecdb33bd8534e67e0bdaeb13f",
"revCount": 295,
"type": "git",
"url": "https://git.lix.systems/lix-project/buildbot-nix.git"
},
"original": {
"ref": "refs/heads/forkos",
"ref": "refs/heads/non-flakes",
"type": "git",
"url": "https://git.lix.systems/lix-project/buildbot-nix.git"
}
@ -126,18 +125,17 @@
"inputs": {
"flake-compat": "flake-compat",
"flake-utils": "flake-utils",
"nix-github-actions": "nix-github-actions",
"nixpkgs": [
"nixpkgs"
],
"stable": "stable"
},
"locked": {
"lastModified": 1731527002,
"narHash": "sha256-dI9I6suECoIAmbS4xcrqF8r2pbmed8WWm5LIF1yWPw8=",
"lastModified": 1711386353,
"narHash": "sha256-gWEpb8Hybnoqb4O4tmpohGZk6+aerAbJpywKcFIiMlg=",
"owner": "zhaofengli",
"repo": "colmena",
"rev": "e3ad42138015fcdf2524518dd564a13145c72ea1",
"rev": "cd65ef7a25cdc75052fbd04b120aeb066c3881db",
"type": "github"
},
"original": {
@ -155,11 +153,11 @@
]
},
"locked": {
"lastModified": 1722960479,
"narHash": "sha256-NhCkJJQhD5GUib8zN9JrmYGMwt4lCRp6ZVNzIiYCl0Y=",
"lastModified": 1702918879,
"narHash": "sha256-tWJqzajIvYcaRWxn+cLUB9L9Pv4dQ3Bfit/YjU5ze3g=",
"owner": "ipetkov",
"repo": "crane",
"rev": "4c6c77920b8d44cd6660c1621dea6b3fc4b4c4f4",
"rev": "7195c00c272fdd92fc74e7d5a0a2844b9fadb2fb",
"type": "github"
},
"original": {
@ -169,12 +167,18 @@
}
},
"crane_2": {
"inputs": {
"nixpkgs": [
"grapevine",
"nixpkgs"
]
},
"locked": {
"lastModified": 1730060262,
"narHash": "sha256-RMgSVkZ9H03sxC+Vh4jxtLTCzSjPq18UWpiM0gq6shQ=",
"lastModified": 1716569590,
"narHash": "sha256-5eDbq8TuXFGGO3mqJFzhUbt5zHVTf5zilQoyW5jnJwo=",
"owner": "ipetkov",
"repo": "crane",
"rev": "498d9f122c413ee1154e8131ace5a35a80d8fa76",
"rev": "109987da061a1bf452f435f1653c47511587d919",
"type": "github"
},
"original": {
@ -215,11 +219,11 @@
"rust-analyzer-src": "rust-analyzer-src"
},
"locked": {
"lastModified": 1730442928,
"narHash": "sha256-U1DWb5c3EfkA7pqx5V1H4AWRA+EaE6UJ0lIRvK1RxgM=",
"lastModified": 1716359173,
"narHash": "sha256-pYcjP6Gy7i6jPWrjiWAVV0BCQp+DdmGaI/k65lBb/kM=",
"owner": "nix-community",
"repo": "fenix",
"rev": "87b4d20f896c99018dde4702a9c6157b516f2a76",
"rev": "b6fc5035b28e36a98370d0eac44f4ef3fd323df6",
"type": "github"
},
"original": {
@ -248,11 +252,11 @@
"flake-compat_2": {
"flake": false,
"locked": {
"lastModified": 1696426674,
"narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
"lastModified": 1673956053,
"narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
"owner": "edolstra",
"repo": "flake-compat",
"rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
"rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
"type": "github"
},
"original": {
@ -316,28 +320,6 @@
}
},
"flake-parts_2": {
"inputs": {
"nixpkgs-lib": [
"grapevine",
"attic",
"nixpkgs"
]
},
"locked": {
"lastModified": 1722555600,
"narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=",
"owner": "hercules-ci",
"repo": "flake-parts",
"rev": "8471fe90ad337a8074e957b69ca4d0089218391d",
"type": "github"
},
"original": {
"owner": "hercules-ci",
"repo": "flake-parts",
"type": "github"
}
},
"flake-parts_3": {
"inputs": {
"nixpkgs-lib": [
"hydra",
@ -359,23 +341,6 @@
"type": "github"
}
},
"flake-parts_4": {
"inputs": {
"nixpkgs-lib": "nixpkgs-lib"
},
"locked": {
"lastModified": 1727826117,
"narHash": "sha256-K5ZLCyfO/Zj9mPFldf3iwS6oZStJcU4tSpiXTMYaaL0=",
"owner": "hercules-ci",
"repo": "flake-parts",
"rev": "3d04084d54bedc3d6b8b736c70ef449225c361b1",
"type": "github"
},
"original": {
"id": "flake-parts",
"type": "indirect"
}
},
"flake-utils": {
"locked": {
"lastModified": 1659877975,
@ -392,15 +357,30 @@
}
},
"flake-utils_2": {
"locked": {
"lastModified": 1667395993,
"narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"flake-utils_3": {
"inputs": {
"systems": "systems_2"
},
"locked": {
"lastModified": 1726560853,
"narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=",
"lastModified": 1710146030,
"narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a",
"rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
"type": "github"
},
"original": {
@ -410,6 +390,21 @@
"type": "github"
}
},
"flake-utils_4": {
"locked": {
"lastModified": 1634851050,
"narHash": "sha256-N83GlSGPJJdcqhUxSCS/WwW5pksYf3VP1M13cDRTSVA=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "c91f3de5adaf1de973b797ef7485e441a65b8935",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"gerrit-dashboard": {
"flake": false,
"locked": {
@ -432,21 +427,20 @@
"crane": "crane_2",
"fenix": "fenix",
"flake-compat": "flake-compat_3",
"flake-utils": "flake-utils_2",
"flake-utils": "flake-utils_3",
"nix-filter": "nix-filter",
"nixpkgs": [
"nixpkgs"
],
"rocksdb": "rocksdb",
"rust-manifest": "rust-manifest"
},
"locked": {
"host": "gitlab.computer.surgery",
"lastModified": 1731820516,
"narHash": "sha256-BJiFq5meQMwOLhVS9/m2HCZoNUIN5u5weJXr94dkIRU=",
"lastModified": 1723576377,
"narHash": "sha256-sTa4XT5xMQkhhLknOfVd433YS1TvkMrE45qAsI1ZB6U=",
"owner": "matrix",
"repo": "grapevine-fork",
"rev": "887e59cf03efb70bfa68b56dda32c411ba6cd5fa",
"rev": "3b99032456700d06dd937db6a85976a8be9d4fa7",
"type": "gitlab"
},
"original": {
@ -486,11 +480,11 @@
]
},
"locked": {
"lastModified": 1728321752,
"narHash": "sha256-GbBAoBF7ZObz0IP+g0LZKxMafpMvNKjTEu9haiZbV54=",
"lastModified": 1724616313,
"narHash": "sha256-9syppf9Gm/6F4wQQAbsf7rGY1DooMsprnsEY/0eaewg=",
"ref": "refs/heads/main",
"rev": "ee1234c15cdcb427dbd4828e0add09d02cd606c9",
"revCount": 4220,
"rev": "44b9a7b95d23e7a8587cb963f00382046707f2db",
"revCount": 4202,
"type": "git",
"url": "https://git.lix.systems/lix-project/hydra.git"
},
@ -511,11 +505,11 @@
"pre-commit-hooks": "pre-commit-hooks"
},
"locked": {
"lastModified": 1728163191,
"narHash": "sha256-SW0IEBsPN1EysqzvfDT+8Kimtzy03O1BxQQm7ZB6fRY=",
"lastModified": 1723919517,
"narHash": "sha256-D6+zmRXzr85p7riphuIrJQqangoJe70XM5jHhMWwXws=",
"ref": "refs/heads/main",
"rev": "ed9b7f4f84fd60ad8618645cc1bae2d686ff0db6",
"revCount": 16323,
"rev": "278fddc317cf0cf4d3602d0ec0f24d1dd281fadb",
"revCount": 16138,
"type": "git",
"url": "https://git.lix.systems/lix-project/lix"
},
@ -526,12 +520,12 @@
},
"nix-eval-jobs": {
"inputs": {
"flake-parts": "flake-parts_3",
"flake-parts": "flake-parts_2",
"lix": [
"hydra",
"lix"
],
"nix-github-actions": "nix-github-actions_3",
"nix-github-actions": "nix-github-actions",
"nixpkgs": [
"hydra",
"nixpkgs"
@ -554,11 +548,11 @@
},
"nix-filter": {
"locked": {
"lastModified": 1730207686,
"narHash": "sha256-SCHiL+1f7q9TAnxpasriP6fMarWE5H43t25F5/9e28I=",
"lastModified": 1710156097,
"narHash": "sha256-1Wvk8UP7PXdf8bCCaEoMnOT1qe5/Duqgj+rL8sRQsSM=",
"owner": "numtide",
"repo": "nix-filter",
"rev": "776e68c1d014c3adde193a18db9d738458cd2ba4",
"rev": "3342559a24e85fc164b295c3444e8a139924675b",
"type": "github"
},
"original": {
@ -589,49 +583,6 @@
}
},
"nix-github-actions": {
"inputs": {
"nixpkgs": [
"colmena",
"nixpkgs"
]
},
"locked": {
"lastModified": 1729742964,
"narHash": "sha256-B4mzTcQ0FZHdpeWcpDYPERtyjJd/NIuaQ9+BV1h+MpA=",
"owner": "nix-community",
"repo": "nix-github-actions",
"rev": "e04df33f62cdcf93d73e9a04142464753a16db67",
"type": "github"
},
"original": {
"owner": "nix-community",
"repo": "nix-github-actions",
"type": "github"
}
},
"nix-github-actions_2": {
"inputs": {
"nixpkgs": [
"grapevine",
"attic",
"nixpkgs"
]
},
"locked": {
"lastModified": 1729742964,
"narHash": "sha256-B4mzTcQ0FZHdpeWcpDYPERtyjJd/NIuaQ9+BV1h+MpA=",
"owner": "nix-community",
"repo": "nix-github-actions",
"rev": "e04df33f62cdcf93d73e9a04142464753a16db67",
"type": "github"
},
"original": {
"owner": "nix-community",
"repo": "nix-github-actions",
"type": "github"
}
},
"nix-github-actions_3": {
"inputs": {
"nixpkgs": [
"hydra",
@ -671,11 +622,11 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1726042813,
"narHash": "sha256-LnNKCCxnwgF+575y0pxUdlGZBO/ru1CtGHIqQVfvjlA=",
"lastModified": 1711401922,
"narHash": "sha256-QoQqXoj8ClGo0sqD/qWKFWezgEwUL0SUh37/vY2jNhc=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "159be5db480d1df880a0135ca0bfed84c2f88353",
"rev": "07262b18b97000d16a4bdb003418bd2fb067a932",
"type": "github"
},
"original": {
@ -685,18 +636,6 @@
"type": "github"
}
},
"nixpkgs-lib": {
"locked": {
"lastModified": 1727825735,
"narHash": "sha256-0xHYkMkeLVQAMa7gvkddbPqpxph+hDzdu1XdGPJR+Os=",
"type": "tarball",
"url": "https://github.com/NixOS/nixpkgs/archive/fb192fec7cc7a4c26d51779e9bab07ce6fa5597a.tar.gz"
},
"original": {
"type": "tarball",
"url": "https://github.com/NixOS/nixpkgs/archive/fb192fec7cc7a4c26d51779e9bab07ce6fa5597a.tar.gz"
}
},
"nixpkgs-regression": {
"locked": {
"lastModified": 1643052045,
@ -715,32 +654,32 @@
},
"nixpkgs-stable": {
"locked": {
"lastModified": 1724316499,
"narHash": "sha256-Qb9MhKBUTCfWg/wqqaxt89Xfi6qTD3XpTzQ9eXi3JmE=",
"lastModified": 1711460390,
"narHash": "sha256-akSgjDZL6pVHEfSE6sz1DNSXuYX6hq+P/1Z5IoYWs7E=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "797f7dc49e0bc7fab4b57c021cdf68f595e47841",
"rev": "44733514b72e732bd49f5511bd0203dea9b9a434",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-24.05",
"ref": "nixos-23.11",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs_2": {
"locked": {
"lastModified": 1731676054,
"narHash": "sha256-OZiZ3m8SCMfh3B6bfGC/Bm4x3qc1m2SVEAlkV6iY7Yg=",
"lastModified": 1723221148,
"narHash": "sha256-7pjpeQlZUNQ4eeVntytU3jkw9dFK3k1Htgk2iuXjaD8=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "5e4fbfb6b3de1aa2872b76d49fafc942626e2add",
"rev": "154bcb95ad51bc257c2ce4043a725de6ca700ef6",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-unstable",
"ref": "nixpkgs-unstable",
"repo": "nixpkgs",
"type": "github"
}
@ -761,23 +700,6 @@
"type": "github"
}
},
"rocksdb": {
"flake": false,
"locked": {
"lastModified": 1730475155,
"narHash": "sha256-u5uuShM2SxHc9/zL4UU56IhCcR/ZQbzde0LgOYS44bM=",
"owner": "facebook",
"repo": "rocksdb",
"rev": "3c27a3dde0993210c5cc30d99717093f7537916f",
"type": "github"
},
"original": {
"owner": "facebook",
"ref": "v9.7.4",
"repo": "rocksdb",
"type": "github"
}
},
"root": {
"inputs": {
"agenix": "agenix",
@ -793,18 +715,17 @@
],
"nix-gerrit": "nix-gerrit",
"nixpkgs": "nixpkgs_2",
"stateless-uptime-kuma": "stateless-uptime-kuma",
"terranix": "terranix"
}
},
"rust-analyzer-src": {
"flake": false,
"locked": {
"lastModified": 1730386175,
"narHash": "sha256-0Uq+/B8eu7pw8B8pxuGdFYKjcVLwNMcHfDxU9sXh7rg=",
"lastModified": 1716107283,
"narHash": "sha256-NJgrwLiLGHDrCia5AeIvZUHUY7xYGVryee0/9D3Ir1I=",
"owner": "rust-lang",
"repo": "rust-analyzer",
"rev": "0ba893e1a00d92557ac91efb771d72eee36ca687",
"rev": "21ec8f523812b88418b2bfc64240c62b3dd967bd",
"type": "github"
},
"original": {
@ -817,47 +738,31 @@
"rust-manifest": {
"flake": false,
"locked": {
"narHash": "sha256-tB9BZB6nRHDk5ELIVlGYlIjViLKBjQl52nC1avhcCwA=",
"narHash": "sha256-aZFye4UrtlcvLHrISldx4g9uGt3thDbVlLMK5keBSj0=",
"type": "file",
"url": "https://static.rust-lang.org/dist/channel-rust-1.81.0.toml"
"url": "https://static.rust-lang.org/dist/channel-rust-1.78.0.toml"
},
"original": {
"type": "file",
"url": "https://static.rust-lang.org/dist/channel-rust-1.81.0.toml"
"url": "https://static.rust-lang.org/dist/channel-rust-1.78.0.toml"
}
},
"stable": {
"locked": {
"lastModified": 1730883749,
"narHash": "sha256-mwrFF0vElHJP8X3pFCByJR365Q2463ATp2qGIrDUdlE=",
"lastModified": 1696039360,
"narHash": "sha256-g7nIUV4uq1TOVeVIDEZLb005suTWCUjSY0zYOlSBsyE=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "dba414932936fde69f0606b4f1d87c5bc0003ede",
"rev": "32dcb45f66c0487e92db8303a798ebc548cadedc",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-24.05",
"ref": "nixos-23.05",
"repo": "nixpkgs",
"type": "github"
}
},
"stateless-uptime-kuma": {
"flake": false,
"locked": {
"lastModified": 1728243069,
"narHash": "sha256-l9fgwesnmFxasCaYUCD7L9bGGJXytLuwtx3CZMgpwJg=",
"ref": "refs/heads/master",
"rev": "880f444ff7862d6127b051cf1a993ad1585b1652",
"revCount": 25,
"type": "git",
"url": "https://git.dgnum.eu/DGNum/stateless-uptime-kuma.git"
},
"original": {
"type": "git",
"url": "https://git.dgnum.eu/DGNum/stateless-uptime-kuma.git"
}
},
"systems": {
"locked": {
"lastModified": 1681028828,
@ -888,38 +793,22 @@
"type": "github"
}
},
"systems_3": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
},
"terranix": {
"inputs": {
"bats-assert": "bats-assert",
"bats-support": "bats-support",
"flake-parts": "flake-parts_4",
"flake-utils": "flake-utils_4",
"nixpkgs": [
"nixpkgs"
],
"systems": "systems_3",
"terranix-examples": "terranix-examples"
},
"locked": {
"lastModified": 1728959489,
"narHash": "sha256-1Pu2j5xsBTuoyga08ZVf+rKp3FOMmJh/0fXen/idOrA=",
"lastModified": 1695406838,
"narHash": "sha256-xiUfVD6rtsVWFotVtUW3Q1nQh4obKzgvpN1wqZuGXvM=",
"owner": "terranix",
"repo": "terranix",
"rev": "7734e2ee6a1472807a33ce1e7da794bed2aaf91c",
"rev": "fc9077ca02ab5681935dbf0ecd725c4d889b9275",
"type": "github"
},
"original": {
@ -930,11 +819,11 @@
},
"terranix-examples": {
"locked": {
"lastModified": 1637156952,
"narHash": "sha256-KqvXIe1yiKOEP9BRYqNQN+LOWPCsWojh0WjEgv5jfEI=",
"lastModified": 1636300201,
"narHash": "sha256-0n1je1WpiR6XfCsvi8ZK7GrpEnMl+DpwhWaO1949Vbc=",
"owner": "terranix",
"repo": "terranix-examples",
"rev": "921680efb8af0f332d8ad73718d53907f9483e24",
"rev": "a934aa1cf88f6bd6c6ddb4c77b77ec6e1660bd5e",
"type": "github"
},
"original": {

View file

@ -2,7 +2,7 @@
description = "Bagel cooking infrastructure";
inputs = {
nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
terranix.url = "github:terranix/terranix";
terranix.inputs.nixpkgs.follows = "nixpkgs";
@ -22,15 +22,12 @@
gerrit-dashboard.url = "git+https://git.lix.systems/the-distro/gerrit-monitoring.git";
gerrit-dashboard.flake = false;
buildbot-nix.url = "git+https://git.lix.systems/lix-project/buildbot-nix.git?ref=refs/heads/forkos";
buildbot-nix.url = "git+https://git.lix.systems/lix-project/buildbot-nix.git?ref=refs/heads/non-flakes";
buildbot-nix.inputs.nixpkgs.follows = "nixpkgs";
channel-scripts.url = "git+https://git.lix.systems/the-distro/channel-scripts.git";
channel-scripts.inputs.nixpkgs.follows = "nixpkgs";
stateless-uptime-kuma.url = "git+https://git.dgnum.eu/DGNum/stateless-uptime-kuma.git";
stateless-uptime-kuma.flake = false;
lix.follows = "hydra/lix";
grapevine = {
@ -58,7 +55,6 @@
inputs.lix.overlays.default
inputs.nix-gerrit.overlays.default
inputs.channel-scripts.overlays.default
(import "${inputs.stateless-uptime-kuma}/overlay.nix")
];
};
terraform = pkgs.opentofu;
@ -68,6 +64,7 @@
./terraform
{
bagel.dnsimple.enable = true;
bagel.gandi.enable = true;
bagel.hydra.enable = true;
}
];
@ -115,90 +112,25 @@
./common
];
floralInfraModules = commonModules ++ [
({ config, lib, ... }: {
# This means that anyone with @floral-infra permissions
# can ssh on root of every machines handled here.
bagel.admins.allowedGroups = [
"floral-infra"
];
# Tag all machines which have local boot as local bootables.
deployment.tags = lib.mkMerge [
[ "floral" ]
(lib.mkIf (config.bagel.baremetal.builders.enable -> !config.bagel.baremetal.builders.netboot)
[ "localboot" ]
)
];
bagel.monitoring.grafana-agent.tenant = "floral";
bagel.secrets.tenant = "floral";
bagel.builders.extra-build-capacity.provider.tenant = "floral";
bagel.services.buildbot.tenant = "floral";
})
];
# These are Floral baremetal builders.
makeBuilder = i:
let
enableNetboot = i >= 6;
in
lib.nameValuePair "builder-${toString i}" {
imports = floralInfraModules;
bagel.baremetal.builders = { enable = true; num = i; netboot = enableNetboot; };
makeBuilder = i: lib.nameValuePair "builder-${toString i}" {
imports = commonModules;
bagel.baremetal.builders = { enable = true; num = i; netboot = i >= 6; };
};
lixInfraModules = commonModules ++ [
{
# This means that anyone with @lix-infra permissions
# can ssh on root of every machines handled here.
bagel.admins.allowedGroups = [
"lix-infra"
];
# Tag all machines which have local boot as local bootables.
# Lix has no netbootable machine.
deployment.tags = [ "localboot" "lix" ];
bagel.monitoring.grafana-agent.tenant = "lix";
bagel.secrets.tenant = "lix";
bagel.builders.extra-build-capacity.provider = {
tenant = "lix";
buildfarmPublicKeys = [
# buildbot.lix.systems SSH key
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDu4cEqZzAI/1vZjSQkTJ4ijIg9nuloOuSKUrnkJIOFn"
];
};
bagel.services.buildbot.tenant = "lix";
}
];
builders = lib.listToAttrs (lib.genList makeBuilder 11);
in {
meta.nixpkgs = systemBits.x86_64-linux.pkgs;
# Add any non-x86_64 native systems here.
# Cross compilation is not supported yet.
meta.nodeNixpkgs =
let
aarch64-systems = systems: lib.genAttrs systems (system: systemBits.aarch64-linux.pkgs);
in
aarch64-systems [
"build01-aarch64-lix"
];
meta.specialArgs.inputs = inputs;
bagel-box.imports = floralInfraModules ++ [ ./hosts/bagel-box ];
meta01.imports = floralInfraModules ++ [ ./hosts/meta01 ];
gerrit01.imports = floralInfraModules ++ [ ./hosts/gerrit01 ];
fodwatch.imports = floralInfraModules ++ [ ./hosts/fodwatch ];
git.imports = floralInfraModules ++ [ ./hosts/git ];
wob-vpn-gw.imports = floralInfraModules ++ [ ./hosts/wob-vpn-gw ];
buildbot.imports = floralInfraModules ++ [ ./hosts/buildbot ];
public01.imports = floralInfraModules ++ [ ./hosts/public01 ];
build-coord.imports = floralInfraModules ++ [ ./hosts/build-coord ];
build01-aarch64-lix.imports = lixInfraModules ++ [ ./hosts/build01-aarch64-lix ];
buildbot-lix.imports = lixInfraModules ++ [ ./hosts/buildbot-lix ];
bagel-box.imports = commonModules ++ [ ./hosts/bagel-box ];
meta01.imports = commonModules ++ [ ./hosts/meta01 ];
gerrit01.imports = commonModules ++ [ ./hosts/gerrit01 ];
fodwatch.imports = commonModules ++ [ ./hosts/fodwatch ];
git.imports = commonModules ++ [ ./hosts/git ];
wob-vpn-gw.imports = commonModules ++ [ ./hosts/wob-vpn-gw ];
buildbot.imports = commonModules ++ [ ./hosts/buildbot ];
public01.imports = commonModules ++ [ ./hosts/public01 ];
build-coord.imports = commonModules ++ [ ./hosts/build-coord ];
} // builders;
hydraJobs = builtins.mapAttrs (n: v: v.config.system.build.netbootDir or v.config.system.build.toplevel) self.nixosConfigurations;

View file

@ -38,7 +38,7 @@
zramSwap = {
enable = true;
memoryPercent = 100;
memoryPercent = 25;
};
networking.useNetworkd = true;

View file

@ -1,27 +0,0 @@
{ ... }: {
networking.hostName = "build01";
networking.domain = "aarch64.lix.systems";
# Those free sweet VMs.
bagel.hardware.oracle-vm = {
enable = true;
system = "aarch64-linux";
};
fileSystems."/" =
{ device = "/dev/disk/by-uuid/a333323c-99f0-4258-8f68-496858d56f71";
fsType = "ext4";
};
fileSystems."/boot" =
{ device = "/dev/disk/by-uuid/3E74-C937";
fsType = "vfat";
};
swapDevices = [ ];
bagel.builders.extra-build-capacity.provider.enable = true;
i18n.defaultLocale = "en_US.UTF-8";
system.stateVersion = "24.05";
deployment.targetHost = "build01.aarch64.lix.systems";
}

View file

@ -1,71 +0,0 @@
# Configuration for a virtual machine in Raito's micro-DC basement.
# 32 vCPU (2014 grade Xeon though)
# 32GB RAM
# 30GB SSD
# 500GB HDD
# All specifications can be upgraded to a certain extent, just ask Raito.
# Hosts the coordinator for Buildbot.
#
# vim: et:ts=2:sw=2:
#
{ lib, modulesPath, ... }: {
networking.hostName = "buildbot";
networking.domain = "lix.systems";
zramSwap.enable = true;
bagel.sysadmin.enable = true;
# Buildbot is proxied.
bagel.raito.v6-proxy-awareness.enable = true;
bagel.hardware.raito-vm = {
enable = true;
networking = {
nat-lan-mac = "BC:24:11:75:62:42";
wan = {
mac = "BC:24:11:B2:5F:2E";
address = "2001:bc8:38ee:100::200/56";
};
};
};
i18n.defaultLocale = "en_US.UTF-8";
bagel.services.buildbot = {
enable = true;
domain = "buildbot.lix.systems";
gerrit =
{
domain = "gerrit.lix.systems";
port = 2022;
username = "buildbot";
};
cors.allowedOrigins = [
"https://*.lix.systems"
];
projects = [
"lix"
"lix-installer"
];
buildSystems = [
"x86_64-linux"
"aarch64-linux"
"aarch64-darwin"
# Too slow.
/* "x86_64-darwin" */
];
# Lix is not allowed to use yet Floral's x86_64 builders for now.
builders = [ ];
};
# This machine does not use /nix from btrfs, and instead uses a store on a bigger disk.
fileSystems."/nix" =
lib.mkForce
{ device = "/dev/disk/by-uuid/1815ca49-d0b0-4b99-8aec-0d790498ba6f";
fsType = "xfs";
neededForBoot = true;
options = [ "relatime" ];
};
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
system.stateVersion = "24.05";
deployment.targetHost = "buildbot.lix.systems";
}

View file

@ -2,7 +2,6 @@
config,
lib,
pkgs,
nodes,
...
}:
{
@ -27,24 +26,7 @@
bagel.services.buildbot = {
enable = true;
domain = "buildbot.forkos.org";
gerrit =
let
cfgGerrit = nodes.gerrit01.config.bagel.services.gerrit;
in
{
domain = cfgGerrit.canonicalDomain;
port = cfgGerrit.port;
username = "buildbot";
};
cors.allowedOrigins = [
"https://*.forkos.org"
];
projects = [
"buildbot-test"
"nixpkgs"
"infra"
];
builders = [ "builder-9" ];
builders = [ "builder-10" ];
};
i18n.defaultLocale = "en_US.UTF-8";

View file

@ -23,9 +23,6 @@
};
};
# Block all these crawlers!!
bagel.services.nginx.crawler-blocker.enable = true;
fileSystems."/gerrit-data" = {
device = "/dev/disk/by-uuid/d1062305-0dea-4740-9a27-b6b1691862a4";
fsType = "ext4";
@ -42,7 +39,7 @@
};
age.secrets.ows-deploy-key = {
file = ../../secrets/floral/ows-deploy-key.age;
file = ../../secrets/ows-deploy-key.age;
mode = "0600";
owner = "git";
group = "git";
@ -124,7 +121,7 @@
};
};
age.secrets.s3-channel-staging-keys.file = ../../secrets/floral/s3-channel-staging-keys.age;
age.secrets.s3-channel-staging-keys.file = ../../secrets/s3-channel-staging-keys.age;
bagel.nixpkgs.channel-scripts = {
enable = true;
otlp.enable = true;

View file

@ -9,11 +9,6 @@
# TODO: make it the default
networking.domain = "infra.forkos.org";
bagel.status = {
enable = true;
domain = "status.forkos.org";
};
bagel.sysadmin.enable = true;
# Newsletter is proxied.
bagel.raito.v6-proxy-awareness.enable = true;

View file

@ -1,10 +1,9 @@
{ lib
, buildGo122Module
, buildGoModule
, fetchFromGitHub
}:
# FIXME: update, remove this pin
buildGo122Module rec {
buildGoModule rec {
pname = "pyroscope";
version = "1.7.1";

View file

@ -1,76 +1,52 @@
let
keys = import common/ssh-keys.nix;
commonKeys = {
# WARNING: `keys.users.*` are *lists*, so you need concatenate them, don't put them into lists!
# Otherwise, agenix will be confused!
global = keys.users.raito;
lix = keys.users.hexchen ++ keys.users.jade;
floral = keys.users.delroth;
};
commonKeys = keys.users.delroth ++ keys.users.raito;
secrets = with keys; {
floral = {
hydra-postgres-key = [ machines.build-coord ];
hydra-s3-credentials = [ machines.build-coord ];
hydra-signing-priv = [ machines.build-coord ];
hydra-ssh-key-priv = [ machines.build-coord ];
hydra-postgres-key = [ machines.build-coord ];
hydra-s3-credentials = [ machines.build-coord ];
hydra-signing-priv = [ machines.build-coord ];
hydra-ssh-key-priv = [ machines.build-coord ];
netbox-environment = [ machines.meta01 ];
mimir-environment = [ machines.meta01 ];
mimir-webhook-url = [ machines.meta01 ];
grafana-oauth-secret = [ machines.meta01 ];
loki-environment = [ machines.meta01 ];
gerrit-prometheus-bearer-token = [ machines.gerrit01 machines.meta01 ];
pyroscope-secrets = [ machines.meta01 ];
tempo-environment = [ machines.meta01 ];
netbox-environment = [ machines.meta01 ];
mimir-environment = [ machines.meta01 ];
mimir-webhook-url = [ machines.meta01 ];
grafana-oauth-secret = [ machines.meta01 ];
loki-environment = [ machines.meta01 ];
gerrit-prometheus-bearer-token = [ machines.gerrit01 machines.meta01 ];
pyroscope-secrets = [ machines.meta01 ];
tempo-environment = [ machines.meta01 ];
buildbot-worker-password = [ machines.buildbot ];
buildbot-oauth-secret = [ machines.buildbot ];
buildbot-workers = [ machines.buildbot ];
# Private SSH key to Gerrit
# ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHx52RUPWzTa2rBA96xcnGjjzAboNN/hm6gW+Q6JiSos
buildbot-service-key = [ machines.buildbot ];
# Signing key for Buildbot's specific cache
buildbot-signing-key = [ machines.buildbot ];
buildbot-remote-builder-key = [ machines.buildbot ];
buildbot-worker-password = [ machines.buildbot ];
buildbot-oauth-secret = [ machines.buildbot ];
buildbot-workers = [ machines.buildbot ];
# Private SSH key to Gerrit
# ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHx52RUPWzTa2rBA96xcnGjjzAboNN/hm6gW+Q6JiSos
buildbot-service-key = [ machines.buildbot ];
# Signing key for Buildbot's specific cache
buildbot-signing-key = [ machines.buildbot ];
buildbot-remote-builder-key = [ machines.buildbot ];
# These are the same password, but nginx wants it in htpasswd format
metrics-push-htpasswd = [ machines.meta01 ];
# Yes, even Lix machines are included in this monitoring infrastructure.
metrics-push-password = builtins.attrValues machines;
# These are the same password, but nginx wants it in htpasswd format
metrics-push-htpasswd = [ machines.meta01 ];
metrics-push-password = builtins.attrValues machines;
ows-deploy-key = [ machines.gerrit01 ];
s3-channel-staging-keys = [ machines.gerrit01 ];
s3-channel-keys = [ machines.gerrit01 ];
ows-deploy-key = [ machines.gerrit01 ];
s3-channel-staging-keys = [ machines.gerrit01 ];
s3-channel-keys = [ machines.gerrit01 ];
postgres-ca-priv = [ machines.bagel-box ];
postgres-tls-priv = [ machines.bagel-box ];
postgres-ca-priv = [ machines.bagel-box ];
postgres-tls-priv = [ machines.bagel-box ];
newsletter-secrets = [ machines.public01 ];
s3-revproxy-api-keys = [ machines.public01 ];
stateless-uptime-kuma-password = [ machines.public01 ];
};
lix = {
buildbot-worker-password = [ machines.buildbot-lix ];
buildbot-oauth-secret = [ machines.buildbot-lix ];
buildbot-workers = [ machines.buildbot-lix ];
# Private SSH key to Gerrit
# ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHx52RUPWzTa2rBA96xcnGjjzAboNN/hm6gW+Q6JiSos
buildbot-service-key = [ machines.buildbot-lix ];
# Signing key for Buildbot's specific cache
buildbot-signing-key = [ machines.buildbot-lix ];
buildbot-remote-builder-key = [ machines.buildbot-lix ];
};
newsletter-secrets = [ machines.public01 ];
s3-revproxy-api-keys = [ machines.public01 ];
stateless-uptime-kuma-password = [ machines.public01 ];
};
mkSecretListFor = tenant:
map (secretName: {
name = "secrets/${tenant}/${secretName}.age";
value.publicKeys = secrets.${tenant}."${secretName}" ++ commonKeys.global ++ commonKeys.${tenant};
}) (builtins.attrNames secrets.${tenant});
in
builtins.listToAttrs (
(mkSecretListFor "floral") ++ (mkSecretListFor "lix")
map (secretName: {
name = "secrets/${secretName}.age";
value.publicKeys = secrets."${secretName}" ++ commonKeys;
}) (builtins.attrNames secrets)
)

View file

@ -1,68 +0,0 @@
age-encryption.org/v1
-> ssh-ed25519 +HUDfA d5f2ESneC0wsoc9rwTjNfNXMBjCbjAQ7euthH2Buq1E
5CynaQ8zhDRBvcmifhCsiDtllztCVAqs8rU36DOxgPw
-> ssh-ed25519 +uvEmw EtYRis2LP0jv1W8mx8vFYNzkgi8OoqnA8cM2huS6NBk
ll1csFIO+hVYk+I0uSVJmlDKj9aTWvf4kaYI5LJcm7w
-> ssh-ed25519 DMaM1w ex4QJN8CG99J15i+yvqGEiEZn9OlGIC+cmLHL4u8ZEI
VXnOv4CGK68q5t6hUV3oKAtxGZ+4FVbrmE1yMn16A0Q
-> ssh-ed25519 sixKXw drXN6+q1y7L7ZU4chTfHfelu5GcTdff+i/UMFV0+3RQ
+8jmgnMh2OpQ3vhAuyQYWslfx7KO84a8KsCpoRD3Yl8
-> ssh-ed25519 aHbF7w Af7NgjZ/Nvh5FHrX2VlF5riTIhJ+fdxTo6OR+8PcNwA
ktKpm/HnOnw2Ym7xee3N1rneEX7+/xDhcp71N1NNHAA
-> ssh-ed25519 87T2Ig 8mEUxJ/5NUvV+qQCDQH2Tm6Ryr5hf4xgsQlqXGf03Fw
EavMcnsg/3EYBLQEBHX+0oTDKq5ZL4vj+mZntPM8UMU
-> ssh-ed25519 Ao+7Wg UphWbatIaa+R1oZbfHazFhrawf0vax/3ZZS7YuX03Hs
dwBbwoV0jpjiKr+nj+CRfUDgDl7ISpsCintVAzHnIFQ
-> ssh-ed25519 wIR2ZA ZM58Nq7eJX9JVeYkoJf+mw8hxhYGoTx042ow1u3mJkw
UtEaf7e4xsPO0ISlIF9LF+GcwTBqw4AXdMO4MASfgLQ
-> ssh-ed25519 oGiV/Q G5KX/Eox+9md0yFRUZvGIsio2gWM17soHsL6H6zEX2g
vI8jPjBAoFF0xhvRRLPzCMSiQOQ0fKuRb3CYVu3KUUo
-> ssh-ed25519 gO3aog p9nZtjzoA0zJM+7Y6R16mpdub3dhu67yOYTUNKRytgI
YL9vAp1+CK7jgmXkB47ufZMz+/swngkdUvEGR1zFZwc
-> ssh-ed25519 N/+Clw 6LzFdtNsWewuJK2r97ZXJbRazvK3raN78UGanR/zWVU
WT0y+sfDP3ffVwRcbYw51ArFR3OzXnoyi9IXwZZKEL8
-> ssh-ed25519 CtkSZw CV0jQ5dIbgFtMxGK1X9b1qJOKmske8VgIPW5NW9mAwc
clv7P3de61nZmXrvbOgL7Llw8ZqBMm2WFqgpznDwKv8
-> ssh-ed25519 keg2lg 3Nk40ByQj8RThj4QDY2BdAkw55mXAJprXQRGjQqGvz0
f8OFszJ8p90crFd+awEE12CNd7b22zgpH2XRKmH/Hf0
-> ssh-ed25519 H885DA GDiJYH+gaC++TSqfoPDOTFcsCZRhEl0EeTeab7tgcWU
kMILmwNMnMS7rgC3kKsAksu4Txn5owPU2y09h4aHKY8
-> ssh-ed25519 Rq7K4Q VCNxGtCSCD2OYSWWwl0+yf189xV3QwRiwo80h4NPTkE
hHkgYHLbISdl/RRdlInp9ub854M9ZKFSXpLgKW2YkmQ
-> ssh-ed25519 vvyRpw XSCCrqEOtvzQRssI0U1DHirKoPgbOUKJxNKnioHhT2Y
HGey1j0Kxae5Qs0aw6eqFziQGiRmNA+lEwbRdf5hhbM
-> ssh-ed25519 aSEktQ mXY70Lgl76J4O5dPdDcIqmJ40EinigDuZrUghpGOq2I
U2qeVFxGCYCEFWU+7vHc5Mu9EuzScowrjnwUyoqPj5U
-> ssh-ed25519 cD6JxA at89poimBZyeeM8CQrxDxN0yCNDT2k04++py1fFycj8
cQV/K5zc5x/oYnJ4N0MX3sTboT4G4ZNvVUVdHuJRzbA
-> ssh-ed25519 ec6XRQ spJtb/xy4k4dmwKz8R2CPhC1WcuNV/rnDT978GkjHHk
KrGEVGts/AhzbRNreqQ/CVanXL3l/9oMWxnpBLj23qU
-> ssh-ed25519 1qYEfw KRkTYlvvnsCIExKQNmCyU7YxnGZsiI03kzecXNpLzUQ
h2YagV7BzlsF7banzwXbOudTdlFzT7LC8PvtxAsX36U
-> ssh-ed25519 2D+APA 4hdYlOnNIT9Q6tyKwXzy+u66Ti2EJopK43Sipebd0As
tuesc9/QcEu4q9bTFJ5zJr0qvgLcmpn4at4cYtHrtbE
-> ssh-ed25519 eTSU6g i1qT6PtepHXnoLCqDbhk86QG+SR9luQaw34a34gy5mw
YE9VBAT5SLW2ECHRU+dMg9na6OQNVRVGuhY8vOdmE/Q
-> ssh-ed25519 j2r2qQ TTTbSB/8UIDmmI3C9+u24PYZNfjl9jGADKHNWIwLfGE
SNDforwii/GFp82TpyOcVIVrZWCe2QQKrjzPA6XA7Jc
-> ssh-ed25519 CyxfgQ P5EiJ54v65Sz1gHuI0s170Z7c1WjcZLlb7NYigElfVs
iYJUGpoE9LBIlv+O1navSSsy3EJ8tusXXX+/QAQvjNI
-> ssh-ed25519 C/bBAQ hlBDpQRkcVCr3B6TCrBjxauhUX6Ndpm0s6x8W4hU6gM
OFG3EuGJkSoEEXhbJ/Tp2DBdnBcs+hzxjNRdvcOSpQs
-> ssh-ed25519 +qVung cGEGpO8NJfpj9ixAH9lhYkPKPDdQWryVxSOhMGQdnWM
+MycbIEab3P/AOS9i/YmPBDXB76hp3xUcWI4VMihV2w
-> ssh-rsa krWCLQ
Zv3dPYERlX1MaVaJTBDwIcjt1yLmu4Z7MovPgjGg01p+XsdBXeepTyOl+gRBwGgo
AW5CIuaChYxtSNJ6nOgSaUpqzILycUF1xE1jROe3MIX2MZ4KGD1qoqcHbiCAng+a
RqYrwAKnNea9FQMVfhYZBkRoYE6ne1R+0G6BoFM/okz24pAAFPBx+sMMhfTkt0uV
kHVx0dgRw1pxa7Na98WH/7E0zp9VuBvVHGXfk1rfW/UQlbIO5RP3nldFoa6OmOWS
JZ022UvjyC1re0KCurka4y+qmaiRKnTBmpIXxJFMwNCAQ8O8SeAQ3DHKHmXNMOIL
ZVICtRRk0uX36AVU8DWDog
-> ssh-ed25519 /vwQcQ kF8+hsA+0Msjd3q0SL52cae5RDqx4ls5kPKnc3UZyms
Q33kIKJL3Vjxu7LQ5l4M3tlEuj+OW4uGh1x+JxthW8A
-> ssh-ed25519 0R97PA gWBH71l6w9upTE0DwqOMSvWXc5VyJiKFAQLaSpWQ43E
IrOrvzEa0bABw6UOpP8pM8WhuRNMaWJ2khljJIKwOS8
-> ssh-ed25519 K3b7BA oS14iav9pSioLecMkOanJz89OJygLugvrnnTs5pKzz8
akupMSiqXussXJyHwFm/f0imKALjowJVqd8/LFcC/58
--- bCJXTEDaKg4FF2dQp2zFMU4Zm0zACekCuDvthcAyncM
&­Ÿ €WaïãàBD R(¯¥Ñ”ufj<>úVÁ8nÆ>ßøëæðZúâ{Idƒ„©,³*„%Ç“È

View file

@ -1,7 +0,0 @@
age-encryption.org/v1
-> ssh-ed25519 Ao+7Wg q7oRHUUlAvD8OUbpPT7d6eLMPWU0YS/verYTDE5BCkY
/87/1uqOvRYeqel9IjnFmGyF9SXUQD8MTgfcj91b/Fs
--- ulIeB91NJ7z/64h9BCLSD9/RW/zwv3m1Zo2ovNuInv8
Îœç}³Óš#épÇ o>ä·*vµ÷ÄåŽs?[¦º´L
<EFBFBD>þz™rý‰?R±Ñó7<Ê
æi!€{X„¾òÓ

Binary file not shown.

View file

@ -1,6 +0,0 @@
age-encryption.org/v1
-> ssh-ed25519 Ao+7Wg EMpfs0EpWwaIKAoUBfEkyAHLIwi6JnGG6RvUWM5LjnU
LKiwUBNc791U/GVRNlRPZE/TEMJjcFFrLruFJhiyiOI
--- 0khp8u+4vHgGyQqP05m473Eo09eyOUZLI5+EK4olzoc
N3(
ª•ûxRq°<71>f<EFBFBD>Ó;ͼ3¬~RˆÓC^ñ +fœš1”®˜xˆ÷ÅëñSØ— hâ £ÖË°GˆÓnYIûµ:7¾!°u×Hþy/‰Øð‰™.¯¤á^¹lC™ôUÈËþ5cž:]ÿNž&'MÎè¶É-˜ÆHF¦D0 cjô ‹Ð~

View file

@ -1,6 +0,0 @@
age-encryption.org/v1
-> ssh-ed25519 Ao+7Wg RPKKoI5l5cYVdSvOxTHCUtwceac4vSi3+vlaqHr8kQg
qbgTHCeQDNM30IJNZ/BU6wgryJwB316H5GWWaYB/wng
--- GuFi3GSRdlBJ5YRjfAVyFDZ+4TH575kFJLnFp5847N0
-èƒÞHÖÜ*x´M7¼t<,4ˆŠÑ^<5E>5@v°<>£€º,ZMÒg=M
» 3výJÄ«ÐÖê¿Nz­8'<^'4&WÂf"Êõ´À›ë\©º»ëêwmzúlAl|+„ÆKš~68ñE­Ýîk•8ø?S&òaMÝ~ž¹ê¿]Vfø ÝJxaõDù¥x

Binary file not shown.

View file

@ -1,40 +0,0 @@
AI2Bot
Ai2Bot-Dolma
Amazonbot
anthropic-ai
Applebot
Applebot-Extended
Bytespider
CCBot
ChatGPT-User
Claude-Web
ClaudeBot
cohere-ai
Diffbot
FacebookBot
facebookexternalhit
FriendlyCrawler
Google-Extended
GoogleOther
GoogleOther-Image
GoogleOther-Video
GPTBot
iaskspider/2.0
ICC-Crawler
ImagesiftBot
img2dataset
ISSCyberRiskCrawler
Kangaroo Bot
Meta-ExternalAgent
Meta-ExternalFetcher
OAI-SearchBot
omgili
omgilibot
PerplexityBot
PetalBot
Scrapy
Sidetrade indexer bot
Timpibot
VelenPublicWebCrawler
Webzio-Extended
YouBot

View file

@ -1,32 +0,0 @@
{ pkgs, config, lib, ... }:
let
inherit (lib) mkEnableOption mkIf mkOption types concatStringsSep mkDefault splitString;
cfg = config.bagel.services.nginx.crawler-blocker;
mkRobotsFile = blockedUAs: pkgs.writeText "robots.txt" ''
${concatStringsSep "\n" (map (ua: "User-agent: ${ua}") blockedUAs)}
Disallow: /
'';
in
{
options = {
bagel.services.nginx.crawler-blocker = {
enable = mkEnableOption "the crawler blocker";
userAgents = mkOption {
type = types.listOf types.str;
default = splitString "\n" (builtins.readFile ./blocked-ua.txt);
};
};
services.nginx.virtualHosts = mkOption {
type = types.attrsOf (types.submodule {
config = {
locations."= /robots.txt" = mkIf cfg.enable (mkDefault {
alias = mkRobotsFile cfg.userAgents;
});
};
});
};
};
}

View file

@ -7,69 +7,15 @@
}:
let
cfg = config.bagel.services.buildbot;
cfgGerrit = nodes.gerrit01.config.bagel.services.gerrit;
ssh-keys = import ../../common/ssh-keys.nix;
freeGbDiskSpace = 20;
extraTenantSpecificBuilders = {
lix = import ./lix.nix {
inherit config nodes;
};
floral = [ ];
}.${cfg.tenant or (throw "${cfg.tenant} is not a known tenant")};
clientId = {
lix = "buildbot";
floral = "forkos-buildbot";
}.${cfg.tenant or (throw "${cfg.tenant} is not a known tenant")};
inherit (lib) mkEnableOption mkOption mkIf types;
in
{
options.bagel.services.buildbot = {
enable = mkEnableOption "Buildbot";
tenant = mkOption {
type = types.enum [ "lix" "floral" ];
description = "Which buildbot tenant to enable";
};
domain = mkOption {
type = types.str;
description = "Domain name for this Buildbot";
};
gerrit = {
domain = mkOption {
type = types.str;
description = "Canonical domain of the Gerrit associated to this Buildbot";
example = [ "cl.forkos.org" ];
};
port = mkOption {
type = types.port;
description = "Gerrit SSH port for this Buildbot";
};
username = mkOption {
type = types.str;
description = "Gerrit service username for this Buildbot";
};
};
cors.allowedOrigins = mkOption {
type = types.listOf types.str;
example = [ "*.forkos.org" ];
description = "Allowed origin for Buildbot and NGINX for CORS without the protocol";
};
buildSystems = mkOption {
type = types.listOf (types.enum [ "x86_64-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" ]);
default = [ "x86_64-linux" ];
example = [ "x86_64-linux" "aarch64-linux" ];
description = "Supported build systems for this buildbot instance.";
};
projects = mkOption {
type = types.listOf types.str;
example = [ "nixpkgs" ];
description = "Static list of projects enabled for Buildbot CI";
};
builders = mkOption {
@ -81,39 +27,28 @@ in
config = mkIf cfg.enable {
networking.firewall.allowedTCPPorts = [ 80 443 ];
bagel.secrets.files = [
"buildbot-worker-password"
"buildbot-oauth-secret"
"buildbot-workers"
"buildbot-service-key"
"buildbot-signing-key"
"buildbot-remote-builder-key"
];
age.secrets.buildbot-worker-password.file = ../../secrets/buildbot-worker-password.age;
age.secrets.buildbot-oauth-secret.file = ../../secrets/buildbot-oauth-secret.age;
age.secrets.buildbot-workers.file = ../../secrets/buildbot-workers.age;
age.secrets.buildbot-service-key.file = ../../secrets/buildbot-service-key.age;
age.secrets.buildbot-signing-key = {
file = ../../secrets/buildbot-signing-key.age;
owner = "buildbot-worker";
group = "buildbot-worker";
};
age.secrets.buildbot-remote-builder-key = {
file = ../../secrets/${cfg.tenant}/buildbot-remote-builder-key.age;
file = ../../secrets/buildbot-remote-builder-key.age;
owner = "buildbot-worker";
group = "buildbot-worker";
};
services.nginx = {
recommendedProxySettings = true;
appendHttpConfig = ''
# Our session stuff is too big with the TWISTED_COOKIE in addition.
# Default is usually 4k or 8k.
large_client_header_buffers 4 16k;
services.nginx.virtualHosts.${cfg.domain} = {
forceSSL = true;
enableACME = true;
extraConfig = ''
add_header Access-Control-Allow-Credentials 'true' always;
add_header Access-Control-Allow-Origin 'https://cl.forkos.org' always;
'';
virtualHosts.${cfg.domain} = {
forceSSL = true;
enableACME = true;
extraConfig = ''
# This is needed so that logged-in users in Buildbot can include their credentials in their requests.
add_header Access-Control-Allow-Credentials 'true' always;
'';
};
};
services.buildbot-nix.worker = {
@ -139,25 +74,30 @@ in
enable = true;
inherit (cfg) domain;
# TODO(raito): is that really necessary when we can just collect buildMachines' systems?
inherit (cfg) buildSystems;
debugging.enable = true;
oauth2 = {
name = "Lix";
inherit clientId;
clientId = "forkos-buildbot";
clientSecretFile = config.age.secrets.buildbot-oauth-secret.path;
resourceEndpoint = "https://identity.lix.systems";
authUri = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/auth";
tokenUri = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/token";
userinfoUri = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/userinfo";
};
# TODO(raito): this is not really necessary, we never have remote buildbot workers.
# we can replace all of this with automatic localworker generation on buildbot-nix side.
workersFile = config.age.secrets.buildbot-workers.path;
# We rely on NGINX to do the CORS dance.
allowedOrigins = cfg.cors.allowedOrigins;
allowedOrigins = [
"*.forkos.org"
];
# TODO(raito): is that really necessary when we can just collect buildMachines' systems?
buildSystems = [
"x86_64-linux"
];
buildMachines = map (n: {
hostName = nodes.${n}.config.networking.fqdn;
@ -171,14 +111,20 @@ in
# Contrary to how Nix works, here we can specify non-base64 public host keys.
publicHostKey = ssh-keys.machines.${n};
}
) cfg.builders ++ extraTenantSpecificBuilders;
) cfg.builders;
gerrit = {
domain = cfgGerrit.canonicalDomain;
# Manually managed account…
# TODO: https://git.lix.systems/the-distro/infra/issues/69
inherit (cfg.gerrit) domain port username;
username = "buildbot";
port = cfgGerrit.port;
privateKeyFile = config.age.secrets.buildbot-service-key.path;
inherit (cfg) projects;
projects = [
"buildbot-test"
"nixpkgs"
"infra"
];
};
evalWorkerCount = 6;
@ -187,47 +133,10 @@ in
signingKeyFile = config.age.secrets.buildbot-signing-key.path;
};
# Make PostgreSQL restart smoother.
systemd.services.postgresql.serviceConfig = {
Restart = "always";
RestartMaxDelaySec = "5m";
RestartSteps = 10;
};
services.postgresql.settings = {
# DB Version: 15
# OS Type: linux
# DB Type: web
# Total Memory (RAM): 64 GB
# CPUs num: 16
# Connections num: 100
# Data Storage: ssd
max_connections = 100;
shared_buffers = "16GB";
effective_cache_size = "48GB";
maintenance_work_mem = "2GB";
checkpoint_completion_target = 0.9;
wal_buffers = "16MB";
default_statistics_target = 100;
random_page_cost = 1.1;
effective_io_concurrency = 200;
work_mem = "41943kB";
huge_pages = "try";
min_wal_size = "1GB";
max_wal_size = "4GB";
max_worker_processes = 16;
max_parallel_workers_per_gather = 4;
max_parallel_workers = 16;
max_parallel_maintenance_workers = 4;
};
nix.settings.keep-derivations = true;
nix.gc = {
automatic = true;
dates = "hourly";
options = ''
--max-freed "$((${toString freeGbDiskSpace} * 1024**3 - 1024 * $(df -P -k /nix/store | tail -n 1 | ${pkgs.gawk}/bin/awk '{ print $4 }')))"
'';
};
};
}

View file

@ -1,50 +0,0 @@
{ config, nodes, ... }:
let
ssh-keys = import ../../common/ssh-keys.nix;
in
[
{
hostName = "build01.aarch64.lix.systems";
maxJobs = 2;
protocol = "ssh-ng";
sshKey = config.age.secrets.buildbot-remote-builder-key.path;
sshUser = "nix";
systems = [ "aarch64-linux" ];
publicHostKey = ssh-keys.machines.build01-aarch64-lix;
supportedFeatures = nodes.build01-aarch64-lix.config.nix.settings.system-features;
}
{
hostName = "build02.aarch64.lix.systems";
maxJobs = 4;
protocol = "ssh-ng";
sshKey = config.age.secrets.buildbot-remote-builder-key.path;
sshUser = "nix";
systems = [ "aarch64-linux" ];
publicHostKey = ssh-keys.machines.build02-aarch64-lix;
# TODO: use build02 features.
supportedFeatures = nodes.build01-aarch64-lix.config.nix.settings.system-features;
}
{
hostName = "build01.aarch64-darwin.lix.systems";
maxJobs = 2;
protocol = "ssh-ng";
sshKey = config.age.secrets.buildbot-remote-builder-key.path;
sshUser = "m1";
systems = [ "aarch64-darwin" "x86_64-darwin" ];
publicHostKey = ssh-keys.machines.build01-aarch64-darwin-lix;
supportedFeatures = [ "big-parallel" ];
}
# a.k.a. https://git.newtype.fr/newtype/newtype-org-configurations/src/branch/main/docs/epyc.md
{
hostName = "epyc.infra.newtype.fr";
# at 256G this could run 64 builds but the machine is shared
# (and historically we used no more than 16 concurrent jobs)
maxJobs = 16;
protocol = "ssh-ng";
sshKey = config.age.secrets.buildbot-remote-builder-key.path;
sshUser = "nix";
systems = [ "x86_64-linux" "i686-linux" ];
publicHostKey = ssh-keys.machines.epyc-newtype-fr;
supportedFeatures = [ "benchmark" "big-parallel" "nixos-test" "kvm" ];
}
]

View file

@ -1,12 +1,10 @@
{
imports = [
./block-crawlers
./gerrit
./channel-scripts
./hydra
./matrix
./monitoring
./uptime-kuma
./netbox
./ofborg
./postgres
@ -15,6 +13,5 @@
./buildbot
./newsletter
./s3-revproxy
./extra-builders
];
}

View file

@ -1,6 +0,0 @@
{
imports = [
# Remote builders
./provider.nix
];
}

View file

@ -1,46 +0,0 @@
## Tenant-specific build capacity.
## This can come from anywhere and is not hold to the same level of responsibility that our build-infra has.
{ pkgs, config, lib, nodes, ... }:
let
inherit (lib) mkIf types mkEnableOption mkOption;
freeGbDiskSpace = 10;
cfg = config.bagel.builders.extra-build-capacity.provider;
in
{
options.bagel.builders.extra-build-capacity.provider = {
enable = mkEnableOption "providing of extra build capacity to other systems";
buildfarmPublicKeys = mkOption {
type = types.listOf types.str;
description = "SSH public keys to allow to connect for remote builds";
};
# TODO: register tenant in some deployment wide module
# so that the consumer side can just automatically generate buildMachines entries.
tenant = mkOption {
type = types.enum [ "lix" ];
};
};
config = mkIf cfg.enable {
users.groups.builders = {};
users.users.nix = {
openssh.authorizedKeys.keys = cfg.buildfarmPublicKeys;
extraGroups = [ "builders" ];
isNormalUser = true;
};
nix.settings.allowed-users = [ "@wheel" "@builders" ];
nix.settings.trusted-users = [ "@builders" ];
nix.gc.automatic = true;
nix.gc.dates = "hourly";
nix.gc.options = ''
--max-freed "$((${toString freeGbDiskSpace} * 1024**3 - 1024 * $(df -P -k /nix/store | tail -n 1 | ${pkgs.gawk}/bin/awk '{ print $4 }')))"
'';
# Bump the open files limit so that non-root users can run NixOS VM tests, if supported at all.
security.pam.loginLimits = [
{ domain = "*"; item = "nofile"; type = "-"; value = "20480"; }
];
};
}

View file

@ -46,7 +46,7 @@ in
config = mkIf cfg.enable {
networking.firewall.allowedTCPPorts = [ cfg.port ];
age.secrets.alloy-push-password.file = ../../secrets/floral/metrics-push-password.age;
age.secrets.alloy-push-password.file = ../../secrets/metrics-push-password.age;
environment.systemPackages = [ jdk
pkgs.git
@ -129,9 +129,7 @@ in
serverId = "9e5216ad-038d-4d74-a4e8-716515834a94";
builtinPlugins = [
# Disable gitiles as it generates too much traffic.
# Prefer git.forkos.org.
# "gitiles"
"gitiles"
"codemirror-editor"
"reviewnotes"
"download-commands"
@ -143,9 +141,7 @@ in
plugins = with pkgs.gerritPlugins; [
oauth
metrics-reporter-prometheus
# Theme plugin
(pkgs.concatText "theme.js" [ ./theme.js ])
# Buildbot checks plugin
# Buildbot checks plugin (writeText because services.gerrit.plugins expects packages)
(pkgs.runCommand "checks.js" {
BASE_URI = builtins.toJSON "https://buildbot.forkos.org";
SUPPORTED_PROJECTS = builtins.toJSON [
@ -330,7 +326,7 @@ in
};
};
age.secrets.gerrit-prometheus-bearer-token.file = ../../secrets/floral/gerrit-prometheus-bearer-token.age;
age.secrets.gerrit-prometheus-bearer-token.file = ../../secrets/gerrit-prometheus-bearer-token.age;
bagel.monitoring.grafana-agent.exporters.gerrit = {
port = 4778; # grrt
bearerTokenFile = config.age.secrets.gerrit-prometheus-bearer-token.path;

View file

@ -57,7 +57,6 @@ in
name = "git-gc-preserve-${name}";
value = {
description = "Git-GC-Preserve Service - ${name}";
path = [ pkgs.util-linux ];
serviceConfig = {
WorkingDirectory = gcConfig.repoPath;
Type = "oneshot";

View file

@ -1,69 +0,0 @@
/* Set up theming for Floral.
* vim: set et ts=2 sw=2:
*/
Gerrit.install((plugin) => {
const stylesheet = new CSSStyleSheet();
stylesheet.replace(`
html {
--header-title-content: 'floral.systems';
--blue-50: #f3f4fb;
--blue-100: #e3e6f6;
--blue-200: #ced5ef;
--blue-300: #acb8e4;
--blue-400: #8495d6;
--blue-500: #6775ca;
--blue-600: #5158bb;
--blue-700: #494bac;
--blue-800: #41408d;
--blue-900: #383870;
--blue-950: #252546;
--coral-50: #fff1f1;
--coral-100: #ffe0e0;
--coral-200: #ffc5c5;
--coral-300: #ff9e9d;
--coral-400: #ff6665;
--coral-500: #fe4a49;
--coral-600: #ec1716;
--coral-700: #c70f0e;
--coral-800: #a41110;
--coral-900: #881514;
--coral-950: #4a0505;
--teal-50: #eefbf5;
--teal-100: #d6f5e5;
--teal-200: #b1e9d0;
--teal-300: #7ed7b5;
--teal-400: #49be95;
--teal-500: #27a27b;
--teal-600: #188162;
--teal-700: #136951;
--teal-800: #125342;
--teal-900: #104437;
--teal-950: #08261f;
--zinc-50: #fafafa;
--zinc-100: #f4f4f5;
--zinc-200: #e4e4e7;
--zinc-300: #d4d4d8;
--zinc-400: #a1a1aa;
--zinc-500: #71717a;
--zinc-600: #52525b;
--zinc-700: #3f3f46;
--zinc-800: #27272a;
--zinc-900: #18181b;
--zinc-950: #09090b;
}
html.lightTheme {
--header-background-color: var(--teal-700);
--header-text-color: var(--coral-50);
}
html.darkTheme {
--header-background-color: var(--teal-900);
--header-text-color: var(--coral-50);
}
`).then(() => {
document.adoptedStyleSheets = [...document.adoptedStyleSheets, stylesheet];
});
});

View file

@ -29,6 +29,10 @@ in
# NGINX should not give up super fast. Things can take time.
proxy_read_timeout 3600;
}
location = /robots.txt {
return 200 'User-agent: *\nAllow: /';
}
'';
};

View file

@ -66,19 +66,17 @@ in {
# does indeed have our public SSH key and are *builders*
# as a simple evaluation preflight check.
bagel.secrets.files = [
"hydra-s3-credentials"
"hydra-postgres-key"
"hydra-signing-priv"
"hydra-ssh-key-priv"
];
age.secrets.hydra-s3-credentials.file = ../../secrets/hydra-s3-credentials.age;
age.secrets.hydra-postgres-key.group = "hydra";
age.secrets.hydra-postgres-key.mode = "0440";
age.secrets.hydra-postgres-key.file = ../../secrets/hydra-postgres-key.age;
age.secrets.hydra-signing-priv.owner = "hydra-queue-runner";
age.secrets.hydra-signing-priv.file = ../../secrets/hydra-signing-priv.age;
age.secrets.hydra-ssh-key-priv.owner = "hydra-queue-runner";
age.secrets.hydra-ssh-key-priv.file = ../../secrets/hydra-ssh-key-priv.age;
systemd.tmpfiles.rules = [
"d /var/cache/hydra 0755 hydra hydra - -"

View file

@ -32,11 +32,6 @@ in
];
server_name = "forkos.org";
database.backend = "rocksdb";
server_discovery = {
server.authority = "matrix.forkos.org:443";
client.base_url = "https://matrix.forkos.org";
};
};
};
@ -53,7 +48,18 @@ in
"forkos.org" = {
forceSSL = true;
enableACME = true;
locations."/.well-known/matrix".proxyPass = "http://grapevine";
locations = {
"= /.well-known/matrix/server".extraConfig = ''
add_header Content-Type application/json;
add_header Access-Control-Allow-Origin *;
return 200 '{"m.server": "matrix.forkos.org:443"}';
'';
"= /.well-known/matrix/client".extraConfig = ''
add_header Content-Type application/json;
add_header Access-Control-Allow-Origin *;
return 200 '{"m.homeserver": {"base_url": "https://matrix.forkos.org/"}, "m.identity_server": {"base_url": "https://matrix.org/"}, "org.matrix.msc3575.proxy": {"url": "https://matrix.forkos.org"}}';
'';
};
};
};
};

View file

@ -12,14 +12,6 @@ in
options.bagel.monitoring.grafana-agent = {
enable = (mkEnableOption "Grafana Agent") // { default = true; };
tenant = mkOption {
description = ''
Which tenant are we enabling Grafana Agent for.
'';
example = "lix";
type = types.enum [ "lix" "floral" ];
};
exporters = mkOption {
description = ''
Set of additional exporters to scrape.
@ -67,7 +59,7 @@ in
};
config = mkIf cfg.enable {
age.secrets.grafana-agent-password.file = ../../secrets/floral/metrics-push-password.age;
age.secrets.grafana-agent-password.file = ../../secrets/metrics-push-password.age;
services.grafana-agent = {
enable = true;
@ -84,10 +76,7 @@ in
};
}
];
global.external_labels = {
hostname = config.networking.hostName;
inherit (cfg) tenant;
};
global.external_labels.hostname = config.networking.hostName;
configs = [
{
name = config.networking.hostName;

View file

@ -22,9 +22,6 @@ steps_building = Gauge("hydra_steps_building", "Number of steps currently buildi
steps_copying_to = Gauge(
"hydra_steps_copying_to", "Number of steps copying inputs to a worker"
)
steps_waiting_for_download_slot = Gauge(
"hydra_steps_waiting_for_download_slot", "Number of steps waiting for download slot"
)
steps_copying_from = Gauge(
"hydra_steps_copying_from", "Number of steps copying outputs from a worker"
)
@ -233,7 +230,6 @@ def update_metrics(status):
steps_active.set({}, status["nrActiveSteps"])
steps_building.set({}, status["nrStepsBuilding"])
steps_copying_to.set({}, status["nrStepsCopyingTo"])
steps_waiting_for_download_slot.set({}, status["nrStepsWaitingForDownloadSlot"])
steps_copying_from.set({}, status["nrStepsCopyingFrom"])
steps_waiting.set({}, status["nrStepsWaiting"])
steps_unsupported.set({}, status["nrUnsupportedSteps"])
@ -338,11 +334,11 @@ async def update_metrics_loop(hydra_url, scrape_interval):
)
update_metrics(response.json())
await asyncio.sleep(scrape_interval)
except Exception as ex:
logging.exception("Failed to update metrics", exc_info=ex)
await asyncio.sleep(scrape_interval)
@click.command()
@click.option("--hydra-url", default="https://hydra.forkos.org/")

View file

@ -23,10 +23,7 @@ in
};
services.postgresql.settings.shared_preload_libraries = "pg_stat_statements";
systemd.services.postgresql.postStart = lib.mkAfter ''
${config.services.postgresql.package}/bin/psql -U postgres -c "CREATE EXTENSION IF NOT EXISTS pg_stat_statements;";
'';
bagel.monitoring.grafana-agent.exporters.postgres.port = 9104;
};
}
}

View file

@ -1,11 +1,11 @@
groups:
- name: ForkOS automation
rules:
- alert: SyncFailedTooOften
expr: 'changes(node_systemd_unit_state{name=~"ows.*.service",state="failed"}[1d]) > 2'
for: 30m
labels:
severity: critical
annotations:
description: On {{ $labels.instance }}, the synchronization job has failed more than twice in the last 24 hours, check if there's a conflict or a stdenv change.
summary: Synchronization job {{ $labels.name }} has failed more than twice in the last 24 hours
- name: ForkOS automation
rules:
- alert: SyncFailedTooOften
expr: 'changes(node_systemd_unit_state{name=~"ows.*.service",state="failed"}[24h]) > 2'
for: 30m
labels:
severity: critical
annotations:
summary: "Synchronization job {{ $labels.name }} has failed more than twice in the last 24 hours"
description: "On {{ $labels.instance }}, the synchronization job has failed more than twice in the last 24 hours, check if there's a conflict or a stdenv change."

View file

@ -1,118 +1,102 @@
groups:
- name: PostgreSQL
rules:
- alert: PostgresqlTableNotAutoVacuumed
expr: '(pg_stat_user_tables_last_autovacuum > 0) and (time() - pg_stat_user_tables_last_autovacuum) > 60 * 60 * 24 * 10'
labels:
severity: warning
annotations:
description: |-
Table {{ $labels.relname }} has not been auto vacuumed for 10 days
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Postgresql table not auto vacuumed (instance {{ $labels.instance }})
- alert: PostgresqlTableNotAutoAnalyzed
expr: '(pg_stat_user_tables_last_autoanalyze > 0) and (time() - pg_stat_user_tables_last_autoanalyze) > 24 * 60 * 60 * 10'
labels:
severity: warning
annotations:
description: |-
Table {{ $labels.relname }} has not been auto analyzed for 10 days
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Postgresql table not auto analyzed (instance {{ $labels.instance }})
- alert: PostgresqlDeadLocks
expr: 'increase(pg_stat_database_deadlocks{datname!~"template.*|postgres"}[1m]) > 5'
labels:
severity: warning
annotations:
description: |-
PostgreSQL has dead-locks
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Postgresql dead locks (instance {{ $labels.instance }})
- alert: PostgresqlHighRollbackRate
expr: 'sum by (namespace, datname) ((rate(pg_stat_database_xact_rollback{datid!="0",datname!~"template.*|postgres"}[3m])) / ((rate(pg_stat_database_xact_rollback{datid!="0",datname!~"template.*|postgres"}[3m])) + (rate(pg_stat_database_xact_commit{datid!="0",datname!~"template.*|postgres"}[3m])))) > 0.02'
labels:
severity: warning
annotations:
description: |-
Ratio of transactions being aborted compared to committed is > 2 %
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Postgresql high rollback rate (instance {{ $labels.instance }})
- alert: PostgresqlHighRateStatementTimeout
expr: 'rate(postgresql_errors_total{type="statement_timeout"}[1m]) > 3'
labels:
severity: critical
annotations:
description: |-
Postgres transactions showing high rate of statement timeouts
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Postgresql high rate statement timeout (instance {{ $labels.instance }})
- alert: PostgresqlHighRateDeadlock
expr: 'increase(postgresql_errors_total{type="deadlock_detected"}[1m]) > 1'
labels:
severity: critical
annotations:
description: |-
Postgres detected deadlocks
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Postgresql high rate deadlock (instance {{ $labels.instance }})
- alert: PostgresqlTooManyDeadTuples
expr: '((pg_stat_user_tables_n_dead_tup > 10000) / (pg_stat_user_tables_n_live_tup + pg_stat_user_tables_n_dead_tup)) >= 0.1'
for: 2m
labels:
severity: warning
annotations:
description: |-
PostgreSQL dead tuples is too large
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Postgresql too many dead tuples (instance {{ $labels.instance }})
- alert: PostgresqlTooManyLocksAcquired
expr: '((sum(pg_locks_count)) / (pg_settings_max_locks_per_transaction * pg_settings_max_connections)) > 0.2'
for: 2m
labels:
severity: critical
annotations:
description: |-
Too many locks acquired on the database. If this alert happens frequently, we may need to increase the postgres setting max_locks_per_transaction.
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Postgresql too many locks acquired (instance {{ $labels.instance }})
- alert: PostgresqlBloatIndexHigh(>80%)
expr: 'pg_bloat_btree_bloat_pct > 80 and on (idxname) (pg_bloat_btree_real_size > 1e+08)'
for: 1h
labels:
severity: warning
annotations:
description: |-
The index {{ $labels.idxname }} is bloated. You should execute `REINDEX INDEX CONCURRENTLY {{ $labels.idxname }};`
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Postgresql bloat index high (> 80%) (instance {{ $labels.instance }})
- alert: PostgresqlBloatTableHigh(>80%)
expr: 'pg_bloat_table_bloat_pct > 80 and on (relname) (pg_bloat_table_real_size > 2e+08)'
for: 1h
labels:
severity: warning
annotations:
description: |-
The table {{ $labels.relname }} is bloated. You should execute `VACUUM {{ $labels.relname }};`
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Postgresql bloat table high (> 80%) (instance {{ $labels.instance }})
- alert: PostgresqlInvalidIndex
expr: 'pg_genaral_index_info_pg_relation_size{indexrelname=~".*ccnew.*"}'
for: 6h
labels:
severity: warning
annotations:
description: |-
The table {{ $labels.relname }} has an invalid index: {{ $labels.indexrelname }}. You should execute `DROP INDEX {{ $labels.indexrelname }};`
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Postgresql invalid index (instance {{ $labels.instance }})
- name: PostgreSQL
rules:
- alert: PostgresqlTableNotAutoVacuumed
expr: '(pg_stat_user_tables_last_autovacuum > 0) and (time() - pg_stat_user_tables_last_autovacuum) > 60 * 60 * 24 * 10'
for: 0m
labels:
severity: warning
annotations:
summary: Postgresql table not auto vacuumed (instance {{ $labels.instance }})
description: "Table {{ $labels.relname }} has not been auto vacuumed for 10 days\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: PostgresqlTableNotAutoAnalyzed
expr: '(pg_stat_user_tables_last_autoanalyze > 0) and (time() - pg_stat_user_tables_last_autoanalyze) > 24 * 60 * 60 * 10'
for: 0m
labels:
severity: warning
annotations:
summary: Postgresql table not auto analyzed (instance {{ $labels.instance }})
description: "Table {{ $labels.relname }} has not been auto analyzed for 10 days\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: PostgresqlDeadLocks
expr: 'increase(pg_stat_database_deadlocks{datname!~"template.*|postgres"}[1m]) > 5'
for: 0m
labels:
severity: warning
annotations:
summary: Postgresql dead locks (instance {{ $labels.instance }})
description: "PostgreSQL has dead-locks\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: PostgresqlHighRollbackRate
expr: 'sum by (namespace,datname) ((rate(pg_stat_database_xact_rollback{datname!~"template.*|postgres",datid!="0"}[3m])) / ((rate(pg_stat_database_xact_rollback{datname!~"template.*|postgres",datid!="0"}[3m])) + (rate(pg_stat_database_xact_commit{datname!~"template.*|postgres",datid!="0"}[3m])))) > 0.02'
for: 0m
labels:
severity: warning
annotations:
summary: Postgresql high rollback rate (instance {{ $labels.instance }})
description: "Ratio of transactions being aborted compared to committed is > 2 %\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: PostgresqlHighRateStatementTimeout
expr: 'rate(postgresql_errors_total{type="statement_timeout"}[1m]) > 3'
for: 0m
labels:
severity: critical
annotations:
summary: Postgresql high rate statement timeout (instance {{ $labels.instance }})
description: "Postgres transactions showing high rate of statement timeouts\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: PostgresqlHighRateDeadlock
expr: 'increase(postgresql_errors_total{type="deadlock_detected"}[1m]) > 1'
for: 0m
labels:
severity: critical
annotations:
summary: Postgresql high rate deadlock (instance {{ $labels.instance }})
description: "Postgres detected deadlocks\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: PostgresqlTooManyDeadTuples
expr: '((pg_stat_user_tables_n_dead_tup > 10000) / (pg_stat_user_tables_n_live_tup + pg_stat_user_tables_n_dead_tup)) >= 0.1'
for: 2m
labels:
severity: warning
annotations:
summary: Postgresql too many dead tuples (instance {{ $labels.instance }})
description: "PostgreSQL dead tuples is too large\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: PostgresqlTooManyLocksAcquired
expr: '((sum (pg_locks_count)) / (pg_settings_max_locks_per_transaction * pg_settings_max_connections)) > 0.20'
for: 2m
labels:
severity: critical
annotations:
summary: Postgresql too many locks acquired (instance {{ $labels.instance }})
description: "Too many locks acquired on the database. If this alert happens frequently, we may need to increase the postgres setting max_locks_per_transaction.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: PostgresqlBloatIndexHigh(>80%)
expr: 'pg_bloat_btree_bloat_pct > 80 and on (idxname) (pg_bloat_btree_real_size > 100000000)'
for: 1h
labels:
severity: warning
annotations:
summary: Postgresql bloat index high (> 80%) (instance {{ $labels.instance }})
description: "The index {{ $labels.idxname }} is bloated. You should execute `REINDEX INDEX CONCURRENTLY {{ $labels.idxname }};`\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: PostgresqlBloatTableHigh(>80%)
expr: 'pg_bloat_table_bloat_pct > 80 and on (relname) (pg_bloat_table_real_size > 200000000)'
for: 1h
labels:
severity: warning
annotations:
summary: Postgresql bloat table high (> 80%) (instance {{ $labels.instance }})
description: "The table {{ $labels.relname }} is bloated. You should execute `VACUUM {{ $labels.relname }};`\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: PostgresqlInvalidIndex
expr: 'pg_genaral_index_info_pg_relation_size{indexrelname=~".*ccnew.*"}'
for: 6h
labels:
severity: warning
annotations:
summary: Postgresql invalid index (instance {{ $labels.instance }})
description: "The table {{ $labels.relname }} has an invalid index: {{ $labels.indexrelname }}. You should execute `DROP INDEX {{ $labels.indexrelname }};`\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"

View file

@ -1,100 +1,76 @@
groups:
- name: Host & hardware
rules:
- alert: HostOutOfMemory
expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 2m
labels:
severity: warning
annotations:
description: |-
Node memory is filling up (< 10% left)
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host out of memory (instance {{ $labels.instance }})
- alert: HostMemoryUnderMemoryPressure
expr: (rate(node_vmstat_pgmajfault[1m]) > 1000) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 2m
labels:
severity: warning
annotations:
description: |-
The node is under heavy memory pressure. High rate of major page faults
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host memory under memory pressure (instance {{ $labels.instance }})
- alert: HostMemoryIsUnderutilized
expr: (100 - (avg_over_time(node_memory_MemAvailable_bytes[30m]) / node_memory_MemTotal_bytes * 100) < 20) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 1w
labels:
severity: info
annotations:
description: |-
Node memory is < 20% for 1 week. Consider reducing memory space. (instance {{ $labels.instance }})
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host Memory is underutilized (instance {{ $labels.instance }})
- alert: HostOutOfDiskSpace
expr: ((node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and on (instance, device, mountpoint) node_filesystem_readonly == 0) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 2m
labels:
severity: warning
annotations:
description: |-
Disk is almost full (< 10% left)
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host out of disk space (instance {{ $labels.instance }})
- alert: HostDiskWillFillIn24Hours
expr: ((node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and on (instance, device, mountpoint) predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs"}[1h], 24 * 3600) < 0 and on (instance, device, mountpoint) node_filesystem_readonly == 0) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 2m
labels:
severity: warning
annotations:
description: |-
Filesystem is predicted to run out of space within the next 24 hours at current write rate
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host disk will fill in 24 hours (instance {{ $labels.instance }})
- alert: HostCpuIsUnderutilized
expr: (100 - (rate(node_cpu_seconds_total{mode="idle"}[30m]) * 100) < 20) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 1w
labels:
severity: info
annotations:
description: |-
CPU load is < 20% for 1 week. Consider reducing the number of CPUs.
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host CPU is underutilized (instance {{ $labels.instance }})
- alert: HostCpuStealNoisyNeighbor
expr: (avg by (instance) (rate(node_cpu_seconds_total{mode="steal"}[5m])) * 100 > 10) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
labels:
severity: warning
annotations:
description: |-
CPU steal is > 10%. A noisy neighbor is killing VM performances or a spot instance may be out of credit.
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host CPU steal noisy neighbor (instance {{ $labels.instance }})
- alert: HostOomKillDetected
expr: (increase(node_vmstat_oom_kill[1m]) > 0) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
labels:
severity: warning
annotations:
description: |-
OOM kill detected
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host OOM kill detected (instance {{ $labels.instance }})
- alert: HostNetworkInterfaceSaturated
expr: ((rate(node_network_receive_bytes_total{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"}[1m]) + rate(node_network_transmit_bytes_total{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"}[1m])) / node_network_speed_bytes{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"} > 0.8 < 10000) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 1m
labels:
severity: warning
annotations:
description: |-
The network interface "{{ $labels.device }}" on "{{ $labels.instance }}" is getting overloaded.
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host Network Interface Saturated (instance {{ $labels.instance }})
- name: Host & hardware
rules:
- alert: HostOutOfMemory
expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 2m
labels:
severity: warning
annotations:
summary: Host out of memory (instance {{ $labels.instance }})
description: "Node memory is filling up (< 10% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostMemoryUnderMemoryPressure
expr: (rate(node_vmstat_pgmajfault[1m]) > 1000) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 2m
labels:
severity: warning
annotations:
summary: Host memory under memory pressure (instance {{ $labels.instance }})
description: "The node is under heavy memory pressure. High rate of major page faults\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostMemoryIsUnderutilized
expr: (100 - (avg_over_time(node_memory_MemAvailable_bytes[30m]) / node_memory_MemTotal_bytes * 100) < 20) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 1w
labels:
severity: info
annotations:
summary: Host Memory is underutilized (instance {{ $labels.instance }})
description: "Node memory is < 20% for 1 week. Consider reducing memory space. (instance {{ $labels.instance }})\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostOutOfDiskSpace
expr: ((node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) node_filesystem_readonly == 0) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 2m
labels:
severity: warning
annotations:
summary: Host out of disk space (instance {{ $labels.instance }})
description: "Disk is almost full (< 10% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostDiskWillFillIn24Hours
expr: ((node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs"}[1h], 24 * 3600) < 0 and ON (instance, device, mountpoint) node_filesystem_readonly == 0) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 2m
labels:
severity: warning
annotations:
summary: Host disk will fill in 24 hours (instance {{ $labels.instance }})
description: "Filesystem is predicted to run out of space within the next 24 hours at current write rate\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostCpuIsUnderutilized
expr: (100 - (rate(node_cpu_seconds_total{mode="idle"}[30m]) * 100) < 20) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 1w
labels:
severity: info
annotations:
summary: Host CPU is underutilized (instance {{ $labels.instance }})
description: "CPU load is < 20% for 1 week. Consider reducing the number of CPUs.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostCpuStealNoisyNeighbor
expr: (avg by(instance) (rate(node_cpu_seconds_total{mode="steal"}[5m])) * 100 > 10) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 0m
labels:
severity: warning
annotations:
summary: Host CPU steal noisy neighbor (instance {{ $labels.instance }})
description: "CPU steal is > 10%. A noisy neighbor is killing VM performances or a spot instance may be out of credit.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostOomKillDetected
expr: (increase(node_vmstat_oom_kill[1m]) > 0) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 0m
labels:
severity: warning
annotations:
summary: Host OOM kill detected (instance {{ $labels.instance }})
description: "OOM kill detected\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostNetworkInterfaceSaturated
expr: ((rate(node_network_receive_bytes_total{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"}[1m]) + rate(node_network_transmit_bytes_total{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"}[1m])) / node_network_speed_bytes{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"} > 0.8 < 10000) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 1m
labels:
severity: warning
annotations:
summary: Host Network Interface Saturated (instance {{ $labels.instance }})
description: "The network interface \"{{ $labels.device }}\" on \"{{ $labels.instance }}\" is getting overloaded.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"

File diff suppressed because it is too large Load diff

View file

@ -20,6 +20,7 @@
"fiscalYearStartMonth": 0,
"gnetId": 9628,
"graphTooltip": 0,
"id": 27,
"links": [],
"liveNow": false,
"panels": [
@ -100,7 +101,6 @@
"graphMode": "none",
"justifyMode": "auto",
"orientation": "horizontal",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"mean"
@ -112,14 +112,14 @@
"textMode": "name",
"wideLayout": true
},
"pluginVersion": "11.2.1",
"pluginVersion": "11.0.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"expr": "pg_static{tenant=\"$tenant\",hostname=\"$hostname\"}",
"expr": "pg_static{hostname=\"$hostname\"}",
"format": "time_series",
"instant": true,
"intervalFactor": 1,
@ -182,7 +182,6 @@
"graphMode": "none",
"justifyMode": "auto",
"orientation": "horizontal",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"mean"
@ -194,14 +193,14 @@
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.2.1",
"pluginVersion": "11.0.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"expr": "pg_postmaster_start_time_seconds{tenant=\"$tenant\",hostname=\"$hostname\"} * 1000",
"expr": "pg_postmaster_start_time_seconds{hostname=\"$hostname\"} * 1000",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "",
@ -262,7 +261,6 @@
"graphMode": "none",
"justifyMode": "auto",
"orientation": "horizontal",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
@ -274,14 +272,14 @@
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.2.1",
"pluginVersion": "11.0.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"expr": "SUM(pg_stat_database_tup_fetched{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"})",
"expr": "SUM(pg_stat_database_tup_fetched{datname=~\"$datname\", hostname=~\"$hostname\"})",
"format": "time_series",
"intervalFactor": 2,
"refId": "A",
@ -342,7 +340,6 @@
"graphMode": "none",
"justifyMode": "auto",
"orientation": "horizontal",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
@ -354,14 +351,14 @@
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.2.1",
"pluginVersion": "11.0.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"expr": "SUM(pg_stat_database_tup_inserted{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"})",
"expr": "SUM(pg_stat_database_tup_inserted{datname=~\"$datname\", hostname=~\"$hostname\"})",
"format": "time_series",
"intervalFactor": 2,
"refId": "A",
@ -422,7 +419,6 @@
"graphMode": "none",
"justifyMode": "auto",
"orientation": "horizontal",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
@ -434,14 +430,14 @@
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.2.1",
"pluginVersion": "11.0.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"expr": "SUM(pg_stat_database_tup_updated{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"})",
"expr": "SUM(pg_stat_database_tup_updated{datname=~\"$datname\", hostname=~\"$hostname\"})",
"format": "time_series",
"intervalFactor": 2,
"refId": "A",
@ -502,7 +498,6 @@
"graphMode": "none",
"justifyMode": "auto",
"orientation": "horizontal",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"mean"
@ -514,14 +509,14 @@
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.2.1",
"pluginVersion": "11.0.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"expr": "pg_settings_max_connections{tenant=\"$tenant\",hostname=\"$hostname\"}",
"expr": "pg_settings_max_connections{hostname=\"$hostname\"}",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
@ -607,7 +602,6 @@
"graphMode": "none",
"justifyMode": "auto",
"orientation": "horizontal",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
@ -619,14 +613,14 @@
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.2.1",
"pluginVersion": "11.0.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"expr": "pg_settings_shared_buffers_bytes{tenant=\"$tenant\",hostname=\"$hostname\"}",
"expr": "pg_settings_shared_buffers_bytes{hostname=\"$hostname\"}",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
@ -686,7 +680,6 @@
"graphMode": "none",
"justifyMode": "auto",
"orientation": "horizontal",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
@ -698,14 +691,14 @@
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.2.1",
"pluginVersion": "11.0.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"expr": "pg_settings_effective_cache_size_bytes{tenant=\"$tenant\",hostname=\"$hostname\"}",
"expr": "pg_settings_effective_cache_size_bytes{hostname=\"$hostname\"}",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
@ -765,7 +758,6 @@
"graphMode": "none",
"justifyMode": "auto",
"orientation": "horizontal",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
@ -777,14 +769,14 @@
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.2.1",
"pluginVersion": "11.0.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"expr": "pg_settings_maintenance_work_mem_bytes{tenant=\"$tenant\",hostname=\"$hostname\"}",
"expr": "pg_settings_maintenance_work_mem_bytes{hostname=\"$hostname\"}",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
@ -844,7 +836,6 @@
"graphMode": "none",
"justifyMode": "auto",
"orientation": "horizontal",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
@ -856,14 +847,14 @@
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.2.1",
"pluginVersion": "11.0.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"expr": "pg_settings_work_mem_bytes{tenant=\"$tenant\",hostname=\"$hostname\"}",
"expr": "pg_settings_work_mem_bytes{hostname=\"$hostname\"}",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "",
@ -925,7 +916,6 @@
"graphMode": "none",
"justifyMode": "auto",
"orientation": "horizontal",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
@ -937,14 +927,14 @@
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.2.1",
"pluginVersion": "11.0.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"expr": "pg_settings_max_wal_size_bytes{tenant=\"$tenant\",hostname=\"$hostname\"}",
"expr": "pg_settings_max_wal_size_bytes{hostname=\"$hostname\"}",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
@ -1004,7 +994,6 @@
"graphMode": "none",
"justifyMode": "auto",
"orientation": "horizontal",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
@ -1016,14 +1005,14 @@
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.2.1",
"pluginVersion": "11.0.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"expr": "pg_settings_random_page_cost{tenant=\"$tenant\",hostname=\"$hostname\"}",
"expr": "pg_settings_random_page_cost{hostname=\"$hostname\"}",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
@ -1083,7 +1072,6 @@
"graphMode": "none",
"justifyMode": "auto",
"orientation": "horizontal",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
@ -1095,7 +1083,7 @@
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.2.1",
"pluginVersion": "11.0.0",
"targets": [
{
"datasource": {
@ -1162,7 +1150,6 @@
"graphMode": "none",
"justifyMode": "auto",
"orientation": "horizontal",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"mean"
@ -1174,14 +1161,14 @@
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.2.1",
"pluginVersion": "11.0.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"expr": "pg_settings_max_worker_processes{tenant=\"$tenant\",hostname=\"$hostname\"}",
"expr": "pg_settings_max_worker_processes{hostname=\"$hostname\"}",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
@ -1241,7 +1228,6 @@
"graphMode": "none",
"justifyMode": "auto",
"orientation": "horizontal",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
@ -1253,14 +1239,14 @@
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.2.1",
"pluginVersion": "11.0.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "mimir"
},
"expr": "pg_settings_max_parallel_workers{tenant=\"$tenant\",hostname=\"$hostname\"}",
"expr": "pg_settings_max_parallel_workers{hostname=\"$hostname\"}",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
@ -1312,7 +1298,6 @@
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "points",
"fillOpacity": 10,
"gradientMode": "none",
@ -1387,7 +1372,7 @@
"type": "prometheus",
"uid": "mimir"
},
"expr": "pg_stat_activity_count{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\", state=\"active\"} !=0",
"expr": "pg_stat_activity_count{datname=~\"$datname\", hostname=~\"$hostname\", state=\"active\"} !=0",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@ -1416,7 +1401,6 @@
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
@ -1490,7 +1474,7 @@
"type": "prometheus",
"uid": "mimir"
},
"expr": "irate(pg_stat_database_xact_commit{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
"expr": "irate(pg_stat_database_xact_commit{hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{datname}} commits",
@ -1501,7 +1485,7 @@
"type": "prometheus",
"uid": "mimir"
},
"expr": "irate(pg_stat_database_xact_rollback{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
"expr": "irate(pg_stat_database_xact_rollback{hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{datname}} rollbacks",
@ -1528,7 +1512,6 @@
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
@ -1602,7 +1585,7 @@
"type": "prometheus",
"uid": "mimir"
},
"expr": "pg_stat_database_tup_updated{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"} != 0",
"expr": "pg_stat_database_tup_updated{datname=~\"$datname\", hostname=~\"$hostname\"} != 0",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{datname}}",
@ -1630,7 +1613,6 @@
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
@ -1704,7 +1686,7 @@
"type": "prometheus",
"uid": "mimir"
},
"expr": "pg_stat_database_tup_fetched{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"} != 0",
"expr": "pg_stat_database_tup_fetched{datname=~\"$datname\", hostname=~\"$hostname\"} != 0",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{datname}}",
@ -1732,7 +1714,6 @@
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
@ -1806,7 +1787,7 @@
"type": "prometheus",
"uid": "mimir"
},
"expr": "pg_stat_database_tup_inserted{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"} != 0",
"expr": "pg_stat_database_tup_inserted{datname=~\"$datname\", hostname=~\"$hostname\"} != 0",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{datname}}",
@ -1834,7 +1815,6 @@
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
@ -1910,7 +1890,7 @@
"type": "prometheus",
"uid": "mimir"
},
"expr": "pg_locks_count{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\", mode=~\"$mode\"} != 0",
"expr": "pg_locks_count{datname=~\"$datname\", hostname=~\"$hostname\", mode=~\"$mode\"} != 0",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{datname}},{{mode}}",
@ -1968,7 +1948,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2010,7 +1991,7 @@
"type": "prometheus",
"uid": "mimir"
},
"expr": "pg_stat_database_tup_returned{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"} != 0",
"expr": "pg_stat_database_tup_returned{datname=~\"$datname\", hostname=~\"$hostname\"} != 0",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{datname}}",
@ -2069,7 +2050,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2110,7 +2092,7 @@
"type": "prometheus",
"uid": "mimir"
},
"expr": "pg_stat_activity_count{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\", state=~\"idle|idle in transaction|idle in transaction (aborted)\"}",
"expr": "pg_stat_activity_count{datname=~\"$datname\", hostname=~\"$hostname\", state=~\"idle|idle in transaction|idle in transaction (aborted)\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{datname}}, s: {{state}}",
@ -2168,7 +2150,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2210,7 +2193,7 @@
"type": "prometheus",
"uid": "mimir"
},
"expr": "pg_stat_database_tup_deleted{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"} != 0",
"expr": "pg_stat_database_tup_deleted{datname=~\"$datname\", hostname=~\"$hostname\"} != 0",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{datname}}",
@ -2269,7 +2252,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2310,7 +2294,7 @@
"type": "prometheus",
"uid": "mimir"
},
"expr": "pg_stat_database_blks_hit{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"} / (pg_stat_database_blks_read{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"} + pg_stat_database_blks_hit{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"})",
"expr": "pg_stat_database_blks_hit{hostname=\"$hostname\", datname=~\"$datname\"} / (pg_stat_database_blks_read{hostname=\"$hostname\", datname=~\"$datname\"} + pg_stat_database_blks_hit{hostname=\"$hostname\", datname=~\"$datname\"})",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{ datname }}",
@ -2367,7 +2351,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2410,7 +2395,7 @@
"type": "prometheus",
"uid": "mimir"
},
"expr": "irate(pg_stat_bgwriter_buffers_backend{tenant=\"$tenant\",hostname=\"$hostname\"}[5m])",
"expr": "irate(pg_stat_bgwriter_buffers_backend{hostname=\"$hostname\"}[5m])",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "buffers_backend",
@ -2421,7 +2406,7 @@
"type": "prometheus",
"uid": "mimir"
},
"expr": "irate(pg_stat_bgwriter_buffers_alloc{tenant=\"$tenant\",hostname=\"$hostname\"}[5m])",
"expr": "irate(pg_stat_bgwriter_buffers_alloc{hostname=\"$hostname\"}[5m])",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "buffers_alloc",
@ -2432,7 +2417,7 @@
"type": "prometheus",
"uid": "mimir"
},
"expr": "irate(pg_stat_bgwriter_buffers_backend_fsync{tenant=\"$tenant\",hostname=\"$hostname\"}[5m])",
"expr": "irate(pg_stat_bgwriter_buffers_backend_fsync{hostname=\"$hostname\"}[5m])",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "backend_fsync",
@ -2443,7 +2428,7 @@
"type": "prometheus",
"uid": "mimir"
},
"expr": "irate(pg_stat_bgwriter_buffers_checkpoint{tenant=\"$tenant\",hostname=\"$hostname\"}[5m])",
"expr": "irate(pg_stat_bgwriter_buffers_checkpoint{hostname=\"$hostname\"}[5m])",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "buffers_checkpoint",
@ -2454,7 +2439,7 @@
"type": "prometheus",
"uid": "mimir"
},
"expr": "irate(pg_stat_bgwriter_buffers_clean{tenant=\"$tenant\",hostname=\"$hostname\"}[5m])",
"expr": "irate(pg_stat_bgwriter_buffers_clean{hostname=\"$hostname\"}[5m])",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "buffers_clean",
@ -2512,7 +2497,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2554,7 +2540,7 @@
"type": "prometheus",
"uid": "mimir"
},
"expr": "irate(pg_stat_database_conflicts{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
"expr": "irate(pg_stat_database_conflicts{hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{datname}} conflicts",
@ -2565,7 +2551,7 @@
"type": "prometheus",
"uid": "mimir"
},
"expr": "irate(pg_stat_database_deadlocks{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
"expr": "irate(pg_stat_database_deadlocks{hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{datname}} deadlocks",
@ -2624,7 +2610,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2666,7 +2653,7 @@
"type": "prometheus",
"uid": "mimir"
},
"expr": "irate(pg_stat_database_temp_bytes{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
"expr": "irate(pg_stat_database_temp_bytes{hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{datname}}",
@ -2723,7 +2710,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2766,7 +2754,7 @@
"type": "prometheus",
"uid": "mimir"
},
"expr": "irate(pg_stat_bgwriter_checkpoint_write_time{tenant=\"$tenant\",hostname=\"$hostname\"}[5m])",
"expr": "irate(pg_stat_bgwriter_checkpoint_write_time{hostname=\"$hostname\"}[5m])",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "write_time - Total amount of time that has been spent in the portion of checkpoint processing where files are written to disk.",
@ -2777,7 +2765,7 @@
"type": "prometheus",
"uid": "mimir"
},
"expr": "irate(pg_stat_bgwriter_checkpoint_sync_time{tenant=\"$tenant\",hostname=\"$hostname\"}[5m])",
"expr": "irate(pg_stat_bgwriter_checkpoint_sync_time{hostname=\"$hostname\"}[5m])",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "sync_time - Total amount of time that has been spent in the portion of checkpoint processing where files are synchronized to disk.",
@ -2878,32 +2866,8 @@
{
"current": {
"selected": false,
"text": "fake",
"value": "fake"
},
"definition": "label_values(tenant)",
"hide": 0,
"includeAll": true,
"label": "Tenant",
"multi": true,
"name": "tenant",
"options": [],
"query": {
"qryType": 1,
"query": "label_values(tenant)",
"refId": "PrometheusVariableQueryEditor-VariableQuery"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"type": "query"
},
{
"current": {
"selected": true,
"text": "buildbot",
"value": "buildbot"
"text": "bagel-box",
"value": "bagel-box"
},
"datasource": {
"type": "prometheus",
@ -2998,6 +2962,7 @@
"from": "now-6h",
"to": "now"
},
"timeRangeUpdatedDuringEditOrView": false,
"timepicker": {
"refresh_intervals": [
"5s",
@ -3028,4 +2993,4 @@
"uid": "000000039",
"version": 1,
"weekStart": ""
}
}

View file

@ -17,7 +17,7 @@ in
config = mkIf cfg.enable {
age.secrets.grafana-oauth-secret = {
file = ../../../secrets/floral/grafana-oauth-secret.age;
file = ../../../secrets/grafana-oauth-secret.age;
owner = "grafana";
};

View file

@ -13,10 +13,10 @@ in
config = mkIf cfg.enable {
age.secrets = {
metrics-push-htpasswd = {
file = ../../../secrets/floral/metrics-push-htpasswd.age;
file = ../../../secrets/metrics-push-htpasswd.age;
owner = "nginx";
};
loki-environment.file = ../../../secrets/floral/loki-environment.age;
loki-environment.file = ../../../secrets/loki-environment.age;
};
services.loki = {

View file

@ -9,15 +9,6 @@ let
inherit (lib) mkEnableOption mkIf;
mimirPort = config.services.mimir.configuration.server.http_listen_port;
alerts = pkgs.runCommand "mimir-alerts-checked" {
src = ./alerts;
nativeBuildInputs = with pkgs; [ prometheus.cli ];
} ''
promtool check rules $src/*
mkdir $out
cp -R $src $out/anonymous/
'';
in
{
options.bagel.services.prometheus.enable = mkEnableOption "Prometheus scraper";
@ -25,11 +16,11 @@ in
config = mkIf cfg.enable {
age.secrets = {
metrics-push-htpasswd = {
file = ../../../secrets/floral/metrics-push-htpasswd.age;
file = ../../../secrets/metrics-push-htpasswd.age;
owner = "nginx";
};
mimir-environment.file = ../../../secrets/floral/mimir-environment.age;
mimir-webhook-url.file = ../../../secrets/floral/mimir-webhook-url.age;
mimir-environment.file = ../../../secrets/mimir-environment.age;
mimir-webhook-url.file = ../../../secrets/mimir-webhook-url.age;
};
services.mimir = {
@ -69,7 +60,10 @@ in
blocks_storage.backend = "s3";
ruler_storage = {
backend = "local";
local.directory = alerts;
local.directory = pkgs.runCommand "mimir-rules" {} ''
mkdir -p $out
ln -s ${./alerts} $out/anonymous
'';
};
alertmanager = {

View file

@ -13,10 +13,10 @@ in
config = mkIf cfg.enable {
age.secrets = {
metrics-push-htpasswd = {
file = ../../../secrets/floral/metrics-push-htpasswd.age;
file = ../../../secrets/metrics-push-htpasswd.age;
owner = "nginx";
};
tempo-environment.file = ../../../secrets/floral/tempo-environment.age;
tempo-environment.file = ../../../secrets/tempo-environment.age;
};
services.tempo = {

View file

@ -15,7 +15,7 @@ in
];
config = mkIf cfg.enable {
age.secrets.pyroscope-secrets.file = ../../../secrets/floral/pyroscope-secrets.age;
age.secrets.pyroscope-secrets.file = ../../../secrets/pyroscope-secrets.age;
services.nginx = {
upstreams.pyroscope = {
servers."127.0.0.1:${toString pyroscopePort}" = {};

View file

@ -20,10 +20,7 @@ in
};
config = mkIf cfg.enable {
# FIXME: why
nixpkgs.config.permittedInsecurePackages = [ pkgs.netbox_3_7.name ];
age.secrets.netbox-environment.file = ../../secrets/floral/netbox-environment.age;
age.secrets.netbox-environment.file = ../../secrets/netbox-environment.age;
services = {
netbox = {
enable = true;

View file

@ -14,7 +14,7 @@ in
};
config = mkIf cfg.enable {
age.secrets.newsletter-secrets.file = ../../secrets/floral/newsletter-secrets.age;
age.secrets.newsletter-secrets.file = ../../secrets/newsletter-secrets.age;
services.listmonk = {
enable = true;
secretFile = config.age.secrets.newsletter-secrets.path;

View file

@ -26,12 +26,10 @@ in {
webroot = "/var/lib/acme/.challenges";
group = "rabbitmq";
};
services.nginx.enable = true;
services.nginx.virtualHosts.${amqpHost}.locations."/.well-known/acme-challenge".root =
"/var/lib/acme/.challenges";
systemd.services.rabbitmq.requires = ["acme-finished-${amqpHost}.target"];
networking.firewall.allowedTCPPorts = [ 80 443 amqpPort ];
networking.firewall.allowedTCPPorts = [ amqpPort ];
};
}

View file

@ -11,7 +11,7 @@ in {
config = lib.mkIf cfg.enable {
age.secrets.postgresql-tls-priv.owner = "postgres";
age.secrets.postgresql-tls-priv.file = ../../secrets/floral/postgres-tls-priv.age;
age.secrets.postgresql-tls-priv.file = ../../secrets/postgres-tls-priv.age;
systemd.tmpfiles.rules = [
"d /var/db 0755 root root - -"
@ -67,7 +67,7 @@ in {
# Provisioned on the server so that CA operations can be done there.
age.secrets.postgresql-ca-priv.owner = "postgres";
age.secrets.postgresql-ca-priv.file = ../../secrets/floral/postgres-ca-priv.age;
age.secrets.postgresql-ca-priv.file = ../../secrets/postgres-ca-priv.age;
users.users.postgres.packages = [
(pkgs.writeShellScriptBin "postgres-mint-new-client" ''

View file

@ -70,7 +70,7 @@ in
];
config = mkIf cfg.enable {
age.secrets.s3-revproxy-api-keys.file = ../../secrets/floral/s3-revproxy-api-keys.age;
age.secrets.s3-revproxy-api-keys.file = ../../secrets/s3-revproxy-api-keys.age;
# For each target, generate an entry that passes it to the s3-revproxy.
services.nginx.virtualHosts = mapAttrs' (subdomain: _: nameValuePair "${subdomain}.${cfg.domain}" (mkProxiedSubdomain subdomain)) cfg.targets;
# this solves garage supporting neither anonymous access nor automatic

View file

@ -1,97 +0,0 @@
{
inputs,
lib,
config,
...
}:
let
cfg = config.bagel.status;
# TODO: pull domains from a central place
subdomains = [
"cl"
"netbox"
"cache"
"grafana"
"hydra"
"loki"
"mimir"
"pyroscope"
"matrix"
"tempo"
"amqp"
"fodwatch"
"git"
"alerts"
"buildbot"
"b"
"postgres"
"news"
];
port = 3001;
in
{
imports = [ "${inputs.stateless-uptime-kuma}/nixos/module.nix" ];
options.bagel.status = {
enable = lib.mkEnableOption "the status page service (uptime-kuma)";
domain = lib.mkOption {
type = lib.types.str;
};
};
config = lib.mkIf cfg.enable {
services.uptime-kuma.enable = true;
services.nginx = {
enable = true;
virtualHosts.${cfg.domain} = {
enableACME = true;
forceSSL = true;
locations."/" = {
proxyPass = "http://127.0.0.1:${builtins.toString port}";
proxyWebsockets = true;
};
};
};
networking.firewall.allowedTCPPorts = [
80
443
];
age.secrets.stateless-uptime-kuma-password.file = ../../secrets/floral/stateless-uptime-kuma-password.age;
statelessUptimeKuma = {
probesConfig = {
monitors = lib.genAttrs subdomains (name: {
type = "http";
url = "https://${name}.forkos.org/";
tags = [];
});
status_pages = {
"forkos" = {
title = "ForkOS";
description = "health of the ForkOS infra";
showTags = true;
publicGroupList = [
{
name = "Services";
weight = 1;
monitorList = lib.genAttrs subdomains (id: {
inherit id;
});
}
];
};
};
settings = {
entryPage = "statusPage-forkos";
};
};
extraFlags = [ "-s" ];
host = "http://localhost:${builtins.toString port}/";
username = "forkos";
passwordFile = config.age.secrets."stateless-uptime-kuma-password".path;
enableService = true;
};
};
}

View file

@ -1,6 +1,7 @@
{
imports = [
./common.nix
./gandi.nix
./dnsimple.nix
./hydra.nix
./state.nix

View file

@ -114,7 +114,6 @@ in
(record "b" 300 "CNAME" "public01.infra.p.forkos.org")
(record "postgres" 300 "CNAME" "bagel-box.infra.p.forkos.org")
(record "news" 3600 "CNAME" "public01.infra.p.forkos.org")
(record "status" 3600 "CNAME" "public01.infra.p.forkos.org")
# S3 in delroth's basement
(record "cache" 300 "AAAA" "2a02:168:6426::12") # smol.delroth.net

115
terraform/gandi.nix Normal file
View file

@ -0,0 +1,115 @@
{ lib, config, ... }:
let
inherit (lib) mkEnableOption mkIf tf genList;
cfg = config.bagel.gandi;
in
{
options.bagel.gandi = {
enable = mkEnableOption "the Gandi DNS configuration";
};
config = mkIf cfg.enable {
terraform.required_providers.gandi = {
version = "~> 2.3.0";
source = "go-gandi/gandi";
};
resource.secret_resource.gandi_pat.lifecycle.prevent_destroy = true;
provider.gandi = {
personal_access_token = tf.ref "resource.secret_resource.gandi_pat.value";
};
resource.gandi_livedns_domain.forkos_org = {
name = "forkos.org";
};
resource.gandi_livedns_record = let
record = name: ttl: type: values: {
inherit name ttl type values;
};
proxyRecords = name: ttl: type: values: [
# kurisu.lahfa.xyz running a sniproxy:
(record name ttl "A" ["163.172.69.160"])
(record name ttl type values)
];
# Creates a extra *.p record pointing to the sniproxy
dualProxyRecords = name: ttl: type: values: lib.flatten [
(record name ttl type values)
(proxyRecords "${name}.p" ttl type values)
];
# TODO: make less fragile and have actual unique and stable names
canonicalName = record: let
name = builtins.replaceStrings ["." "@"] ["_" "_root_"] record.name;
in
"forkos_org_${record.type}_${name}";
forkosRecords = records:
builtins.listToAttrs (map (record: {
name = canonicalName record;
value = record // {
zone = tf.ref "resource.gandi_livedns_domain.forkos_org.id";
};
}) (lib.flatten records));
in forkosRecords ([
# (record "@" 300 "A" ["163.172.69.160"])
(record "@" 300 "AAAA" ["2001:bc8:38ee:100:1000::20"])
(dualProxyRecords "bagel-box.infra" 300 "AAAA" ["2001:bc8:38ee:100:100::1"])
(dualProxyRecords "gerrit01.infra" 300 "AAAA" ["2001:bc8:38ee:100:1000::10"])
(dualProxyRecords "meta01.infra" 300 "AAAA" ["2001:bc8:38ee:100:1000::20"])
(dualProxyRecords "fodwatch.infra" 300 "AAAA" ["2001:bc8:38ee:100:1000::30"])
# git.infra.forkos.org exposes opensshd
(dualProxyRecords "git.infra" 300 "AAAA" ["2001:bc8:38ee:100:1000::41"])
# git.p.forkos.org exposes forgejo ssh server.
(proxyRecords "git.p" 300 "AAAA" ["2001:bc8:38ee:100:1000::40"])
(dualProxyRecords "buildbot.infra" 300 "AAAA" ["2001:bc8:38ee:100:1000::50"])
(dualProxyRecords "public01.infra" 300 "AAAA" ["2001:bc8:38ee:100:1000::60"])
(record "cl" 300 "CNAME" ["gerrit01.infra.p"])
(record "fodwatch" 300 "CNAME" ["fodwatch.infra.p"])
# git.p.forkos.org is the proxy variant of the Forgejo server.
(record "git" 300 "CNAME" ["git.p"])
(record "netbox" 300 "CNAME" ["meta01.infra.p"])
(record "amqp" 300 "CNAME" ["bagel-box.infra.p"])
(record "grafana" 300 "CNAME" ["meta01.infra.p"])
(record "hydra" 300 "CNAME" ["build-coord.wob01.infra.p"])
(record "loki" 300 "CNAME" ["meta01.infra.p"])
(record "mimir" 300 "CNAME" ["meta01.infra.p"])
(record "pyroscope" 300 "CNAME" ["meta01.infra.p"])
(record "tempo" 300 "CNAME" ["meta01.infra.p"])
(record "matrix" 300 "CNAME" ["meta01.infra.p"])
(record "alerts" 300 "CNAME" ["meta01.infra.p"])
(record "buildbot" 300 "CNAME" ["buildbot.infra.p"])
(record "b" 300 "CNAME" ["public01.infra.p"])
(record "postgres" 300 "CNAME" ["bagel-box.infra.p"])
(record "news" 3600 "CNAME" ["public01.infra.p"])
# S3 in delroth's basement
(record "cache" 300 "AAAA" ["2a02:168:6426::12"]) # smol.delroth.net
(record "cache" 300 "A" ["195.39.247.161"]) # sni proxy
(record "vpn-gw.wob01.infra" 300 "AAAA" [ "2a01:584:11::2" ])
(dualProxyRecords "build-coord.wob01.infra" 300 "AAAA" [ "2a01:584:11::1:11" ])
# TODO: do not hardcode, just reuse the Colmena hive module outputs to generate all the required details.
]
++ (map (index: record "builder-${toString index}.wob01.infra" 300 "AAAA" [ "2a01:584:11::1:${toString index}" ]) (genList lib.id 11))
++ (
let
# FIXME: figure out a way to poke `config.services.s3-revproxy` and
# automate the DNS part away?
buckets = [
"channels"
"releases"
"channel-scripts-test"
];
in
map (bucket: record "${bucket}" 300 "CNAME" [ "public01.infra.p" ]) buckets
));
};
}