Compare commits

..

1 commit

Author SHA1 Message Date
Janik Haag 464a726664
fix(builders/netboot): make "normal" evaluation pass
Without this patch running `colmena build` will run into a few assertion
errors for machines that have `config.bagel.baremetal.builders.netboot == true`
set. This is due to an assertion check in the initrd module making sure
there is a mount point for `/`. This can be trivially fixed by just
setting the mount point to the real world value, which is a tmpfs with
64GB assigned.

We also set `deployment.targetHost` to a domain that will
never resolve in the public internet, to make sure nobody applies these
machines by hand. It would have been nice to throw a error whenever
`colmena apply` gets executed for one of these hosts, but doing so would
defeat the purpose of this patch, because the colmena `build` and `apply`
argument both evaluate the exact same code paths and thus colmena
`build` would error again.

The motivation behind this was, so we could run `colmena build` in CI
in the future, and to not scare of new contributors with random build
failures when they first try to build the machines.

The proper solution would be to exclude all the network booted builders
from the regular colmena hive that is exposed to the cli, but this is
too many yaks to shave for now.
2024-09-23 23:13:33 +02:00
3 changed files with 12 additions and 101 deletions

View file

@ -135,7 +135,7 @@ in
{ address = "2a01:584:11::1:${toString cfg.num}"; prefixLength = 64; }
];
networking.defaultGateway6 = { interface = "uplink"; address = "2a01:584:11::1"; };
deployment.targetHost = "2a01:584:11::1:${toString cfg.num}";
deployment.targetHost = lib.mkIf (!cfg.netboot) "2a01:584:11::1:${toString cfg.num}";
deployment.tags = [ "builders" ];
# Why can't we have nice things? https://bugs.openjdk.org/browse/JDK-8170568

View file

@ -21,13 +21,22 @@ in
'';
};
# machines with the netboot module enabled should only be updated by appliying wob-vpn-gw and rebooting
deployment.targetHost = "invalid.example.com";
# fixes initrd eval warning, and allows `colmena build` to succed
fileSystems."/" = {
device = "none";
fsType = "tmpfs";
options = [ "defaults" "size=64G" "mode=755" ];
};
system.build = {
# Build a kernel and initramfs which will download the IPXE script from hydra using
# u-root pxeboot tool and kexec into the final netbooted system.
notipxe = import (modulesPath + "/..") {
system = "x86_64-linux";
configuration =
configuration =
{ pkgs, config, ... }:
{
@ -57,7 +66,7 @@ in
script = ''
ln -sf /dev/console /dev/tty
until ${pkgs.iputils}/bin/ping -c 1 hydra.forkos.org; do sleep 1; done
${pkgs.u-root}/bin/pxeboot -v -ipv4=false -file https://hydra.forkos.org/job/infra/main/${node.config.networking.hostName}/latest/download-by-type/file/ipxe
${pkgs.u-root}/bin/pxeboot -v -ipv4=false -file https://hydra.forkos.org/job/infra/main/${node.config.networking.hostName}/latest/download-by-type/file/ipxe
'';
};
boot.initrd.systemd.contents."/etc/ssl/certs/ca-certificates.crt".source = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";

View file

@ -45,103 +45,5 @@ in
resource.dnsimple_zone.vzfdfp_de = {
name = "vzfdfp.de";
};
resource.dnsimple_zone_record = let
# https://registry.terraform.io/providers/dnsimple/dnsimple/latest/docs/resources/zone_record
canonicalName = zoneName: record: let
# TODO: make less fragile and have actual unique and stable names
normalize = builtins.replaceStrings ["." "@"] ["_" "_root_"];
zone = normalize zoneName;
name = normalize record.name;
in "${zone}_${record.type}_${name}";
record = name: ttl: type: value: {
inherit name ttl type value;
};
proxyRecords = name: ttl: type: value: [
# kurisu.lahfa.xyz running a sniproxy:
(record name ttl "A" "163.172.69.160")
(record name ttl type value)
];
# Creates a extra *.p record pointing to the sniproxy
dualProxyRecords = name: ttl: type: value: lib.flatten [
(record name ttl type value)
(proxyRecords "${name}.p" ttl type value)
];
domain = zoneName: records:
builtins.listToAttrs (map (record: {
name = canonicalName zoneName record;
value = record // {
zone_name = zoneName;
};
}
) (lib.flatten records));
zones = domains: lib.zipAttrs (lib.mapAttrsToList (zoneName: records: domain zoneName records) domains);
in zones {
"forkos.org" = ([
# (record "@" 300 "A" "163.172.69.160")
(record "@" 300 "AAAA" "2001:bc8:38ee:100:1000::20")
(dualProxyRecords "bagel-box.infra" 300 "AAAA" "2001:bc8:38ee:100:100::1")
(dualProxyRecords "gerrit01.infra" 300 "AAAA" "2001:bc8:38ee:100:1000::10")
(dualProxyRecords "meta01.infra" 300 "AAAA" "2001:bc8:38ee:100:1000::20")
(dualProxyRecords "fodwatch.infra" 300 "AAAA" "2001:bc8:38ee:100:1000::30")
# git.infra.forkos.org exposes opensshd
(dualProxyRecords "git.infra" 300 "AAAA" "2001:bc8:38ee:100:1000::41")
# git.p.forkos.org exposes forgejo ssh server.
(proxyRecords "git.p" 300 "AAAA" "2001:bc8:38ee:100:1000::40")
(dualProxyRecords "buildbot.infra" 300 "AAAA" "2001:bc8:38ee:100:1000::50")
(dualProxyRecords "public01.infra" 300 "AAAA" "2001:bc8:38ee:100:1000::60")
(record "cl" 300 "CNAME" "gerrit01.infra.p.forkos.org")
(record "fodwatch" 300 "CNAME" "fodwatch.infra.p.forkos.org")
# git.p.forkos.org is the proxy variant of the Forgejo server.
(record "git" 300 "CNAME" "git.p.forkos.org")
(record "netbox" 300 "CNAME" "meta01.infra.p.forkos.org")
(record "amqp" 300 "CNAME" "bagel-box.infra.p.forkos.org")
(record "grafana" 300 "CNAME" "meta01.infra.p.forkos.org")
(record "hydra" 300 "CNAME" "build-coord.wob01.infra.p.forkos.org")
(record "loki" 300 "CNAME" "meta01.infra.p.forkos.org")
(record "mimir" 300 "CNAME" "meta01.infra.p.forkos.org")
(record "pyroscope" 300 "CNAME" "meta01.infra.p.forkos.org")
(record "tempo" 300 "CNAME" "meta01.infra.p.forkos.org")
(record "matrix" 300 "CNAME" "meta01.infra.p.forkos.org")
(record "alerts" 300 "CNAME" "meta01.infra.p.forkos.org")
(record "buildbot" 300 "CNAME" "buildbot.infra.p.forkos.org")
(record "b" 300 "CNAME" "public01.infra.p.forkos.org")
(record "postgres" 300 "CNAME" "bagel-box.infra.p.forkos.org")
(record "news" 3600 "CNAME" "public01.infra.p.forkos.org")
# S3 in delroth's basement
(record "cache" 300 "AAAA" "2a02:168:6426::12") # smol.delroth.net
(record "cache" 300 "A" "195.39.247.161") # sni proxy
(record "vpn-gw.wob01.infra" 300 "AAAA" "2a01:584:11::2")
(dualProxyRecords "build-coord.wob01.infra" 300 "AAAA" "2a01:584:11::1:11")
# TODO: do not hardcode, just reuse the Colmena hive module outputs to generate all the required details.
]
++ (map (index: record "builder-${toString index}.wob01.infra" 300 "AAAA" "2a01:584:11::1:${toString index}") (genList lib.id 11))
++ (
let
# FIXME: figure out a way to poke `config.services.s3-revproxy` and
# automate the DNS part away?
buckets = [
"channels"
"releases"
"channel-scripts-test"
];
in
map (bucket: record "${bucket}" 300 "CNAME" "public01.infra.p.forkos.org") buckets
));
"flowery.systems" = [
(record "" 300 "ALIAS" "news.forkos.org")
];
"vzfdfp.de" = [
];
};
};
}