infra/flake.nix

{
  description = "Bagel cooking infrastructure";

  inputs = {
    nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";

    terranix.url = "github:terranix/terranix";
    terranix.inputs.nixpkgs.follows = "nixpkgs";

    agenix.url = "github:ryantm/agenix";
    agenix.inputs.nixpkgs.follows = "nixpkgs";

    colmena.url = "github:zhaofengli/colmena";
    colmena.inputs.nixpkgs.follows = "nixpkgs";

    hydra.url = "git+https://git.lix.systems/lix-project/hydra.git";
    hydra.inputs.nixpkgs.follows = "nixpkgs";

    nix-gerrit.url = "git+https://git.lix.systems/the-distro/nix-gerrit.git";
    nix-gerrit.inputs.nixpkgs.follows = "nixpkgs";

    gerrit-dashboard.url = "git+https://git.lix.systems/the-distro/gerrit-monitoring.git";
    gerrit-dashboard.flake = false;

    buildbot-nix.url = "git+https://git.lix.systems/lix-project/buildbot-nix.git?ref=refs/heads/faster-depinfo";
    buildbot-nix.inputs.nixpkgs.follows = "nixpkgs";

    channel-scripts.url = "git+https://git.lix.systems/the-distro/channel-scripts.git";
    channel-scripts.inputs.nixpkgs.follows = "nixpkgs";

    stateless-uptime-kuma.url = "git+https://git.dgnum.eu/DGNum/stateless-uptime-kuma.git";
    stateless-uptime-kuma.flake = false;

    lix.follows = "hydra/lix";

    grapevine = {
      type = "gitlab";
      host = "gitlab.computer.surgery";
      owner = "matrix";
      repo = "grapevine-fork";
      inputs.nixpkgs.follows = "nixpkgs";
    };
  };

  outputs = { self, nixpkgs, terranix, colmena, ... } @ inputs:
  let
    supportedSystems = [ "x86_64-linux" "aarch64-linux" ];
    forEachSystem = f: builtins.listToAttrs (map (system: {
      name = system;
      value = f system;
    }) supportedSystems);
    systemBits = forEachSystem (system: rec {
      inherit system;
      pkgs = import nixpkgs {
        localSystem = system;
        overlays = [
          inputs.hydra.overlays.default
          inputs.lix.overlays.default
          inputs.nix-gerrit.overlays.default
          inputs.channel-scripts.overlays.default
          (import "${inputs.stateless-uptime-kuma}/overlay.nix")
        ];
      };
      terraform = pkgs.opentofu;
      terraformCfg = terranix.lib.terranixConfiguration {
        inherit system;
        modules = [
          ./terraform
          {
            bagel.dnsimple.enable = true;
            bagel.hydra.enable = true;
          }
        ];
      };
    });
    forEachSystem' = f: forEachSystem (system: (f systemBits.${system}));
    inherit (nixpkgs) lib;
  in
  {
    apps = forEachSystem' ({ system, pkgs, terraformCfg, terraform, ... }: {
      tf = {
        type = "app";
        program = toString (pkgs.writers.writeBash "tf" ''
            set -eo pipefail
            ln -snf ${terraformCfg} config.tf.json
            exec ${lib.getExe terraform} "$@"
          '');
      };

      default = self.apps.${system}.tf;
    });

    devShells = forEachSystem' ({ system, pkgs, ... }: {
      default = pkgs.mkShell {
        packages = [
          inputs.agenix.packages.${system}.agenix

          pkgs.opentofu

          (pkgs.callPackage ./lib/colmena-wrapper.nix { })
        ];
      };
    });

    nixosConfigurations = (colmena.lib.makeHive self.outputs.colmena).nodes;

    colmena = let
      commonModules = [
        inputs.agenix.nixosModules.default
        inputs.hydra.nixosModules.hydra
        inputs.buildbot-nix.nixosModules.buildbot-coordinator
        inputs.buildbot-nix.nixosModules.buildbot-worker

        ./services
        ./common
      ];

      floralInfraModules = commonModules ++ [
        ({ config, lib, ... }: {
          # This means that anyone with @floral-infra permissions
          # can ssh on root of every machines handled here.
          bagel.admins.allowedGroups = [
            "floral-infra"
          ];

          # Tag all machines which have local boot as local bootables.
          deployment.tags = lib.mkMerge [
            [ "floral" ]
            # All nodes that can be local booted, including baremetal nodes.
            (lib.mkIf (config.bagel.baremetal.enable -> !config.bagel.baremetal.netboot)
              [ "localboot" ]
            )
            # Only baremetal nodes that can be local booted.
            (lib.mkIf (config.bagel.baremetal.enable && !config.bagel.baremetal.netboot)
              [ "bm-localboot" ]
            )
          ];

          bagel.monitoring.grafana-agent.tenant = "floral";
          bagel.secrets.tenant = "floral";
          bagel.builders.extra-build-capacity.provider.tenant = "floral";
          bagel.services.buildbot.tenant = "floral";
        })
      ];

      # These are Floral baremetal builders.
      makeColoBaremetal = i:
      let
        enableNetboot = i >= 6;
      in
      # bm for baremetal.
      lib.nameValuePair "bm-${toString i}" {
        imports = floralInfraModules;
        bagel.baremetal = { enable = true; num = i; netboot = enableNetboot; };
      };

      setFlavorForNode = flavor: i: node: {
        imports = [
          node
        ];

        bagel.baremetal.${flavor} = {
          enable = true;
          num = i;
        };
      };
      setFlavorForNodes = flavor: { ranges }: nodes:
      let
        setFlavor = setFlavorForNode flavor;
        # Test if i is in [range.start, range.end[.
        inRange = i: range: i >= range.start && i < range.end;
        # Perform an enumeration from values to indexes.
        reverseEnumerate = list: lib.listToAttrs (lib.zipListsWith (i: x: lib.nameValuePair x i) (lib.range 0 (lib.length list - 1)) list);
        # Filter all nodes by the range data for this flavor.
        filteredNodes = lib.filterAttrs (name: node: lib.any (r: inRange node.bagel.baremetal.num r) ranges) nodes;
        # The enumeration from baremetal numbers to flavor-specific numbers.
        indexes = lib.traceValSeq (reverseEnumerate (map (n: toString n.bagel.baremetal.num) (builtins.attrValues filteredNodes)));
      in
        # Build a new attrset with a new second-level index for the specific flavor.
        lib.mapAttrs (name: node: setFlavor indexes.${toString node.bagel.baremetal.num} node) filteredNodes;

      closedOpenInterval = a: b: { start = a; end = b; };
      interval = a: b: closedOpenInterval a b;
      setStorage = setFlavorForNodes "storage" {
        # List the indexes for the baremetal nodes which are supposed to be storage nodes.
        ranges = [ (interval 5 6) ];
      };
      setBuilders = setFlavorForNodes "builders" {
        # List the indexes for the baremetal nodes which are supposed to be builders nodes.
        ranges = [ (interval 0 5) (interval 6 10) ];
      };

      lixInfraModules = commonModules ++ [
        {
          # This means that anyone with @lix-infra permissions
          # can ssh on root of every machines handled here.
          bagel.admins.allowedGroups = [
            "lix-infra"
          ];

          # Tag all machines which have local boot as local bootables.
          # Lix has no netbootable machine.
          deployment.tags = [ "localboot" "lix" ];

          bagel.monitoring.grafana-agent.tenant = "lix";
          bagel.secrets.tenant = "lix";
          bagel.builders.extra-build-capacity.provider = {
            tenant = "lix";
            buildfarmPublicKeys = [
              # buildbot.lix.systems SSH key
              "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDu4cEqZzAI/1vZjSQkTJ4ijIg9nuloOuSKUrnkJIOFn"
            ];
          };
          bagel.services.buildbot.tenant = "lix";
        }
      ];

      checkForNonReuse = setOfNodes:
      let
        setOfNodesNames = lib.mapAttrs (name: nodes: builtins.attrNames nodes) setOfNodes;
      in
      lib.all (set:
        lib.all (anotherSet: set != anotherSet -> lib.intersectLists setOfNodesNames.${set} setOfNodesNames.${anotherSet} == []) (builtins.attrNames setOfNodes)
      ) (builtins.attrNames setOfNodes);

      baremetalNodes =
      let
        allNodes = lib.listToAttrs (lib.genList makeColoBaremetal 11);
        perRoles = {
          storageNodes = setStorage allNodes;
          builderNodes = setBuilders allNodes;
        };
      in
        assert (lib.assertMsg (checkForNonReuse perRoles) "A baremetal node is simultaneously storage and builder, please review the ranges.");
        lib.foldl (a: b: a // b) { } (builtins.attrValues perRoles);

    in {
      meta.nixpkgs = systemBits.x86_64-linux.pkgs;
      # Add any non-x86_64 native systems here.
      # Cross compilation is not supported yet.
      meta.nodeNixpkgs =
        let
          aarch64-systems = systems: lib.genAttrs systems (system: systemBits.aarch64-linux.pkgs);
        in
        aarch64-systems [
          "build01-aarch64-lix"
        ];
      meta.specialArgs.inputs = inputs;

      bagel-box.imports = floralInfraModules ++ [ ./hosts/bagel-box ];
      meta01.imports = floralInfraModules ++ [ ./hosts/meta01 ];
      gerrit01.imports = floralInfraModules ++ [ ./hosts/gerrit01 ];
      fodwatch.imports = floralInfraModules ++ [ ./hosts/fodwatch ];
      git.imports = floralInfraModules ++ [ ./hosts/git ];
      wob-vpn-gw.imports = floralInfraModules ++ [ ./hosts/wob-vpn-gw ];
      buildbot.imports = floralInfraModules ++ [ ./hosts/buildbot ];
      public01.imports = floralInfraModules ++ [ ./hosts/public01 ];
      build-coord.imports = floralInfraModules ++ [ ./hosts/build-coord ];

      build01-aarch64-lix.imports = lixInfraModules ++ [ ./hosts/build01-aarch64-lix ];
      buildbot-lix.imports = lixInfraModules ++ [ ./hosts/buildbot-lix ];
    } // baremetalNodes;

    hydraJobs = builtins.mapAttrs (n: v: v.config.system.build.netbootDir or v.config.system.build.toplevel) self.nixosConfigurations;
    buildbotJobs = builtins.mapAttrs (_: v: v.config.system.build.toplevel) self.nixosConfigurations;
  };
}