infra/flake.nix

{
  description = "Bagel cooking infrastructure";

  inputs = {
    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";

    terranix.url = "github:terranix/terranix";
    terranix.inputs.nixpkgs.follows = "nixpkgs";

    agenix.url = "github:ryantm/agenix";
    agenix.inputs.nixpkgs.follows = "nixpkgs";

    colmena.url = "github:zhaofengli/colmena";
    colmena.inputs.nixpkgs.follows = "nixpkgs";

    hydra.url = "git+https://git.lix.systems/lix-project/hydra.git";
    hydra.inputs.nixpkgs.follows = "nixpkgs";

    nix-gerrit.url = "git+https://git.lix.systems/the-distro/nix-gerrit.git?ref=refs/heads/bump-minor-3_10";
    nix-gerrit.inputs.nixpkgs.follows = "nixpkgs";

    nix-forgejo.url = "git+https://git.lix.systems/the-distro/nix-forgejo.git";
    nix-forgejo.flake = false;

    ofborg.url = "git+https://git.lix.systems/the-distro/ofborg.git?ref=refs/heads/vcs-generalization";
    ofborg.flake = false;

    gerrit-dashboard.url = "git+https://git.lix.systems/the-distro/gerrit-monitoring.git";
    gerrit-dashboard.flake = false;

    buildbot-nix.url = "git+https://git.lix.systems/lix-project/buildbot-nix.git?ref=refs/heads/forkos";
    buildbot-nix.inputs.nixpkgs.follows = "nixpkgs";

    channel-scripts.url = "git+https://git.lix.systems/the-distro/channel-scripts.git";
    channel-scripts.inputs.nixpkgs.follows = "nixpkgs";

    stateless-uptime-kuma.url = "git+https://git.dgnum.eu/DGNum/stateless-uptime-kuma.git";
    stateless-uptime-kuma.flake = false;

    lix.follows = "hydra/lix";

    grapevine = {
      type = "gitlab";
      host = "gitlab.computer.surgery";
      owner = "matrix";
      repo = "grapevine-fork";
      inputs.nixpkgs.follows = "nixpkgs";
    };
  };

  outputs = { self, nixpkgs, terranix, colmena, ofborg, ... } @ inputs:
  let
    supportedSystems = [ "x86_64-linux" "aarch64-linux" ];
    forEachSystem = f: builtins.listToAttrs (map (system: {
      name = system;
      value = f system;
    }) supportedSystems);
    systemBits = forEachSystem (system: rec {
      inherit system;
      pkgs = import nixpkgs {
        localSystem = system;
        overlays = [
          inputs.hydra.overlays.default
          inputs.lix.overlays.default
          inputs.nix-gerrit.overlays.default
          inputs.channel-scripts.overlays.default
          (import inputs.ofborg {
            pkgs = import nixpkgs { localSystem = system; };
          }).overlay
          (import "${inputs.stateless-uptime-kuma}/overlay.nix")
        ];
      };
      terraform = pkgs.opentofu;
      terraformCfg = terranix.lib.terranixConfiguration {
        inherit system;
        modules = [
          ./terraform
          {
            bagel.dnsimple.enable = true;
            bagel.hydra.enable = true;
          }
        ];
      };
    });
    forEachSystem' = f: forEachSystem (system: (f systemBits.${system}));
    inherit (nixpkgs) lib;
    # ForkOS' library functions.
    flib = import ./lib { inherit (nixpkgs) lib; };
    inherit (flib) singleton;
  in
  {
    apps = forEachSystem' ({ system, pkgs, terraformCfg, terraform, ... }: {
      tf = {
        type = "app";
        program = toString (pkgs.writers.writeBash "tf" ''
            set -eo pipefail
            ln -snf ${terraformCfg} config.tf.json
            exec ${lib.getExe terraform} "$@"
          '');
      };

      default = self.apps.${system}.tf;
    });

    devShells = forEachSystem' ({ system, pkgs, ... }: {
      default = pkgs.mkShell {
        packages = [
          inputs.agenix.packages.${system}.agenix

          pkgs.opentofu

          (pkgs.callPackage ./lib/colmena-wrapper.nix { })
        ];
      };
    });

    nixosConfigurations = (colmena.lib.makeHive self.outputs.colmena).nodes;

    colmena = let
      commonModules = [
        inputs.agenix.nixosModules.default
        inputs.hydra.nixosModules.hydra
        inputs.buildbot-nix.nixosModules.buildbot-coordinator
        inputs.buildbot-nix.nixosModules.buildbot-worker

        ./services
        ./common
      ];

      floralInfraModules = commonModules ++ [
        ({ config, lib, ... }: {
          # This means that anyone with @floral-infra permissions
          # can ssh on root of every machines handled here.
          bagel.admins.allowedGroups = [
            "floral-infra"
          ];

          # Tag all machines which have local boot as local bootables.
          deployment.tags = lib.mkMerge [
            [ "floral" ]
            # All nodes that can be local booted, including baremetal nodes.
            (lib.mkIf (config.bagel.baremetal.enable -> !config.bagel.baremetal.netboot)
              [ "localboot" ]
            )
            # Only baremetal nodes that can be local booted.
            (lib.mkIf (config.bagel.baremetal.enable && !config.bagel.baremetal.netboot)
              [ "bm-localboot" ]
            )
          ];

          bagel.monitoring.grafana-agent.tenant = "floral";
          bagel.secrets.tenant = "floral";
          bagel.builders.extra-build-capacity.provider.tenant = "floral";
          bagel.services.buildbot.tenant = "floral";
        })
      ];

      # These are Floral baremetal builders.
      makeColoBaremetal = i:
      let
        enableNetboot = i >= 6;
      in
      # bm for baremetal.
      lib.nameValuePair "bm-${toString i}" {
        imports = floralInfraModules;
        bagel.baremetal = { enable = true; num = i; netboot = enableNetboot; };
      };

      # Given the data of:
      # - a selector function to filter NixOS nodes
      # - a module factory function to extend a NixOS configuration
      # this will return a function that will take a set of nodes and project it to the filtered
      # nodes augmented with the module factory function.
      # Composing twice the projector should have no effect.
      # `mkSystem :: { renumberedIndex: int, node: NixOS configuration } → NixOS configuration`
      mkProjector = { selector, mkSystem }: nodes:
      let
        # Select all the nodes using the selector.
        selectedNodes = lib.filterAttrs (_: node: selector node.bagel.baremetal.num) nodes;
      in
        # Re-map selected nodes and renumber them in some iteration order
        # and apply the module extension function.
        flib.renumber
          # Indexing function
          (node: node.bagel.baremetal.num)
          # Renumbering function
          (renumberedIndex: node: mkSystem { inherit renumberedIndex node; })
          selectedNodes;

      # Current map:
      # builders: [4, 10].
      # storage: [5]
      # build-coord: [11].

      # Set of projectors that will take a generic baremetal node
      # and reconfigure it for a specific role.
      projectors = {
        storage = {
          # Selectors are just fancy functions that can filter based on the index information.
          # It is possible to construct a range filter to express a collection of intervals,
          # e.g. select 0→4 & 6→8 & 12→15.

          # For now, we will only use pointwise as we have very few machines.
          selector = flib.mkPointwiseFilter [ 5 ];
          mkSystem = { renumberedIndex, node }:
          {
            imports = [ node ];
            bagel.baremetal.storage = {
              enable = true;
              num = renumberedIndex;
            };
          };
        };
        builders = {
          selector = flib.mkPointwiseFilter [ 4 10 ];
          mkSystem = { renumberedIndex, node }: {
            imports = [ node ];
            bagel.baremetal.builders = {
              enable = true;
              num = renumberedIndex;
            };
          };
        };
      };
      project = role: mkProjector projectors.${role};

      lixInfraModules = commonModules ++ [
        {
          # This means that anyone with @lix-infra permissions
          # can ssh on root of every machines handled here.
          bagel.admins.allowedGroups = [
            "lix-infra"
          ];

          # Tag all machines which have local boot as local bootables.
          # Lix has no netbootable machine.
          deployment.tags = [ "localboot" "lix" ];

          bagel.monitoring.grafana-agent.tenant = "lix";
          bagel.secrets.tenant = "lix";
          bagel.builders.extra-build-capacity.provider = {
            tenant = "lix";
            buildfarmPublicKeys = [
              # buildbot.lix.systems SSH key
              "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDu4cEqZzAI/1vZjSQkTJ4ijIg9nuloOuSKUrnkJIOFn"
            ];
          };
          bagel.services.buildbot.tenant = "lix";
        }
      ];

      baremetalNodes =
      let
        # We consider all possible baremetal systems and we filter out a subset that is activated.
        # To configure the set of used machines, configure the `setXYZ` role setter selectors.
        allNodes = lib.listToAttrs (lib.genList makeColoBaremetal 11);
        perRoles = {
          # Project in the sense of linear algebra projectors.
          # We are projecting allNodes on the set of storage nodes.
          # (remember, a projector is a linear function such that p^2 = p).
          storageNodes = project "storage" allNodes;
          builderNodes = project "builders" allNodes;
          # buildCoordinatorNodes = setBuildCoordinators allNodes;
        };
      in
      # TODO: compute what are the offender nodes and their simultaneous roles.
      assert (lib.assertMsg (flib.isValidPartition perRoles) "A baremetal node is simultaneously storage, builder and build coordinator, please review the ranges.");
      # Merge all roles together into one big attribute set of nodes.
      flib.chainAttrs perRoles;

    in {
      meta.nixpkgs = systemBits.x86_64-linux.pkgs;
      # Add any non-x86_64 native systems here.
      # Cross compilation is not supported yet.
      meta.nodeNixpkgs =
        let
          aarch64-systems = systems: lib.genAttrs systems (system: systemBits.aarch64-linux.pkgs);
        in
        aarch64-systems [
          "build01-aarch64-lix"
        ];
      meta.specialArgs.inputs = inputs;

      bagel-box.imports = floralInfraModules ++ [ ./hosts/bagel-box ];
      meta01.imports = floralInfraModules ++ [ ./hosts/meta01 ];
      gerrit01.imports = floralInfraModules ++ [ ./hosts/gerrit01 ];
      fodwatch.imports = floralInfraModules ++ [ ./hosts/fodwatch ];
      git.imports = floralInfraModules ++ [ ./hosts/git ];
      wob-vpn-gw.imports = floralInfraModules ++ [ ./hosts/wob-vpn-gw ];
      buildbot.imports = floralInfraModules ++ [ ./hosts/buildbot ];
      public01.imports = floralInfraModules ++ [ ./hosts/public01 ];
      build-coord.imports = floralInfraModules ++ [ ./hosts/build-coord ];

      build01-aarch64-lix.imports = lixInfraModules ++ [ ./hosts/build01-aarch64-lix ];
      buildbot-lix.imports = lixInfraModules ++ [ ./hosts/buildbot-lix ];
    } // baremetalNodes;

    hydraJobs = builtins.mapAttrs (n: v: v.config.system.build.netbootDir or v.config.system.build.toplevel) self.nixosConfigurations;
    buildbotJobs = builtins.mapAttrs (_: v: v.config.system.build.toplevel) self.nixosConfigurations;
  };
}