flake.nix

@@ -199,6 +199,8 @@
 
       build01-aarch64-lix.imports = lixInfraModules ++ [ ./hosts/build01-aarch64-lix ];
       buildbot-lix.imports = lixInfraModules ++ [ ./hosts/buildbot-lix ];
+      # This is Lix's Garage S3.
+      cache-lix.imports = lixInfraModules ++ [ ./hosts/cache-lix ];
     } // builders;
 
     hydraJobs = builtins.mapAttrs (n: v: v.config.system.build.netbootDir or v.config.system.build.toplevel) self.nixosConfigurations;

hosts/cache-lix/default.nix

@@ -0,0 +1,125 @@
+# Configuration for a virtual machine in Raito's micro-DC basement.
+# 8 vCPU (2014 grade Xeon though)
+# 8GB RAM
+# 100GB SSD
+# 1TB HDD
+# All specifications can be upgraded to a certain extent, just ask Raito.
+# Hosts the Garage S3 instance for the Lix project.
+# Our "binary cache".
+#
+# vim: et:ts=2:sw=2:
+#
+{ config, pkgs, lib, ... }: {
+  networking.hostName = "cache";
+  networking.domain = "lix.systems";
+
+  system.stateVersion = "24.05";
+
+  zramSwap.enable = true;
+  i18n.defaultLocale = "en_US.UTF-8";
+
+  # All the objects are stored there.
+  # Metadata is on the fast SSD.
+  fileSystems."/data" = {
+    device = "/dev/disk/by-label/data";
+    fsType = "ext4";
+  };
+
+  bagel.s3 = {
+    rootDomain = "lix.systems";
+    garage.enable = true;
+    web = {
+      buckets = [
+        "install"
+        "cache"
+        "releases"
+        "docs"
+      ];
+      subdomains = {
+        "cache.lix.systems" = "cache";
+        "install.lix.systems" = "install";
+      };
+    };
+    reverse-proxy = {
+      enable = true;
+      buckets = [
+        "docs"
+        "releases"
+        "install"
+      ];
+      web = {
+        "releases.lix.systems" = "releases";
+        "docs.lix.systems" = "docs";
+      };
+    };
+  };
+
+  # Fix up the manual path so it enables having multiple manuals
+  services.nginx.virtualHosts."docs.lix.systems".locations.${''~ ^/manual/nightly(/[^\s]*)$''} =
+  {
+    extraConfig = ''
+      return 301 /manual/lix/nightly$1;
+    '';
+  };
+
+  systemd.network.links."10-nat-lan".matchConfig.MACAddress = "BC:24:11:1E:7C:9B";
+  systemd.network.networks."10-wan".networkConfig.Address = [ "2001:bc8:38ee:100::210/56" ];
+  systemd.network.links."10-wan".matchConfig.MACAddress = "BC:24:11:42:72:79";
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+
+  # Debugging stuff.
+  virtualisation.vmVariant = {
+    systemd.network.enable = lib.mkForce false;
+    networking.useDHCP = true;
+
+    environment.systemPackages = with pkgs; [
+      tcpdump
+    ];
+
+    system.activationScripts.agenixInstall.text = lib.mkForce ''
+      echo "lol gotcha"
+      _agenix_generation="$(basename "$(readlink ${config.age.secretsDir})" || echo 0)"
+      (( ++_agenix_generation ))
+      p=${config.age.secretsMountPoint}/$_agenix_generation
+      > $p/garage
+      echo "GARAGE_RPC_SECRET=$(${lib.getExe pkgs.openssl.bin} rand -hex 32)" >> $p/garage
+      echo "GARAGE_METRICS_TOKEN=$(${lib.getExe pkgs.openssl.bin} rand -base64 32)" >> $p/garage
+      echo "GARAGE_ADMIN_TOKEN=$(${lib.getExe pkgs.openssl.bin} rand -base64 32)" >> $p/garage
+
+      ln -sfT /var/secrets/garage-s3-api-key $p/s3-revproxy-api-key-env
+      ln -sfT $p ${config.age.secretsDir}
+    '';
+    virtualisation.forwardPorts = [
+      {
+        from = "host";
+        guest.port = 443;
+        host.port = 4043;
+        proto = "tcp";
+      }
+      {
+        from = "host";
+        guest.port = 1337;
+        host.port = 1337;
+        proto = "tcp";
+      }
+      {
+        from = "host";
+        guest.port = 22;
+        host.port = 2022;
+        proto = "tcp";
+      }
+    ];
+
+    security.acme = {
+      defaults.server = "http://127.0.0.1/failfailfail";
+      extraLegoFlags = ["--lol-fail"];
+      extraLegoRenewFlags = ["--lol-fail"];
+      extraLegoRunFlags = ["--lol-fail"];
+    };
+
+    systemd.tmpfiles.rules = [
+      "d /data/s3 700 garage - - -"
+    ];
+  };
+}

services/lix-s3/default.nix

@@ -0,0 +1,33 @@
+{ config, lib, ... }:
+let
+  inherit (lib) mkOption types;
+  cfg = config.bagel.s3;
+in
+{
+  options.bagel.s3 = {
+    rootDomain = mkOption {
+      type = types.str;
+    };
+
+    webRootDomain = mkOption {
+      type = types.str;
+      default = "web.${cfg.rootDomain}";
+    };
+
+    s3RootDomain = mkOption {
+      type = types.str;
+      default = "s3.${cfg.rootDomain}";
+    };
+  };
+
+  imports = [
+    # Generic frontend stuff
+    # e.g. static redirects, etc.
+    ./web.nix
+    # Garage implementation for our S3
+    # In the future, we could be using Ceph store.
+    ./garage.nix
+    # S3 reverse proxy
+    ./s3-revproxy.nix
+  ];
+}

services/lix-s3/garage-ephemeral-key.nix

@@ -0,0 +1,63 @@
+{ wrap }:
+{ lib, pkgs, config, ... }:
+let
+  garage-ephemeral-key = pkgs.writers.writePython3Bin
+    "garage-ephemeral-key"
+    { libraries = [ pkgs.python3.pkgs.requests ]; }
+    (builtins.readFile ./garage_ephemeral_key.py);
+
+  # the usual copy pasta of systemd-analyze security satisfying rules
+  containment = {
+    DynamicUser = true;
+    CapabilityBoundingSet = "";
+    NoNewPrivileges = true;
+    PrivateTmp = true;
+    PrivateUsers = true;
+    PrivateDevices = true;
+    ProtectHome = true;
+    ProtectClock = true;
+    ProtectProc = "noaccess";
+    ProcSubset = "pid";
+    UMask = "0077";
+    ProtectKernelLogs = true;
+    ProtectKernelModules = true;
+    ProtectKernelTunables = true;
+    ProtectControlGroups = true;
+    ProtectHostname = true;
+    RestrictSUIDSGID = true;
+    RestrictRealtime = true;
+    RestrictNamespaces = true;
+    LockPersonality = true;
+    RemoveIPC = true;
+    SystemCallFilter = [ "@system-service" "~@privileged" ];
+    RestrictAddressFamilies = [ "AF_INET" "AF_INET6" ];
+    MemoryDenyWriteExecute = true;
+    SystemCallArchitectures = "native";
+  };
+in
+{
+  _file = ./garage-ephemeral-key.nix;
+
+  environment.systemPackages = [
+    (wrap garage-ephemeral-key "garage-ephemeral-key")
+  ];
+
+  # Clean expired ephemeral keys every 2 minutes
+  systemd.timers.garage-ephemeral-key-clean = {
+    wantedBy = [ "multi-user.target" ];
+    timerConfig = {
+      # Every 2 minutes.
+      OnCalendar = "*-*-* *:00/2";
+    };
+  };
+
+  systemd.services.garage-ephemeral-key-clean = {
+    after = [ "garage.service" ];
+    wants = [ "garage.service" ];
+    serviceConfig = {
+      ExecStart = "${lib.getExe garage-ephemeral-key} clean";
+
+      EnvironmentFile = config.age.secrets.garage.path;
+    } // containment;
+  };
+}

services/lix-s3/garage.nix

@@ -0,0 +1,106 @@
+{ config, pkgs, lib, ... }:
+let
+  cfg = config.bagel.s3.garage;
+  inherit (lib) mkEnableOption mkIf mkOption types;
+  # TODO: send me back upstream to cl.forkos.org.
+  wrap = pkg: name: pkgs.writeShellScriptBin name ''
+    set -a
+    [[ -z "''${GARAGE_RPC_SECRET:-}" ]] && source ${config.age.secrets.garage.path}
+    set +a
+    exec ${lib.getExe pkg} "$@"
+  '';
+  # TODO: generalize this idea
+  rootDomains = {
+    lix = "lix.systems";
+    floral = "floral.systems";
+  };
+in
+{
+  options.bagel.s3.garage = {
+    enable = mkEnableOption "the Garage implementation of S3";
+    tenant = mkOption {
+      type = types.enum [ "lix" "floral" ];
+    };
+
+    api.address = mkOption {
+      type = types.str;
+      default = "127.0.0.1:3900";
+    };
+
+    rootDomain = mkOption {
+      type = types.str;
+      default = rootDomains.${cfg.tenant};
+    };
+
+    dataDir = mkOption {
+      type = types.str;
+    };
+
+    metadataDir = mkOption {
+      default = "/var/lib/garage/metadata";
+    };
+  };
+
+  imports = [
+    (import ./garage-ephemeral-key.nix { inherit wrap; })
+  ];
+
+  config = mkIf cfg.enable {
+    bagel.secrets.files = [
+      "garage-environment"
+    ];
+
+    services.garage = {
+      enable = true;
+      # Slightly evil, but we have to wrap it here so the garage cli Just
+      # Works(tm) from the shell. It will no-op in the wrapper if the env is
+      # already set.
+      package = wrap pkgs.garage_0_9 "garage";
+
+      settings = {
+        data_dir = cfg.dataDir;
+        metadata_dir = cfg.metadataDir;
+
+        db_engine = "lmdb";
+
+        replication_mode = "none";
+        compression_level = 7;
+
+        rpc_bind_addr = "[::]:3901";
+        rpc_public_addr = "127.0.0.1:3901";
+
+        s3_api = {
+          s3_region = "garage";
+          api_bind_addr = cfg.api.address;
+          root_domain = ".s3.${cfg.rootDomain}";
+        };
+
+        s3_web = {
+          bind_addr = "127.0.0.1:3902";
+          root_domain = ".web.${cfg.rootDomain}";
+          index = "index.html";
+        };
+
+        k2v_api.api_bind_addr = "[::]:3904";
+        # FIXME(raito): api bind address should be secured.
+        # admin.api_bind_addr = "[${wnlib.generateIPv6Address "monitoring" "cache"}]:3903";
+      };
+
+      environmentFile = config.age.secrets.garage-environment.path;
+    };
+
+    systemd.services.garage.serviceConfig = {
+      User = "garage";
+      ReadWriteDirectories = [
+        cfg.dataDir
+      ];
+      StateDirectory = "garage";
+    };
+
+    users.users.garage = {
+      isSystemUser = true;
+      group = "garage";
+    };
+    users.groups.garage = { };
+  };
+}

services/lix-s3/s3-revproxy.nix

@@ -0,0 +1,106 @@
+{ lib, config, ... }:
+let
+  inherit (lib) mkOption mkIf types mapAttrs;
+  cfgParent = config.bagel.s3;
+  cfg = config.bagel.s3.reverse-proxy;
+  mkTarget = { name, bucket ? name }: {
+    mount = {
+      host = "${name}.${cfgParent.webRootDomain}";
+      path = [ "/" ];
+    };
+    actions.GET = {
+      enabled = true;
+      config = {
+        # e.g. /2.90 will 404, so it will redirect to /2.90/ if it is a directory
+        redirectWithTrailingSlashForNotFoundFile = true;
+        indexDocument = "index.html";
+      };
+    };
+
+    bucket = {
+      name = bucket;
+      region = "garage";
+      s3Endpoint = "https://${cfgParent.s3RootDomain}";
+      credentials = {
+        accessKey.env = "AWS_ACCESS_KEY_ID";
+        secretKey.env = "AWS_SECRET_KEY";
+      };
+    };
+  };
+in
+{
+  options.bagel.s3.reverse-proxy = {
+    targets = mkOption {
+      type = types.attrsOf (types.submodule ({ name, ... }: {
+        bucket = mkOption {
+          type = types.str;
+          default = name;
+        };
+      }));
+      default = { };
+    };
+
+    port = mkOption {
+      type = types.port;
+      default = 10652;
+    };
+  };
+
+  config = mkIf cfg.enable {
+    age.secrets.s3-revproxy-api-key-env.file = ./s3-revproxy-env.age;
+    # this solves garage supporting neither anonymous access nor automatic
+    # directory indexing by simply ignoring garage's web server and replacing it
+    # with overengineered golang instead.
+    services.s3-revproxy = {
+      enable = true;
+      settings = {
+        templates = {
+          helpers = [ ./s3-revproxy-templates/_helpers.tpl ];
+          notFoundError = {
+            headers = {
+              "Content-Type" = "{{ template \"main.headers.contentType\" . }}";
+            };
+            status = "404";
+          };
+          folderList = {
+            path = ./s3-revproxy-templates/folder-list.tpl;
+            headers = {
+              "Content-Type" = "{{ template \"main.headers.contentType\" . }}";
+            };
+            # empty s3 directories are not real and cannot hurt you.
+            # due to redirectWithTrailingSlashForNotFoundFile, garbage file names
+            # get redirected as folders, which then appear as empty, yielding
+            # poor UX.
+            status = ''
+              {{- if eq (len .Entries) 0 -}}
+                404
+              {{- else -}}
+                200
+              {{- end -}}
+            '';
+          };
+        };
+        /* For metrics and debugging (e.g. pulling the config)
+        internalServer = {
+          listenAddr = "127.0.0.1";
+          port = 1337;
+        };
+        */
+        server = {
+          listenAddr = "127.0.0.1";
+          port = cfg.port;
+
+          # it's going right into nginx, so no point
+          compress.enabled = false;
+          cors = {
+            enabled = true;
+            allowMethods = [ "GET" ];
+            allowOrigins = [ "*" ];
+          };
+        };
+        targets = mapAttrs mkTarget cfg.targets;
+      };
+      environmentFile = config.age.secrets.s3-revproxy-api-key-env.path;
+    };
+  };
+}

services/lix-s3/web.nix

@@ -0,0 +1,81 @@
+{ config, lib, ... }:
+# TODO: move to wildcard TLS.
+let
+  cfgParent = config.bagel.s3;
+  cfg = config.bagel.s3.web;
+
+  buckets = [ "install" "cache" "releases" "docs" ];
+  mkWebLocationBlock = host: {
+    proxyPass = "http://127.0.0.1:3902";
+    extraConfig = ''
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header Host ${host};
+    '';
+  };
+  mkDirectSubdomain = subdomain: {
+    enableACME = true;
+    forceSSL = true;
+    locations."/" = mkWebLocationBlock "${subdomain}.${webHost}";
+  };
+
+  # Makes a subdomain that gets proxied through s3-proxy to provide directory
+  # listings and reasonable 404 pages.
+  # This is not used on cache, since there a directory listing for cache is a
+  # liability at best.
+  mkProxiedSubdomain = subdomain: {
+    enableACME = true;
+    forceSSL = true;
+    locations."/" = {
+      recommendedProxySettings = true;
+      proxyPass = "http://127.0.0.1:${toString s3RevproxyPort}/";
+    };
+  };
+in
+{
+  options.bagel.s3.web = {
+
+  };
+
+  config = {
+    services.nginx = {
+      enable = true;
+
+      virtualHosts = {
+        ${host} = {
+          enableACME = true;
+          forceSSL = true;
+
+          serverAliases = builtins.map (b: "${b}.${host}") buckets;
+
+          locations."/" = {
+            proxyPass = "http://127.0.0.1:3900";
+
+            extraConfig = ''
+              proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+              proxy_set_header Host $host;
+
+              # Disable buffering to a temporary file.
+              proxy_max_temp_file_size 0;
+              client_max_body_size 20G;
+            '';
+          };
+        };
+
+        ${webHost} = {
+          enableACME = true;
+          forceSSL = true;
+          locations."/" = mkWebLocationBlock "$host";
+
+          # Create a subdomain for each bucket; and include special aliases
+          # for our special buckets 'cache' and 'install'.
+          serverAliases =
+            (builtins.map (b: "${b}.${webHost}") buckets);
+        };
+      };
+    };
+
+    networking.firewall.allowedTCPPorts = [
+      80 443
+    ];
+  };
+}