From a3f6f573a2a23f41053ffffea0916bdf2a653b04 Mon Sep 17 00:00:00 2001 From: Raito Bezarius Date: Mon, 7 Oct 2024 15:21:03 +0200 Subject: [PATCH] feat: add cache.lix.systems Introduces a full-fleged self-hosted S3 NixOS module with various knobs to perform NGINX-based redirects and s3-revproxy endpoints. Signed-off-by: Raito Bezarius --- flake.nix | 2 + hosts/cache-lix/default.nix | 125 +++++++++++++++++++++++ services/lix-s3/default.nix | 33 ++++++ services/lix-s3/garage-ephemeral-key.nix | 63 ++++++++++++ services/lix-s3/garage.nix | 106 +++++++++++++++++++ services/lix-s3/s3-revproxy.nix | 106 +++++++++++++++++++ services/lix-s3/web.nix | 81 +++++++++++++++ 7 files changed, 516 insertions(+) create mode 100644 hosts/cache-lix/default.nix create mode 100644 services/lix-s3/default.nix create mode 100644 services/lix-s3/garage-ephemeral-key.nix create mode 100644 services/lix-s3/garage.nix create mode 100644 services/lix-s3/s3-revproxy.nix create mode 100644 services/lix-s3/web.nix diff --git a/flake.nix b/flake.nix index 75c44e8..8b3cde7 100644 --- a/flake.nix +++ b/flake.nix @@ -199,6 +199,8 @@ build01-aarch64-lix.imports = lixInfraModules ++ [ ./hosts/build01-aarch64-lix ]; buildbot-lix.imports = lixInfraModules ++ [ ./hosts/buildbot-lix ]; + # This is Lix's Garage S3. + cache-lix.imports = lixInfraModules ++ [ ./hosts/cache-lix ]; } // builders; hydraJobs = builtins.mapAttrs (n: v: v.config.system.build.netbootDir or v.config.system.build.toplevel) self.nixosConfigurations; diff --git a/hosts/cache-lix/default.nix b/hosts/cache-lix/default.nix new file mode 100644 index 0000000..064fdff --- /dev/null +++ b/hosts/cache-lix/default.nix @@ -0,0 +1,125 @@ +# Configuration for a virtual machine in Raito's micro-DC basement. +# 8 vCPU (2014 grade Xeon though) +# 8GB RAM +# 100GB SSD +# 1TB HDD +# All specifications can be upgraded to a certain extent, just ask Raito. +# Hosts the Garage S3 instance for the Lix project. +# Our "binary cache". +# +# vim: et:ts=2:sw=2: +# +{ config, pkgs, lib, ... }: { + networking.hostName = "cache"; + networking.domain = "lix.systems"; + + system.stateVersion = "24.05"; + + zramSwap.enable = true; + i18n.defaultLocale = "en_US.UTF-8"; + + # All the objects are stored there. + # Metadata is on the fast SSD. + fileSystems."/data" = { + device = "/dev/disk/by-label/data"; + fsType = "ext4"; + }; + + bagel.s3 = { + rootDomain = "lix.systems"; + garage.enable = true; + web = { + buckets = [ + "install" + "cache" + "releases" + "docs" + ]; + subdomains = { + "cache.lix.systems" = "cache"; + "install.lix.systems" = "install"; + }; + }; + reverse-proxy = { + enable = true; + buckets = [ + "docs" + "releases" + "install" + ]; + web = { + "releases.lix.systems" = "releases"; + "docs.lix.systems" = "docs"; + }; + }; + }; + + # Fix up the manual path so it enables having multiple manuals + services.nginx.virtualHosts."docs.lix.systems".locations.${''~ ^/manual/nightly(/[^\s]*)$''} = + { + extraConfig = '' + return 301 /manual/lix/nightly$1; + ''; + }; + + systemd.network.links."10-nat-lan".matchConfig.MACAddress = "BC:24:11:1E:7C:9B"; + systemd.network.networks."10-wan".networkConfig.Address = [ "2001:bc8:38ee:100::210/56" ]; + systemd.network.links."10-wan".matchConfig.MACAddress = "BC:24:11:42:72:79"; + + nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; + + # Debugging stuff. + virtualisation.vmVariant = { + systemd.network.enable = lib.mkForce false; + networking.useDHCP = true; + + environment.systemPackages = with pkgs; [ + tcpdump + ]; + + system.activationScripts.agenixInstall.text = lib.mkForce '' + echo "lol gotcha" + _agenix_generation="$(basename "$(readlink ${config.age.secretsDir})" || echo 0)" + (( ++_agenix_generation )) + p=${config.age.secretsMountPoint}/$_agenix_generation + > $p/garage + echo "GARAGE_RPC_SECRET=$(${lib.getExe pkgs.openssl.bin} rand -hex 32)" >> $p/garage + echo "GARAGE_METRICS_TOKEN=$(${lib.getExe pkgs.openssl.bin} rand -base64 32)" >> $p/garage + echo "GARAGE_ADMIN_TOKEN=$(${lib.getExe pkgs.openssl.bin} rand -base64 32)" >> $p/garage + + ln -sfT /var/secrets/garage-s3-api-key $p/s3-revproxy-api-key-env + ln -sfT $p ${config.age.secretsDir} + ''; + virtualisation.forwardPorts = [ + { + from = "host"; + guest.port = 443; + host.port = 4043; + proto = "tcp"; + } + { + from = "host"; + guest.port = 1337; + host.port = 1337; + proto = "tcp"; + } + { + from = "host"; + guest.port = 22; + host.port = 2022; + proto = "tcp"; + } + ]; + + security.acme = { + defaults.server = "http://127.0.0.1/failfailfail"; + extraLegoFlags = ["--lol-fail"]; + extraLegoRenewFlags = ["--lol-fail"]; + extraLegoRunFlags = ["--lol-fail"]; + }; + + systemd.tmpfiles.rules = [ + "d /data/s3 700 garage - - -" + ]; + }; +} diff --git a/services/lix-s3/default.nix b/services/lix-s3/default.nix new file mode 100644 index 0000000..dbfac27 --- /dev/null +++ b/services/lix-s3/default.nix @@ -0,0 +1,33 @@ +{ config, lib, ... }: +let + inherit (lib) mkOption types; + cfg = config.bagel.s3; +in +{ + options.bagel.s3 = { + rootDomain = mkOption { + type = types.str; + }; + + webRootDomain = mkOption { + type = types.str; + default = "web.${cfg.rootDomain}"; + }; + + s3RootDomain = mkOption { + type = types.str; + default = "s3.${cfg.rootDomain}"; + }; + }; + + imports = [ + # Generic frontend stuff + # e.g. static redirects, etc. + ./web.nix + # Garage implementation for our S3 + # In the future, we could be using Ceph store. + ./garage.nix + # S3 reverse proxy + ./s3-revproxy.nix + ]; +} diff --git a/services/lix-s3/garage-ephemeral-key.nix b/services/lix-s3/garage-ephemeral-key.nix new file mode 100644 index 0000000..b318960 --- /dev/null +++ b/services/lix-s3/garage-ephemeral-key.nix @@ -0,0 +1,63 @@ +{ wrap }: +{ lib, pkgs, config, ... }: +let + garage-ephemeral-key = pkgs.writers.writePython3Bin + "garage-ephemeral-key" + { libraries = [ pkgs.python3.pkgs.requests ]; } + (builtins.readFile ./garage_ephemeral_key.py); + + # the usual copy pasta of systemd-analyze security satisfying rules + containment = { + DynamicUser = true; + CapabilityBoundingSet = ""; + NoNewPrivileges = true; + PrivateTmp = true; + PrivateUsers = true; + PrivateDevices = true; + ProtectHome = true; + ProtectClock = true; + ProtectProc = "noaccess"; + ProcSubset = "pid"; + UMask = "0077"; + ProtectKernelLogs = true; + ProtectKernelModules = true; + ProtectKernelTunables = true; + ProtectControlGroups = true; + ProtectHostname = true; + RestrictSUIDSGID = true; + RestrictRealtime = true; + RestrictNamespaces = true; + LockPersonality = true; + RemoveIPC = true; + SystemCallFilter = [ "@system-service" "~@privileged" ]; + RestrictAddressFamilies = [ "AF_INET" "AF_INET6" ]; + MemoryDenyWriteExecute = true; + SystemCallArchitectures = "native"; + }; +in +{ + _file = ./garage-ephemeral-key.nix; + + environment.systemPackages = [ + (wrap garage-ephemeral-key "garage-ephemeral-key") + ]; + + # Clean expired ephemeral keys every 2 minutes + systemd.timers.garage-ephemeral-key-clean = { + wantedBy = [ "multi-user.target" ]; + timerConfig = { + # Every 2 minutes. + OnCalendar = "*-*-* *:00/2"; + }; + }; + + systemd.services.garage-ephemeral-key-clean = { + after = [ "garage.service" ]; + wants = [ "garage.service" ]; + serviceConfig = { + ExecStart = "${lib.getExe garage-ephemeral-key} clean"; + + EnvironmentFile = config.age.secrets.garage.path; + } // containment; + }; +} diff --git a/services/lix-s3/garage.nix b/services/lix-s3/garage.nix new file mode 100644 index 0000000..9997dda --- /dev/null +++ b/services/lix-s3/garage.nix @@ -0,0 +1,106 @@ +{ config, pkgs, lib, ... }: +let + cfg = config.bagel.s3.garage; + inherit (lib) mkEnableOption mkIf mkOption types; + # TODO: send me back upstream to cl.forkos.org. + wrap = pkg: name: pkgs.writeShellScriptBin name '' + set -a + [[ -z "''${GARAGE_RPC_SECRET:-}" ]] && source ${config.age.secrets.garage.path} + set +a + exec ${lib.getExe pkg} "$@" + ''; + # TODO: generalize this idea + rootDomains = { + lix = "lix.systems"; + floral = "floral.systems"; + }; +in +{ + options.bagel.s3.garage = { + enable = mkEnableOption "the Garage implementation of S3"; + tenant = mkOption { + type = types.enum [ "lix" "floral" ]; + }; + + api.address = mkOption { + type = types.str; + default = "127.0.0.1:3900"; + }; + + rootDomain = mkOption { + type = types.str; + default = rootDomains.${cfg.tenant}; + }; + + dataDir = mkOption { + type = types.str; + }; + + metadataDir = mkOption { + default = "/var/lib/garage/metadata"; + }; + }; + + imports = [ + (import ./garage-ephemeral-key.nix { inherit wrap; }) + ]; + + config = mkIf cfg.enable { + bagel.secrets.files = [ + "garage-environment" + ]; + + services.garage = { + enable = true; + # Slightly evil, but we have to wrap it here so the garage cli Just + # Works(tm) from the shell. It will no-op in the wrapper if the env is + # already set. + package = wrap pkgs.garage_0_9 "garage"; + + settings = { + data_dir = cfg.dataDir; + metadata_dir = cfg.metadataDir; + + db_engine = "lmdb"; + + replication_mode = "none"; + compression_level = 7; + + rpc_bind_addr = "[::]:3901"; + rpc_public_addr = "127.0.0.1:3901"; + + s3_api = { + s3_region = "garage"; + api_bind_addr = cfg.api.address; + root_domain = ".s3.${cfg.rootDomain}"; + }; + + s3_web = { + bind_addr = "127.0.0.1:3902"; + root_domain = ".web.${cfg.rootDomain}"; + index = "index.html"; + }; + + k2v_api.api_bind_addr = "[::]:3904"; + # FIXME(raito): api bind address should be secured. + # admin.api_bind_addr = "[${wnlib.generateIPv6Address "monitoring" "cache"}]:3903"; + }; + + environmentFile = config.age.secrets.garage-environment.path; + }; + + systemd.services.garage.serviceConfig = { + User = "garage"; + ReadWriteDirectories = [ + cfg.dataDir + ]; + StateDirectory = "garage"; + }; + + users.users.garage = { + isSystemUser = true; + group = "garage"; + }; + users.groups.garage = { }; + }; +} diff --git a/services/lix-s3/s3-revproxy.nix b/services/lix-s3/s3-revproxy.nix new file mode 100644 index 0000000..37a974e --- /dev/null +++ b/services/lix-s3/s3-revproxy.nix @@ -0,0 +1,106 @@ +{ lib, config, ... }: +let + inherit (lib) mkOption mkIf types mapAttrs; + cfgParent = config.bagel.s3; + cfg = config.bagel.s3.reverse-proxy; + mkTarget = { name, bucket ? name }: { + mount = { + host = "${name}.${cfgParent.webRootDomain}"; + path = [ "/" ]; + }; + actions.GET = { + enabled = true; + config = { + # e.g. /2.90 will 404, so it will redirect to /2.90/ if it is a directory + redirectWithTrailingSlashForNotFoundFile = true; + indexDocument = "index.html"; + }; + }; + + bucket = { + name = bucket; + region = "garage"; + s3Endpoint = "https://${cfgParent.s3RootDomain}"; + credentials = { + accessKey.env = "AWS_ACCESS_KEY_ID"; + secretKey.env = "AWS_SECRET_KEY"; + }; + }; + }; +in +{ + options.bagel.s3.reverse-proxy = { + targets = mkOption { + type = types.attrsOf (types.submodule ({ name, ... }: { + bucket = mkOption { + type = types.str; + default = name; + }; + })); + default = { }; + }; + + port = mkOption { + type = types.port; + default = 10652; + }; + }; + + config = mkIf cfg.enable { + age.secrets.s3-revproxy-api-key-env.file = ./s3-revproxy-env.age; + # this solves garage supporting neither anonymous access nor automatic + # directory indexing by simply ignoring garage's web server and replacing it + # with overengineered golang instead. + services.s3-revproxy = { + enable = true; + settings = { + templates = { + helpers = [ ./s3-revproxy-templates/_helpers.tpl ]; + notFoundError = { + headers = { + "Content-Type" = "{{ template \"main.headers.contentType\" . }}"; + }; + status = "404"; + }; + folderList = { + path = ./s3-revproxy-templates/folder-list.tpl; + headers = { + "Content-Type" = "{{ template \"main.headers.contentType\" . }}"; + }; + # empty s3 directories are not real and cannot hurt you. + # due to redirectWithTrailingSlashForNotFoundFile, garbage file names + # get redirected as folders, which then appear as empty, yielding + # poor UX. + status = '' + {{- if eq (len .Entries) 0 -}} + 404 + {{- else -}} + 200 + {{- end -}} + ''; + }; + }; + /* For metrics and debugging (e.g. pulling the config) + internalServer = { + listenAddr = "127.0.0.1"; + port = 1337; + }; + */ + server = { + listenAddr = "127.0.0.1"; + port = cfg.port; + + # it's going right into nginx, so no point + compress.enabled = false; + cors = { + enabled = true; + allowMethods = [ "GET" ]; + allowOrigins = [ "*" ]; + }; + }; + targets = mapAttrs mkTarget cfg.targets; + }; + environmentFile = config.age.secrets.s3-revproxy-api-key-env.path; + }; + }; +} diff --git a/services/lix-s3/web.nix b/services/lix-s3/web.nix new file mode 100644 index 0000000..3a76a69 --- /dev/null +++ b/services/lix-s3/web.nix @@ -0,0 +1,81 @@ +{ config, lib, ... }: +# TODO: move to wildcard TLS. +let + cfgParent = config.bagel.s3; + cfg = config.bagel.s3.web; + + buckets = [ "install" "cache" "releases" "docs" ]; + mkWebLocationBlock = host: { + proxyPass = "http://127.0.0.1:3902"; + extraConfig = '' + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host ${host}; + ''; + }; + mkDirectSubdomain = subdomain: { + enableACME = true; + forceSSL = true; + locations."/" = mkWebLocationBlock "${subdomain}.${webHost}"; + }; + + # Makes a subdomain that gets proxied through s3-proxy to provide directory + # listings and reasonable 404 pages. + # This is not used on cache, since there a directory listing for cache is a + # liability at best. + mkProxiedSubdomain = subdomain: { + enableACME = true; + forceSSL = true; + locations."/" = { + recommendedProxySettings = true; + proxyPass = "http://127.0.0.1:${toString s3RevproxyPort}/"; + }; + }; +in +{ + options.bagel.s3.web = { + + }; + + config = { + services.nginx = { + enable = true; + + virtualHosts = { + ${host} = { + enableACME = true; + forceSSL = true; + + serverAliases = builtins.map (b: "${b}.${host}") buckets; + + locations."/" = { + proxyPass = "http://127.0.0.1:3900"; + + extraConfig = '' + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host; + + # Disable buffering to a temporary file. + proxy_max_temp_file_size 0; + client_max_body_size 20G; + ''; + }; + }; + + ${webHost} = { + enableACME = true; + forceSSL = true; + locations."/" = mkWebLocationBlock "$host"; + + # Create a subdomain for each bucket; and include special aliases + # for our special buckets 'cache' and 'install'. + serverAliases = + (builtins.map (b: "${b}.${webHost}") buckets); + }; + }; + }; + + networking.firewall.allowedTCPPorts = [ + 80 443 + ]; + }; +}