feat: add cache.lix.systems

Introduces a full-fleged self-hosted S3 NixOS module with various knobs
to perform NGINX-based redirects and s3-revproxy endpoints.

Signed-off-by: Raito Bezarius <masterancpp@gmail.com>
This commit is contained in:
raito 2024-10-07 15:21:03 +02:00
parent decc9963ee
commit a3f6f573a2
7 changed files with 516 additions and 0 deletions

View file

@ -199,6 +199,8 @@
build01-aarch64-lix.imports = lixInfraModules ++ [ ./hosts/build01-aarch64-lix ]; build01-aarch64-lix.imports = lixInfraModules ++ [ ./hosts/build01-aarch64-lix ];
buildbot-lix.imports = lixInfraModules ++ [ ./hosts/buildbot-lix ]; buildbot-lix.imports = lixInfraModules ++ [ ./hosts/buildbot-lix ];
# This is Lix's Garage S3.
cache-lix.imports = lixInfraModules ++ [ ./hosts/cache-lix ];
} // builders; } // builders;
hydraJobs = builtins.mapAttrs (n: v: v.config.system.build.netbootDir or v.config.system.build.toplevel) self.nixosConfigurations; hydraJobs = builtins.mapAttrs (n: v: v.config.system.build.netbootDir or v.config.system.build.toplevel) self.nixosConfigurations;

125
hosts/cache-lix/default.nix Normal file
View file

@ -0,0 +1,125 @@
# Configuration for a virtual machine in Raito's micro-DC basement.
# 8 vCPU (2014 grade Xeon though)
# 8GB RAM
# 100GB SSD
# 1TB HDD
# All specifications can be upgraded to a certain extent, just ask Raito.
# Hosts the Garage S3 instance for the Lix project.
# Our "binary cache".
#
# vim: et:ts=2:sw=2:
#
{ config, pkgs, lib, ... }: {
networking.hostName = "cache";
networking.domain = "lix.systems";
system.stateVersion = "24.05";
zramSwap.enable = true;
i18n.defaultLocale = "en_US.UTF-8";
# All the objects are stored there.
# Metadata is on the fast SSD.
fileSystems."/data" = {
device = "/dev/disk/by-label/data";
fsType = "ext4";
};
bagel.s3 = {
rootDomain = "lix.systems";
garage.enable = true;
web = {
buckets = [
"install"
"cache"
"releases"
"docs"
];
subdomains = {
"cache.lix.systems" = "cache";
"install.lix.systems" = "install";
};
};
reverse-proxy = {
enable = true;
buckets = [
"docs"
"releases"
"install"
];
web = {
"releases.lix.systems" = "releases";
"docs.lix.systems" = "docs";
};
};
};
# Fix up the manual path so it enables having multiple manuals
services.nginx.virtualHosts."docs.lix.systems".locations.${''~ ^/manual/nightly(/[^\s]*)$''} =
{
extraConfig = ''
return 301 /manual/lix/nightly$1;
'';
};
systemd.network.links."10-nat-lan".matchConfig.MACAddress = "BC:24:11:1E:7C:9B";
systemd.network.networks."10-wan".networkConfig.Address = [ "2001:bc8:38ee:100::210/56" ];
systemd.network.links."10-wan".matchConfig.MACAddress = "BC:24:11:42:72:79";
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
# Debugging stuff.
virtualisation.vmVariant = {
systemd.network.enable = lib.mkForce false;
networking.useDHCP = true;
environment.systemPackages = with pkgs; [
tcpdump
];
system.activationScripts.agenixInstall.text = lib.mkForce ''
echo "lol gotcha"
_agenix_generation="$(basename "$(readlink ${config.age.secretsDir})" || echo 0)"
(( ++_agenix_generation ))
p=${config.age.secretsMountPoint}/$_agenix_generation
> $p/garage
echo "GARAGE_RPC_SECRET=$(${lib.getExe pkgs.openssl.bin} rand -hex 32)" >> $p/garage
echo "GARAGE_METRICS_TOKEN=$(${lib.getExe pkgs.openssl.bin} rand -base64 32)" >> $p/garage
echo "GARAGE_ADMIN_TOKEN=$(${lib.getExe pkgs.openssl.bin} rand -base64 32)" >> $p/garage
ln -sfT /var/secrets/garage-s3-api-key $p/s3-revproxy-api-key-env
ln -sfT $p ${config.age.secretsDir}
'';
virtualisation.forwardPorts = [
{
from = "host";
guest.port = 443;
host.port = 4043;
proto = "tcp";
}
{
from = "host";
guest.port = 1337;
host.port = 1337;
proto = "tcp";
}
{
from = "host";
guest.port = 22;
host.port = 2022;
proto = "tcp";
}
];
security.acme = {
defaults.server = "http://127.0.0.1/failfailfail";
extraLegoFlags = ["--lol-fail"];
extraLegoRenewFlags = ["--lol-fail"];
extraLegoRunFlags = ["--lol-fail"];
};
systemd.tmpfiles.rules = [
"d /data/s3 700 garage - - -"
];
};
}

View file

@ -0,0 +1,33 @@
{ config, lib, ... }:
let
inherit (lib) mkOption types;
cfg = config.bagel.s3;
in
{
options.bagel.s3 = {
rootDomain = mkOption {
type = types.str;
};
webRootDomain = mkOption {
type = types.str;
default = "web.${cfg.rootDomain}";
};
s3RootDomain = mkOption {
type = types.str;
default = "s3.${cfg.rootDomain}";
};
};
imports = [
# Generic frontend stuff
# e.g. static redirects, etc.
./web.nix
# Garage implementation for our S3
# In the future, we could be using Ceph store.
./garage.nix
# S3 reverse proxy
./s3-revproxy.nix
];
}

View file

@ -0,0 +1,63 @@
{ wrap }:
{ lib, pkgs, config, ... }:
let
garage-ephemeral-key = pkgs.writers.writePython3Bin
"garage-ephemeral-key"
{ libraries = [ pkgs.python3.pkgs.requests ]; }
(builtins.readFile ./garage_ephemeral_key.py);
# the usual copy pasta of systemd-analyze security satisfying rules
containment = {
DynamicUser = true;
CapabilityBoundingSet = "";
NoNewPrivileges = true;
PrivateTmp = true;
PrivateUsers = true;
PrivateDevices = true;
ProtectHome = true;
ProtectClock = true;
ProtectProc = "noaccess";
ProcSubset = "pid";
UMask = "0077";
ProtectKernelLogs = true;
ProtectKernelModules = true;
ProtectKernelTunables = true;
ProtectControlGroups = true;
ProtectHostname = true;
RestrictSUIDSGID = true;
RestrictRealtime = true;
RestrictNamespaces = true;
LockPersonality = true;
RemoveIPC = true;
SystemCallFilter = [ "@system-service" "~@privileged" ];
RestrictAddressFamilies = [ "AF_INET" "AF_INET6" ];
MemoryDenyWriteExecute = true;
SystemCallArchitectures = "native";
};
in
{
_file = ./garage-ephemeral-key.nix;
environment.systemPackages = [
(wrap garage-ephemeral-key "garage-ephemeral-key")
];
# Clean expired ephemeral keys every 2 minutes
systemd.timers.garage-ephemeral-key-clean = {
wantedBy = [ "multi-user.target" ];
timerConfig = {
# Every 2 minutes.
OnCalendar = "*-*-* *:00/2";
};
};
systemd.services.garage-ephemeral-key-clean = {
after = [ "garage.service" ];
wants = [ "garage.service" ];
serviceConfig = {
ExecStart = "${lib.getExe garage-ephemeral-key} clean";
EnvironmentFile = config.age.secrets.garage.path;
} // containment;
};
}

106
services/lix-s3/garage.nix Normal file
View file

@ -0,0 +1,106 @@
{ config, pkgs, lib, ... }:
let
cfg = config.bagel.s3.garage;
inherit (lib) mkEnableOption mkIf mkOption types;
# TODO: send me back upstream to cl.forkos.org.
wrap = pkg: name: pkgs.writeShellScriptBin name ''
set -a
[[ -z "''${GARAGE_RPC_SECRET:-}" ]] && source ${config.age.secrets.garage.path}
set +a
exec ${lib.getExe pkg} "$@"
'';
# TODO: generalize this idea
rootDomains = {
lix = "lix.systems";
floral = "floral.systems";
};
in
{
options.bagel.s3.garage = {
enable = mkEnableOption "the Garage implementation of S3";
tenant = mkOption {
type = types.enum [ "lix" "floral" ];
};
api.address = mkOption {
type = types.str;
default = "127.0.0.1:3900";
};
rootDomain = mkOption {
type = types.str;
default = rootDomains.${cfg.tenant};
};
dataDir = mkOption {
type = types.str;
};
metadataDir = mkOption {
default = "/var/lib/garage/metadata";
};
};
imports = [
(import ./garage-ephemeral-key.nix { inherit wrap; })
];
config = mkIf cfg.enable {
bagel.secrets.files = [
"garage-environment"
];
services.garage = {
enable = true;
# Slightly evil, but we have to wrap it here so the garage cli Just
# Works(tm) from the shell. It will no-op in the wrapper if the env is
# already set.
package = wrap pkgs.garage_0_9 "garage";
settings = {
data_dir = cfg.dataDir;
metadata_dir = cfg.metadataDir;
db_engine = "lmdb";
replication_mode = "none";
compression_level = 7;
rpc_bind_addr = "[::]:3901";
rpc_public_addr = "127.0.0.1:3901";
s3_api = {
s3_region = "garage";
api_bind_addr = cfg.api.address;
root_domain = ".s3.${cfg.rootDomain}";
};
s3_web = {
bind_addr = "127.0.0.1:3902";
root_domain = ".web.${cfg.rootDomain}";
index = "index.html";
};
k2v_api.api_bind_addr = "[::]:3904";
# FIXME(raito): api bind address should be secured.
# admin.api_bind_addr = "[${wnlib.generateIPv6Address "monitoring" "cache"}]:3903";
};
environmentFile = config.age.secrets.garage-environment.path;
};
systemd.services.garage.serviceConfig = {
User = "garage";
ReadWriteDirectories = [
cfg.dataDir
];
StateDirectory = "garage";
};
users.users.garage = {
isSystemUser = true;
group = "garage";
};
users.groups.garage = { };
};
}

View file

@ -0,0 +1,106 @@
{ lib, config, ... }:
let
inherit (lib) mkOption mkIf types mapAttrs;
cfgParent = config.bagel.s3;
cfg = config.bagel.s3.reverse-proxy;
mkTarget = { name, bucket ? name }: {
mount = {
host = "${name}.${cfgParent.webRootDomain}";
path = [ "/" ];
};
actions.GET = {
enabled = true;
config = {
# e.g. /2.90 will 404, so it will redirect to /2.90/ if it is a directory
redirectWithTrailingSlashForNotFoundFile = true;
indexDocument = "index.html";
};
};
bucket = {
name = bucket;
region = "garage";
s3Endpoint = "https://${cfgParent.s3RootDomain}";
credentials = {
accessKey.env = "AWS_ACCESS_KEY_ID";
secretKey.env = "AWS_SECRET_KEY";
};
};
};
in
{
options.bagel.s3.reverse-proxy = {
targets = mkOption {
type = types.attrsOf (types.submodule ({ name, ... }: {
bucket = mkOption {
type = types.str;
default = name;
};
}));
default = { };
};
port = mkOption {
type = types.port;
default = 10652;
};
};
config = mkIf cfg.enable {
age.secrets.s3-revproxy-api-key-env.file = ./s3-revproxy-env.age;
# this solves garage supporting neither anonymous access nor automatic
# directory indexing by simply ignoring garage's web server and replacing it
# with overengineered golang instead.
services.s3-revproxy = {
enable = true;
settings = {
templates = {
helpers = [ ./s3-revproxy-templates/_helpers.tpl ];
notFoundError = {
headers = {
"Content-Type" = "{{ template \"main.headers.contentType\" . }}";
};
status = "404";
};
folderList = {
path = ./s3-revproxy-templates/folder-list.tpl;
headers = {
"Content-Type" = "{{ template \"main.headers.contentType\" . }}";
};
# empty s3 directories are not real and cannot hurt you.
# due to redirectWithTrailingSlashForNotFoundFile, garbage file names
# get redirected as folders, which then appear as empty, yielding
# poor UX.
status = ''
{{- if eq (len .Entries) 0 -}}
404
{{- else -}}
200
{{- end -}}
'';
};
};
/* For metrics and debugging (e.g. pulling the config)
internalServer = {
listenAddr = "127.0.0.1";
port = 1337;
};
*/
server = {
listenAddr = "127.0.0.1";
port = cfg.port;
# it's going right into nginx, so no point
compress.enabled = false;
cors = {
enabled = true;
allowMethods = [ "GET" ];
allowOrigins = [ "*" ];
};
};
targets = mapAttrs mkTarget cfg.targets;
};
environmentFile = config.age.secrets.s3-revproxy-api-key-env.path;
};
};
}

81
services/lix-s3/web.nix Normal file
View file

@ -0,0 +1,81 @@
{ config, lib, ... }:
# TODO: move to wildcard TLS.
let
cfgParent = config.bagel.s3;
cfg = config.bagel.s3.web;
buckets = [ "install" "cache" "releases" "docs" ];
mkWebLocationBlock = host: {
proxyPass = "http://127.0.0.1:3902";
extraConfig = ''
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host ${host};
'';
};
mkDirectSubdomain = subdomain: {
enableACME = true;
forceSSL = true;
locations."/" = mkWebLocationBlock "${subdomain}.${webHost}";
};
# Makes a subdomain that gets proxied through s3-proxy to provide directory
# listings and reasonable 404 pages.
# This is not used on cache, since there a directory listing for cache is a
# liability at best.
mkProxiedSubdomain = subdomain: {
enableACME = true;
forceSSL = true;
locations."/" = {
recommendedProxySettings = true;
proxyPass = "http://127.0.0.1:${toString s3RevproxyPort}/";
};
};
in
{
options.bagel.s3.web = {
};
config = {
services.nginx = {
enable = true;
virtualHosts = {
${host} = {
enableACME = true;
forceSSL = true;
serverAliases = builtins.map (b: "${b}.${host}") buckets;
locations."/" = {
proxyPass = "http://127.0.0.1:3900";
extraConfig = ''
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $host;
# Disable buffering to a temporary file.
proxy_max_temp_file_size 0;
client_max_body_size 20G;
'';
};
};
${webHost} = {
enableACME = true;
forceSSL = true;
locations."/" = mkWebLocationBlock "$host";
# Create a subdomain for each bucket; and include special aliases
# for our special buckets 'cache' and 'install'.
serverAliases =
(builtins.map (b: "${b}.${webHost}") buckets);
};
};
};
networking.firewall.allowedTCPPorts = [
80 443
];
};
}