feat: add cache.lix.systems
Introduces a full-fleged self-hosted S3 NixOS module with various knobs to perform NGINX-based redirects and s3-revproxy endpoints. Signed-off-by: Raito Bezarius <masterancpp@gmail.com>
This commit is contained in:
parent
decc9963ee
commit
a3f6f573a2
|
|
@ -199,6 +199,8 @@
|
|||
|
||||
build01-aarch64-lix.imports = lixInfraModules ++ [ ./hosts/build01-aarch64-lix ];
|
||||
buildbot-lix.imports = lixInfraModules ++ [ ./hosts/buildbot-lix ];
|
||||
# This is Lix's Garage S3.
|
||||
cache-lix.imports = lixInfraModules ++ [ ./hosts/cache-lix ];
|
||||
} // builders;
|
||||
|
||||
hydraJobs = builtins.mapAttrs (n: v: v.config.system.build.netbootDir or v.config.system.build.toplevel) self.nixosConfigurations;
|
||||
|
|
|
|||
125
hosts/cache-lix/default.nix
Normal file
125
hosts/cache-lix/default.nix
Normal file
|
|
@ -0,0 +1,125 @@
|
|||
# Configuration for a virtual machine in Raito's micro-DC basement.
|
||||
# 8 vCPU (2014 grade Xeon though)
|
||||
# 8GB RAM
|
||||
# 100GB SSD
|
||||
# 1TB HDD
|
||||
# All specifications can be upgraded to a certain extent, just ask Raito.
|
||||
# Hosts the Garage S3 instance for the Lix project.
|
||||
# Our "binary cache".
|
||||
#
|
||||
# vim: et:ts=2:sw=2:
|
||||
#
|
||||
{ config, pkgs, lib, ... }: {
|
||||
networking.hostName = "cache";
|
||||
networking.domain = "lix.systems";
|
||||
|
||||
system.stateVersion = "24.05";
|
||||
|
||||
zramSwap.enable = true;
|
||||
i18n.defaultLocale = "en_US.UTF-8";
|
||||
|
||||
# All the objects are stored there.
|
||||
# Metadata is on the fast SSD.
|
||||
fileSystems."/data" = {
|
||||
device = "/dev/disk/by-label/data";
|
||||
fsType = "ext4";
|
||||
};
|
||||
|
||||
bagel.s3 = {
|
||||
rootDomain = "lix.systems";
|
||||
garage.enable = true;
|
||||
web = {
|
||||
buckets = [
|
||||
"install"
|
||||
"cache"
|
||||
"releases"
|
||||
"docs"
|
||||
];
|
||||
subdomains = {
|
||||
"cache.lix.systems" = "cache";
|
||||
"install.lix.systems" = "install";
|
||||
};
|
||||
};
|
||||
reverse-proxy = {
|
||||
enable = true;
|
||||
buckets = [
|
||||
"docs"
|
||||
"releases"
|
||||
"install"
|
||||
];
|
||||
web = {
|
||||
"releases.lix.systems" = "releases";
|
||||
"docs.lix.systems" = "docs";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
# Fix up the manual path so it enables having multiple manuals
|
||||
services.nginx.virtualHosts."docs.lix.systems".locations.${''~ ^/manual/nightly(/[^\s]*)$''} =
|
||||
{
|
||||
extraConfig = ''
|
||||
return 301 /manual/lix/nightly$1;
|
||||
'';
|
||||
};
|
||||
|
||||
systemd.network.links."10-nat-lan".matchConfig.MACAddress = "BC:24:11:1E:7C:9B";
|
||||
systemd.network.networks."10-wan".networkConfig.Address = [ "2001:bc8:38ee:100::210/56" ];
|
||||
systemd.network.links."10-wan".matchConfig.MACAddress = "BC:24:11:42:72:79";
|
||||
|
||||
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
|
||||
|
||||
# Debugging stuff.
|
||||
virtualisation.vmVariant = {
|
||||
systemd.network.enable = lib.mkForce false;
|
||||
networking.useDHCP = true;
|
||||
|
||||
environment.systemPackages = with pkgs; [
|
||||
tcpdump
|
||||
];
|
||||
|
||||
system.activationScripts.agenixInstall.text = lib.mkForce ''
|
||||
echo "lol gotcha"
|
||||
_agenix_generation="$(basename "$(readlink ${config.age.secretsDir})" || echo 0)"
|
||||
(( ++_agenix_generation ))
|
||||
p=${config.age.secretsMountPoint}/$_agenix_generation
|
||||
> $p/garage
|
||||
echo "GARAGE_RPC_SECRET=$(${lib.getExe pkgs.openssl.bin} rand -hex 32)" >> $p/garage
|
||||
echo "GARAGE_METRICS_TOKEN=$(${lib.getExe pkgs.openssl.bin} rand -base64 32)" >> $p/garage
|
||||
echo "GARAGE_ADMIN_TOKEN=$(${lib.getExe pkgs.openssl.bin} rand -base64 32)" >> $p/garage
|
||||
|
||||
ln -sfT /var/secrets/garage-s3-api-key $p/s3-revproxy-api-key-env
|
||||
ln -sfT $p ${config.age.secretsDir}
|
||||
'';
|
||||
virtualisation.forwardPorts = [
|
||||
{
|
||||
from = "host";
|
||||
guest.port = 443;
|
||||
host.port = 4043;
|
||||
proto = "tcp";
|
||||
}
|
||||
{
|
||||
from = "host";
|
||||
guest.port = 1337;
|
||||
host.port = 1337;
|
||||
proto = "tcp";
|
||||
}
|
||||
{
|
||||
from = "host";
|
||||
guest.port = 22;
|
||||
host.port = 2022;
|
||||
proto = "tcp";
|
||||
}
|
||||
];
|
||||
|
||||
security.acme = {
|
||||
defaults.server = "http://127.0.0.1/failfailfail";
|
||||
extraLegoFlags = ["--lol-fail"];
|
||||
extraLegoRenewFlags = ["--lol-fail"];
|
||||
extraLegoRunFlags = ["--lol-fail"];
|
||||
};
|
||||
|
||||
systemd.tmpfiles.rules = [
|
||||
"d /data/s3 700 garage - - -"
|
||||
];
|
||||
};
|
||||
}
|
||||
33
services/lix-s3/default.nix
Normal file
33
services/lix-s3/default.nix
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
{ config, lib, ... }:
|
||||
let
|
||||
inherit (lib) mkOption types;
|
||||
cfg = config.bagel.s3;
|
||||
in
|
||||
{
|
||||
options.bagel.s3 = {
|
||||
rootDomain = mkOption {
|
||||
type = types.str;
|
||||
};
|
||||
|
||||
webRootDomain = mkOption {
|
||||
type = types.str;
|
||||
default = "web.${cfg.rootDomain}";
|
||||
};
|
||||
|
||||
s3RootDomain = mkOption {
|
||||
type = types.str;
|
||||
default = "s3.${cfg.rootDomain}";
|
||||
};
|
||||
};
|
||||
|
||||
imports = [
|
||||
# Generic frontend stuff
|
||||
# e.g. static redirects, etc.
|
||||
./web.nix
|
||||
# Garage implementation for our S3
|
||||
# In the future, we could be using Ceph store.
|
||||
./garage.nix
|
||||
# S3 reverse proxy
|
||||
./s3-revproxy.nix
|
||||
];
|
||||
}
|
||||
63
services/lix-s3/garage-ephemeral-key.nix
Normal file
63
services/lix-s3/garage-ephemeral-key.nix
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
{ wrap }:
|
||||
{ lib, pkgs, config, ... }:
|
||||
let
|
||||
garage-ephemeral-key = pkgs.writers.writePython3Bin
|
||||
"garage-ephemeral-key"
|
||||
{ libraries = [ pkgs.python3.pkgs.requests ]; }
|
||||
(builtins.readFile ./garage_ephemeral_key.py);
|
||||
|
||||
# the usual copy pasta of systemd-analyze security satisfying rules
|
||||
containment = {
|
||||
DynamicUser = true;
|
||||
CapabilityBoundingSet = "";
|
||||
NoNewPrivileges = true;
|
||||
PrivateTmp = true;
|
||||
PrivateUsers = true;
|
||||
PrivateDevices = true;
|
||||
ProtectHome = true;
|
||||
ProtectClock = true;
|
||||
ProtectProc = "noaccess";
|
||||
ProcSubset = "pid";
|
||||
UMask = "0077";
|
||||
ProtectKernelLogs = true;
|
||||
ProtectKernelModules = true;
|
||||
ProtectKernelTunables = true;
|
||||
ProtectControlGroups = true;
|
||||
ProtectHostname = true;
|
||||
RestrictSUIDSGID = true;
|
||||
RestrictRealtime = true;
|
||||
RestrictNamespaces = true;
|
||||
LockPersonality = true;
|
||||
RemoveIPC = true;
|
||||
SystemCallFilter = [ "@system-service" "~@privileged" ];
|
||||
RestrictAddressFamilies = [ "AF_INET" "AF_INET6" ];
|
||||
MemoryDenyWriteExecute = true;
|
||||
SystemCallArchitectures = "native";
|
||||
};
|
||||
in
|
||||
{
|
||||
_file = ./garage-ephemeral-key.nix;
|
||||
|
||||
environment.systemPackages = [
|
||||
(wrap garage-ephemeral-key "garage-ephemeral-key")
|
||||
];
|
||||
|
||||
# Clean expired ephemeral keys every 2 minutes
|
||||
systemd.timers.garage-ephemeral-key-clean = {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
timerConfig = {
|
||||
# Every 2 minutes.
|
||||
OnCalendar = "*-*-* *:00/2";
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services.garage-ephemeral-key-clean = {
|
||||
after = [ "garage.service" ];
|
||||
wants = [ "garage.service" ];
|
||||
serviceConfig = {
|
||||
ExecStart = "${lib.getExe garage-ephemeral-key} clean";
|
||||
|
||||
EnvironmentFile = config.age.secrets.garage.path;
|
||||
} // containment;
|
||||
};
|
||||
}
|
||||
106
services/lix-s3/garage.nix
Normal file
106
services/lix-s3/garage.nix
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
{ config, pkgs, lib, ... }:
|
||||
let
|
||||
cfg = config.bagel.s3.garage;
|
||||
inherit (lib) mkEnableOption mkIf mkOption types;
|
||||
# TODO: send me back upstream to cl.forkos.org.
|
||||
wrap = pkg: name: pkgs.writeShellScriptBin name ''
|
||||
set -a
|
||||
[[ -z "''${GARAGE_RPC_SECRET:-}" ]] && source ${config.age.secrets.garage.path}
|
||||
set +a
|
||||
exec ${lib.getExe pkg} "$@"
|
||||
'';
|
||||
# TODO: generalize this idea
|
||||
rootDomains = {
|
||||
lix = "lix.systems";
|
||||
floral = "floral.systems";
|
||||
};
|
||||
in
|
||||
{
|
||||
options.bagel.s3.garage = {
|
||||
enable = mkEnableOption "the Garage implementation of S3";
|
||||
tenant = mkOption {
|
||||
type = types.enum [ "lix" "floral" ];
|
||||
};
|
||||
|
||||
api.address = mkOption {
|
||||
type = types.str;
|
||||
default = "127.0.0.1:3900";
|
||||
};
|
||||
|
||||
rootDomain = mkOption {
|
||||
type = types.str;
|
||||
default = rootDomains.${cfg.tenant};
|
||||
};
|
||||
|
||||
dataDir = mkOption {
|
||||
type = types.str;
|
||||
};
|
||||
|
||||
metadataDir = mkOption {
|
||||
default = "/var/lib/garage/metadata";
|
||||
};
|
||||
};
|
||||
|
||||
imports = [
|
||||
(import ./garage-ephemeral-key.nix { inherit wrap; })
|
||||
];
|
||||
|
||||
config = mkIf cfg.enable {
|
||||
bagel.secrets.files = [
|
||||
"garage-environment"
|
||||
];
|
||||
|
||||
services.garage = {
|
||||
enable = true;
|
||||
# Slightly evil, but we have to wrap it here so the garage cli Just
|
||||
# Works(tm) from the shell. It will no-op in the wrapper if the env is
|
||||
# already set.
|
||||
package = wrap pkgs.garage_0_9 "garage";
|
||||
|
||||
settings = {
|
||||
data_dir = cfg.dataDir;
|
||||
metadata_dir = cfg.metadataDir;
|
||||
|
||||
db_engine = "lmdb";
|
||||
|
||||
replication_mode = "none";
|
||||
compression_level = 7;
|
||||
|
||||
rpc_bind_addr = "[::]:3901";
|
||||
rpc_public_addr = "127.0.0.1:3901";
|
||||
|
||||
s3_api = {
|
||||
s3_region = "garage";
|
||||
api_bind_addr = cfg.api.address;
|
||||
root_domain = ".s3.${cfg.rootDomain}";
|
||||
};
|
||||
|
||||
s3_web = {
|
||||
bind_addr = "127.0.0.1:3902";
|
||||
root_domain = ".web.${cfg.rootDomain}";
|
||||
index = "index.html";
|
||||
};
|
||||
|
||||
k2v_api.api_bind_addr = "[::]:3904";
|
||||
# FIXME(raito): api bind address should be secured.
|
||||
# admin.api_bind_addr = "[${wnlib.generateIPv6Address "monitoring" "cache"}]:3903";
|
||||
};
|
||||
|
||||
environmentFile = config.age.secrets.garage-environment.path;
|
||||
};
|
||||
|
||||
systemd.services.garage.serviceConfig = {
|
||||
User = "garage";
|
||||
ReadWriteDirectories = [
|
||||
cfg.dataDir
|
||||
];
|
||||
StateDirectory = "garage";
|
||||
};
|
||||
|
||||
users.users.garage = {
|
||||
isSystemUser = true;
|
||||
group = "garage";
|
||||
};
|
||||
users.groups.garage = { };
|
||||
};
|
||||
}
|
||||
106
services/lix-s3/s3-revproxy.nix
Normal file
106
services/lix-s3/s3-revproxy.nix
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
{ lib, config, ... }:
|
||||
let
|
||||
inherit (lib) mkOption mkIf types mapAttrs;
|
||||
cfgParent = config.bagel.s3;
|
||||
cfg = config.bagel.s3.reverse-proxy;
|
||||
mkTarget = { name, bucket ? name }: {
|
||||
mount = {
|
||||
host = "${name}.${cfgParent.webRootDomain}";
|
||||
path = [ "/" ];
|
||||
};
|
||||
actions.GET = {
|
||||
enabled = true;
|
||||
config = {
|
||||
# e.g. /2.90 will 404, so it will redirect to /2.90/ if it is a directory
|
||||
redirectWithTrailingSlashForNotFoundFile = true;
|
||||
indexDocument = "index.html";
|
||||
};
|
||||
};
|
||||
|
||||
bucket = {
|
||||
name = bucket;
|
||||
region = "garage";
|
||||
s3Endpoint = "https://${cfgParent.s3RootDomain}";
|
||||
credentials = {
|
||||
accessKey.env = "AWS_ACCESS_KEY_ID";
|
||||
secretKey.env = "AWS_SECRET_KEY";
|
||||
};
|
||||
};
|
||||
};
|
||||
in
|
||||
{
|
||||
options.bagel.s3.reverse-proxy = {
|
||||
targets = mkOption {
|
||||
type = types.attrsOf (types.submodule ({ name, ... }: {
|
||||
bucket = mkOption {
|
||||
type = types.str;
|
||||
default = name;
|
||||
};
|
||||
}));
|
||||
default = { };
|
||||
};
|
||||
|
||||
port = mkOption {
|
||||
type = types.port;
|
||||
default = 10652;
|
||||
};
|
||||
};
|
||||
|
||||
config = mkIf cfg.enable {
|
||||
age.secrets.s3-revproxy-api-key-env.file = ./s3-revproxy-env.age;
|
||||
# this solves garage supporting neither anonymous access nor automatic
|
||||
# directory indexing by simply ignoring garage's web server and replacing it
|
||||
# with overengineered golang instead.
|
||||
services.s3-revproxy = {
|
||||
enable = true;
|
||||
settings = {
|
||||
templates = {
|
||||
helpers = [ ./s3-revproxy-templates/_helpers.tpl ];
|
||||
notFoundError = {
|
||||
headers = {
|
||||
"Content-Type" = "{{ template \"main.headers.contentType\" . }}";
|
||||
};
|
||||
status = "404";
|
||||
};
|
||||
folderList = {
|
||||
path = ./s3-revproxy-templates/folder-list.tpl;
|
||||
headers = {
|
||||
"Content-Type" = "{{ template \"main.headers.contentType\" . }}";
|
||||
};
|
||||
# empty s3 directories are not real and cannot hurt you.
|
||||
# due to redirectWithTrailingSlashForNotFoundFile, garbage file names
|
||||
# get redirected as folders, which then appear as empty, yielding
|
||||
# poor UX.
|
||||
status = ''
|
||||
{{- if eq (len .Entries) 0 -}}
|
||||
404
|
||||
{{- else -}}
|
||||
200
|
||||
{{- end -}}
|
||||
'';
|
||||
};
|
||||
};
|
||||
/* For metrics and debugging (e.g. pulling the config)
|
||||
internalServer = {
|
||||
listenAddr = "127.0.0.1";
|
||||
port = 1337;
|
||||
};
|
||||
*/
|
||||
server = {
|
||||
listenAddr = "127.0.0.1";
|
||||
port = cfg.port;
|
||||
|
||||
# it's going right into nginx, so no point
|
||||
compress.enabled = false;
|
||||
cors = {
|
||||
enabled = true;
|
||||
allowMethods = [ "GET" ];
|
||||
allowOrigins = [ "*" ];
|
||||
};
|
||||
};
|
||||
targets = mapAttrs mkTarget cfg.targets;
|
||||
};
|
||||
environmentFile = config.age.secrets.s3-revproxy-api-key-env.path;
|
||||
};
|
||||
};
|
||||
}
|
||||
81
services/lix-s3/web.nix
Normal file
81
services/lix-s3/web.nix
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
{ config, lib, ... }:
|
||||
# TODO: move to wildcard TLS.
|
||||
let
|
||||
cfgParent = config.bagel.s3;
|
||||
cfg = config.bagel.s3.web;
|
||||
|
||||
buckets = [ "install" "cache" "releases" "docs" ];
|
||||
mkWebLocationBlock = host: {
|
||||
proxyPass = "http://127.0.0.1:3902";
|
||||
extraConfig = ''
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header Host ${host};
|
||||
'';
|
||||
};
|
||||
mkDirectSubdomain = subdomain: {
|
||||
enableACME = true;
|
||||
forceSSL = true;
|
||||
locations."/" = mkWebLocationBlock "${subdomain}.${webHost}";
|
||||
};
|
||||
|
||||
# Makes a subdomain that gets proxied through s3-proxy to provide directory
|
||||
# listings and reasonable 404 pages.
|
||||
# This is not used on cache, since there a directory listing for cache is a
|
||||
# liability at best.
|
||||
mkProxiedSubdomain = subdomain: {
|
||||
enableACME = true;
|
||||
forceSSL = true;
|
||||
locations."/" = {
|
||||
recommendedProxySettings = true;
|
||||
proxyPass = "http://127.0.0.1:${toString s3RevproxyPort}/";
|
||||
};
|
||||
};
|
||||
in
|
||||
{
|
||||
options.bagel.s3.web = {
|
||||
|
||||
};
|
||||
|
||||
config = {
|
||||
services.nginx = {
|
||||
enable = true;
|
||||
|
||||
virtualHosts = {
|
||||
${host} = {
|
||||
enableACME = true;
|
||||
forceSSL = true;
|
||||
|
||||
serverAliases = builtins.map (b: "${b}.${host}") buckets;
|
||||
|
||||
locations."/" = {
|
||||
proxyPass = "http://127.0.0.1:3900";
|
||||
|
||||
extraConfig = ''
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header Host $host;
|
||||
|
||||
# Disable buffering to a temporary file.
|
||||
proxy_max_temp_file_size 0;
|
||||
client_max_body_size 20G;
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
${webHost} = {
|
||||
enableACME = true;
|
||||
forceSSL = true;
|
||||
locations."/" = mkWebLocationBlock "$host";
|
||||
|
||||
# Create a subdomain for each bucket; and include special aliases
|
||||
# for our special buckets 'cache' and 'install'.
|
||||
serverAliases =
|
||||
(builtins.map (b: "${b}.${webHost}") buckets);
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
networking.firewall.allowedTCPPorts = [
|
||||
80 443
|
||||
];
|
||||
};
|
||||
}
|
||||
Loading…
Reference in a new issue