diff --git a/hosts/public01/default.nix b/hosts/public01/default.nix index 0844dec..b1430a0 100755 --- a/hosts/public01/default.nix +++ b/hosts/public01/default.nix @@ -10,7 +10,7 @@ networking.domain = "infra.forkos.org"; bagel.sysadmin.enable = true; - # Buildbot is proxied. + # Newsletter is proxied. bagel.raito.v6-proxy-awareness.enable = true; bagel.newsletter = { enable = true; @@ -27,6 +27,17 @@ }; }; + bagel.services.s3-revproxy = { + enable = true; + domain = "forkos.org"; + s3.apiUrl = "s3.delroth.net"; + targets = { + channels = "bagel-channels"; + releases = "bagel-releases"; + channel-scripts-test = "bagel-channel-scripts-test"; + }; + }; + i18n.defaultLocale = "en_US.UTF-8"; system.stateVersion = "24.05"; diff --git a/overlays/default.nix b/overlays/default.nix index bf4af66..860f6b9 100644 --- a/overlays/default.nix +++ b/overlays/default.nix @@ -3,5 +3,6 @@ iusb-spoof = final.callPackage ./iusb-spoof.nix {}; u-root = final.callPackage ./u-root {}; pyroscope = final.callPackage ./pyroscope {}; + s3-revproxy = final.callPackage ./s3-revproxy {}; }) ] diff --git a/overlays/s3-revproxy/default.nix b/overlays/s3-revproxy/default.nix new file mode 100644 index 0000000..c7969c1 --- /dev/null +++ b/overlays/s3-revproxy/default.nix @@ -0,0 +1,39 @@ +# Originally written by Jade Lovelace for Lix. +{ lib, buildGoModule, fetchFromGitHub }: +buildGoModule rec { + pname = "s3-revproxy"; + version = "4.15.0"; + + src = fetchFromGitHub { + owner = "oxyno-zeta"; + repo = "s3-proxy"; + rev = "v${version}"; + hash = "sha256-q0cfAo8Uz7wtKljmSDaJ320bjg2yXydvvxubAsMKzbc="; + }; + + vendorHash = "sha256-dOwNQtTfOCQcjgNBV/FeWdwbW9xi1OK5YD7PBPPDKOQ="; + + ldflags = [ + "-X github.com/oxyno-zeta/s3-proxy/pkg/s3-proxy/version.Version=${version}" + "-X github.com/oxyno-zeta/s3-proxy/pkg/s3-proxy/version.Metadata=" + ]; + + postPatch = '' + # Refer to the included templates in the package instead of cwd-relative + sed -i "s#Path = \"templates/#Path = \"$out/share/s3-revproxy/templates/#" pkg/s3-proxy/config/config.go + ''; + + postInstall = '' + mkdir -p $out/share/s3-revproxy + cp -r templates/ $out/share/s3-revproxy/templates + ''; + + meta = { + description = "S3 Reverse Proxy with GET, PUT and DELETE methods and authentication (OpenID Connect and Basic Auth)"; + homepage = "https://oxyno-zeta.github.io/s3-proxy"; + # hm, not having a maintainers entry is kind of inconvenient + maintainers = [ ]; + licenses = lib.licenses.asl20; + mainProgram = "s3-proxy"; + }; +} diff --git a/services/default.nix b/services/default.nix index f205ca8..9d009b5 100644 --- a/services/default.nix +++ b/services/default.nix @@ -11,5 +11,6 @@ ./baremetal-builder ./buildbot ./newsletter + ./s3-revproxy ]; } diff --git a/services/s3-revproxy/default.nix b/services/s3-revproxy/default.nix new file mode 100644 index 0000000..b456fb1 --- /dev/null +++ b/services/s3-revproxy/default.nix @@ -0,0 +1,131 @@ +# Originally written by Jade Lovelace for Lix. +# Adapted for ForkOS. +{ lib, config, ... }: +let + inherit (lib) mkEnableOption mkIf types mkOption mapAttrs mapAttrs' nameValuePair; + cfg = config.bagel.services.s3-revproxy; + s3RevproxyPort = 10652; + mkTarget = { name, bucket ? name }: { + mount = { + host = "${name}.${cfg.domain}"; + path = [ "/" ]; + }; + actions.GET = { + enabled = true; + config = { + # e.g. /2.90 will 404, so it will redirect to /2.90/ if it is a directory + redirectWithTrailingSlashForNotFoundFile = true; + indexDocument = "index.html"; + }; + }; + + bucket = { + name = bucket; + region = "garage"; + s3Endpoint = "https://${cfg.s3.apiUrl}"; + credentials = { + accessKey.env = "AWS_ACCESS_KEY_ID"; + secretKey.env = "AWS_SECRET_KEY"; + }; + }; + }; + # Makes a subdomain that gets proxied through s3-proxy to provide directory + # listings and reasonable 404 pages. + # This is not used on cache, since there a directory listing for cache is a + # liability at best. + mkProxiedSubdomain = subdomain: { + enableACME = true; + forceSSL = true; + locations."/" = { + recommendedProxySettings = true; + proxyPass = "http://127.0.0.1:${toString s3RevproxyPort}/"; + }; + }; +in +{ + options.bagel.services.s3-revproxy = { + enable = mkEnableOption "a S3 reverse proxy"; + + domain = mkOption { + type = types.str; + }; + + s3 = { + apiUrl = mkOption { + type = types.str; + description = "An URL to the S3 API"; + }; + }; + + targets = mkOption { + type = types.attrsOf types.str; + description = '' + A mapping between a nice name and a bucket name. + ''; + }; + }; + + imports = [ + ./module.nix + ]; + + config = mkIf cfg.enable { + age.secrets.s3-revproxy-api-keys.file = ../../secrets/s3-revproxy-api-keys.age; + # For each target, generate an entry that passes it to the s3-revproxy. + services.nginx.virtualHosts = mapAttrs' (subdomain: _: nameValuePair "${subdomain}.${cfg.domain}" (mkProxiedSubdomain subdomain)) cfg.targets; + # this solves garage supporting neither anonymous access nor automatic + # directory indexing by simply ignoring garage's web server and replacing it + # with overengineered golang instead. + services.s3-revproxy = { + enable = true; + settings = { + templates = { + helpers = [ ./s3-revproxy-templates/_helpers.tpl ]; + notFoundError = { + headers = { + "Content-Type" = "{{ template \"main.headers.contentType\" . }}"; + }; + status = "404"; + }; + folderList = { + path = ./s3-revproxy-templates/folder-list.tpl; + headers = { + "Content-Type" = "{{ template \"main.headers.contentType\" . }}"; + }; + # empty s3 directories are not real and cannot hurt you. + # due to redirectWithTrailingSlashForNotFoundFile, garbage file names + # get redirected as folders, which then appear as empty, yielding + # poor UX. + status = '' + {{- if eq (len .Entries) 0 -}} + 404 + {{- else -}} + 200 + {{- end -}} + ''; + }; + }; + /* For metrics and debugging (e.g. pulling the config) + internalServer = { + listenAddr = "127.0.0.1"; + port = 1337; + }; + */ + server = { + listenAddr = "127.0.0.1"; + port = s3RevproxyPort; + + # it's going right into nginx, so no point + compress.enabled = false; + cors = { + enabled = true; + allowMethods = [ "GET" ]; + allowOrigins = [ "*" ]; + }; + }; + targets = mapAttrs (name: bucket: mkTarget { inherit name bucket; }) cfg.targets; + }; + environmentFile = config.age.secrets.s3-revproxy-api-keys.path; + }; + }; +} diff --git a/services/s3-revproxy/module.nix b/services/s3-revproxy/module.nix new file mode 100644 index 0000000..424d6dd --- /dev/null +++ b/services/s3-revproxy/module.nix @@ -0,0 +1,69 @@ +# Originally, written by Jade Lovelace for Lix. +{ config, pkgs, lib, ... }: +let cfg = config.services.s3-revproxy; + settingsGenerator = pkgs.formats.yaml {}; + # Needs to be in a directory, so we might as well implement autoreload, why not! + configFile = settingsGenerator.generate "config.yaml" cfg.settings; + + inherit (lib) types; +in +{ + options.services.s3-revproxy = { + enable = lib.mkEnableOption "s3 reverse proxy"; + package = lib.mkPackageOption pkgs "s3-revproxy" {}; + settings = lib.mkOption { + default = { }; + type = settingsGenerator.type; + description = '' + Settings to use for the service. See the documentation at https://oxyno-zeta.github.io/s3-proxy/configuration/structure/ + ''; + }; + + environmentFile = lib.mkOption { + type = types.nullOr types.path; + default = null; + description = '' + Environment file to use for s3-revproxy. + ''; + }; + }; + + config = lib.mkIf cfg.enable { + environment.etc."s3-revproxy/config.yaml".source = configFile; + systemd.services.s3-revproxy = { + wantedBy = [ "multi-user.target" ]; + + serviceConfig = { + ExecStart = "${lib.getExe cfg.package} --config /etc/s3-revproxy"; + + DynamicUser = true; + CapabilityBoundingSet = ""; + NoNewPrivileges = true; + PrivateTmp = true; + PrivateUsers = true; + PrivateDevices = true; + ProtectHome = true; + ProtectClock = true; + ProtectProc = "noaccess"; + ProcSubset = "pid"; + UMask = "0077"; + ProtectKernelLogs = true; + ProtectKernelModules = true; + ProtectKernelTunables = true; + ProtectControlGroups = true; + ProtectHostname = true; + RestrictSUIDSGID = true; + RestrictRealtime = true; + RestrictNamespaces = true; + LockPersonality = true; + RemoveIPC = true; + SystemCallFilter = [ "@system-service" "~@privileged" ]; + RestrictAddressFamilies = [ "AF_INET" "AF_INET6" ]; + MemoryDenyWriteExecute = true; + SystemCallArchitectures = "native"; + + EnvironmentFile = lib.optionals (cfg.environmentFile != null) [ cfg.environmentFile ]; + }; + }; + }; +} diff --git a/services/s3-revproxy/s3-revproxy-templates/_helpers.tpl b/services/s3-revproxy/s3-revproxy-templates/_helpers.tpl new file mode 100644 index 0000000..662932b --- /dev/null +++ b/services/s3-revproxy/s3-revproxy-templates/_helpers.tpl @@ -0,0 +1,37 @@ +{{/* SPDX-License-Identifier: Apache-2.0 */}} +{{/* SPDX-FileCopyrightText: 2024 s3-proxy contributors */}} + +{{- /* This function will allow to get user identifier. */ -}} +{{- define "main.userIdentifier" -}} +{{- if .User -}} +{{- .User.GetIdentifier -}} +{{- end -}} +{{- end -}} + + +{{- /* This function will allow to get the content type header from "Accept" header */ -}} +{{- define "main.headers.contentType" -}} +{{- if contains "application/json" (.Request.Header.Get "Accept") -}} +application/json; charset=utf-8 +{{- else -}} +text/html; charset=utf-8 +{{- end -}} +{{- end -}} + +{{- /* This will forge the json output of an error */ -}} +{{- define "main.body.errorJsonBody" -}} +{"error": {{ .Error.Error | toJson }}} +{{- end -}} + +{{- define "notFoundErrorBody" -}} + {{- if contains "application/json" (.Request.Header.Get "Accept") -}} + {{ template "main.body.errorJsonBody" . }} + {{- else -}} + + + +

Not Found {{ .Request.URL.Path }}

+ + + {{- end -}} +{{- end -}} diff --git a/services/s3-revproxy/s3-revproxy-templates/folder-list.tpl b/services/s3-revproxy/s3-revproxy-templates/folder-list.tpl new file mode 100644 index 0000000..d1ac3d9 --- /dev/null +++ b/services/s3-revproxy/s3-revproxy-templates/folder-list.tpl @@ -0,0 +1,52 @@ +{{/* SPDX-License-Identifier: Apache-2.0 */}} +{{/* SPDX-FileCopyrightText: 2024 s3-proxy contributors */}} +{{- $root := . -}} +{{- if eq (len .Entries) 0 -}} + {{- include "notFoundErrorBody" . -}} +{{- else -}} + +{{- if contains "application/json" (.Request.Header.Get "Accept") -}} +[ + {{- $maxLen := len $root.Entries -}} + {{- range $index, $entry := $root.Entries -}} + {"name": {{ $entry.Name | toJson -}} + ,"etag": {{ $entry.ETag | toJson -}} + ,"type": {{ $entry.Type | toJson -}} + ,"size": {{ $entry.Size | toJson -}} + ,"path": {{ $entry.Path | toJson -}} + ,"lastModified": {{ $entry.LastModified | date "2006-01-02T15:04:05Z07:00" | toJson -}} + }{{- if ne $index (sub $maxLen 1) -}},{{- end -}} + {{- end -}} +] +{{- else -}} + + + +

Index of {{ .Request.URL.Path }}

+ + + + + + + + + + + + + + + {{- range .Entries }} + + + + + + {{- end }} + +
EntrySizeLast modified
.. - -
{{ .Name }}{{- if eq .Type "FOLDER" -}} - {{- else -}}{{ .Size | humanSize }}{{- end -}}{{ .LastModified }}
+ + + {{- end -}} +{{- end -}} diff --git a/services/s3-revproxy/s3-revproxy-templates/not-found-error.tpl b/services/s3-revproxy/s3-revproxy-templates/not-found-error.tpl new file mode 100644 index 0000000..da4af03 --- /dev/null +++ b/services/s3-revproxy/s3-revproxy-templates/not-found-error.tpl @@ -0,0 +1,3 @@ +{{/* SPDX-License-Identifier: Apache-2.0 */}} +{{/* SPDX-FileCopyrightText: 2024 s3-proxy contributors */}} +{{- include "notFoundErrorBody" . -}}