chore: add lorri to prevent direnv from blocking, closes #147

Otherwise, we will have issues with this exporter.

Signed-off-by: Raito Bezarius <masterancpp@gmail.com>

.envrc

@@ -1,2 +1,2 @@
 # shellcheck shell=bash
-use flake
+eval "$(lorri direnv --flake .)" 

common/admins.nix

@@ -19,6 +19,7 @@ in
       "thubrecht"
       "winter"
       "yuka"
+      "ckie"
     ];
 
     lix-infra.members = [
@@ -39,5 +40,6 @@ in
     "thubrecht"
     "winter"
     "yuka"
+    "ckie"
   ] (name: {});
 }

common/ssh-keys.nix

@@ -63,5 +63,6 @@
     thubrecht = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPM1jpXR7BWQa7Sed7ii3SbvIPRRlKb3G91qC0vOwfJn" ];
     yuka = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIxQ3NYBi8v1f/vhxLKDcA6upmX0pctRDbnK6SER5OUR yureka" ];
     winter = [ "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIH/LDRUG+U+++UmlxvA2kspioTjktQZ8taDcHq8gVlkfAAAABHNzaDo=" ];
+    ckie = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIH3uTwzSSMAPg84fwbNp2cq9+BdLFeA1VzDGth4zCAbz https://mei.puppycat.house" ];
   };
 }

flake.lock

@@ -87,16 +87,16 @@
         "treefmt-nix": "treefmt-nix"
       },
       "locked": {
-        "lastModified": 1728307353,
-        "narHash": "sha256-eVSDu52qZn48c1HfDlH79JkDIzuE7qyY0ipPMbrpYzE=",
-        "ref": "refs/heads/forkos",
-        "rev": "60860d308404efc14cff66513f9e8e4a002756c3",
-        "revCount": 299,
+        "lastModified": 1728837991,
+        "narHash": "sha256-+jXVHPmX9eUtH2JhMKye0Tm2KMQTmD8FlHHfbcaXMOI=",
+        "ref": "refs/heads/bring-back-old-gerrit-reporting",
+        "rev": "879e9cdcdf2d7e6566ee512d015acc4d23f35517",
+        "revCount": 302,
         "type": "git",
         "url": "https://git.lix.systems/lix-project/buildbot-nix.git"
       },
       "original": {
-        "ref": "refs/heads/forkos",
+        "ref": "refs/heads/bring-back-old-gerrit-reporting",
         "type": "git",
         "url": "https://git.lix.systems/lix-project/buildbot-nix.git"
       }

flake.nix

@@ -22,7 +22,7 @@
     gerrit-dashboard.url = "git+https://git.lix.systems/the-distro/gerrit-monitoring.git";
     gerrit-dashboard.flake = false;
 
-    buildbot-nix.url = "git+https://git.lix.systems/lix-project/buildbot-nix.git?ref=refs/heads/forkos";
+    buildbot-nix.url = "git+https://git.lix.systems/lix-project/buildbot-nix.git?ref=refs/heads/bring-back-old-gerrit-reporting";
     buildbot-nix.inputs.nixpkgs.follows = "nixpkgs";
 
     channel-scripts.url = "git+https://git.lix.systems/the-distro/channel-scripts.git";
@@ -93,6 +93,9 @@
     devShells = forEachSystem' ({ system, pkgs, ... }: {
       default = pkgs.mkShell {
         packages = [
+          pkgs.direnv
+          pkgs.lorri
+
           inputs.agenix.packages.${system}.agenix
 
           pkgs.opentofu

hosts/buildbot-lix/default.nix

@@ -38,7 +38,9 @@
       port = 2022;
       username = "buildbot";
     };
-    cors.allowedOrigin = "*.lix.systems";
+    cors.allowedOrigins = [
+      "https://*.lix.systems"
+    ];
     projects = [
       "lix"
       "lix-installer"

hosts/buildbot/default.nix

@@ -36,13 +36,15 @@
         port = cfgGerrit.port;
         username = "buildbot";
       };
-    cors.allowedOrigin = "*.forkos.org";
+    cors.allowedOrigins = [
+      "https://*.forkos.org"
+    ];
     projects = [
       "buildbot-test"
       "nixpkgs"
       "infra"
     ];
-    builders = [ "builder-10" ];
+    builders = [ "builder-9" ];
   };
 
   i18n.defaultLocale = "en_US.UTF-8";

hosts/gerrit01/default.nix

@@ -23,6 +23,9 @@
     };
   };
 
+  # Block all these crawlers!!
+  bagel.services.nginx.crawler-blocker.enable = true;
+
   fileSystems."/gerrit-data" = {
     device = "/dev/disk/by-uuid/d1062305-0dea-4740-9a27-b6b1691862a4";
     fsType = "ext4";

services/block-crawlers/blocked-ua.txt

@@ -0,0 +1,40 @@
+AI2Bot
+Ai2Bot-Dolma
+Amazonbot
+anthropic-ai
+Applebot
+Applebot-Extended
+Bytespider
+CCBot
+ChatGPT-User
+Claude-Web
+ClaudeBot
+cohere-ai
+Diffbot
+FacebookBot
+facebookexternalhit
+FriendlyCrawler
+Google-Extended
+GoogleOther
+GoogleOther-Image
+GoogleOther-Video
+GPTBot
+iaskspider/2.0
+ICC-Crawler
+ImagesiftBot
+img2dataset
+ISSCyberRiskCrawler
+Kangaroo Bot
+Meta-ExternalAgent
+Meta-ExternalFetcher
+OAI-SearchBot
+omgili
+omgilibot
+PerplexityBot
+PetalBot
+Scrapy
+Sidetrade indexer bot
+Timpibot
+VelenPublicWebCrawler
+Webzio-Extended
+YouBot

services/block-crawlers/default.nix

@@ -0,0 +1,32 @@
+{ pkgs, config, lib, ... }:
+let
+  inherit (lib) mkEnableOption mkIf mkOption types concatStringsSep mkDefault splitString;
+  cfg = config.bagel.services.nginx.crawler-blocker;
+  mkRobotsFile = blockedUAs: pkgs.writeText "robots.txt" ''
+    ${concatStringsSep "\n" (map (ua: "User-agent: ${ua}") blockedUAs)}
+    Disallow: /
+  '';
+in
+{
+  options = {
+    bagel.services.nginx.crawler-blocker = {
+      enable = mkEnableOption "the crawler blocker";
+
+      userAgents = mkOption {
+        type = types.listOf types.str;
+        default = splitString "\n" (builtins.readFile ./blocked-ua.txt);
+      };
+    };
+
+    services.nginx.virtualHosts = mkOption {
+      type = types.attrsOf (types.submodule {
+        config = {
+          locations."= /robots.txt" = mkIf cfg.enable (mkDefault {
+            alias = mkRobotsFile cfg.userAgents;
+          });
+        };
+      });
+    };
+  };
+}
+

services/buildbot/default.nix

@@ -53,9 +53,9 @@ in
       };
     };
 
-    cors.allowedOrigin = mkOption {
-      type = types.str;
-      example = "*.forkos.org";
+    cors.allowedOrigins = mkOption {
+      type = types.listOf types.str;
+      example = [ "*.forkos.org" ];
       description = "Allowed origin for Buildbot and NGINX for CORS without the protocol";
     };
 
@@ -100,6 +100,7 @@ in
     };
 
     services.nginx = {
+      recommendedProxySettings = true;
       appendHttpConfig = ''
         # Our session stuff is too big with the TWISTED_COOKIE in addition.
         # Default is usually 4k or 8k.
@@ -109,8 +110,8 @@ in
         forceSSL = true;
         enableACME = true;
         extraConfig = ''
+          # This is needed so that logged-in users in Buildbot can include their credentials in their requests.
           add_header Access-Control-Allow-Credentials 'true' always;
-          add_header Access-Control-Allow-Origin 'https://${cfg.cors.allowedOrigin}' always;
         '';
       };
     };
@@ -155,9 +156,8 @@ in
       # we can replace all of this with automatic localworker generation on buildbot-nix side.
       workersFile = config.age.secrets.buildbot-workers.path;
 
-      allowedOrigins = [
-        cfg.cors.allowedOrigin
-      ];
+      # We rely on NGINX to do the CORS dance.
+      allowedOrigins = cfg.cors.allowedOrigins;
 
       buildMachines = map (n: {
           hostName = nodes.${n}.config.networking.fqdn;

services/default.nix

@@ -1,5 +1,6 @@
 {
   imports = [
+    ./block-crawlers
     ./gerrit
     ./channel-scripts
     ./hydra

services/gerrit/default.nix

@@ -129,7 +129,9 @@ in
       serverId = "9e5216ad-038d-4d74-a4e8-716515834a94";
 
       builtinPlugins = [
-        "gitiles"
+        # Disable gitiles as it generates too much traffic.
+        # Prefer git.forkos.org.
+        # "gitiles"
         "codemirror-editor"
         "reviewnotes"
         "download-commands"

services/gerrit/www.nix

@@ -29,10 +29,6 @@ in
             # NGINX should not give up super fast. Things can take time.
             proxy_read_timeout 3600;
           }
-
-          location = /robots.txt {
-            return 200 'User-agent: *\nAllow: /';
-          }
         '';
       };
 

services/matrix/default.nix

@@ -32,6 +32,11 @@ in
           ];
           server_name = "forkos.org";
           database.backend = "rocksdb";
+
+          server_discovery = {
+            server.authority = "matrix.forkos.org:443";
+            client.base_url = "https://matrix.forkos.org";
+          };
         };
       };
 
@@ -48,18 +53,7 @@ in
           "forkos.org" = {
             forceSSL = true;
             enableACME = true;
-            locations = {
-              "= /.well-known/matrix/server".extraConfig = ''
-                add_header Content-Type application/json;
-                add_header Access-Control-Allow-Origin *;
-                return 200 '{"m.server": "matrix.forkos.org:443"}';
-              '';
-              "= /.well-known/matrix/client".extraConfig = ''
-                add_header Content-Type application/json;
-                add_header Access-Control-Allow-Origin *;
-                return 200 '{"m.homeserver": {"base_url": "https://matrix.forkos.org/"}, "m.identity_server": {"base_url": "https://matrix.org/"}, "org.matrix.msc3575.proxy": {"url": "https://matrix.forkos.org"}}';
-              '';
-            };
+            locations."/.well-known/matrix".proxyPass = "http://grapevine";
           };
         };
       };

services/monitoring/exporters/postgres.nix

@@ -23,7 +23,10 @@ in
     };
 
     services.postgresql.settings.shared_preload_libraries = "pg_stat_statements";
+    systemd.services.postgresql.postStart = lib.mkAfter ''
+      ${config.services.postgresql.package}/bin/psql -U postgres -c "CREATE EXTENSION IF NOT EXISTS pg_stat_statements;";
+    '';
 
     bagel.monitoring.grafana-agent.exporters.postgres.port = 9104;
   };
-}
+}