Compare commits

..

No commits in common. "2ae7672fad07cdb9a3c184aa37df5107071f0387" and "97bee26977b25319905a6c0c485b0d6a1b520323" have entirely different histories.

16 changed files with 36 additions and 117 deletions

2
.envrc
View file

@ -1,2 +1,2 @@
# shellcheck shell=bash
eval "$(lorri direnv --flake .)"
use flake

View file

@ -19,7 +19,6 @@ in
"thubrecht"
"winter"
"yuka"
"ckie"
];
lix-infra.members = [
@ -40,6 +39,5 @@ in
"thubrecht"
"winter"
"yuka"
"ckie"
] (name: {});
}

View file

@ -63,6 +63,5 @@
thubrecht = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPM1jpXR7BWQa7Sed7ii3SbvIPRRlKb3G91qC0vOwfJn" ];
yuka = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIxQ3NYBi8v1f/vhxLKDcA6upmX0pctRDbnK6SER5OUR yureka" ];
winter = [ "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIH/LDRUG+U+++UmlxvA2kspioTjktQZ8taDcHq8gVlkfAAAABHNzaDo=" ];
ckie = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIH3uTwzSSMAPg84fwbNp2cq9+BdLFeA1VzDGth4zCAbz https://mei.puppycat.house" ];
};
}

View file

@ -87,16 +87,16 @@
"treefmt-nix": "treefmt-nix"
},
"locked": {
"lastModified": 1728837991,
"narHash": "sha256-+jXVHPmX9eUtH2JhMKye0Tm2KMQTmD8FlHHfbcaXMOI=",
"ref": "refs/heads/bring-back-old-gerrit-reporting",
"rev": "879e9cdcdf2d7e6566ee512d015acc4d23f35517",
"revCount": 302,
"lastModified": 1728307353,
"narHash": "sha256-eVSDu52qZn48c1HfDlH79JkDIzuE7qyY0ipPMbrpYzE=",
"ref": "refs/heads/forkos",
"rev": "60860d308404efc14cff66513f9e8e4a002756c3",
"revCount": 299,
"type": "git",
"url": "https://git.lix.systems/lix-project/buildbot-nix.git"
},
"original": {
"ref": "refs/heads/bring-back-old-gerrit-reporting",
"ref": "refs/heads/forkos",
"type": "git",
"url": "https://git.lix.systems/lix-project/buildbot-nix.git"
}

View file

@ -22,7 +22,7 @@
gerrit-dashboard.url = "git+https://git.lix.systems/the-distro/gerrit-monitoring.git";
gerrit-dashboard.flake = false;
buildbot-nix.url = "git+https://git.lix.systems/lix-project/buildbot-nix.git?ref=refs/heads/bring-back-old-gerrit-reporting";
buildbot-nix.url = "git+https://git.lix.systems/lix-project/buildbot-nix.git?ref=refs/heads/forkos";
buildbot-nix.inputs.nixpkgs.follows = "nixpkgs";
channel-scripts.url = "git+https://git.lix.systems/the-distro/channel-scripts.git";
@ -93,9 +93,6 @@
devShells = forEachSystem' ({ system, pkgs, ... }: {
default = pkgs.mkShell {
packages = [
pkgs.direnv
pkgs.lorri
inputs.agenix.packages.${system}.agenix
pkgs.opentofu

View file

@ -38,9 +38,7 @@
port = 2022;
username = "buildbot";
};
cors.allowedOrigins = [
"https://*.lix.systems"
];
cors.allowedOrigin = "*.lix.systems";
projects = [
"lix"
"lix-installer"

View file

@ -36,15 +36,13 @@
port = cfgGerrit.port;
username = "buildbot";
};
cors.allowedOrigins = [
"https://*.forkos.org"
];
cors.allowedOrigin = "*.forkos.org";
projects = [
"buildbot-test"
"nixpkgs"
"infra"
];
builders = [ "builder-9" ];
builders = [ "builder-10" ];
};
i18n.defaultLocale = "en_US.UTF-8";

View file

@ -23,9 +23,6 @@
};
};
# Block all these crawlers!!
bagel.services.nginx.crawler-blocker.enable = true;
fileSystems."/gerrit-data" = {
device = "/dev/disk/by-uuid/d1062305-0dea-4740-9a27-b6b1691862a4";
fsType = "ext4";

View file

@ -1,40 +0,0 @@
AI2Bot
Ai2Bot-Dolma
Amazonbot
anthropic-ai
Applebot
Applebot-Extended
Bytespider
CCBot
ChatGPT-User
Claude-Web
ClaudeBot
cohere-ai
Diffbot
FacebookBot
facebookexternalhit
FriendlyCrawler
Google-Extended
GoogleOther
GoogleOther-Image
GoogleOther-Video
GPTBot
iaskspider/2.0
ICC-Crawler
ImagesiftBot
img2dataset
ISSCyberRiskCrawler
Kangaroo Bot
Meta-ExternalAgent
Meta-ExternalFetcher
OAI-SearchBot
omgili
omgilibot
PerplexityBot
PetalBot
Scrapy
Sidetrade indexer bot
Timpibot
VelenPublicWebCrawler
Webzio-Extended
YouBot

View file

@ -1,32 +0,0 @@
{ pkgs, config, lib, ... }:
let
inherit (lib) mkEnableOption mkIf mkOption types concatStringsSep mkDefault splitString;
cfg = config.bagel.services.nginx.crawler-blocker;
mkRobotsFile = blockedUAs: pkgs.writeText "robots.txt" ''
${concatStringsSep "\n" (map (ua: "User-agent: ${ua}") blockedUAs)}
Disallow: /
'';
in
{
options = {
bagel.services.nginx.crawler-blocker = {
enable = mkEnableOption "the crawler blocker";
userAgents = mkOption {
type = types.listOf types.str;
default = splitString "\n" (builtins.readFile ./blocked-ua.txt);
};
};
services.nginx.virtualHosts = mkOption {
type = types.attrsOf (types.submodule {
config = {
locations."= /robots.txt" = mkIf cfg.enable (mkDefault {
alias = mkRobotsFile cfg.userAgents;
});
};
});
};
};
}

View file

@ -53,9 +53,9 @@ in
};
};
cors.allowedOrigins = mkOption {
type = types.listOf types.str;
example = [ "*.forkos.org" ];
cors.allowedOrigin = mkOption {
type = types.str;
example = "*.forkos.org";
description = "Allowed origin for Buildbot and NGINX for CORS without the protocol";
};
@ -100,7 +100,6 @@ in
};
services.nginx = {
recommendedProxySettings = true;
appendHttpConfig = ''
# Our session stuff is too big with the TWISTED_COOKIE in addition.
# Default is usually 4k or 8k.
@ -110,8 +109,8 @@ in
forceSSL = true;
enableACME = true;
extraConfig = ''
# This is needed so that logged-in users in Buildbot can include their credentials in their requests.
add_header Access-Control-Allow-Credentials 'true' always;
add_header Access-Control-Allow-Origin 'https://${cfg.cors.allowedOrigin}' always;
'';
};
};
@ -156,8 +155,9 @@ in
# we can replace all of this with automatic localworker generation on buildbot-nix side.
workersFile = config.age.secrets.buildbot-workers.path;
# We rely on NGINX to do the CORS dance.
allowedOrigins = cfg.cors.allowedOrigins;
allowedOrigins = [
cfg.cors.allowedOrigin
];
buildMachines = map (n: {
hostName = nodes.${n}.config.networking.fqdn;

View file

@ -1,6 +1,5 @@
{
imports = [
./block-crawlers
./gerrit
./channel-scripts
./hydra

View file

@ -129,9 +129,7 @@ in
serverId = "9e5216ad-038d-4d74-a4e8-716515834a94";
builtinPlugins = [
# Disable gitiles as it generates too much traffic.
# Prefer git.forkos.org.
# "gitiles"
"gitiles"
"codemirror-editor"
"reviewnotes"
"download-commands"

View file

@ -29,6 +29,10 @@ in
# NGINX should not give up super fast. Things can take time.
proxy_read_timeout 3600;
}
location = /robots.txt {
return 200 'User-agent: *\nAllow: /';
}
'';
};

View file

@ -32,11 +32,6 @@ in
];
server_name = "forkos.org";
database.backend = "rocksdb";
server_discovery = {
server.authority = "matrix.forkos.org:443";
client.base_url = "https://matrix.forkos.org";
};
};
};
@ -53,7 +48,18 @@ in
"forkos.org" = {
forceSSL = true;
enableACME = true;
locations."/.well-known/matrix".proxyPass = "http://grapevine";
locations = {
"= /.well-known/matrix/server".extraConfig = ''
add_header Content-Type application/json;
add_header Access-Control-Allow-Origin *;
return 200 '{"m.server": "matrix.forkos.org:443"}';
'';
"= /.well-known/matrix/client".extraConfig = ''
add_header Content-Type application/json;
add_header Access-Control-Allow-Origin *;
return 200 '{"m.homeserver": {"base_url": "https://matrix.forkos.org/"}, "m.identity_server": {"base_url": "https://matrix.org/"}, "org.matrix.msc3575.proxy": {"url": "https://matrix.forkos.org"}}';
'';
};
};
};
};

View file

@ -23,9 +23,6 @@ in
};
services.postgresql.settings.shared_preload_libraries = "pg_stat_statements";
systemd.services.postgresql.postStart = lib.mkAfter ''
${config.services.postgresql.package}/bin/psql -U postgres -c "CREATE EXTENSION IF NOT EXISTS pg_stat_statements;";
'';
bagel.monitoring.grafana-agent.exporters.postgres.port = 9104;
};