Compare commits

...

2 commits

Author SHA1 Message Date
Ilya K 71dacb5b2c Clean up SSH key dupes, add Maxine 2024-07-05 17:20:50 +03:00
Ilya K c3b4cc03dc Add Loki + Promtail setup 2024-07-05 17:20:22 +03:00
12 changed files with 214 additions and 12 deletions

View file

@ -1,13 +1,16 @@
{ let
users.users.root.openssh.authorizedKeys.keys = [ keys = import ./ssh-keys.nix;
# delroth in {
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII3tjB4KYDok3KlWxdBp/yEmqhhmybd+w0VO4xUwLKKV" users.users.root.openssh.authorizedKeys.keys =
# raito keys.users.delroth ++
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcEkYM1r8QVNM/G5CxJInEdoBCWjEHHDdHlzDYNSUIdHHsn04QY+XI67AdMCm8w30GZnLUIj5RiJEWXREUApby0GrfxGGcy8otforygfgtmuUKAUEHdU2MMwrQI7RtTZ8oQ0USRGuqvmegxz3l5caVU7qGvBllJ4NUHXrkZSja2/51vq80RF4MKkDGiz7xUTixI2UcBwQBCA/kQedKV9G28EH+1XfvePqmMivZjl+7VyHsgUVj9eRGA1XWFw59UPZG8a7VkxO/Eb3K9NF297HUAcFMcbY6cPFi9AaBgu3VC4eetDnoN/+xT1owiHi7BReQhGAy/6cdf7C/my5ehZwD" keys.users.k900 ++
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE0xMwWedkKosax9+7D2OlnMxFL/eV4CvFZLsbLptpXr" keys.users.raito ++
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU" keys.users.maxine ++
"ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJFsZ7PMDt80tYXHyScQajNhqH4wuYg/o0OxfOHaZD4rXuT0VIKflKH1M9LslfHWIEH3XNeqhQOziH9r+Ny5JcM=" [
# k900 # more raito
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOi9vgVGs+S5kEsUqHPvyMMh1Q9gqL4TcbHoe5d73tun" "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcEkYM1r8QVNM/G5CxJInEdoBCWjEHHDdHlzDYNSUIdHHsn04QY+XI67AdMCm8w30GZnLUIj5RiJEWXREUApby0GrfxGGcy8otforygfgtmuUKAUEHdU2MMwrQI7RtTZ8oQ0USRGuqvmegxz3l5caVU7qGvBllJ4NUHXrkZSja2/51vq80RF4MKkDGiz7xUTixI2UcBwQBCA/kQedKV9G28EH+1XfvePqmMivZjl+7VyHsgUVj9eRGA1XWFw59UPZG8a7VkxO/Eb3K9NF297HUAcFMcbY6cPFi9AaBgu3VC4eetDnoN/+xT1owiHi7BReQhGAy/6cdf7C/my5ehZwD"
]; "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE0xMwWedkKosax9+7D2OlnMxFL/eV4CvFZLsbLptpXr"
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU"
"ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJFsZ7PMDt80tYXHyScQajNhqH4wuYg/o0OxfOHaZD4rXuT0VIKflKH1M9LslfHWIEH3XNeqhQOziH9r+Ny5JcM="
];
} }

View file

@ -2,10 +2,13 @@
machines = { machines = {
bagel-box = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAsO4bNqY04uG13Pg3ubHfRDssTphDLzZ4YUniE5/p+M"; bagel-box = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAsO4bNqY04uG13Pg3ubHfRDssTphDLzZ4YUniE5/p+M";
meta01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM5t9gYorOWgpCFDJgb24pyCKIabGpeI2H/UfdvXODcT"; meta01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM5t9gYorOWgpCFDJgb24pyCKIabGpeI2H/UfdvXODcT";
gerrit01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA+eSZu+u9sCynrMlsmFzQHLIELQAuVg0Cs1pBvwb4+A";
}; };
users = { users = {
delroth = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII3tjB4KYDok3KlWxdBp/yEmqhhmybd+w0VO4xUwLKKV" ]; delroth = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII3tjB4KYDok3KlWxdBp/yEmqhhmybd+w0VO4xUwLKKV" ];
raito = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICaw9ihTG7ucB8P38XdalEWev8+q96e2yNm4B+/I9IJp" ]; raito = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICaw9ihTG7ucB8P38XdalEWev8+q96e2yNm4B+/I9IJp" ];
k900 = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOi9vgVGs+S5kEsUqHPvyMMh1Q9gqL4TcbHoe5d73tun" ];
maxine = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILpWQfhNFdrxMTP/1DwBVuk49f3df9iH7Tbdu8ltIKjr" ];
}; };
} }

View file

@ -23,6 +23,7 @@
}; };
bagel.meta.monitoring.address = "meta01.infra.forkos.org"; bagel.meta.monitoring.address = "meta01.infra.forkos.org";
bagel.services.prometheus.enable = true; bagel.services.prometheus.enable = true;
bagel.services.loki.enable = true;
bagel.services.grafana.enable = true; bagel.services.grafana.enable = true;
i18n.defaultLocale = "fr_FR.UTF-8"; i18n.defaultLocale = "fr_FR.UTF-8";

View file

@ -9,6 +9,11 @@ let
netbox-environment = [ machines.meta01 ]; netbox-environment = [ machines.meta01 ];
mimir-environment = [ machines.meta01 ]; mimir-environment = [ machines.meta01 ];
grafana-oauth-secret = [ machines.meta01 ]; grafana-oauth-secret = [ machines.meta01 ];
loki-environment = [ machines.meta01 ];
# These are the same password, but nginx wants it in htpasswd format
loki-htpasswd = [ machines.meta01 ];
promtail-password = builtins.attrValues machines;
}; };
in in
builtins.listToAttrs ( builtins.listToAttrs (

View file

@ -0,0 +1,9 @@
age-encryption.org/v1
-> ssh-ed25519 j2r2qQ w0lLquFUUcmEZ/Fh1YSt85tAJkBwavORQbwMr7gMqF4
J4T+EHm1uHbCZkAUNoNcB9uGSz082mFL8+dkCnvYQnM
-> ssh-ed25519 K3b7BA 28bJZgBPPc2KIE5+b8LJuQ5L4YAiRAJzucEuOqXHdVM
7hKENFr8QX0jpwuuQEjGFrUywJuhL1Tdi2V4/gR8JWE
--- GSPZxz39TMMWv0qhotNgnXa5679Q7VK8JGjQjI7A8oM
J²\@F“N• ³å2®ô¨w×!¯1Vf»§˜Ž·ÞO²CÓw®®V°£šÌº.^݆ 7<C2A0>w‡n4äàdW-Ö¾"@0¨ú¹EÏ¿·°ck,]M}x<>øÌ<C3B8>¡Ûy°[×ÁJ:!è‘ !ø螀c¬
Bë¹R
nøê€þÀáÆ^9í¤M<ú

View file

@ -0,0 +1,7 @@
age-encryption.org/v1
-> ssh-ed25519 j2r2qQ nLWy3WcVJWCl3rXkhcSbp1joqmkk06QnxhCZ4UtSvmw
iQ+Hx/vhiFgkWfbxHwGjxMBEqzyGww4/9do3W7V/y1Y
-> ssh-ed25519 K3b7BA RkF2ADcjOGtivl9MrhO/HFwxlTAkbFHWL3iinUldMiM
7q/zdVTMLevukZjkHtcN88iYzfTLvq2s3QdkgsFSO9M
--- 1b2HiK06vJPqBgHVDD0QELOtfkl7/rlgGS9uI1mSbus
„uܧoL;õå¬" 4¦Û»Z¼˜@§öãƒÐ3+93Q4óÄ o•ŒØwé“„6<>M-²DkJn´;ñ*g <0A>Yœ75ËSò)Ù°©

View file

@ -0,0 +1,12 @@
age-encryption.org/v1
-> ssh-ed25519 +HUDfA ZUM0ACC/NIekvX1PkCiXTHaTeE3ybudmY3piHw2iekQ
cHj94FIR6gNJ3Hw9FI7K15OYgxbjkajGtCftD+2Mr8c
-> ssh-ed25519 2D+APA tzlyOnAXnLxXO/47n45sFPiJF3FXd98UU5ajPhD2wSs
P8ZdUiBeME17SU2BpMgOq4plyAqgzLOQWHa1+Q7cjYo
-> ssh-ed25519 j2r2qQ 3OikD9JOmug7kdPAPz+JT/ryB6xBQhu2+cwS9h5sKGI
XiIuxOyey2I6hmqabUCPzLc85q/1r9OwVGjHWYNQsp0
-> ssh-ed25519 K3b7BA Bdqcqt4GgLzuSiEnIyImDiOQGwyIhhozRXMmNrp7glI
65joZcnl0Hqe90Th2EdVgbcxUJFpy3fOgk6oPiSHh2A
--- 6x6BFNypc+u3DpsHX3SajwEy1TqsAtbFei0ddRpEoBg
äªUG¾xj4»®Îþ‡b=óžóñ¼Rd<52>3sHYÝ Ô<>*Qµ9Ã6n34&äw»~h!§ ^„[êš

View file

@ -2,5 +2,6 @@
imports = [ imports = [
./exporters ./exporters
./lgtm ./lgtm
./promtail.nix
]; ];
} }

View file

@ -1,6 +1,7 @@
{ {
imports = [ imports = [
./grafana.nix ./grafana.nix
./loki.nix
./prometheus.nix ./prometheus.nix
]; ];
} }

View file

@ -93,6 +93,13 @@ in
access = "proxy"; access = "proxy";
url = "http://127.0.0.1:9009/prometheus"; url = "http://127.0.0.1:9009/prometheus";
} }
{
name = "Loki";
type = "loki";
uid = "loki";
access = "proxy";
url = "http://127.0.0.1:9090/";
}
]; ];
}; };
}; };

View file

@ -0,0 +1,100 @@
{
config,
lib,
...
}:
let
cfg = config.bagel.services.loki;
inherit (lib) mkEnableOption mkIf;
in
{
options.bagel.services.loki.enable = mkEnableOption "Loki storage";
config = mkIf cfg.enable {
age.secrets = {
loki-htpasswd = {
file = ../../../secrets/loki-htpasswd.age;
owner = "nginx";
};
loki-environment.file = ../../../secrets/loki-environment.age;
};
services.loki = {
enable = true;
extraFlags = ["--config.expand-env"];
configuration = {
server = {
http_listen_port = 9090;
grpc_listen_port = 9096;
# 16M
grpc_server_max_recv_msg_size = 16777216;
grpc_server_max_send_msg_size = 16777216;
};
auth_enabled = false;
common = {
storage.s3 = {
endpoint = "s3.delroth.net";
region = "garage";
bucketnames = "bagel-loki";
secret_access_key = "\${S3_KEY}"; # This is a secret injected via an environment variable
access_key_id = "\${S3_KEY_ID}";
s3forcepathstyle = true;
};
ring.kvstore.store = "memberlist";
replication_factor = 1;
};
memberlist = {
bind_port = 7947;
advertise_port = 7947;
};
storage_config.tsdb_shipper = {
active_index_directory = "/var/lib/loki/index";
cache_location = "/var/lib/loki/cache";
};
compactor = {
working_directory = "/var/lib/loki/compactor";
compaction_interval = "10m";
retention_enabled = true;
retention_delete_delay = "1s";
retention_delete_worker_count = 150;
delete_request_store = "filesystem";
};
limits_config.retention_period = "1w";
schema_config = {
configs = [
{
from = "2024-07-01";
store = "tsdb";
object_store = "s3";
schema = "v13";
index = {
prefix = "index_";
period = "24h";
};
}
];
};
};
};
systemd.services.loki.serviceConfig.EnvironmentFile = [ config.age.secrets.loki-environment.path ];
services.nginx.virtualHosts."loki.forkos.org" = {
enableACME = true;
forceSSL = true;
locations."/loki/api/v1/push" = {
proxyPass = "http://localhost:${toString config.services.loki.configuration.server.http_listen_port}";
basicAuthFile = config.age.secrets.loki-htpasswd.path;
};
};
};
}

View file

@ -0,0 +1,53 @@
{
config,
lib,
...
}:
let
cfg = config.bagel.monitoring.promtail;
inherit (lib) mkEnableOption mkIf;
in
{
options.bagel.monitoring.promtail.enable = (mkEnableOption "Promtail log export") // { default = true; };
config = mkIf cfg.enable {
age.secrets.promtail-password = {
file = ../../secrets/promtail-password.age;
owner = "promtail";
};
services.promtail = {
enable = true;
configuration = {
server.disable = true;
clients = [
{
url = "https://loki.forkos.org/loki/api/v1/push";
basic_auth = {
username = "promtail";
password_file = config.age.secrets.promtail-password.path;
};
}
];
scrape_configs = [
{
job_name = "system";
journal = {
max_age = "12h";
labels = {
job = "systemd-journal";
host = config.networking.hostName;
};
};
relabel_configs = [
{
source_labels = [ "__journal__systemd_unit" ];
target_label = "unit";
}
];
}
];
};
};
};
}