Compare commits
7 commits
ddd0c365ae
...
d26dd9f2a6
Author | SHA1 | Date | |
---|---|---|---|
Ilya K | d26dd9f2a6 | ||
Ilya K | 461e63b683 | ||
Ilya K | 8d22df82bd | ||
Ilya K | 22c87569b7 | ||
Pierre Bourdon | 30859b2872 | ||
Pierre Bourdon | 0c68a23275 | ||
raito | 8dc7ee9864 |
1
.gitignore
vendored
1
.gitignore
vendored
|
@ -2,3 +2,4 @@ result
|
|||
.gcroots
|
||||
config.tf.json
|
||||
.direnv
|
||||
.terraform
|
||||
|
|
22
flake.nix
22
flake.nix
|
@ -31,7 +31,12 @@
|
|||
terraform = pkgs.opentofu;
|
||||
terraformCfg = terranix.lib.terranixConfiguration {
|
||||
inherit system;
|
||||
modules = [ ];
|
||||
modules = [
|
||||
./terraform
|
||||
{
|
||||
bagel.hydra.enable = true;
|
||||
}
|
||||
];
|
||||
};
|
||||
in
|
||||
{
|
||||
|
@ -46,6 +51,16 @@
|
|||
${lib.getExe terraform} apply
|
||||
'');
|
||||
};
|
||||
plan = {
|
||||
type = "app";
|
||||
program = toString (pkgs.writers.writeBash "plan" ''
|
||||
set -eo pipefail
|
||||
rm -f config.tf.json
|
||||
cp ${terraformCfg} config.tf.json
|
||||
${lib.getExe terraform} init
|
||||
${lib.getExe terraform} plan
|
||||
'');
|
||||
};
|
||||
# nix run ".#destroy"
|
||||
destroy = {
|
||||
type = "app";
|
||||
|
@ -56,13 +71,16 @@
|
|||
${lib.getExe terraform} destroy
|
||||
'');
|
||||
};
|
||||
|
||||
default = self.apps.${system}.apply;
|
||||
};
|
||||
apps.${system}.default = self.apps.${system}.apply;
|
||||
|
||||
devShells.${system}.default = pkgs.mkShell {
|
||||
packages = [
|
||||
inputs.agenix.packages.${system}.agenix
|
||||
inputs.colmena.packages.${system}.colmena
|
||||
|
||||
pkgs.opentofu
|
||||
];
|
||||
};
|
||||
|
||||
|
|
|
@ -40,7 +40,6 @@
|
|||
hydra.enable = true;
|
||||
hydra.dbi = "dbi:Pg:dbname=hydra;user=hydra";
|
||||
};
|
||||
bagel.meta.monitoring.address = "bagel-box.delroth.net";
|
||||
|
||||
security.acme.acceptTerms = true;
|
||||
security.acme.defaults.email = "bagel@delroth.net";
|
||||
|
|
|
@ -24,7 +24,6 @@
|
|||
};
|
||||
};
|
||||
};
|
||||
bagel.meta.monitoring.address = "gerrit01.infra.forkos.org";
|
||||
|
||||
fileSystems."/gerrit-data" = {
|
||||
device = "/dev/disk/by-uuid/d1062305-0dea-4740-9a27-b6b1691862a4";
|
||||
|
|
|
@ -24,8 +24,6 @@
|
|||
};
|
||||
};
|
||||
|
||||
bagel.meta.monitoring.address = "fodwatch.infra.forkos.org";
|
||||
|
||||
i18n.defaultLocale = "en_US.UTF-8";
|
||||
|
||||
system.stateVersion = "24.05";
|
||||
|
|
|
@ -21,7 +21,6 @@
|
|||
enable = true;
|
||||
domain = "netbox.forkos.org";
|
||||
};
|
||||
bagel.meta.monitoring.address = "meta01.infra.forkos.org";
|
||||
bagel.services.prometheus.enable = true;
|
||||
bagel.services.loki.enable = true;
|
||||
bagel.services.grafana.enable = true;
|
||||
|
|
100
services/monitoring/agent.nix
Normal file
100
services/monitoring/agent.nix
Normal file
|
@ -0,0 +1,100 @@
|
|||
{
|
||||
config,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
let
|
||||
cfg = config.bagel.monitoring.grafana-agent;
|
||||
inherit (lib) mkEnableOption mkOption mkIf types;
|
||||
passwordAsCredential = "\${CREDENTIALS_DIRECTORY}/password";
|
||||
in
|
||||
{
|
||||
options.bagel.monitoring.grafana-agent = {
|
||||
enable = (mkEnableOption "Grafana Agent") // { default = true; };
|
||||
|
||||
exporters = mkOption {
|
||||
description = "List of all exporters to scrape";
|
||||
type = types.listOf (types.submodule {
|
||||
options.port = mkOption {
|
||||
description = "Exporter port";
|
||||
type = types.int;
|
||||
};
|
||||
});
|
||||
default = [];
|
||||
};
|
||||
};
|
||||
|
||||
config = mkIf cfg.enable {
|
||||
age.secrets.grafana-agent-password.file = ../../secrets/metrics-push-password.age;
|
||||
|
||||
services.grafana-agent = {
|
||||
enable = true;
|
||||
credentials.password = config.age.secrets.grafana-agent-password.path;
|
||||
settings = {
|
||||
metrics = {
|
||||
global.remote_write = [
|
||||
{
|
||||
url = "https://mimir.forkos.org/api/v1/push";
|
||||
basic_auth = {
|
||||
username = "promtail";
|
||||
password_file = passwordAsCredential;
|
||||
};
|
||||
}
|
||||
];
|
||||
configs = [
|
||||
{
|
||||
name = config.networking.hostName;
|
||||
scrape_configs = [
|
||||
{
|
||||
job_name = config.networking.hostName;
|
||||
static_configs = [
|
||||
{ targets = map (e: "localhost:" + (toString e.port)) config.bagel.monitoring.grafana-agent.exporters; }
|
||||
];
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
};
|
||||
logs = {
|
||||
global.clients = [
|
||||
{
|
||||
url = "https://loki.forkos.org/loki/api/v1/push";
|
||||
basic_auth = {
|
||||
username = "promtail";
|
||||
password_file = passwordAsCredential;
|
||||
};
|
||||
}
|
||||
];
|
||||
configs = [
|
||||
{
|
||||
name = "journald";
|
||||
scrape_configs = [
|
||||
{
|
||||
job_name = "system";
|
||||
journal = {
|
||||
max_age = "12h";
|
||||
labels = {
|
||||
job = "systemd-journal";
|
||||
host = config.networking.hostName;
|
||||
};
|
||||
};
|
||||
relabel_configs = [
|
||||
{
|
||||
source_labels = [ "__journal__systemd_unit" ];
|
||||
target_label = "unit";
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
positions_directory = "\${STATE_DIRECTORY}/positions";
|
||||
};
|
||||
integrations.node_exporter.enable_collectors = [
|
||||
"processes"
|
||||
"systemd"
|
||||
];
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
|
@ -2,6 +2,6 @@
|
|||
imports = [
|
||||
./exporters
|
||||
./lgtm
|
||||
./promtail.nix
|
||||
./agent.nix
|
||||
];
|
||||
}
|
|
@ -17,6 +17,6 @@ in
|
|||
listenAddress = "0.0.0.0";
|
||||
};
|
||||
|
||||
bagel.meta.monitoring.exporters = [ { port = 9102; } ];
|
||||
bagel.monitoring.grafana-agent.exporters = [ { port = 9102; } ];
|
||||
};
|
||||
}
|
||||
|
|
|
@ -1,37 +1,7 @@
|
|||
{
|
||||
config,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
let
|
||||
inherit (lib) mkOption types;
|
||||
in
|
||||
{
|
||||
imports = [
|
||||
./cadvisor.nix
|
||||
./node.nix
|
||||
./nginx.nix
|
||||
./postgres.nix
|
||||
];
|
||||
|
||||
options.bagel = {
|
||||
meta.monitoring = {
|
||||
address = mkOption {
|
||||
description = "Node's public address";
|
||||
type = types.str;
|
||||
};
|
||||
exporters = mkOption {
|
||||
description = "List of all exporters to scrape";
|
||||
type = types.listOf (types.submodule {
|
||||
options.port = mkOption {
|
||||
description = "Exporter port";
|
||||
type = types.int;
|
||||
};
|
||||
});
|
||||
default = [];
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
config.networking.firewall.allowedTCPPorts = map (e: e.port) config.bagel.meta.monitoring.exporters;
|
||||
}
|
|
@ -30,7 +30,7 @@ in
|
|||
];
|
||||
};
|
||||
|
||||
bagel.meta.monitoring.exporters = [
|
||||
bagel.monitoring.grafana-agent.exporters = [
|
||||
{ port = 9103; }
|
||||
];
|
||||
};
|
||||
|
|
|
@ -1,25 +0,0 @@
|
|||
{
|
||||
config,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
let
|
||||
cfg = config.bagel.monitoring.exporters.node;
|
||||
inherit (lib) mkEnableOption mkIf;
|
||||
in
|
||||
{
|
||||
options.bagel.monitoring.exporters.node.enable = (mkEnableOption "Standard node_exporter") // { default = true; };
|
||||
|
||||
config = mkIf cfg.enable {
|
||||
services.prometheus.exporters.node = {
|
||||
enable = true;
|
||||
enabledCollectors = [
|
||||
"processes"
|
||||
"systemd"
|
||||
];
|
||||
port = 9101;
|
||||
};
|
||||
|
||||
bagel.meta.monitoring.exporters = [ { port = 9101; } ];
|
||||
};
|
||||
}
|
|
@ -24,7 +24,7 @@ in
|
|||
|
||||
services.postgresql.settings.shared_preload_libraries = "pg_stat_statements";
|
||||
|
||||
bagel.meta.monitoring.exporters = [
|
||||
bagel.monitoring.grafana-agent.exporters = [
|
||||
{ port = 9104; }
|
||||
];
|
||||
};
|
||||
|
|
5
services/monitoring/lgtm/alerts/demo.yaml
Normal file
5
services/monitoring/lgtm/alerts/demo.yaml
Normal file
|
@ -0,0 +1,5 @@
|
|||
groups:
|
||||
- name: Demo alerts
|
||||
rules:
|
||||
- alert: Demo alert
|
||||
expr: 1
|
|
@ -2,6 +2,6 @@
|
|||
imports = [
|
||||
./grafana.nix
|
||||
./loki.nix
|
||||
./prometheus.nix
|
||||
./mimir.nix
|
||||
];
|
||||
}
|
|
@ -92,6 +92,7 @@ in
|
|||
uid = "mimir";
|
||||
access = "proxy";
|
||||
url = "http://127.0.0.1:9009/prometheus";
|
||||
isDefault = true;
|
||||
}
|
||||
{
|
||||
name = "Loki";
|
||||
|
@ -100,6 +101,17 @@ in
|
|||
access = "proxy";
|
||||
url = "http://127.0.0.1:9090/";
|
||||
}
|
||||
{
|
||||
name = "Mimir Alertmanager";
|
||||
type = "alertmanager";
|
||||
uid = "mimir-alertmanager";
|
||||
access = "proxy";
|
||||
url = "http://127.0.0.1:9009/";
|
||||
jsonData = {
|
||||
handleGrafanaManagedAlerts = true;
|
||||
implementation = "mimir";
|
||||
};
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
|
|
|
@ -12,8 +12,8 @@ in
|
|||
|
||||
config = mkIf cfg.enable {
|
||||
age.secrets = {
|
||||
loki-htpasswd = {
|
||||
file = ../../../secrets/loki-htpasswd.age;
|
||||
metrics-push-htpasswd = {
|
||||
file = ../../../secrets/metrics-push-htpasswd.age;
|
||||
owner = "nginx";
|
||||
};
|
||||
loki-environment.file = ../../../secrets/loki-environment.age;
|
||||
|
@ -93,7 +93,7 @@ in
|
|||
forceSSL = true;
|
||||
locations."/loki/api/v1/push" = {
|
||||
proxyPass = "http://localhost:${toString config.services.loki.configuration.server.http_listen_port}";
|
||||
basicAuthFile = config.age.secrets.loki-htpasswd.path;
|
||||
basicAuthFile = config.age.secrets.metrics-push-htpasswd.path;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
|
92
services/monitoring/lgtm/mimir.nix
Normal file
92
services/monitoring/lgtm/mimir.nix
Normal file
|
@ -0,0 +1,92 @@
|
|||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
let
|
||||
cfg = config.bagel.services.prometheus;
|
||||
inherit (lib) mkEnableOption mkIf;
|
||||
|
||||
mimirPort = config.services.mimir.configuration.server.http_listen_port;
|
||||
in
|
||||
{
|
||||
options.bagel.services.prometheus.enable = mkEnableOption "Prometheus scraper";
|
||||
|
||||
config = mkIf cfg.enable {
|
||||
age.secrets = {
|
||||
metrics-push-htpasswd = {
|
||||
file = ../../../secrets/metrics-push-htpasswd.age;
|
||||
owner = "nginx";
|
||||
};
|
||||
mimir-environment.file = ../../../secrets/mimir-environment.age;
|
||||
};
|
||||
|
||||
services.mimir = {
|
||||
enable = true;
|
||||
extraFlags = ["--config.expand-env=true"];
|
||||
configuration = {
|
||||
target = "all,alertmanager";
|
||||
|
||||
multitenancy_enabled = false;
|
||||
|
||||
common.storage = {
|
||||
backend = "s3";
|
||||
s3 = {
|
||||
endpoint = "s3.delroth.net";
|
||||
bucket_name = "bagel-mimir";
|
||||
secret_access_key = "\${S3_KEY}"; # This is a secret injected via an environment variable
|
||||
access_key_id = "\${S3_KEY_ID}";
|
||||
};
|
||||
};
|
||||
|
||||
server = {
|
||||
http_listen_port = 9009;
|
||||
grpc_server_max_recv_msg_size = 104857600;
|
||||
grpc_server_max_send_msg_size = 104857600;
|
||||
grpc_server_max_concurrent_streams = 1000;
|
||||
};
|
||||
|
||||
ingester.ring.replication_factor = 1;
|
||||
|
||||
blocks_storage.backend = "s3";
|
||||
ruler_storage = {
|
||||
backend = "local";
|
||||
local.directory = pkgs.runCommand "mimir-rules" {} ''
|
||||
mkdir -p $out
|
||||
ln -s ${./alerts} $out/anonymous
|
||||
'';
|
||||
};
|
||||
|
||||
alertmanager = {
|
||||
sharding_ring.replication_factor = 1;
|
||||
fallback_config_file = pkgs.writers.writeYAML "alertmanager.yaml" {
|
||||
route = {
|
||||
group_by = ["alertname"];
|
||||
receiver = "matrix";
|
||||
};
|
||||
receivers = [
|
||||
{
|
||||
name = "matrix";
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
alertmanager_storage.backend = "filesystem";
|
||||
|
||||
ruler.alertmanager_url = "http://localhost:${toString mimirPort}/alertmanager";
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services.mimir.serviceConfig.EnvironmentFile = [ config.age.secrets.mimir-environment.path ];
|
||||
|
||||
services.nginx.virtualHosts."mimir.forkos.org" = {
|
||||
enableACME = true;
|
||||
forceSSL = true;
|
||||
locations."/api/v1/push" = {
|
||||
proxyPass = "http://localhost:${toString mimirPort}";
|
||||
basicAuthFile = config.age.secrets.metrics-push-htpasswd.path;
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
|
@ -1,83 +0,0 @@
|
|||
{
|
||||
config,
|
||||
lib,
|
||||
nodes,
|
||||
...
|
||||
}:
|
||||
let
|
||||
cfg = config.bagel.services.prometheus;
|
||||
inherit (lib) mkEnableOption mkIf;
|
||||
|
||||
forEachMachine = fn: map fn (builtins.attrValues nodes);
|
||||
|
||||
allMetas = forEachMachine (machine: {
|
||||
name = machine.config.networking.hostName;
|
||||
address = machine.config.bagel.meta.monitoring.address or null;
|
||||
exporters = machine.config.bagel.meta.monitoring.exporters or [];
|
||||
});
|
||||
|
||||
scrapableMetas = builtins.filter (m: m.address != null && m.exporters != []) allMetas;
|
||||
|
||||
toJobConfig = m: {
|
||||
job_name = m.name;
|
||||
static_configs = [
|
||||
{ targets = map (e: m.address + ":" + (toString e.port)) m.exporters; }
|
||||
];
|
||||
};
|
||||
|
||||
jobConfigs = map toJobConfig scrapableMetas;
|
||||
in
|
||||
{
|
||||
options.bagel.services.prometheus.enable = mkEnableOption "Prometheus scraper";
|
||||
|
||||
config = mkIf cfg.enable {
|
||||
age.secrets.mimir-environment.file = ../../../secrets/mimir-environment.age;
|
||||
|
||||
services.prometheus = {
|
||||
enable = true;
|
||||
enableAgentMode = true;
|
||||
listenAddress = "127.0.0.1";
|
||||
port = 9001;
|
||||
globalConfig.scrape_interval = "15s";
|
||||
scrapeConfigs = jobConfigs;
|
||||
remoteWrite = [
|
||||
{ url = "http://localhost:9009/api/v1/push"; }
|
||||
];
|
||||
};
|
||||
|
||||
services.mimir = {
|
||||
enable = true;
|
||||
extraFlags = ["--config.expand-env=true"];
|
||||
configuration = {
|
||||
multitenancy_enabled = false;
|
||||
|
||||
common.storage = {
|
||||
backend = "s3";
|
||||
s3 = {
|
||||
endpoint = "s3.delroth.net";
|
||||
bucket_name = "bagel-mimir";
|
||||
secret_access_key = "\${S3_KEY}"; # This is a secret injected via an environment variable
|
||||
access_key_id = "\${S3_KEY_ID}";
|
||||
};
|
||||
};
|
||||
|
||||
server = {
|
||||
http_listen_port = 9009;
|
||||
grpc_server_max_recv_msg_size = 104857600;
|
||||
grpc_server_max_send_msg_size = 104857600;
|
||||
grpc_server_max_concurrent_streams = 1000;
|
||||
};
|
||||
|
||||
ingester.ring.replication_factor = 1;
|
||||
|
||||
blocks_storage.backend = "s3";
|
||||
ruler_storage = {
|
||||
backend = "local";
|
||||
local.directory = ./alerts;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services.mimir.serviceConfig.EnvironmentFile = [ config.age.secrets.mimir-environment.path ];
|
||||
};
|
||||
}
|
|
@ -1,53 +0,0 @@
|
|||
{
|
||||
config,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
let
|
||||
cfg = config.bagel.monitoring.promtail;
|
||||
inherit (lib) mkEnableOption mkIf;
|
||||
in
|
||||
{
|
||||
options.bagel.monitoring.promtail.enable = (mkEnableOption "Promtail log export") // { default = true; };
|
||||
|
||||
config = mkIf cfg.enable {
|
||||
age.secrets.promtail-password = {
|
||||
file = ../../secrets/promtail-password.age;
|
||||
owner = "promtail";
|
||||
};
|
||||
|
||||
services.promtail = {
|
||||
enable = true;
|
||||
configuration = {
|
||||
server.disable = true;
|
||||
clients = [
|
||||
{
|
||||
url = "https://loki.forkos.org/loki/api/v1/push";
|
||||
basic_auth = {
|
||||
username = "promtail";
|
||||
password_file = config.age.secrets.promtail-password.path;
|
||||
};
|
||||
}
|
||||
];
|
||||
scrape_configs = [
|
||||
{
|
||||
job_name = "system";
|
||||
journal = {
|
||||
max_age = "12h";
|
||||
labels = {
|
||||
job = "systemd-journal";
|
||||
host = config.networking.hostName;
|
||||
};
|
||||
};
|
||||
relabel_configs = [
|
||||
{
|
||||
source_labels = [ "__journal__systemd_unit" ];
|
||||
target_label = "unit";
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
6
terraform/default.nix
Normal file
6
terraform/default.nix
Normal file
|
@ -0,0 +1,6 @@
|
|||
{
|
||||
imports = [
|
||||
./hydra.nix
|
||||
./state.nix
|
||||
];
|
||||
}
|
72
terraform/hydra.nix
Normal file
72
terraform/hydra.nix
Normal file
|
@ -0,0 +1,72 @@
|
|||
{ lib, config, ... }:
|
||||
let
|
||||
inherit (lib) mkEnableOption mkIf types mkOption;
|
||||
cfg = config.bagel.hydra;
|
||||
in
|
||||
{
|
||||
options.bagel.hydra = {
|
||||
enable = mkEnableOption "the Hydra jobsets";
|
||||
};
|
||||
|
||||
config = mkIf cfg.enable {
|
||||
terraform.required_providers.hydra = {
|
||||
version = "~> 0.1";
|
||||
source = "DeterminateSystems/hydra";
|
||||
};
|
||||
|
||||
provider.hydra = {
|
||||
host = "https://hydra.bagel.delroth.net";
|
||||
# username/password are provided via HYDRA_USERNAME/HYDRA_PASSWORD
|
||||
};
|
||||
|
||||
resource.hydra_project.forkos = {
|
||||
name = "forkos";
|
||||
display_name = "ForkOS";
|
||||
description = "ForkOS packages collection";
|
||||
homepage = "https://cl.forkos.org";
|
||||
owner = "raito";
|
||||
enabled = true;
|
||||
visible = true;
|
||||
};
|
||||
|
||||
resource.hydra_jobset.raito-nixos-rolling-small = {
|
||||
project = config.resource.hydra_project.forkos.name;
|
||||
state = "enabled";
|
||||
visible = true;
|
||||
name = "raito-nixos-rolling-small";
|
||||
type = "legacy";
|
||||
description = "master branch for raito-nixos";
|
||||
|
||||
nix_expression = {
|
||||
file = "pkgs/top-level/release.nix";
|
||||
input = "nixpkgs";
|
||||
};
|
||||
|
||||
check_interval = 0;
|
||||
scheduling_shares = 3000;
|
||||
keep_evaluations = 3;
|
||||
|
||||
email_notifications = false;
|
||||
|
||||
input = [
|
||||
{
|
||||
name = "nixpkgs";
|
||||
type = "git";
|
||||
value = "https://cl.forkos.org/nixpkgs";
|
||||
notify_committers = false;
|
||||
}
|
||||
{
|
||||
name = "officialRelease";
|
||||
type = "boolean";
|
||||
value = "false";
|
||||
notify_committers = false;
|
||||
}
|
||||
{
|
||||
name = "supportedSystems";
|
||||
type = "nix";
|
||||
value = ''[ "x86_64-linux" ]'';
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
}
|
21
terraform/state.nix
Normal file
21
terraform/state.nix
Normal file
|
@ -0,0 +1,21 @@
|
|||
{
|
||||
# We use terraform.backend.s3 directly instead of the type-checked Terranix
|
||||
# backend.s3 options. The latter does not support setting arbitrary s3
|
||||
# endpoints.
|
||||
#
|
||||
# Note: currently requires the user to provide AWS_ACCESS_KEY_ID as well as
|
||||
# AWS_SECRET_ACCESS_KEY in their environment variables.
|
||||
|
||||
terraform.backend.s3 = {
|
||||
endpoints.s3 = "s3.delroth.net";
|
||||
region = "garage";
|
||||
bucket = "bagel-terraform-state";
|
||||
key = "state";
|
||||
|
||||
# It's just a dump Garage server, don't try to be smart.
|
||||
skip_credentials_validation = true;
|
||||
skip_region_validation = true;
|
||||
skip_requesting_account_id = true;
|
||||
skip_metadata_api_check = true;
|
||||
};
|
||||
}
|
Loading…
Reference in a new issue