Remove custom labels added to logs during parsing

Promtail was configured to create labels for nearly every key in the
logs. This was done to support easier label-based querying. Loki
however is not optimized to  work with labels having a high cardinality.
This led to failures in Loki, if it had to handle a high number of
logs. In addition, the high number of labels led to a huge number of
chunks being created, mostly just containing a single log entry,
making querying and storage very inefficient.

This change removes all custom made labels, except for the
gerrit_version label. Logs should rather be queried using the grep-
like syntax of LogQL for which Loki is optimized.

Change-Id: I70e2a3ff4f640bc6f5d08d50212958a7bca2eae1
This commit is contained in:
Thomas Draebing 2020-03-23 11:53:13 +01:00
parent ab26ebb833
commit 1d6a3dcc5e

View file

@ -29,18 +29,8 @@ scrape_configs:
pipeline_stages:
- json:
expressions:
level:
timestamp: '"@timestamp"'
exception: '"exception"'
thread: thread_name
logger: logger_name
class:
message:
- json:
source: exception
expressions:
exception_message:
exception_class:
- template:
source: timestamp
template: '{{ Replace .Value "," "." 1 }}'
@ -57,12 +47,6 @@ scrape_configs:
source: message
expression: "Gerrit Code Review (?P<gerrit_version>.*) ready"
- labels:
level:
exception_message:
exception_class:
thread:
logger:
class:
gerrit_version:
- job_name: gerrit_httpd
static_configs:
@ -76,23 +60,12 @@ scrape_configs:
- json:
expressions:
timestamp:
thread:
user:
method:
status:
protocol:
- template:
source: timestamp
template: '{{ Replace .Value "," "." 1 }}'
- timestamp:
source: timestamp
format: '02/Jan/2006:15:04:05.999 -0700'
- labels:
thread:
user:
method:
status:
protocol:
- job_name: gerrit_sshd
static_configs:
- targets:
@ -105,20 +78,9 @@ scrape_configs:
- json:
expressions:
timestamp:
session:
thread:
user:
account_id:
status:
- template:
source: timestamp
template: '{{ Replace .Value "," "." 1 }}'
- timestamp:
source: timestamp
format: 2006-01-02 15:04:05.999 -0700
- labels:
session:
thread:
user:
account_id:
status: