使用 Vector 将 PostgreSQL 日志输出为 Prometheus 指标

2022-05-12 07:33:54.285 UTC  2672031 @ from  [vxid: txid:0] [] LOG:  checkpoint complete: wrote 64 buffers (0.0%); 0 WAL file(s) added, 0 removed, 10 recycled; write=6.266
s, sync=0.004 s, total=6.285 s; sync files=10, longest=0.003 s, average=0.001 s; distance=163840 kB, estimate=163844 kB
1.
2.

pipeline_stages:
  - multiline:
      firstline: '^\[\d{4}-\d{2}-\d{2} \d{1,2}:\d{2}:\d{2}\]'
  - regex:
      expression: '^(?P\[\d{4}-\d{2}-\d{2} \d{1,2}:\d{2}:\d{2}\]) (?P(?s:.*))$'
  - metrics:
     log_lines_total:
       type: Counter
       description: "total number of log lines"
       prefix: pg_custom_
       max_idle_duration: 24h
       config:
         match_all: true
         action: inc
     log_bytes_total:
       type: Counter
       description: "total bytes of log lines"
       prefix: pg_custom_
       max_idle_duration: 24h
       config:
         match_all: true
         count_entry_bytes: true
         action: add
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.

# playbook-vector.yaml
---
- name: Setup vector
 hosts:
   - pg
 become: yes
 vars:
   arch: amd64
   version: 0.18.1
   vector_template: files/40-vector.toml
   vector_config_file: /etc/vector/vector.toml
 tasks:
   - name: Setup install vector
     become: yes
     apt:
       deb: "https://packages.timber.io/vector/{{ version }}/vector-{{ version }}-{{ arch }}.deb"
       install_recommends: yes
     notify:
       - restart vector

   - name: Copy config
     copy:
       src: "{{ vector_template }}"
       dest: "{{ vector_config_file }}"
       mode: 0644
       owner: vector
       group: vector
     notify: restart vector

   - name: Start Vector
     service:
       state: started
       enabled: yes
       name: vector

 handlers:
   - name: restart vector
     service:
       state: restarted
       daemon_reload: yes
       name: vector
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.
34.
35.
36.
37.
38.
39.
40.
41.

# vector.toml
[sources.postgres_logs.multiline]
start_pattern = '^\d{4}-[0-1]\d-[0-3]\d \d+:\d+:\d+\.\d+ [A-Z]{3}'
mode = "halt_before"
condition_pattern = '^\d{4}-[0-1]\d-[0-3]\d \d+:\d+:\d+\.\d+ [A-Z]{3}'
timeout_ms = 1000
1.
2.
3.
4.
5.
6.

# vector.toml
[transforms.postgres_remap]
type   = "remap"
inputs = [ "postgres_logs" ]
source = """. |= parse_regex!(.message, r'^(?P\\d{4}-[0-1]\\d-[0-3]\\d \\d+:\\d+:\\d+\\.\\d+ [A-Z]{3})  (?P\\d+) (?P(\\[\\w+\\]@\\w+|@|\\w+@\\w+)) from (?P(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}|\\[\\w+\\]|\\s*)) (?P\\[\\w+:.+:\\d+\\]) (?P(\\[\\]|\\[\\w.+\\])) (?P.*[A-Z]):  (?P.*)$')del(.timestamp)message_parts, err = split(.message, ", ", limit: 2)structured = parse_key_value(message_parts[1], key_value_delimiter: ":", field_delimiter: ",") ?? {}message = message_parts[0]. = merge(., structured)del(."please try the setup again")del(.message)"""
1.
2.
3.
4.
5.

# vector.toml
[transforms.postgres_filter]
type      = "filter"
inputs    = [ "postgres_remap" ]
condition = '.level == "ERROR" || .level == "FATAL"'
1.
2.
3.
4.
5.

# vector.toml
[transforms.postgres_metric]
type   = "log_to_metric"
inputs = [ "postgres_filter" ]

  [[transforms.postgres_metric.metrics]]
  type      = "counter"
  field     = "level"
  name      = "error_total"
  namespace = "pg_log"

    [transforms.postgres_metric.metrics.tags]
    level = "{{level}}"
    host  = "{{host}}"
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.

[sinks.postgres_export_metric]
type              = "prometheus_exporter"
inputs            = [ "postgres_metric" ]
address           = "0.0.0.0:9598"
default_namespace = "pg_log"
1.
2.
3.
4.
5.

scrape_configs:
  - job_name: custom-pg-log-exporter
    static_configs:
      - targets: ['10.10.10.2:9598', '10.10.10.3:9598', '10.10.10.4:9598']
1.
2.
3.
4.

- alert: PgErrorCountChangeWarning
    expr: |        increase(pg_log_error_total{level="ERROR"}[30m]) > 0
    for: 10m
    labels:
      severity: warning
    annotations:
      summary: The amount of errors in pg host {{$labels.host}} log has changed to {{$value}}
      description: |       There are errors in the PostgreSQL logs on the {{$labels.host}} server.
- alert: PgErrorCountChangeCritical
    expr: |        increase(pg_log_error_total{level="FATAL"}[30m]) > 0
    for: 10m
    labels:
      severity: critical
    annotations:
      summary: The amount of fatal errors in pg host {{$labels.host}} log has changed to {{$value}}
      description: |
        There are fatal errors in the PostgreSQL logs on the {{$labels.host}} server.
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.