使用 Vector 将 PostgreSQL 日志输出为 Prometheus 指标

2022-05-12 07:33:54.285 UTC  2672031 @ from  [vxid: txid:0] [] LOG:  checkpoint complete: wrote 64 buffers (0.0%); 0 WAL file(s) added, 0 removed, 10 recycled; write=6.266
s, sync=0.004 s, total=6.285 s; sync files=10, longest=0.003 s, average=0.001 s; distance=163840 kB, estimate=163844 kB

pipeline_stages:
  - multiline:
      firstline: '^\[\d{4}-\d{2}-\d{2} \d{1,2}:\d{2}:\d{2}\]'
  - regex:
      expression: '^(?P\[\d{4}-\d{2}-\d{2} \d{1,2}:\d{2}:\d{2}\]) (?P(?s:.*))$'
  - metrics:
     log_lines_total:
       type: Counter
       description: "total number of log lines"
       prefix: pg_custom_
       max_idle_duration: 24h
       config:
         match_all: true
         action: inc
     log_bytes_total:
       type: Counter
       description: "total bytes of log lines"
       prefix: pg_custom_
       max_idle_duration: 24h
       config:
         match_all: true
         count_entry_bytes: true
         action: add

# playbook-vector.yaml
---
- name: Setup vector
 hosts:
   - pg
 become: yes
 vars:
   arch: amd64
   version: 0.18.1
   vector_template: files/40-vector.toml
   vector_config_file: /etc/vector/vector.toml
 tasks:
   - name: Setup install vector
     become: yes
     apt:
       deb: "https://packages.timber.io/vector/{{ version }}/vector-{{ version }}-{{ arch }}.deb"
       install_recommends: yes
     notify:
       - restart vector

   - name: Copy config
     copy:
       src: "{{ vector_template }}"
       dest: "{{ vector_config_file }}"
       mode: 0644
       owner: vector
       group: vector
     notify: restart vector

   - name: Start Vector
     service:
       state: started
       enabled: yes
       name: vector

 handlers:
   - name: restart vector
     service:
       state: restarted
       daemon_reload: yes
       name: vector

# vector.toml
[sources.postgres_logs.multiline]
start_pattern = '^\d{4}-[0-1]\d-[0-3]\d \d+:\d+:\d+\.\d+ [A-Z]{3}'
mode = "halt_before"
condition_pattern = '^\d{4}-[0-1]\d-[0-3]\d \d+:\d+:\d+\.\d+ [A-Z]{3}'
timeout_ms = 1000

# vector.toml
[transforms.postgres_remap]
type   = "remap"
inputs = [ "postgres_logs" ]
source = """. |= parse_regex!(.message, r'^(?P\\d{4}-[0-1]\\d-[0-3]\\d \\d+:\\d+:\\d+\\.\\d+ [A-Z]{3})  (?P\\d+) (?P(\\[\\w+\\]@\\w+|@|\\w+@\\w+)) from (?P(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}|\\[\\w+\\]|\\s*)) (?P\\[\\w+:.+:\\d+\\]) (?P(\\[\\]|\\[\\w.+\\])) (?P.*[A-Z]):  (?P.*)$')del(.timestamp)message_parts, err = split(.message, ", ", limit: 2)structured = parse_key_value(message_parts[1], key_value_delimiter: ":", field_delimiter: ",") ?? {}message = message_parts[0]. = merge(., structured)del(."please try the setup again")del(.message)"""

# vector.toml
[transforms.postgres_filter]
type      = "filter"
inputs    = [ "postgres_remap" ]
condition = '.level == "ERROR" || .level == "FATAL"'

# vector.toml
[transforms.postgres_metric]
type   = "log_to_metric"
inputs = [ "postgres_filter" ]

  [[transforms.postgres_metric.metrics]]
  type      = "counter"
  field     = "level"
  name      = "error_total"
  namespace = "pg_log"

    [transforms.postgres_metric.metrics.tags]
    level = "{{level}}"
    host  = "{{host}}"

[sinks.postgres_export_metric]
type              = "prometheus_exporter"
inputs            = [ "postgres_metric" ]
address           = "0.0.0.0:9598"
default_namespace = "pg_log"

scrape_configs:
  - job_name: custom-pg-log-exporter
    static_configs:
      - targets: ['10.10.10.2:9598', '10.10.10.3:9598', '10.10.10.4:9598']

- alert: PgErrorCountChangeWarning
    expr: |        increase(pg_log_error_total{level="ERROR"}[30m]) > 0
    for: 10m
    labels:
      severity: warning
    annotations:
      summary: The amount of errors in pg host {{$labels.host}} log has changed to {{$value}}
      description: |       There are errors in the PostgreSQL logs on the {{$labels.host}} server.
- alert: PgErrorCountChangeCritical
    expr: |        increase(pg_log_error_total{level="FATAL"}[30m]) > 0
    for: 10m
    labels:
      severity: critical
    annotations:
      summary: The amount of fatal errors in pg host {{$labels.host}} log has changed to {{$value}}
      description: |
        There are fatal errors in the PostgreSQL logs on the {{$labels.host}} server.