监控神器Prometheus（11）

systemctl enable consul-server1 consul-server2 consul-server3
systemctl stop consul-server1 consul-server2 consul-server3
systemctl restart consul-server1 consul-server2 consul-server3
systemctl status consul-server1 consul-server2 consul-server3
1.
2.
3.
4.

# 环境变量
cat >> /etc/profile << EOF
export CONSUL_HTTP_TOKEN='your_token'
EOF

# consul members  --token='your_token'
Node     Address              Status  Type    Build  Protocol  DC          Segment
server1  172.26.42.229:8301   alive   server  1.6.0  2         prometheus  <all>
server2  172.26.42.229:28301  alive   server  1.6.0  2         prometheus  <all>
server3  172.26.42.229:38301  alive   server  1.6.0  2         prometheus  <all>

# 验证集群UI
在页面http://127.0.0.1:8500/ui/prometheus/acls/tokens 输入配置中的 master token，再刷新界面可以在services和nodes中查看到信息

# 验证API，通过在header中增加x-consul-token则可返回节点列表
curl http://127.0.0.1:8500/v1/catalog/nodes -H 'x-consul-token: ${CONSUL_HTTP_TOKEN}'
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.

# 创建目录&赋权
mkdir -p /var/log/consul/
chown -R syslog.syslog /var/log/consul/

# 创建日志配置文件
cat >/etc/rsyslog.d/consul.conf <<EOF
local0.* /var/log/consul/consul.log
EOF

# 修改默认配置文件中的以下内容
vim /etc/rsyslog.d/50-default.conf

# 变更前
*.*;auth,authpriv.none          -/var/log/syslog

# 变更后
*.*;auth,authpriv.none,local0.none          -/var/log/syslog

# 重启rsyslog让配置生效。
$ systemctl restart rsyslog

# 创建日志轮循规则
$ cat >/etc/logrotate.d/consul <<EOF
/var/log/consul/*log {
missingok
compress
notifempty
daily
rotate 5
create 0600 root root
}
EOF

# 在Systemd启动脚本中加入`-syslog`参数
sed -i 's@ExecStart=/usr/local/bin/consul agent@ExecStart=/usr/local/bin/consul agent -syslog@g' /lib/systemd/system/consul-server{1..3}.service

# 重启服务
systemctl restart consul-server1 consul-server2 consul-server3

# 查看输出日志，对于加入ELK的配置就不过多描述了，如果想了解，加入我们的qq群与微信群咨询相关解决方案。
tail -f /var/log/consul/consul.log
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.
34.
35.
36.
37.
38.
39.
40.
41.

# 基于AWS EC2 REDIS 发现规则

cat >> /data/prometheus/conf/prometheus.yml <<EOF
  - job_name: 'ec2_exporter'
      consul_sd_configs:
          - server: 172.26.42.229:8500
          token: '${CONSUL_HTTP_TOKEN}'
          services: ['node_exporter']
      relabel_configs:
          - source_labels: [__address__]
          regex: 172.26.42.229:8300
          action: drop
          - source_labels: [__meta_consul_tags]
          regex: ".*,prod,.*"
          replacement: prod
          action: replace
          target_label: env

  - job_name: 'redis_exporter'
      consul_sd_configs:
          - server: 172.26.42.229:28500
          token: '${CONSUL_HTTP_TOKEN}'
          services: ['redis_exporter']
      relabel_configs:
          - source_labels: [__address__]
          regex: 172.26.42.229:28300
          action: drop
          - source_labels: [__meta_consul_tags]
          regex: ".*,prod,.*"
          replacement: prod
          action: replace
          target_label: env
  - job_name: 'mysql_exporter'
      consul_sd_configs:
          - server: 172.26.42.229:38500
          token: '${CONSUL_HTTP_TOKEN}'
          services: ['mysql_exporter']
      relabel_configs:
          - source_labels: [__address__]
          regex: 172.26.42.229:38300
          action: drop
          - source_labels: [__meta_consul_tags]
          regex: ".*,prod,.*"
          replacement: prod
          action: replace
          target_label: env
EOF


# registered nginx01 service to consul1
curl -H "x-consul-token: ${CONSUL_HTTP_TOKEN}" -X PUT -d '{"ID": "node_exporter01", "Name": "node_exporter", "Address": "172.26.42.229", "Port": 9100, "Tags": ["prod"], "EnableTagOverride": false}' \
http://172.26.42.229:28500/v1/agent/service/register

# registered redis01 to consul2
curl -H "x-consul-token: ${CONSUL_HTTP_TOKEN}" -X PUT -d '{"ID": "redis_exporter01", "Name": "redis_exporter", "Address": "172.26.42.229", "Port": 9121, "Tags": ["prod"], "EnableTagOverride": false}' \
http://172.26.42.229:28500/v1/agent/service/register

# registered mysql01 to consul2
curl -H "x-consul-token: ${CONSUL_HTTP_TOKEN}" -X PUT -d '{"ID": "mysql_exporter01", "Name": "mysql_exporter", "Address": "172.26.42.229", "Port": 9105, "Tags": ["prod"], "EnableTagOverride": false}' \
http://172.26.42.229:38500/v1/agent/service/register 

# delete nginx01 service 
curl -X PUT -H "x-consul-token: ${CONSUL_HTTP_TOKEN}" http://172.26.42.229:8500/v1/agent/service/deregister/node_exporter01

# delete redis01 service
curl -X PUT -H "x-consul-token: ${CONSUL_HTTP_TOKEN}" http://172.26.42.229:28500/v1/agent/service/deregister/redis_exporter01

# delete mysql_exporter01 service
curl -X PUT -H "x-consul-token: ${CONSUL_HTTP_TOKEN}" http://172.26.42.229:28500/v1/agent/service/deregister/mysql_exporter01
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.
34.
35.
36.
37.
38.
39.
40.
41.
42.
43.
44.
45.
46.
47.
48.
49.
50.
51.
52.
53.
54.
55.
56.
57.
58.
59.
60.
61.
62.
63.
64.
65.
66.
67.
68.
69.