监控神器Prometheus（11）

启动服务

systemctl enable consul-server1 consul-server2 consul-server3
systemctl stop consul-server1 consul-server2 consul-server3
systemctl restart consul-server1 consul-server2 consul-server3
systemctl status consul-server1 consul-server2 consul-server3

查看集群
返回空节点是正常的，因为开启了ACL，所以访问的时候需要加入token

# 环境变量
cat >> /etc/profile << EOF
export CONSUL_HTTP_TOKEN='your_token'
EOF

# consul members  --token='your_token'
Node     Address              Status  Type    Build  Protocol  DC          Segment
server1  172.26.42.229:8301   alive   server  1.6.0  2         prometheus  <all>
server2  172.26.42.229:28301  alive   server  1.6.0  2         prometheus  <all>
server3  172.26.42.229:38301  alive   server  1.6.0  2         prometheus  <all>

# 验证集群UI
在页面http://127.0.0.1:8500/ui/prometheus/acls/tokens 输入配置中的 master token，再刷新界面可以在services和nodes中查看到信息

# 验证API，通过在header中增加x-consul-token则可返回节点列表
curl http://127.0.0.1:8500/v1/catalog/nodes -H 'x-consul-token: ${CONSUL_HTTP_TOKEN}'

将Consul日志加入Syslog
此处为可选项，如果你需要单独将日志输出到ELK，那么此项配置非常有必要，因为默认的日志都打到syslog中了。

# 创建目录&赋权
mkdir -p /var/log/consul/
chown -R syslog.syslog /var/log/consul/

# 创建日志配置文件
cat >/etc/rsyslog.d/consul.conf <<EOF
local0.* /var/log/consul/consul.log
EOF

# 修改默认配置文件中的以下内容
vim /etc/rsyslog.d/50-default.conf

# 变更前
*.*;auth,authpriv.none          -/var/log/syslog

# 变更后
*.*;auth,authpriv.none,local0.none          -/var/log/syslog

# 重启rsyslog让配置生效。
$ systemctl restart rsyslog

# 创建日志轮循规则
$ cat >/etc/logrotate.d/consul <<EOF
/var/log/consul/*log {
missingok
compress
notifempty
daily
rotate 5
create 0600 root root
}
EOF

# 在Systemd启动脚本中加入`-syslog`参数
sed -i 's@ExecStart=/usr/local/bin/consul agent@ExecStart=/usr/local/bin/consul agent -syslog@g' /lib/systemd/system/consul-server{1..3}.service

# 重启服务
systemctl restart consul-server1 consul-server2 consul-server3

# 查看输出日志，对于加入ELK的配置就不过多描述了，如果想了解，加入我们的qq群与微信群咨询相关解决方案。
tail -f /var/log/consul/consul.log

FAQ: 如果集群加入失败，清除/data/consul/server{1..3}/data/目录下的数据，重启服务即可。

Prometheus集成Consul

# 基于AWS EC2 REDIS 发现规则

cat >> /data/prometheus/conf/prometheus.yml <<EOF
  - job_name: 'ec2_exporter'
      consul_sd_configs:
          - server: 172.26.42.229:8500
          token: '${CONSUL_HTTP_TOKEN}'
          services: ['node_exporter']
      relabel_configs:
          - source_labels: [__address__]
          regex: 172.26.42.229:8300
          action: drop
          - source_labels: [__meta_consul_tags]
          regex: ".*,prod,.*"
          replacement: prod
          action: replace
          target_label: env

  - job_name: 'redis_exporter'
      consul_sd_configs:
          - server: 172.26.42.229:28500
          token: '${CONSUL_HTTP_TOKEN}'
          services: ['redis_exporter']
      relabel_configs:
          - source_labels: [__address__]
          regex: 172.26.42.229:28300
          action: drop
          - source_labels: [__meta_consul_tags]
          regex: ".*,prod,.*"
          replacement: prod
          action: replace
          target_label: env
  - job_name: 'mysql_exporter'
      consul_sd_configs:
          - server: 172.26.42.229:38500
          token: '${CONSUL_HTTP_TOKEN}'
          services: ['mysql_exporter']
      relabel_configs:
          - source_labels: [__address__]
          regex: 172.26.42.229:38300
          action: drop
          - source_labels: [__meta_consul_tags]
          regex: ".*,prod,.*"
          replacement: prod
          action: replace
          target_label: env
EOF


# registered nginx01 service to consul1
curl -H "x-consul-token: ${CONSUL_HTTP_TOKEN}" -X PUT -d '{"ID": "node_exporter01", "Name": "node_exporter", "Address": "172.26.42.229", "Port": 9100, "Tags": ["prod"], "EnableTagOverride": false}' \
http://172.26.42.229:28500/v1/agent/service/register

# registered redis01 to consul2
curl -H "x-consul-token: ${CONSUL_HTTP_TOKEN}" -X PUT -d '{"ID": "redis_exporter01", "Name": "redis_exporter", "Address": "172.26.42.229", "Port": 9121, "Tags": ["prod"], "EnableTagOverride": false}' \
http://172.26.42.229:28500/v1/agent/service/register

# registered mysql01 to consul2
curl -H "x-consul-token: ${CONSUL_HTTP_TOKEN}" -X PUT -d '{"ID": "mysql_exporter01", "Name": "mysql_exporter", "Address": "172.26.42.229", "Port": 9105, "Tags": ["prod"], "EnableTagOverride": false}' \
http://172.26.42.229:38500/v1/agent/service/register 

# delete nginx01 service 
curl -X PUT -H "x-consul-token: ${CONSUL_HTTP_TOKEN}" http://172.26.42.229:8500/v1/agent/service/deregister/node_exporter01

# delete redis01 service
curl -X PUT -H "x-consul-token: ${CONSUL_HTTP_TOKEN}" http://172.26.42.229:28500/v1/agent/service/deregister/redis_exporter01

# delete mysql_exporter01 service
curl -X PUT -H "x-consul-token: ${CONSUL_HTTP_TOKEN}" http://172.26.42.229:28500/v1/agent/service/deregister/mysql_exporter01

欢迎大家关注我的公众号ID:k8stech

文章转自公众号：Kubernetes技术栈