监控神器Prometheus（3）

#创建服务相关文件目录
ALERT_PATH='/data/alertmanager'
mkdir -p ${ALERT_PATH}/{bin,conf,logs,templates}
cd /usr/local/src/
wget https://github.com/prometheus/alertmanager/releases/download/v0.18.0/alertmanager-0.18.0.linux-amd64.tar.gz && tar -xvf alertmanager-0.18.0.linux-amd64.tar.gz
cp alertmanager-0.18.0.linux-amd64/alertmanager /data/prometheus/alertmanager/bin/
cp alertmanager-0.18.0.linux-amd64/amtool /data/prometheus/alertmanager/bin/
chown -R prometheus.prometheus /data/alertmanager

# Setting Variables
cat >> /etc/profile <<EOF
PATH=/data/alertmanager/bin:$PATH:$HOME/bin
EOF
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.

cat > /data/alertmanager/conf/alertmanager.yml <<EOF
global:
  resolve_timeout: 5m
  # smtp配置
  smtp_from: "prom-alert@example.com"
  smtp_smarthost: 'email-smtp.us-west-2.amazonaws.com:465'
  smtp_auth_username: "user"
  smtp_auth_password: "pass"
  smtp_require_tls: true
templates:
  - '/data/alertmanager/templates/*.tmpl'
route:
  receiver: test1
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  group_by: [alertname]
  routes:
# ads webhook
  - receiver: test1
    group_wait: 10s
    match:
      team: ads
# ops webhook
  - receiver: test2
    group_wait: 10s
    match:
      team: operations
receivers:
- name: test1
  email_configs:
  - to: '9935226@qq.com'
    headers: { Subject: "[ads] 报警邮件"} # 接收邮件的标题
  webhook_configs:
  - url: http://localhost:8060/dingtalk/ads/send

- name: test2
  email_configs:
  - to: '9935226@qq.com,deniss.wang@gmail.com'
    send_resolved: true
    headers: { Subject: "[ops] 报警邮件"} # 接收邮件的标题
  webhook_configs:
  - url: http://localhost:8060/dingtalk/ops/send
    # wx config
  wechat_configs:
  - corp_id: 'wwxxxxxxxxxxxxxx'
    api_url: 'https://qyapi.weixin.qq.com/cgi-bin/'
    send_resolved: true
    to_party: '2'
    agent_id: '1000002'
    api_secret: '1FvHxuGbbG35FYsuW0YyI4czWY/.2'
EOF
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.
34.
35.
36.
37.
38.
39.
40.
41.
42.
43.
44.
45.
46.
47.
48.
49.
50.
51.
52.

# alertmanager.service
cat >/lib/systemd/system/alertmanager.service<<\EOF
[Unit]
Description=Prometheus: the alerting system
Documentation=http://prometheus.io/docs/
After=prometheus.service

[Service]
ExecStart=/data/alertmanager/bin/alertmanager --config.file=/data/alertmanager/conf/alertmanager.yml
Restart=always
StartLimitInterval=0
RestartSec=10

[Install]
WantedBy=multi-user.target
EOF

# 启动服务
systemctl enable alertmanager.service
systemctl start alertmanager.service
systemctl status alertmanager.service

# 查看端口是否正常
netstat -anplt|grep 9093
tcp6       0      0 :::9093                 :::*                    LISTEN      1547/alertmanager
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.

cd /usr/local/src/
wget https://github.com/timonwong/prometheus-webhook-dingtalk/releases/download/v0.3.0/prometheus-webhook-dingtalk-0.3.0.linux-amd64.tar.gz
tar -zxvf prometheus-webhook-dingtalk-0.3.0.linux-amd64.tar.gz
mv prometheus-webhook-dingtalk-0.3.0.linux-amd64 /data/alertmanager/webhook-dingtalk
# 创建Systemd webhook-dingtalk 服务
cat > /etc/systemd/system/webhook-dingtalk.service << EOF
[Unit]
Description=webhook-dingding
Documentation=https://prometheus.io/
After=network.target

[Service]
Type=simple
User=prometheus
ExecStart=/data/alertmanager/webhook-dingtalk/prometheus-webhook-dingtalk \
--ding.profile="ads=https://oapi.dingtalk.com/robot/send?access_token=284de68124e97420a2ee8ae1b8f12fabe3213213213" \
--ding.profile="ops=https://oapi.dingtalk.com/robot/send?access_token=8bce3bd11f7040d57d44caa5b6ef9417eab24e1123123123213" 
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF
# 启动服务
systemctl enable webhook-dingtalk
systemctl start webhook-dingtalk
systemctl status webhook-dingtalk
# 查看端口是否正常
netstat -anplt|grep 8060
tcp6       0      0 :::8060                 :::*                    LISTEN      1635/prometheus-web
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.

# Pull Image
docker pull timonwong/prometheus-webhook-dingtalk:v0.3.0
# 启动
docker run -d --restart always -p 8060:8060 timonwong/prometheus-webhook-dingtalk:v0.3.0 --ding.profile="ads=https://oapi.dingtalk.com/robot/send?access_token=284de68124e97420a2ee8ae1b8f12fabe3213213213" \
--ding.profile="ops=https://oapi.dingtalk.com/robot/send?access_token=8bce3bd11f7040d57d44caa5b6ef9417eab24e1123123123213"
1.
2.
3.
4.
5.

# Ubuntu
wget https://dl.grafana.com/oss/release/grafana_6.4.0_amd64.deb
sudo dpkg -i grafana_6.4.0_amd64.deb

# Centos7
yum localinstall -y https://dl.grafana.com/oss/release/grafana-6.4.4-1.x86_64.rpm

# 安装拼图、Consul数据源插件
grafana-cli plugins install grafana-piechart-panel
grafana-cli plugins install sbueringer-consul-datasource
systemctl restart grafana-server.service

# 启动服务
systemctl enable grafana-server.service
systemctl start grafana-server.service
systemctl status grafana-server.service

# nginx 配置
cat > /etc/nginx/conf.d/grafana.k8stech.net.conf << EOF
server {
    listen       80;
    server_name  grafana.k8stech.net;

    location / {
            try_files $uri @grafana;
    }

    location @grafana {
            internal;
            proxy_pass http://localhost:3000;
    }
}
EOF
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.