监控逻辑：
定时监控服务运行->发现监控服务异常->写入错误值1到文件->node_exporter采集文件状态值->通过Prometheus->Grafana设置预警规则->Grafana设置告警推送->人员修复->定时监控服务运行->发现监控服务正常->写入正确值0到文件->node_exporter采集文件状态值->通过Prometheus->Grafana设置预警规则->Grafana设置告警推送【OK】
--collector.textfile.directory=./textfile_collector 定义文件目录,读取 middleware.prom
zhongts 的个人博客
nohup ./node_exporter \
--web.listen-address=:9100 \
--collector.textfile.directory=./textfile_collector \
> node_exporter.log 2>&1 &
#!/bin/bash
# mkdir /server/app/node_exporter-1.7.0.linux-amd64/monitor/
# vi /server/app/node_exporter-1.7.0.linux-amd64/monitor/server_monitor.sh
# chmod +x /server/app/node_exporter-1.7.0.linux-amd64/monitor/server_monitor.sh
# mkdir /server/app/node_exporter-1.7.0.linux-amd64/textfile_collector
# chmod 755 /server/app/node_exporter-1.7.0.linux-amd64/textfile_collector
# 服务监控
# */5 * * * * /server/app/node_exporter-1.7.0.linux-amd64/monitor/server_monitor.sh
# 日志与指标路径:q：：:：:
LOG_FILE="/server/app/node_exporter-1.7.0.linux-amd64/monitor/service_monitor.log"
TEXT_DIR="/server/app/node_exporter-1.7.0.linux-amd64/textfile_collector"
tmp_file="${TEXT_DIR}/middleware.tmp"
prom_file="${TEXT_DIR}/middleware.prom"
# 初始化临时文件，清空上一轮指标
> "$tmp_file"
# 日志函数
log() {
local now=$(date "+%Y-%m-%d %H:%M:%S")
echo "[$now] $1" >> "$LOG_FILE"
echo "[$now] $1"
}
# 服务名称,容器ID
SERVICE_LIST=(
msc-yunwei_bridge,cfda59f506ba
msc-yunwei_bridgeapp,48c86ce44007
infrst-chengdu_lifeline-data-app,7f16c80c8033
msc-yunwei_tunnel,df6a6b2306bd
infrst-chengdu_lifeline-third-party-api,1c867fbd315a
infrst-chengdu_lifeline-model-app,90e1b59a351f
dlls_lifeline-bridge-weigh-app,b468c31bdfe6
)
# 检测服务
check_service() {
local item="$1"
local s_name=$(echo "$item" | cut -d',' -f1)
local s_cid=$(echo "$item" | cut -d',' -f2)
if docker ps --filter "id=${s_cid}" | grep -q .; then
log "正常：$s_name 运行正常，容器ID：$s_cid"
echo "service_status1{svc=\"$s_name\"} 0" >> "$tmp_file"
else
log "异常：$s_name 容器已停止，执行重启"
# docker restart "$s_cid"
sleep 10
# 重启后二次校验
if docker ps --filter "id=${s_cid}" | grep -q .; then
log "正常：$s_name 重启成功，容器ID：$s_cid"
echo "service_status1{svc=\"$s_name\"} 0" >> "$tmp_file"
else
log "严重异常：$s_name 重启失败，需人工排查，容器ID：$s_cid"
echo "service_status1{svc=\"$s_name\"} 1" >> "$tmp_file"
fi
fi
}
log "===== 本轮所有服务检测开始 ====="
for line in "${SERVICE_LIST[@]}"; do
check_service "$line"
done
# 替换prom文件
{
echo '# HELP service_status1 服务运行状态 0正常 1异常'
echo '# TYPE service_status1 gauge'
cat "$tmp_file"
} > "${TEXT_DIR}/final.tmp"
mv -f "${TEXT_DIR}/final.tmp" "$prom_file"
log "===== 本轮所有服务检测完毕 ====="
#!/bin/bash
# /data/server/service/server_monitor.sh
# mkdir /root/node_exporter-1.7.0.linux-amd64/textfile_collector
# chmod 755 /root/node_exporter-1.7.0.linux-amd64/textfile_collector
# 服务监控
# */5 * * * * /data/server/service/server_monitor.sh
SERVICE_LIST=(
"/data/server/teuip/,teuip"
"/data/server/service/dataops-webapi-main-2.1/bin/,dataops-webapi-main"
"/data/server/service/admin-webapi-main-2.1/bin/,admin-webapi-main"
"/data/server/service/datasecurity-webapi-main-2.1/bin/,datasecurity-webapi-main"
"/data/server/service/datalog-2.1/bin/,datalog"
"/data/server/service/dataquality-webapi-main-2.1/bin/,dataquality-webapi-main"
"/data/server/service/dataman-webapi-main-2.1/bin/,dataman-webapi-main"
"/data/server/service/dataservice-webapi-main-2.1/bin/,dataservice-webapi-main"
"/data/server/service/springgateway-2.1/bin/,springgateway"
"/data/server/service/dataprocess-webapi-main-2.1/bin/,dataprocess-webapi-main"
)
# 日志与指标路径
LOG_FILE="/data/server/service_monitor.log"
TEXT_DIR="/root/node_exporter-1.7.0.linux-amd64/textfile_collector"
tmp_file="${TEXT_DIR}/middleware.tmp"
prom_file="${TEXT_DIR}/middleware.prom"
# 初始化临时文件，清空上一轮指标
> "$tmp_file"
# 日志函数
log() {
local now=$(date "+%Y-%m-%d %H:%M:%S")
echo "[$now] $1" >> "$LOG_FILE"
echo "[$now] $1"
}
# 检测服务
check_service() {
local item="$1"
# 逗号分割 目录、进程关键字
local s_dir=$(echo "$item" | cut -d',' -f1)
local s_key=$(echo "$item" | cut -d',' -f2)
local s_name="$s_key"
# 切换目录
log "===== 检测服务：$s_name ====="
cd "$s_dir" || {
log "错误：目录 $s_dir 无法进入，跳过"
echo "service_status{svc=\"$s_name\"} 1" >> "$tmp_file"
return
}
# 获取PID
local pid=$(./start.sh status 2>/dev/null | grep -i "pid is" | awk '{print $NF}')
# 双重判断：PID非空 && 进程存在
if [ -n "$pid" ] && ps -p "$pid" > /dev/null 2>&1; then
log "正常：$s_name 运行正常，PID:$pid"
echo "service_status{svc=\"$s_name\"} 0" >> "$tmp_file"
else
log "异常：$s_name 已宕机，执行重启"
# 重启命令
# ./start.sh restart
sleep 15
# 重启后再次校验
local new_pid=$(./start.sh status 2>/dev/null | grep -i "pid is" | awk '{print $NF}')
if [ -n "正常：$new_pid" ] && ps -p "$new_pid" > /dev/null 2>&1; then
# 刷新采集
log "正常：$s_name 重启成功，新PID:$new_pid"
echo "service_status{svc=\"$s_name\"} 0" >> "$tmp_file"
else
log "错误：$s_name 重启失败，正在通知人工排查。"
echo "service_status{svc=\"$s_name\"} 1" >> "$tmp_file"
fi
fi
}
log "===== 本轮所有服务检测开始 ====="
# 循环检测所有业务服务
for line in "${SERVICE_LIST[@]}"; do
check_service "$line"
done
# 替换prom文件
{
echo '# HELP service_status 服务运行状态 0正常 1异常'
echo '# TYPE service_status gauge'
cat "$tmp_file"
} > "${TEXT_DIR}/final.tmp"
mv -f "${TEXT_DIR}/final.tmp" "$prom_file"
log "===== 本轮所有服务检测完毕 ====="