# 1. 通知升级（避免重复通知）
define serviceescalation {
    hostgroup_name          all-servers
    service_description     *
    first_notification      1
    last_notification       3
    notification_interval   5
    contact_groups          l1-ops
    escalation_options      c,r
}

define serviceescalation {
    hostgroup_name          all-servers
    service_description     *
    first_notification      4
    last_notification       0
    notification_interval   30
    contact_groups          l2-ops
    escalation_options      c
}

# 2. 依赖关系（避免级联告警）
define servicedependency {
    host_name                       db-master
    service_description             MySQL
    dependent_host_name             web-server-01
    dependent_service_description   HTTP
    execution_failure_criteria      w,u,c
    notification_failure_criteria   w,u,c
}

# 3. 维护窗口（变更期间静默）
# 通过脚本批量设置
./schedule_maintenance.sh webservers 120 admin "Planned upgrade v2.1"

# 4. 仅通知 CRITICAL（过滤低级别告警）
define service {
    ...
    notification_options    c,r   # 仅通知 CRITICAL 和 RECOVERY
}

# 初始化 Git 仓库
cd /etc/nagios
git init
git add .
git commit -m "Initial Nagios configuration"

# 配置变更
vim /etc/nagios/hosts/production/webservers.cfg
nagios -v /etc/nagios/nagios.cfg
git add .
git commit -m "feat: add web-server-03 to monitoring"

# 配置回滚
git log --oneline
git revert <commit-hash>
nagios -v /etc/nagios/nagios.cfg
systemctl restart nagios

# 定期备份
#!/bin/bash
# backup_nagios.sh
BACKUP_DIR="/backup/nagios/$(date +%Y%m%d)"
mkdir -p $BACKUP_DIR
tar czf $BACKUP_DIR/nagios-config-$(date +%Y%m%d).tar.gz /etc/nagios/
find /backup/nagios/ -mtime +90 -delete

# 1. 文件描述符限制
# /etc/security/limits.conf
nagios soft nofile 65535
nagios hard nofile 65535

# 2. 内核参数优化
# /etc/sysctl.conf
net.core.somaxconn = 65535
net.ipv4.tcp_max_syn_backlog = 65535
net.ipv4.tcp_tw_reuse = 1

# 3. I/O 调度器
# SSD 使用 noop/deadline
echo noop > /sys/block/sda/queue/scheduler

# 4. 文件系统优化
# 使用 noatime 挂载选项
# /etc/fstab
/dev/sda1 / ext4 defaults,noatime 0 1

# /usr/local/nagios/etc/nagios.cfg

# 1. 事件循环优化
event_broker_options=-1    # 启用事件代理

# 2. 检查调度优化
service_inter_check_delay_method=s  # 自动调度
service_interleave_factor=s         # 自动交错
host_inter_check_delay=5.0          # 主机检查间隔

# 3. 状态更新优化
status_update_interval=10           # 状态文件更新间隔
object_cache_file=/dev/shm/objects.cache  # 使用内存文件系统

# 4. 日志优化
log_level=LOG_NONE           # 生产环境减少日志
log_rotation_method=d        # 日志轮转
log_archive_path=/var/log/nagios/archives/
archive_logs=1

# 5. 通知优化
notification_timeout=30      # 通知超时

# 6. 被动检查优化
check_result_path=/dev/shm/checkresults  # 使用 tmpfs
max_check_result_file_age=3600

# 1. 分级检查间隔
# 关键服务：1 分钟
define service {
    use                     critical-service  # 自定义模板
    host_name               trade-server-01
    service_description     Trade API
    check_command           check_http!443!/api/trade
}

# 一般服务：5 分钟
define service {
    use                     generic-service
    host_name               web-server-01
    service_description     HTTP
    check_command           check_http!80!/
}

# 次要服务：15 分钟
define service {
    use                     low-priority-service
    host_name               dev-server-01
    service_description     Log Rotation
    check_command           check_logrotate
}

# 2. 使用被动检查替代频繁主动检查
define service {
    use                     generic-service
    host_name               external-api
    service_description     API Health
    passive_checks_enabled  1
    active_checks_enabled   0
    check_freshness         1
    freshness_threshold     300
    check_command           check_dummy!2!"No heartbeat received"
}

# 3. 避免检查风暴
# 错误：同时检查 100 台服务器的同一服务
# 正确：使用 check_interleave_factor 分散

# 关键性能指标
# 1. 检查延迟
grep "service_check_latency" /var/log/nagios/nagios.log | awk '{print $NF}' | sort -n | tail -10

# 2. 检查执行时间
grep "service_check_execution_time" /var/log/nagios/nagios.log | awk '{print $NF}' | sort -n | tail -10

# 3. 每秒检查数
grep "SERVICE ALERT" /var/log/nagios/nagios.log | wc -l

# 4. 进程资源使用
ps aux | grep nagios
top -p $(pgrep nagios)

# 5. 性能基准
# < 500 服务：单核 1G 内存足够
# 500-2000 服务：2 核 4G 内存
# 2000-5000 服务：4 核 8G 内存
# > 5000 服务：考虑分布式架构

#!/bin/bash
# sync_nagios_config.sh - 配置同步脚本

PRIMARY="nagios-master"
SECONDARY="nagios-slave"
CONFIG_DIR="/etc/nagios"
LOG_FILE="/var/log/nagios/sync.log"

log() {
    echo "$(date '+%Y-%m-%d %H:%M:%S') $1" >> $LOG_FILE
}

# 从主节点同步配置
log "Starting config sync from $PRIMARY"

rsync -avz --delete \
    ${PRIMARY}:${CONFIG_DIR}/ ${CONFIG_DIR}/ \
    --exclude='var/' \
    --exclude='rw/' \
    --exclude='*.pid' \
    --exclude='status.dat' \
    --exclude='retention.dat' \
    >> $LOG_FILE 2>&1

if [ $? -eq 0 ]; then
    log "Config sync successful"

    # 验证配置
    /usr/local/nagios/bin/nagios -v ${CONFIG_DIR}/nagios.cfg
    if [ $? -eq 0 ]; then
        log "Config validation passed, reloading Nagios"
        systemctl reload nagios
    else
        log "ERROR: Config validation failed"
    fi
else
    log "ERROR: Config sync failed"
fi

# /etc/keepalived/keepalived.conf (主节点)
vrrp_script check_nagios {
    script "/usr/local/bin/check_nagios.sh"
    interval 2
    weight 20
}

vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 51
    priority 100
    advert_int 1

    authentication {
        auth_type PASS
        auth_pass nagios_ha
    }

    virtual_ipaddress {
        10.0.0.100/24
    }

    track_script {
        check_nagios
    }

    notify_master "/usr/local/bin/notify_master.sh"
    notify_backup "/usr/local/bin/notify_backup.sh"
}

# check_nagios.sh
#!/bin/bash
if ! pgrep -x nagios > /dev/null; then
    exit 1
fi
if ! curl -sf -u nagiosadmin:password http://localhost/nagios/cgi-bin/status.cgi > /dev/null; then
    exit 1
fi
exit 0

# 1. 最小权限原则
# Nagios 使用专用低权限用户运行
useradd -r -s /bin/bash nagios

# 2. 文件权限
chmod 600 /etc/nagios/resource.cfg    # 敏感信息
chmod 640 /etc/nagios/nagios.cfg      # 配置文件
chmod 750 /usr/local/nagios/etc/      # 配置目录

# 3. SELinux 启用
setenforce 1
# 或配置 SELinux 策略

# 4. 定期更新
yum update -y
# 关注 CVE 安全公告

# 1. 启用 HTTPS
# Apache SSL 配置
<VirtualHost *:443>
    SSLEngine on
    SSLCertificateFile /etc/pki/tls/certs/nagios.crt
    SSLCertificateKeyFile /etc/pki/tls/private/nagios.key
</VirtualHost>

# 2. 强密码策略
htpasswd -c -B /etc/nagios/htpasswd.users nagiosadmin

# 3. IP 白名单
<Location "/nagios">
    Require ip 10.0.0.0/8
    Require ip 172.16.0.0/12
    Require ip 192.168.0.0/16
</Location>

# 4. 禁用目录列表
Options -Indexes

# 5. 启用安全头
Header always set X-Content-Type-Options nosniff
Header always set X-Frame-Options SAMEORIGIN
Header always set Content-Security-Policy "default-src 'self'"

# 1. SSL/TLS 强制启用
ssl_version=TLSv1.2+
ssl_cipher_list=HIGH:!aNULL:!MD5

# 2. IP 白名单
allowed_hosts=10.0.0.10,10.0.0.11

# 3. 禁用动态参数（生产环境）
dont_blame_nrpe=0

# 4. 防火墙限制
firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="10.0.0.10" port protocol="tcp" port="5666" accept'

#!/bin/bash
# auto_discover_and_monitor.sh - 自动发现并添加监控

NETWORK="192.168.1.0/24"
OUTPUT_DIR="/etc/nagios/hosts/auto_discovered"

mkdir -p $OUTPUT_DIR

# 自动发现存活主机
for IP in $(nmap -sn $NETWORK | grep "Nmap scan report" | awk '{print $5}'); do
    HOSTNAME=$(dig +short -x $IP | sed 's/\.$//')
    if [ -z "$HOSTNAME" ]; then
        HOSTNAME="auto-$(echo $IP | tr '.' '-')"
    fi

    # 检查是否已存在
    if ! grep -q "$HOSTNAME" /etc/nagios/hosts/**/*.cfg 2>/dev/null; then
        echo "Discovered new host: $HOSTNAME ($IP)"
        cat > $OUTPUT_DIR/${HOSTNAME}.cfg << EOF
define host {
    use             linux-server
    host_name       $HOSTNAME
    alias           Auto-discovered host
    address         $IP
    check_command   check_ping
}
EOF
    fi
done

# 验证并重载
nagios -v /etc/nagios/nagios.cfg && systemctl reload nagios

#!/usr/bin/env python3
# generate_nagios_config.py - 从 CMDB 生成 Nagios 配置

import json
import yaml

def load_hosts_from_cmdb():
    """从 CMDB 加载主机信息"""
    # 示例数据，实际应从 API 获取
    return [
        {"hostname": "web-01", "ip": "192.168.1.10", "type": "web", "env": "prod"},
        {"hostname": "db-01", "ip": "192.168.1.20", "type": "database", "env": "prod"},
        {"hostname": "web-02", "ip": "192.168.1.11", "type": "web", "env": "staging"},
    ]

def generate_host_config(host):
    """生成主机配置"""
    template = {
        "web": "linux-server-web",
        "database": "linux-server-db",
        "default": "linux-server"
    }

    return f"""define host {{
    use             {template.get(host['type'], template['default'])}
    host_name       {host['hostname']}
    alias           {host['hostname']} ({host['env']})
    address         {host['ip']}
    hostgroups      {host['type']}s-{host['env']}
}}
"""

def generate_service_config(host):
    """根据主机类型生成服务配置"""
    configs = {
        "web": f"""define service {{
    use                     generic-service
    host_name               {host['hostname']}
    service_description     HTTP
    check_command           check_http!80!/
}}

define service {{
    use                     generic-service
    host_name               {host['hostname']}
    service_description     HTTPS
    check_command           check_https!443!/
}}
""",
        "database": f"""define service {{
    use                     generic-service
    host_name               {host['hostname']}
    service_description     MySQL
    check_command           check_mysql_health!connection-time
}}
"""
    }
    return configs.get(host['type'], "")

def main():
    hosts = load_hosts_from_cmdb()

    with open("/etc/nagios/hosts/auto_generated.cfg", "w") as f:
        for host in hosts:
            f.write(generate_host_config(host))
            f.write(generate_service_config(host))

    print(f"Generated config for {len(hosts)} hosts")

if __name__ == "__main__":
    main()

# ansible/playbooks/nagios_deploy.yml
---
- name: Deploy Nagios Configuration
  hosts: nagios_servers
  become: yes
  vars:
    nagios_config_dir: /etc/nagios
    nagios_hosts:
      - hostname: web-01
        address: 192.168.1.10
        template: linux-server
      - hostname: db-01
        address: 192.168.1.20
        template: linux-server

  tasks:
    - name: Install Nagios
      yum:
        name: nagios
        state: present

    - name: Generate host configs
      template:
        src: templates/host.cfg.j2
        dest: "{{ nagios_config_dir }}/hosts/{{ item.hostname }}.cfg"
      loop: "{{ nagios_hosts }}"
      notify: Reload Nagios

    - name: Validate config
      command: nagios -v {{ nagios_config_dir }}/nagios.cfg
      register: config_check
      failed_when: config_check.rc != 0

  handlers:
    - name: Reload Nagios
      service:
        name: nagios
        state: reloaded

# 每日检查
□ 检查 Nagios 服务状态
□ 检查告警数量和趋势
□ 检查磁盘空间（日志、性能数据）
□ 检查 Web 界面可访问性
□ 检查关键服务状态

# 每周检查
□ 分析告警趋势报告
□ 检查配置变更记录
□ 清理过期日志和性能数据
□ 验证备份完整性
□ 检查 NRPE 节点状态

# 每月检查
□ 审查告警策略有效性
□ 更新监控覆盖率评估
□ 性能数据分析和容量规划
□ 安全漏洞扫描
□ 应急预案演练

#!/bin/bash
# nagios_maintenance.sh - 定期维护脚本

LOG_FILE="/var/log/nagios/maintenance.log"
NAGIOS_DIR="/usr/local/nagios"

log() {
    echo "$(date '+%Y-%m-%d %H:%M:%S') $1" >> $LOG_FILE
}

log "Starting maintenance tasks"

# 1. 清理旧日志（保留 90 天）
find $NAGIOS_DIR/var/archives/ -name "*.log" -mtime +90 -delete
log "Cleaned old log files"

# 2. 清理旧性能数据（保留 180 天）
find $NAGIOS_DIR/var/perfdata/ -name "*.rrd" -mtime +180 -delete
log "Cleaned old perfdata files"

# 3. 压缩日志
gzip $NAGIOS_DIR/var/archives/nagios-$(date -d 'yesterday' +%m-%d-%Y).log 2>/dev/null
log "Compressed yesterday's log"

# 4. 检查磁盘空间
USAGE=$(df -h /var/log/nagios | tail -1 | awk '{print $5}' | tr -d '%')
if [ $USAGE -gt 90 ]; then
    log "WARNING: Disk usage is ${USAGE}%"
    # 发送告警
    echo "Nagios disk usage warning: ${USAGE}%" | mail -s "Nagios Disk Alert" [email protected]
fi

# 5. 验证配置
$NAGIOS_DIR/bin/nagios -v $NAGIOS_DIR/etc/nagios.cfg > /dev/null 2>&1
if [ $? -ne 0 ]; then
    log "ERROR: Config validation failed"
fi

# 6. 备份配置
tar czf /backup/nagios/config-$(date +%Y%m%d).tar.gz $NAGIOS_DIR/etc/
log "Backup completed"

log "Maintenance tasks completed"

# 故障复盘报告

## 基本信息
- 故障时间：2024-01-01 12:00 - 13:30
- 故障级别：P1
- 影响范围：Web 服务部分用户不可用
- 负责人：张三

## 故障描述
[简要描述故障现象]

## 时间线
- 12:00 告警触发：Web Server 01 HTTP CRITICAL
- 12:05 值班人员响应，开始排查
- 12:15 定位问题：磁盘空间满
- 12:30 清理磁盘空间
- 13:00 服务恢复正常
- 13:30 确认监控恢复

## 根本原因
[分析根本原因]

## 处理措施
[已采取的临时和永久措施]

## 改进项
| 改进项 | 负责人 | 截止日期 |
|--------|--------|----------|
| 增加磁盘空间告警阈值 | 张三 | 2024-01-05 |
| 实施日志自动清理 | 李四 | 2024-01-10 |
| 增加磁盘容量监控 | 王五 | 2024-01-15 |

## 经验教训
[总结经验教训]