# 获取完整状态
curl -s 'localhost:4001/status?pretty' | python3 -m json.tool

{
    "build": {
        "branch": "master",
        "commit": "abc1234",
        "version": "v8.36.5"
    },
    "store": {
        "raft_state": "Leader",
        "node_id": "node1",
        "db_conf": {
            "fk_constraints": true,
            "wal": true,
            "on_disk": false
        },
        "num_raft_peers": 2,
        "num_open_connections": 5,
        "applied_index": 12345,
        "commit_index": 12345,
        "last_log_index": 12345,
        "last_log_term": 3,
        "last_snapshot_index": 12000,
        "last_snapshot_term": 2,
        "last_contact": "0s",
        "term": 3,
        "num_snaps": 5,
        "db_size": 1048576
    },
    "runtime": {
        "GOARCH": "amd64",
        "GOOS": "linux",
        "GOMAXPROCS": 4,
        "num_cpu": 8,
        "num_goroutine": 42,
        "version": "go1.21.5"
    }
}

# 就绪检查（HTTP 200 = 就绪，其他 = 未就绪）
curl -s -o /dev/null -w "%{http_code}" localhost:4001/status/ready

# Leader 检查
curl -s -o /dev/null -w "%{http_code}" localhost:4001/status/leader

curl -s 'localhost:4001/nodes?pretty'

#!/bin/bash
# rqlite-exporter.sh — rqlite 指标采集脚本（输出 Prometheus 格式）
NODES=("localhost:4001" "localhost:4011" "localhost:4021")

echo "# HELP rqlite_raft_state Current Raft state (0=unknown, 1=follower, 2=candidate, 3=leader)"
echo "# TYPE rqlite_raft_state gauge"
echo "# HELP rqlite_applied_index Last applied Raft log index"
echo "# TYPE rqlite_applied_index gauge"
echo "# HELP rqlite_commit_index Last committed Raft log index"
echo "# TYPE rqlite_commit_index gauge"
echo "# HELP rqlite_num_peers Number of Raft peers"
echo "# TYPE rqlite_num_peers gauge"
echo "# HELP rqlite_db_size Database size in bytes"
echo "# TYPE rqlite_db_size gauge"
echo "# HELP rqlite_num_snapshots Number of snapshots"
echo "# TYPE rqlite_num_snapshots gauge"
echo "# HELP rqlite_num_open_connections Number of open connections"
echo "# TYPE rqlite_num_open_connections gauge"
echo "# HELP rqlite_up Whether the node is up (1=up, 0=down)"
echo "# TYPE rqlite_up gauge"

for node in "${NODES[@]}"; do
    id=$(echo "$node" | tr ':' '_')
    status=$(curl -s --connect-timeout 3 "http://$node/status" 2>/dev/null)
    
    if [ $? -ne 0 ] || [ -z "$status" ]; then
        echo "rqlite_up{node=\"$node\"} 0"
        continue
    fi
    
    echo "rqlite_up{node=\"$node\"} 1"
    
    # 解析状态
    raft_state=$(echo "$status" | python3 -c "import json,sys; d=json.load(sys.stdin); print(d['store']['raft_state'])" 2>/dev/null)
    
    case "$raft_state" in
        "Leader")    state_val=3 ;;
        "Follower")  state_val=1 ;;
        "Candidate") state_val=2 ;;
        *)           state_val=0 ;;
    esac
    
    echo "rqlite_raft_state{node=\"$node\",state=\"$raft_state\"} $state_val"
    
    echo "$status" | python3 -c "
import json, sys
d = json.load(sys.stdin)
s = d.get('store', {})
node = '$node'
print(f'rqlite_applied_index{{node=\"{node}\"}} {s.get(\"applied_index\", 0)}')
print(f'rqlite_commit_index{{node=\"{node}\"}} {s.get(\"commit_index\", 0)}')
print(f'rqlite_num_peers{{node=\"{node}\"}} {s.get(\"num_raft_peers\", 0)}')
print(f'rqlite_db_size{{node=\"{node}\"}} {s.get(\"db_size\", 0)}')
print(f'rqlite_num_snapshots{{node=\"{node}\"}} {s.get(\"num_snaps\", 0)}')
" 2>/dev/null
done

# prometheus.yml
global:
  scrape_interval: 15s
  evaluation_interval: 15s

scrape_configs:
  # 直接抓取 rqlite 状态（需要自定义 exporter）
  - job_name: 'rqlite'
    static_configs:
      - targets:
          - 'localhost:4001'
          - 'localhost:4011'
          - 'localhost:4021'
        labels:
          cluster: 'rqlite-prod'
    
    # 使用自定义脚本 exporter
    metrics_path: /metrics
    scrape_interval: 30s

# rqlite-alerts.yml
groups:
  - name: rqlite_alerts
    rules:
      # 节点宕机
      - alert: RqliteNodeDown
        expr: rqlite_up == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "rqlite 节点 {{ $labels.node }} 已宕机"
          description: "节点 {{ $labels.node }} 已超过 1 分钟无法访问"
      
      # 无 Leader
      - alert: RqliteNoLeader
        expr: count(rqlite_raft_state{state="Leader"}) == 0
        for: 30s
        labels:
          severity: critical
        annotations:
          summary: "rqlite 集群无 Leader"
          description: "当前集群中没有任何 Leader 节点"
      
      # 多个 Leader（脑裂）
      - alert: RqliteMultipleLeaders
        expr: count(rqlite_raft_state{state="Leader"}) > 1
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "rqlite 集群存在多个 Leader"
          description: "检测到 {{ $value }} 个 Leader，可能存在脑裂"
      
      # Raft 日志落后
      - alert: RqliteReplicationLag
        expr: (max(rqlite_applied_index) - rqlite_applied_index) > 1000
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "rqlite 节点 {{ $labels.node }} 复制落后"
          description: "节点落后 {{ $value }} 条日志"
      
      # 数据库大小增长过快
      - alert: RqliteDBSizeGrowing
        expr: increase(rqlite_db_size[1h]) > 104857600  # 100MB/h
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "rqlite 数据库增长过快"
          description: "过去 1 小时数据库增长了 {{ $value }} bytes"

# docker-compose-monitoring.yml
version: "3.8"

services:
  prometheus:
    image: prom/prometheus:latest
    container_name: prometheus
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - ./rqlite-alerts.yml:/etc/prometheus/rqlite-alerts.yml
      - prometheus-data:/prometheus
    networks:
      - monitoring

  grafana:
    image: grafana/grafana:latest
    container_name: grafana
    ports:
      - "3000:3000"
    environment:
      GF_SECURITY_ADMIN_PASSWORD: admin
    volumes:
      - grafana-data:/var/lib/grafana
    networks:
      - monitoring
    depends_on:
      - prometheus

  alertmanager:
    image: prom/alertmanager:latest
    container_name: alertmanager
    ports:
      - "9093:9093"
    volumes:
      - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml
    networks:
      - monitoring

volumes:
  prometheus-data:
  grafana-data:

networks:
  monitoring:
    driver: bridge

{
    "dashboard": {
        "title": "rqlite 集群监控",
        "panels": [
            {
                "title": "节点状态",
                "type": "stat",
                "targets": [{
                    "expr": "rqlite_up",
                    "legendFormat": "{{ node }}"
                }],
                "fieldConfig": {
                    "defaults": {
                        "mappings": [
                            {"type": "value", "options": {"0": {"text": "离线", "color": "red"}}},
                            {"type": "value", "options": {"1": {"text": "在线", "color": "green"}}}
                        ]
                    }
                }
            },
            {
                "title": "Raft 角色",
                "type": "stat",
                "targets": [{
                    "expr": "rqlite_raft_state",
                    "legendFormat": "{{ node }} - {{ state }}"
                }]
            },
            {
                "title": "Applied Index 趋势",
                "type": "timeseries",
                "targets": [{
                    "expr": "rate(rqlite_applied_index[5m])",
                    "legendFormat": "{{ node }}"
                }]
            },
            {
                "title": "数据库大小",
                "type": "timeseries",
                "targets": [{
                    "expr": "rqlite_db_size",
                    "legendFormat": "{{ node }}"
                }],
                "fieldConfig": {
                    "defaults": {
                        "unit": "bytes"
                    }
                }
            }
        ]
    }
}

# Docker 日志查看
docker logs rqlite1 --tail 100 -f

# systemd 日志查看
journalctl -u rqlited -f

# 日志级别（rqlite 目前不支持动态调整日志级别）

# Promtail 配置（Loki 日志收集）
scrape_configs:
  - job_name: rqlite
    docker_sd_configs:
      - filters:
          - name: label
            values: ["app=rqlite"]
    pipeline_stages:
      - regex:
          expression: '\[(?P<component>\w+)\] (?P<level>\w+): (?P<message>.*)'
      - labels:
          component:
          level: