#!/bin/bash
# prometheus_stack_setup.sh - Prometheus监控栈部署

# 安装Prometheus
echo "安装Prometheus..."
wget https://github.com/prometheus/prometheus/releases/download/v2.47.0/prometheus-2.47.0.linux-amd64.tar.gz
tar xvf prometheus-2.47.0.linux-amd64.tar.gz
mv prometheus-2.47.0.linux-amd64 /opt/prometheus

# 配置Prometheus
cat > /opt/prometheus/prometheus.yml << EOF
global:
  scrape_interval: 15s
  evaluation_interval: 15s

alerting:
  alertmanagers:
    - static_configs:
        - targets:
            - alertmanager:9093

rule_files:
  - "slo_rules.yml"
  - "alert_rules.yml"

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  - job_name: 'node-exporter'
    static_configs:
      - targets: ['node-exporter:9100']

  - job_name: 'my-app'
    static_configs:
      - targets: ['my-app:8080']
EOF

# 创建systemd服务
cat > /etc/systemd/system/prometheus.service << EOF
[Unit]
Description=Prometheus Monitoring
After=network.target

[Service]
ExecStart=/opt/prometheus/prometheus --config.file=/opt/prometheus/prometheus.yml
Restart=always
User=prometheus

[Install]
WantedBy=multi-user.target
EOF

# 安装Grafana
echo "安装Grafana..."
apt-get install -y grafana

# 启动服务
systemctl daemon-reload
systemctl enable prometheus
systemctl start prometheus
systemctl enable grafana-server
systemctl start grafana-server

echo "Prometheus监控栈部署完成"

#!/bin/bash
# incident_handler.sh - 故障处理脚本

set -euo pipefail

INCIDENT_ID=$(date +%Y%m%d%H%M%S)
LOG_FILE="/var/log/incidents/${INCIDENT_ID}.log"
ALERT_THRESHOLD_CPU=85
ALERT_THRESHOLD_MEM=90

log() {
    local level="$1"
    local message="$2"
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] [$level] $message" | tee -a "$LOG_FILE"
}

# 初始化
mkdir -p /var/log/incidents
log "INFO" "===== 故障处理开始 ====="
log "INFO" "故障ID: $INCIDENT_ID"

# 检查系统资源
check_resources() {
    log "INFO" "检查系统资源..."
    
    CPU_USAGE=$(top -bn1 | grep "Cpu(s)" | awk '{print $2}' | cut -d'%' -f1)
    MEM_USAGE=$(free | grep Mem | awk '{printf "%.0f", $3/$2 * 100}')
    
    log "INFO" "CPU使用率: ${CPU_USAGE}%"
    log "INFO" "内存使用率: ${MEM_USAGE}%"
    
    if (( $(echo "$CPU_USAGE > $ALERT_THRESHOLD_CPU" | bc -l) )); then
        log "WARN" "CPU使用率过高!"
    fi
    
    if (( $(echo "$MEM_USAGE > $ALERT_THRESHOLD_MEM" | bc -l) )); then
        log "WARN" "内存使用率过高!"
    fi
}

# 检查服务状态
check_services() {
    log "INFO" "检查服务状态..."
    
    SERVICES=("nginx" "mysql" "redis" "app")
    
    for service in "${SERVICES[@]}"; do
        if systemctl is-active --quiet "$service"; then
            log "INFO" "服务 $service: 运行正常"
        else
            log "ERROR" "服务 $service: 运行异常!"
            
            # 尝试自动重启
            log "INFO" "尝试重启服务 $service..."
            systemctl restart "$service"
            
            sleep 5
            
            if systemctl is-active --quiet "$service"; then
                log "INFO" "服务 $service 重启成功"
            else
                log "ERROR" "服务 $service 重启失败，需要人工介入"
            fi
        fi
    done
}

# 健康检查
health_check() {
    log "INFO" "执行健康检查..."
    
    ENDPOINTS=("http://localhost:8080/health" "http://localhost:8080/ready")
    
    for endpoint in "${ENDPOINTS[@]}"; do
        if curl -sf "$endpoint" &>/dev/null; then
            log "INFO" "健康检查 $endpoint: 通过"
        else
            log "ERROR" "健康检查 $endpoint: 失败"
        fi
    done
}

# 故障定级
classify_incident() {
    local severity="$1"
    
    case "$severity" in
        1)
            log "CRITICAL" "故障定级: P1 - 严重故障，影响大量用户"
            ;;
        2)
            log "ERROR" "故障定级: P2 - 中等故障，影响部分用户"
            ;;
        3)
            log "WARN" "故障定级: P3 - 轻微故障，影响少量用户"
            ;;
    esac
}

# 主函数
main() {
    local severity="${1:-3}"
    
    check_resources
    check_services
    health_check
    classify_incident "$severity"
    
    log "INFO" "===== 故障处理完成 ====="
}

main "$@"

#!/bin/bash
# canary_deploy.sh - 灰度发布脚本

set -euo pipefail

APP_NAME="myapp"
NEW_VERSION="1.2.0"
REPLICAS=10
CANARY_PERCENT=10

log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"
}

# 更新灰度版本
update_canary() {
    log "更新灰度版本..."
    
    kubectl set image deployment/$APP_NAME $APP_NAME=$APP_NAME:$NEW_VERSION
    kubectl patch deployment/$APP_NAME -p '{"spec":{"replicas":'$CANARY_PERCENT'}}'
    
    log "灰度版本已更新到 ${NEW_VERSION}，副本数: ${CANARY_PERCENT}%"
}

# 观察灰度
observe_canary() {
    log "观察灰度指标..."
    
    for i in {1..30}; do
        ERROR_RATE=$(kubectl exec -it deploy/$APP_NAME -- curl -s http://localhost:8080/metrics | grep error_rate | awk '{print $2}')
        
        log "当前错误率: $ERROR_RATE"
        
        if (( $(echo "$ERROR_RATE > 0.05" | bc -l) )); then
            log "错误率过高，自动回滚!"
            rollback_canary
            exit 1
        fi
        
        sleep 10
    done
}

# 完整发布
full_deploy() {
    log "执行完整发布..."
    
    kubectl patch deployment/$APP_NAME -p '{"spec":{"replicas":'$REPLICAS'}}'
    
    log "发布完成!"
}

# 回滚
rollback_canary() {
    log "回滚灰度版本..."
    
    kubectl rollout undo deployment/$APP_NAME
    kubectl patch deployment/$APP_NAME -p '{"spec":{"replicas":'$REPLICAS'}}'
    
    log "回滚完成!"
}

# 主函数
main() {
    log "===== 开始灰度发布 ====="
    log "应用: $APP_NAME"
    log "新版本: $NEW_VERSION"
    
    update_canary
    observe_canary
    full_deploy
    
    log "===== 灰度发布完成 ====="
}

main

#!/bin/bash
# auto_health_check.sh - 自动化健康检查脚本

set -euo pipefail

CHECK_INTERVAL=60
LOG_FILE="/var/log/health_check.log"

log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >> "$LOG_FILE"
}

# 检查服务健康
check_service_health() {
    local service="$1"
    local endpoint="$2"
    local expected_code="${3:-200}"
    
    local actual_code=$(curl -s -o /dev/null -w "%{http_code}" "$endpoint" 2>/dev/null || echo "000")
    
    if [[ "$actual_code" == "$expected_code" ]]; then
        log "OK: $service 健康检查通过"
        return 0
    else
        log "FAIL: $service 健康检查失败 (期望: $expected_code, 实际: $actual_code)"
        return 1
    fi
}

# 检查系统资源
check_system_resources() {
    local cpu_usage=$(top -bn1 | grep "Cpu(s)" | awk '{print $2}' | cut -d'%' -f1)
    local mem_usage=$(free | grep Mem | awk '{printf "%.0f", $3/$2 * 100}')
    local disk_usage=$(df -h / | tail -1 | awk '{print $5}' | sed 's/%//')
    
    log "系统资源 - CPU: ${cpu_usage}%, 内存: ${mem_usage}%, 磁盘: ${disk_usage}%"
    
    if (( $(echo "$cpu_usage > 90" | bc -l) )); then
        log "WARN: CPU使用率过高"
    fi
    
    if (( mem_usage > 90 )); then
        log "WARN: 内存使用率过高"
    fi
    
    if (( disk_usage > 90 )); then
        log "WARN: 磁盘使用率过高"
    fi
}

# 检查端口监听
check_port_listening() {
    local port="$1"
    
    if netstat -tuln 2>/dev/null | grep -q ":$port "; then
        log "OK: 端口 $port 正常监听"
        return 0
    else
        log "FAIL: 端口 $port 未监听"
        return 1
    fi
}

# 主函数
main() {
    log "===== 健康检查开始 ====="
    
    # 检查关键服务
    check_service_health "nginx" "http://localhost:80" || true
    check_service_health "app" "http://localhost:8080/health" || true
    
    # 检查系统资源
    check_system_resources
    
    # 检查关键端口
    check_port_listening 80
    check_port_listening 443
    check_port_listening 3306
    check_port_listening 6379
    
    log "===== 健康检查完成 ====="
}

# 循环执行
while true; do
    main
    sleep "$CHECK_INTERVAL"
done

#!/bin/bash
# auto_scaling.sh - 基于指标的自动扩缩容脚本

set -euo pipefail

NAMESPACE="default"
DEPLOYMENT="myapp"
MIN_REPLICAS=3
MAX_REPLICAS=10
CPU_THRESHOLD=70
MEM_THRESHOLD=80

log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"
}

# 获取当前副本数
get_current_replicas() {
    kubectl get deployment $DEPLOYMENT -n $NAMESPACE -o jsonpath='{.spec.replicas}'
}

# 获取CPU使用率
get_cpu_usage() {
    kubectl top pods -n $NAMESPACE -l app=$DEPLOYMENT --no-headers 2>/dev/null | \
        awk '{sum+=$2; count++} END {print count>0 ? sum/count : 0}' | sed 's/%//'
}

# 获取内存使用率
get_mem_usage() {
    kubectl top pods -n $NAMESPACE -l app=$DEPLOYMENT --no-headers 2>/dev/null | \
        awk '{sum+=$3; count++} END {print count>0 ? sum/count : 0}' | sed 's/%//'
}

# 扩容
scale_up() {
    local current=$1
    local new=$((current + 1))
    
    if [[ $new -le $MAX_REPLICAS ]]; then
        log "扩容: $current -> $new"
        kubectl scale deployment $DEPLOYMENT -n $NAMESPACE --replicas=$new
    fi
}

# 缩容
scale_down() {
    local current=$1
    local new=$((current - 1))
    
    if [[ $new -ge $MIN_REPLICAS ]]; then
        log "缩容: $current -> $new"
        kubectl scale deployment $DEPLOYMENT -n $NAMESPACE --replicas=$new
    fi
}

# 主函数
main() {
    local current=$(get_current_replicas)
    local cpu=$(get_cpu_usage)
    local mem=$(get_mem_usage)
    
    log "当前副本数: $current, CPU: ${cpu}%, 内存: ${mem}%"
    
    # 基于CPU扩容
    if (( $(echo "$cpu > $CPU_THRESHOLD" | bc -l) )); then
        log "CPU使用率超过阈值 ($CPU_THRESHOLD%)"
        scale_up $current
    # 基于内存扩容
    elif (( $(echo "$mem > $MEM_THRESHOLD" | bc -l) )); then
        log "内存使用率超过阈值 ($MEM_THRESHOLD%)"
        scale_up $current
    # 资源使用低，缩容
    elif (( $(echo "$cpu < 20 && $mem < 40" | bc -l) )); then
        log "资源使用率低，考虑缩容"
        scale_down $current
    fi
}

main

#!/bin/bash
# sre_metrics.sh - SRE指标收集脚本

set -euo pipefail

METRICS_FILE="/var/log/sre_metrics.json"
DATE=$(date +%Y-%m-%d)

log_metrics() {
    local mttr=$1
    local mtbf=$2
    local deploy_freq=$3
    local change_fail_rate=$4
    
    cat > "$METRICS_FILE" << EOF
{
    "date": "$DATE",
    "mttr": $mttr,
    "mtbf": $mtbf,
    "deploy_frequency": $deploy_freq,
    "change_failure_rate": $change_fail_rate,
    "timestamp": "$(date -Iseconds)"
}
EOF
    
    echo "SRE指标已记录到 $METRICS_FILE"
    cat "$METRICS_FILE"
}

# 计算MTTR（示例）
calculate_mttr() {
    local total_incident_time=120
    local incident_count=4
    echo "scale=2; $total_incident_time / $incident_count" | bc
}

# 计算MTBF（示例）
calculate_mtbf() {
    local total_uptime=43200
    local incident_count=2
    echo "scale=2; $total_uptime / ($incident_count + 1)" | bc
}

# 计算部署频率（示例）
calculate_deploy_freq() {
    local deploy_count=50
    local days=30
    echo "scale=2; $deploy_count / $days" | bc
}

# 计算变更失败率（示例）
calculate_change_fail_rate() {
    local failed_changes=2
    local total_changes=50
    echo "scale=4; $failed_changes / $total_changes" | bc
}

# 主函数
main() {
    local mttr=$(calculate_mttr)
    local mtbf=$(calculate_mtbf)
    local deploy_freq=$(calculate_deploy_freq)
    local change_fail_rate=$(calculate_change_fail_rate)
    
    log_metrics $mttr $mtbf $deploy_freq $change_fail_rate
}

main