# 1. 检查kubelet进程状态
systemctl status kubelet

# 2. 查看kubelet启动日志
journalctl -u kubelet -n 200 --no-pager

# 3. 如果进程不存在，手动启动
systemctl start kubelet

# 4. 查看kubelet配置文件
cat /var/lib/kubelet/config.yaml

# 5. 检查kubelet服务配置
systemctl cat kubelet

# 1. 检查CNI插件Pod状态
kubectl get pods -n kube-system -l k8s-app=flannel
kubectl get pods -n kube-system -l k8s-app=calico-node

# 2. 检查flanneld服务状态
systemctl status flanneld

# 3. 查看flannel日志
journalctl -u flanneld -n 100 --no-pager

# 4. 检查flannel接口
ip link show flannel.1

# 5. 重启CNI插件（如需要）
kubectl delete pod -n kube-system -l k8s-app=flannel
kubectl rollout restart daemonset -n kube-system flanneld

# 检查flannel网络配置
cat /var/run/flannel/subnet.env

# 查看flannel日志
tail -f /var/log/flannel/flannel.log

# 重启flannel后验证
systemctl restart flanneld
ip link show flannel.1

# 1. 检查磁盘空间
df -h
df -i  # 检查inode

# 2. 检查内存使用
free -m
cat /proc/meminfo | grep -E "MemAvailable|MemFree"

# 3. 检查文件描述符
lsof | wc -l
cat /proc/sys/fs/file-nr

# 4. 检查进程数
ps aux | wc -l
cat /proc/sys/kernel/pid_max

# 清理旧日志
journalctl --vacuum-size=500M
rm -rf /var/log/*.gz

# 清理Docker资源
docker system prune -af
docker image prune -af

# 清理kubelet缓存
rm -rf /var/lib/kubelet/cache/*

# 1. 检查kubelet证书过期时间
openssl x509 -in /var/lib/kubelet/pki/cert.crt -noout -dates

# 2. 检查kubelet配置文件
cat /var/lib/kubelet/config.yaml

# 3. 查看kubelet日志中的证书错误
journalctl -u kubelet -n 100 | grep -i cert

# 4. 如果证书过期，更新证书
kubeadm certs renew all
systemctl restart kubelet

# 使用kubeadm管理证书时
kubeadm certs check-expiration
kubeadm certs renew all

# 重启kubelet使新证书生效
systemctl restart kubelet

# 1. 检查swap状态
swapon -s
free -m

# 2. 临时关闭swap
swapoff -a

# 3. 永久关闭swap（注释掉/etc/fstab中的swap行）
sed -i '/swap/s/^/#/' /etc/fstab

# 4. 重启kubelet
systemctl restart kubelet

# 1. 检查系统时间
timedatectl

# 2. 检查时区
timedatectl status

# 3. 同步时间
ntpdate pool.ntp.org
# 或
chronyc -a makestep

# 4. 启用NTP服务
systemctl enable --now chronyd

#!/bin/bash
# node-recovery.sh - K8s节点NotReady自动恢复脚本

NODE_NAME=$(hostname)
LOG_FILE="/var/log/node-recovery.log"

log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a $LOG_FILE
}

check_kubelet() {
    if ! systemctl is-active --quiet kubelet; then
        log "Kubelet is not running, starting..."
        systemctl start kubelet
        sleep 10
    fi
}

check_flannel() {
    if ! systemctl is-active --quiet flanneld; then
        log "Flannel is not running, restarting..."
        systemctl restart flanneld
        sleep 5
    fi
}

check_disk() {
    DISK_USAGE=$(df -h / | awk 'NR==2 {print $5}' | sed 's/%//')
    if [ $DISK_USAGE -gt 85 ]; then
        log "Disk usage is high ($DISK_USAGE%), cleaning up..."
        docker system prune -af > /dev/null 2>&1
        journalctl --vacuum-size=200M > /dev/null 2>&1
    fi
}

check_swap() {
    if swapon -s | grep -q "/dev"; then
        log "Swap is enabled, disabling..."
        swapoff -a
    fi
}

main() {
    log "Starting node recovery for $NODE_NAME"
    check_kubelet
    check_flannel
    check_disk
    check_swap
    log "Recovery completed"
}

main

#!/bin/bash
# node-health-check.sh - 节点健康检查脚本

check_all() {
    echo "=== 节点健康检查 ==="
    echo ""

    echo "[1] Kubelet状态:"
    systemctl is-active kubelet
    echo ""

    echo "[2] Flannel状态:"
    systemctl is-active flanneld
    ip link show flannel.1 | grep -q UP && echo "flannel.1 is UP" || echo "flannel.1 is DOWN"
    echo ""

    echo "[3] 磁盘使用:"
    df -h / | awk 'NR==2 {print "使用率: " $5}'
    echo ""

    echo "[4] 内存使用:"
    free -m | awk 'NR==2 {print "使用率: " $3/$2*100 "%"}'
    echo ""

    echo "[5] Swap状态:"
    swapon -s | grep -v Filename || echo "Swap未启用"
    echo ""

    echo "[6] Docker状态:"
    systemctl is-active docker
    echo ""

    echo "[7] 时间同步:"
    timedatectl status | grep "System clock"
}

check_all

#!/bin/bash
# node-init.sh - K8s节点初始化脚本

# 关闭swap
swapoff -a
sed -i '/swap/s/^/#/' /etc/fstab

# 设置内核参数
cat >> /etc/sysctl.conf << EOF
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
vm.swappiness = 0
vm.max_map_count = 262144
EOF

sysctl -p

# 增加文件描述符限制
cat >> /etc/security/limits.conf << EOF
* soft nofile 655360
* hard nofile 655360
* soft nproc 655360
* hard nproc 655360
EOF

# 关闭防火墙（或配置正确规则）
systemctl disable --now firewalld

# 安装NTP
yum install -y chrony
systemctl enable --now chronyd

# 设置journal大小
mkdir -p /etc/systemd/journald.conf.d
cat > /etc/systemd/journald.conf.d/size.conf << EOF
[Journal]
SystemMaxUse=500M
SystemMaxFileSize=50M
MaxRetentionSec=7day
EOF

systemctl restart systemd-journald