本文基于 Ubuntu 22.04 和 Kubernetes 1.28，演示了通过多 Master 节点配合 etcd 集群构建高可用控制平面的方案。使用 Keepalived 提供虚拟 IP（VIP），Nginx 作为负载均衡器分发 API Server 请求。环境准备包括安装依赖、配置内核参数及 containerd。后续步骤涵盖 kubeadm 初始化集群、加入节点及 Flannel 网络插件部署，确保集群稳定运行。
随着业务规模扩大和系统稳定性要求的不断提升，单 Master 的 Kubernetes 集群在生产环境中已经难以满足高可用需求。一旦 Master 节点发生故障，集群将面临 API 不可用、调度停止等风险，严重影响业务运行。
Kubernetes 官方推荐通过多 Master + etcd 集群的方式构建高可用控制平面，但在多 Master 架构下，如何为 kube-apiserver 提供统一、稳定的访问入口，成为集群高可用部署中的关键问题。
在生产实践中，常见的解决方案是通过 Keepalived + Nginx：
本篇将基于 Kubernetes 1.28 版本，详细介绍一种生产可落地的高可用部署方案，通过多 Master + 多 etcd 架构、Keepalived 提供 VIP、Nginx 实现 kube-apiserver 的负载均衡，构建一个稳定、可靠、可扩展的 Kubernetes 高可用集群。
本方案采用三 Master + 两节点负载均衡的经典高可用架构：
无论单个 Master、单个 etcd 节点，还是单个负载均衡节点发生故障，集群都可以自动切换，确保 Kubernetes API 服务持续可用。
| 服务器 | 主机名 | IP 地址 | 主要组件 |
|---|---|---|---|
| master-01 + etcd-01 节点 | k8s-master-1 | 192.168.24.31 | apiserver controller-manager scheduler proxy etcd |
| master-02 + etcd-02 节点 | k8s-master-2 | 192.168.24.32 | apiserver controller-manager scheduler proxy etcd |
| master-03 + etcd-03 节点 | k8s-master-3 | 192.168.24.33 | apiserver controller-manager scheduler proxy etcd |
| node-01 节点 | k8s-node-1 | 192.168.24.34 | kubelet proxy（资源有限，本 demo 只使用一个 node 节点） |
| nginx + keepalive-01 节点 | keepalived-1 | 192.168.24.35 | nginx keepalived（主） |
| nginx + keepalive-02 节点 | keepalived-2 | 192.168.24.36 | nginx keepalived（备） |
apt install iproute2 net-tools ntpdate tcpdump telnet traceroute nfs-kernel-server nfs-common lrzsz tree openssl libssl-dev libpcre3 libpcre3-dev zlib1g-dev gcc openssh-server iotop unzip vim -y
# 配置主机名
hostnamectl set-hostname k8s-master-1
hostnamectl set-hostname k8s-master-2
hostnamectl set-hostname k8s-master-3
hostnamectl set-hostname k8s-node-1
# 配置 hosts 文件
cat >> /etc/hosts <<EOF
192.168.24.31 k8s-master-1
192.168.24.32 k8s-master-2
192.168.24.33 k8s-master-3
192.168.24.34 k8s-node-1
EOF
# 关闭 swap 分区
swapoff -a
sed -i '/swap/d' /etc/fstab
# 加载内核模块
cat > /etc/modules-load.d/containerd.conf <<EOF
overlay
br_netfilter
EOF
modprobe overlay
modprobe br_netfilter
# 加载 sysctl 参数
cat > /etc/sysctl.d/kubernetes.conf <<EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
net.ipv4.ip_nonlocal_bind=1
EOF
sysctl --system
# 安装 containerd（K8s 1.24 版本以后已经不使用 docker 作为容器运行时了）
apt update
apt install -y containerd
# 生成配置文件
mkdir -p /etc/containerd
containerd config default | tee /etc/containerd/config.toml
# 修改 cgroup 驱动和镜像地址
sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
sed -i 's#sandbox_image = "registry.k8s.io/pause:3.8"#sandbox_image = "registry.aliyuncs.com/google_containers/pause:3.9"#' /etc/containerd/config.toml
# 启动 containerd
systemctl restart containerd
systemctl enable containerd
containerd --version
# 添加阿里云 GPG KEY
mkdir -p /etc/apt/keyrings
curl -fsSL https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.28/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
# 添加阿里云 K8S 源
cat > /etc/apt/sources.list.d/kubernetes.list <<EOF
deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] \
https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.28/deb/ /
EOF
# 更新并安装
apt update
apt install -y kubelet kubeadm kubectl
# 配置开机自启
systemctl enable kubelet --now
systemctl status kubelet
# 可以看到状态是 auto-restart，正常
# 安装 nginx
# 安装必备组件
sudo apt install curl gnupg2 ca-certificates lsb-release ubuntu-keyring
# 导入官方 nginx 签名密钥
curl https://nginx.org/keys/nginx_signing.key | gpg --dearmor | sudo tee /usr/share/keyrings/nginx-archive-keyring.gpg >/dev/null
# 设置 apt 存储库
echo "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] \
http://nginx.org/packages/ubuntu `lsb_release -cs` nginx" | sudo tee /etc/apt/sources.list.d/nginx.list
# 使用主线 nginx 包
echo "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] \
http://nginx.org/packages/mainline/ubuntu `lsb_release -cs` nginx" | sudo tee /etc/apt/sources.list.d/nginx.list
# 设置存储库优先级
echo -e "Package: *\nPin: origin nginx.org\nPin: release o=nginx\nPin-Priority: 900\n" | sudo tee /etc/apt/preferences.d/99nginx
# 更新源并安装 nginx
apt update
apt install nginx
# 配置 nginx
vim /etc/nginx/nginx.conf
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" ''$status $body_bytes_sent "$http_referer" ''"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
sendfile on;
#tcp_nopush on;
keepalive_timeout 65;
#gzip on;
include /etc/nginx/conf.d/*.conf;
}
stream {
upstream kube_apiserver {
server 192.168.24.31:6443;
server 192.168.24.32:6443;
server 192.168.24.33:6443;
}
server {
listen 6443;
proxy_pass kube_apiserver;
}
}
# 启动 nginx
nginx -t && nginx
# 验证 nginx 端口是否启动
ss -tnlp
# 安装 keepalived
apt install keepalived
# 配置 keepalived (192.168.24.35 - MASTER)
vim /etc/keepalived/keepalived.conf
vrrp_instance VI_1 {
state MASTER
interface ens160
virtual_router_id 51
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1234
}
virtual_ipaddress {
192.168.24.30
}
}
# 配置 keepalived (192.168.24.36 - BACKUP)
vim /etc/keepalived/keepalived.conf
vrrp_instance VI_1 {
state BACKUP
interface ens160
virtual_router_id 51
priority 90
advert_int 1
authentication {
auth_type PASS
auth_pass 1234
}
virtual_ipaddress {
192.168.24.30
}
}
# 启动 keepalived
systemctl enable keepalived --now
# 查看 VIP 是否生效
ip a
# 初始化集群（VIP:6443）
kubeadm init --control-plane-endpoint "192.168.24.30:6443" --upload-certs \
--pod-network-cidr=10.244.0.0/16 \
--kubernetes-version=1.28.15 \
--image-repository=registry.aliyuncs.com/google_containers
# 其他 master 节点加入集群（类似如下命令，初始化完成后会有提示）
kubeadm join 192.168.24.30:6443 --token odmvxr.yvxk22il5749xkdc \
--discovery-token-ca-cert-hash sha256:40c0f9fdb7f426a18a5d90e30bb65de4d6115d2dab27c78b20194e7424c5afdc \
--control-plane \
--certificate-key 382d1d4669d7149422b6a6aace780e3d38965c5450888403b8d0b3bc871434f3
# node 节点加入集群（类似如下命令，初始化完成后会有提示）
kubeadm join 192.168.24.30:6443 \
--token odmvxr.yvxk22il5749xkdc \
--discovery-token-ca-cert-hash sha256:40c0f9fdb7f426a18a5d90e30bb65de4d6115d2dab27c78b20194e7424c5afdc
# 验证 K8S 集群部署情况（如果出现 NotReady，是没有安装网络插件导致的，往后看）
root@k8s-master-1:~# kubectl get node -A
NAME STATUS ROLES AGE VERSION
k8s-master-1 Ready control-plane 3d21h v1.28.15
k8s-master-2 Ready control-plane 3d21h v1.28.15
k8s-master-3 Ready control-plane 3d21h v1.28.15
k8s-node-1 Ready <none> 3d21h v1.28.15
root@k8s-master-1:~# kubectl get pod -A -owide
NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
default demo-6bb9f6fc6-k9ckz 1/1 Running 0 2d22h 10.244.3.25 k8s-node-1 <none><none>
default demo-6bb9f6fc6-pz598 1/1 Running 0 2d22h 10.244.3.24 k8s-node-1 <none><none>
ingress-nginx ingress-nginx-controller-pzz9k 1/1 Running 0 2d22h 192.168.24.34 k8s-node-1 <none><none>
kube-flannel kube-flannel-ds-bchvv 1/1 Running 0 3d6h 192.168.24.32 k8s-master-2 <none><none>
kube-flannel kube-flannel-ds-tqw7l 1/1 Running 0 3d6h 192.168.24.34 k8s-node-1 <none><none>
kube-flannel kube-flannel-ds-xx895 1/1 Running 0 3d6h 192.168.24.31 k8s-master-1 <none><none>
kube-flannel kube-flannel-ds-z9bcz 1/1 Running 0 3d6h 192.168.24.33 k8s-master-3 <none><none>
kube-system coredns-66f779496c-d2c8d 1/1 Running 0 3d21h 10.244.2.3 k8s-master-3 <none><none>
kube-system coredns-66f779496c-wrvwp 1/1 Running 0 3d21h 10.244.2.2 k8s-master-3 <none><none>
kube-system dashboard-metrics-scraper-64fbc9c6c7-9phmb 1/1 Running 0 3d6h 10.244.3.3 k8s-node-1 <none><none>
kube-system etcd-k8s-master-1 1/1 Running 0 3d21h 192.168.24.31 k8s-master-1 <none><none>
kube-system etcd-k8s-master-2 1/1 Running 0 3d21h 192.168.24.32 k8s-master-2 <none><none>
kube-system etcd-k8s-master-3 1/1 Running 0 3d21h 192.168.24.33 k8s-master-3 <none><none>
kube-system kube-apiserver-k8s-master-1 1/1 Running 0 3d21h 192.168.24.31 k8s-master-1 <none><none>
kube-system kube-apiserver-k8s-master-2 1/1 Running 0 3d21h 192.168.24.32 k8s-master-2 <none><none>
kube-system kube-apiserver-k8s-master-3 1/1 Running 0 3d21h 192.168.24.33 k8s-master-3 <none><none>
kube-system kube-controller-manager-k8s-master-1 1/1 Running 1(3d21h ago) 3d21h 192.168.24.31 k8s-master-1 <none><none>
kube-system kube-controller-manager-k8s-master-2 1/1 Running 0 3d21h 192.168.24.32 k8s-master-2 <none><none>
kube-system kube-controller-manager-k8s-master-3 1/1 Running 0 3d21h 192.168.24.33 k8s-master-3 <none><none>
kube-system kube-proxy-fftrh 1/1 Running 0 3d21h 192.168.24.34 k8s-node-1 <none><none>
kube-system kube-proxy-hl9sl 1/1 Running 0 3d21h 192.168.24.32 k8s-master-2 <none><none>
kube-system kube-proxy-ksh6p 1/1 Running 0 3d21h 192.168.24.33 k8s-master-3 <none><none>
kube-system kube-proxy-r72c6 1/1 Running 0 3d21h 192.168.24.31 k8s-master-1 <none><none>
kube-system kube-scheduler-k8s-master-1 1/1 Running 1(3d21h ago) 3d21h 192.168.24.31 k8s-master-1 <none><none>
kube-system kube-scheduler-k8s-master-2 1/1 Running 1(4h28m ago) 3d21h 192.168.24.32 k8s-master-2 <none><none>
kube-system kube-scheduler-k8s-master-3 1/1 Running 0 3d21h 192.168.24.33 k8s-master-3 <none><none>
kube-system kubernetes-dashboard-78d7494969-wmd7l 1/1 Running 0 3d6h 10.244.3.2 k8s-node-1 <none><none>
# 这里以安装 Flannel 为例
mkdir -p /root/kubernetes/cni
vim /root/kubernetes/cni/kube-flannel.yaml
---
kind: Namespace
apiVersion: v1
metadata:
name: kube-flannel
labels:
k8s-app: flannel
pod-security.kubernetes.io/enforce: privileged
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
labels:
k8s-app: flannel
name: flannel
rules:
- apiGroups:
- ""
resources:
- pods
verbs:
- get
- apiGroups:
- ""
resources:
- nodes
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- nodes/status
verbs:
-
# 运行 flannel pod
# 耐心等待 Pod 启动后再使用 kubectl get node 就能看到是 Ready 状态了
cd /root/kubernetes/cni && kubectl apply -f .
# 如果遇到镜像拉取失败报错 X509，是因为这里使用的是自签证书的 Harbor 仓库，各位需根据自己的实际情况进行镜像更换
# 排查命令
kubectl get pod -n kube-flannel
kubectl describe pod -n kube-falnnel kube-flannel-ds-xxxxx
systemctl status containerd
# 解决方案（将 harbor 服务器的证书文件复制到 k8s 服务器后更新证书，重启 containerd 即可）
scp -rp root@harbor:/data/cert/ca.crt k8s:/usr/local/share/ca-certificates/harbor-ca.crt
sudo update-ca-certificates
ll /etc/ssl/certs/harbor-ca.pem
systemctl restart containerd
本文基于 Ubuntu 22.04 + Kubernetes 1.28，完整演示了一套生产可落地的 Kubernetes 高可用部署方案，通过多 Master + Keepalived + Nginx 架构，成功构建了一个稳定、可靠的高可用集群控制平面。
微信公众号「极客日志」，在微信中扫描左侧二维码关注。展示文案：极客日志 zeeklog
将字符串编码和解码为其 Base64 格式表示形式即可。 在线工具，Base64 字符串编码/解码在线工具，online
将字符串、文件或图像转换为其 Base64 表示形式。 在线工具，Base64 文件转换器在线工具，online
将 Markdown（GFM）转为 HTML 片段，浏览器内 marked 解析；与 HTML转Markdown 互为补充。 在线工具，Markdown转HTML在线工具，online
将 HTML 片段转为 GitHub Flavored Markdown，支持标题、列表、链接、代码块与表格等；浏览器内处理，可链接预填。 在线工具，HTML转Markdown在线工具，online
通过删除不必要的空白来缩小和压缩JSON。 在线工具，JSON 压缩在线工具，online
将JSON字符串修饰为友好的可读格式。 在线工具，JSON美化和格式化在线工具，online