基于 LVM 部署 MinIO 分布式高可用集群 | 极客日志

基于 LVM 部署 MinIO 分布式高可用集群 | 极客日志

主机名	ip	操作系统版本	备注
LB	192.168.48.88	openEuler 24.03 (LTS-SP2)	负载均衡节点，haproxy
minio1	192.168.48.100	Ubuntu 22.04.5 LTS	minio
minio2	192.168.48.101	Ubuntu 22.04.5 LTS	minio
minio3	192.168.48.102	Ubuntu 22.04.5 LTS	minio

root@minio1:~# cat /etc/hosts
127.0.0.1 localhost
192.168.48.100 minio1
192.168.48.101 minio2
192.168.48.102 minio3

root@minio1:~# lsblk
NAME         MAJ:MIN RM SIZE RO TYPE MOUNTPOINTS
loop0          7:0    0 63.9M  1 loop /snap/core20/2318
loop1          7:1    0   87M  1 loop /snap/lxd/29351
loop2          7:2    0 38.8M  1 loop /snap/snapd/21759
sda            8:0    0 300G  0 disk
├─sda1         8:1    0    1M  0 part
├─sda2         8:2    0    2G  0 part /boot
└─sda3         8:3    0 298G  0 part
└─ubuntu--vg-ubuntu--lv 253:0    0 100G  0 lvm /
sr0           11:0    1    2G  0 rom

root@minio1:~# pvs
PV         VG       Fmt Attr PSize   PFree
/dev/sda3  ubuntu-vg lvm2 a-- <298.00g <198.00g

root@minio1:~# vgs
VG        #PV #LV #SN Attr   VSize   VFree
ubuntu-vg   1   1   0 wz--n- <298.00g <198.00g

root@minio1:~# lvcreate -n minio -L 50G ubuntu-vg
Logical volume "minio" created.

root@minio1:~# lvs
LV      VG       Attr     LSize   Pool Origin Data% Meta% Move Log Cpy%Sync Convert
minio   ubuntu-vg -wi-a----- 50.00g 
utbuntu-lv ubuntu-vg -wi-ao---- 100.00g

root@minio1:~# mkfs.ext4 /dev/ubuntu-vg/minio
root@minio1:~# echo /dev/ubuntu-vg/minio /data/ ext4 defaults 0 0 >> /etc/fstab
root@minio1:~# mkdir /data/
root@minio1:~# mount -a
root@minio1:~# mkdir -p /data/minio{1..4}
root@minio1:~# useradd -r -s /sbin/nologin minio
root@minio1:~# chown -R minio: /data/minio*

root@minio1:~# wget https://dl.minio.org.cn/server/minio/release/linux-amd64/archive/minio.RELEASE.2024-11-07T00-52-20Z
root@minio1:~# install minio.RELEASE.2024-11-07T00-52-20Z /usr/local/bin/minio

root@minio1:~# cat > /etc/default/minio << 'EOF'
# MinIO 环境变量配置文件

# MinIO 管理员账号
MINIO_ROOT_USER=minioadmin

# MinIO 管理员密码（生产环境请使用强密码）
MINIO_ROOT_PASSWORD=minioadmin

# MinIO 存储卷配置 - 分布式模式
# 格式：http://节点名：端口/数据目录
# 这里配置了 3 个节点，每个节点 4 个磁盘
MINIO_VOLUMES='http://minio{1...3}:9000/data/minio{1...4}'

# MinIO 启动选项
# --console-address :9001 指定 Web 控制台端口
MINIO_OPTS='--console-address :9001'

# Prometheus 认证类型
MINIO_PROMETHEUS_AUTH_TYPE="public"
EOF

cat > /lib/systemd/system/minio.service << 'EOF'
[Unit]
Description=MinIO Distributed Storage
Documentation=https://docs.min.io
Wants=network-online.target
After=network-online.target

[Service]
WorkingDirectory=/usr/local
User=minio
Group=minio
EnvironmentFile=-/etc/default/minio
ExecStartPre=/bin/bash -c "if [ -z \"${MINIO_VOLUMES}\" ]; then echo \"Variable MINIO_VOLUMES not set in /etc/default/minio\"; exit 1; fi"
ExecStart=/usr/local/bin/minio server $MINIO_OPTS $MINIO_VOLUMES
Restart=always
RestartSec=5
LimitNOFILE=1048576
TasksMax=infinity
OOMScoreAdjust=-1000

[Install]
WantedBy=multi-user.target
EOF

root@minio1:~# systemctl daemon-reload
root@minio1:~# systemctl enable --now minio.service
Created symlink /etc/systemd/system/multi-user.target.wants/minio.service → /lib/systemd/system/minio.service.

root@minio1:~# systemctl status minio.service
● minio.service - MinIO Distributed Storage
     Loaded: loaded (/lib/systemd/system/minio.service; enabled; vendor preset: enabled)
     Active: active (running) since Sun 2026-03-01 23:13:49 UTC; 18s ago
       Docs: https://docs.min.io
   Process: 1511 ExecStartPre=/bin/bash -c if [ -z "${MINIO_VOLUMES}" ]; then echo "Var>
     Main PID: 1520 (minio)
       Tasks: 13
       Memory: 71.8M
         CPU: 613ms
       CGroup: /system.slice/minio.service
               └─1520 /usr/local/bin/minio server --console-address :9001 http://minio{1..> 

Mar 01 23:13:56 minio1 minio[1520]: MinIO Object Storage Server
Mar 01 23:13:56 minio1 minio[1520]: Copyright: 2015-2026 MinIO, Inc.
Mar 01 23:13:56 minio1 minio[1520]: License: GNU AGPLv3 - https://www.gnu.org/licenses/a>
Mar 01 23:13:56 minio1 minio[1520]: Version: RELEASE.2024-11-07T00-52-20Z (go1.23.3 linu>
Mar 01 23:13:56 minio1 minio[1520]: API: http://192.168.48.100:9000 http://127.0.0.1:90>
Mar 01 23:13:56 minio1 minio[1520]: WebUI: http://192.168.48.100:9001 http://127.0.0.1:9>
Mar 01 23:13:56 minio1 minio[1520]: Docs: https://docs.min.io
Mar 01 23:13:56 minio1 minio[1520]: ---------------------------
Mar 01 23:13:56 minio1 minio[1520]: WARN: Detected default credentials 'minioadmin:minio>
Mar 01 23:13:56 minio1 minio[1520]: INFO: IAM load(startup) finished. (duration: 5.37050>

# 查看端口
root@minio1:~# ss -ntlp |grep minio
LISTEN 0     4096       127.0.0.1:9000     0.0.0.0:*   users:(("minio",pid=1520,fd=8))
LISTEN 0     4096           [::1]:9000         [::]:*   users:(("minio",pid=1520,fd=10))
LISTEN 0     4096             *:9001           *:*   users:(("minio",pid=1520,fd=29))
LISTEN 0     4096             *:9000           *:*   users:(("minio",pid=1520,fd=9))

apt install -y haproxy

cat > /etc/haproxy/haproxy.cfg << 'EOF'
global log /dev/log local0 log /dev/log local1 notice chroot /var/lib/haproxy stats socket /var/lib/haproxy/admin.sock mode 660 level admin stats timeout 30s user haproxy group haproxy daemon
# 默认 SSL 配置
# ssl-default-bind-ciphers PROFILE=SYSTEM
# ssl-default-server-ciphers PROFILE=SYSTEM
defaults log global mode http option httplog option dontlognull timeout connect 5000 timeout client 50000 timeout server 50000
# HAProxy 监控页面
listen stats bind 0.0.0.0:9999 mode http log global stats enable stats uri /haproxy-status stats auth admin:123456 stats refresh 30s stats show-node stats admin if LOCALHOST
# MinIO API 负载均衡
listen minio bind 192.168.48.88:9000 mode http log global option httplog option forwardfor option httpchk http-check send meth GET uri /minio/health/live ver HTTP/1.1 hdr Host minio.local http-check expect status 200 balance roundrobin timeout connect 5s timeout server 30s timeout client 30s server minio1 192.168.48.100:9000 check inter 3000 fall 2 rise 5 server minio2 192.168.48.101:9000 check inter 3000 fall 2 rise 5 server minio3 192.168.48.102:9000 check inter 3000 fall 2 rise 5
# MinIO Console 负载均衡
listen minio_console bind 192.168.48.88:9001 mode http log global option httplog option forwardfor option httpchk GET /login balance roundrobin timeout connect 5s timeout server 30s timeout client 30s server minio1-console 192.168.48.100:9001 check inter 3000 fall 2 rise 5 server minio2-console 192.168.48.101:9001 check inter 3000 fall 2 rise 5 server minio3-console 192.168.48.102:9001 check inter 3000 fall 2 rise 5
EOF

systemctl daemon-reload || systemctl enable haproxy --now
systemctl status haproxy