Docker-ceph集群

环境

Centos7.9

所有节点Host

cat >> /etc/hosts << EOF
172.19.0.17 ceph-node1
172.19.0.14 ceph-node2
172.19.0.3 ceph-node3
EOF

所有节点 Hostname

hostnamectl set-hostname ceph-node1 # 节点一
hostnamectl set-hostname ceph-node2 # 节点二
hostnamectl set-hostname ceph-node3 # 节点三

所有节点安装 Docker

mkdir -p /data/ceph/{admin,data,etc,lib,logs}
curl -fsSL https://get.docker.com | bash -s docker --mirror Aliyun
systemctl daemon-reload
systemctl restart docker
systemctl enable docker

所有节点挂载云盘

chmod +x *.sh

lsblk |grep vdb
mkfs.xfs -f /dev/vdb
mkdir /data/ceph/data/osd && mount /dev/vdb /data/ceph/data/osd
df -Th

主节点执行

cd /data/ceph/admin

cat > start_mon.sh << EOF
#!/bin/bash
docker run -d --net=host \
    --name=mon \
    -v /etc/localtime:/etc/localtime \
    -v /data/ceph/etc:/etc/ceph \
    -v /data/ceph/lib:/var/lib/ceph \
    -v /data/ceph/logs:/var/log/ceph \
    -e MON_IP=172.19.0.17 \
    -e CEPH_PUBLIC_NETWORK=172.19.0.0/16 \
    ceph/daemon:latest-nautilus  mon
EOF


cat > start_osd.sh << EOF
#!/bin/bash
docker run -d \
    --name=osd \
    --net=host \
    --restart=always \
    --privileged=true \
    --pid=host \
    -v /etc/localtime:/etc/localtime \
    -v /data/ceph/etc:/etc/ceph \
    -v /data/ceph/lib:/var/lib/ceph \
    -v /data/ceph/logs:/var/log/ceph \
    -v /data/ceph/data/osd:/var/lib/ceph/osd \
    ceph/daemon:latest-nautilus  osd_directory  
EOF


cat > start_mgr.sh << EOF
#!/bin/bash
docker run -d --net=host  \
  --name=mgr \
  -v /etc/localtime:/etc/localtime \
  -v /data/ceph/etc:/etc/ceph \
  -v /data/ceph/lib:/var/lib/ceph \
  -v /data/ceph/logs:/var/log/ceph \
  ceph/daemon:latest-nautilus mgr
EOF



cat > start_mds.sh << EOF
#!/bin/bash
docker run -d \
   --net=host \
   --name=mds \
   --privileged=true \
   -v /etc/localtime:/etc/localtime \
   -v /data/ceph/etc:/etc/ceph \
   -v /data/ceph/lib:/var/lib/ceph \
   -v /data/ceph/logs:/var/log/ceph \
   -e CEPHFS_CREATE=0 \
   -e CEPHFS_METADATA_POOL_PG=512 \
   -e CEPHFS_DATA_POOL_PG=512 \
   ceph/daemon:latest-nautilus  mds
EOF


cat >  start_rgw.sh << EOF
#!/bin/bash
docker run \
   -d --net=host \
   --name=rgw \
   -v /data/ceph/lib:/var/lib/ceph/ \
   -v /data/ceph/etc:/etc/ceph \
   -v /etc/localtime:/etc/localtime \
   ceph/daemon:latest-nautilus  rgw
EOF

主节点目录详情

[root@VM-200-17-centos ~]# chmod +x /data/ceph/admin/*.sh
[root@VM-200-17-centos ~]# yum install tree -y
[root@VM-200-17-centos ~]# tree /data/ceph
.
|-- admin
|   |-- start_mds.sh
|   |-- start_mgr.sh
|   |-- start_mon.sh
|   |-- start_rgw.sh
|   `-- start_osd.sh
|-- data
|   `-- osd
|-- etc
|-- lib
`-- logs

执行(主节点执行)

[root@VM-200-17-centos ~]# sh /data/ceph/admin/start_mon.sh
[root@VM-200-17-centos ~]# docker ps
CONTAINER ID   IMAGE                         COMMAND                  CREATED       STATUS       PORTS     NAMES
2d287f3c5f73   ceph/daemon:latest-nautilus   "/opt/ceph-container…"   3 hours ago   Up 3 hours             mon

[root@VM-200-17-centos ~]# vi /data/ceph/etc/ceph.conf

[global]
fsid = 171912aa-2b67-42e9-a988-37615b91f3e2
mon initial members = ceph-node1
mon host = 172.19.0.17,172.19.0.14,172.19.0.3
public network = 172.19.0.0/16
cluster network = 172.19.0.0/16
osd journal size = 100
# 容忍更多的时钟误差
mon clock drift allowed = 2
mon clock drift warn backoff = 30
mon_max_pg_per_osd = 1000
# 推送到各节点：
# 允许删除pool
mon_allow_pool_delete = true
osd max object name len = 256
osd max object namespace len = 64

[mgr]
# 开启WEB仪表盘
mgr modules = dashboard
[client.rgw.ceph1]
# 设置rgw网关的web访问端口
rgw_frontends = "civetweb port=7480"

复制文件到其他节点（主节点执行）

注意修改ceph.conf： mon initial members 的名字 注意修改 start_mon.sh 脚本 IP 地址

ssh-keygen
ssh-copy-id -i ceph-node2
ssh-copy-id -i ceph-node3
scp -r /data/ceph ceph-node2:/data
scp -r /data/ceph ceph-node3:/data

执行启动(所有节点)

# 启动 osd、mgr、rgw
sh /data/ceph/admin/start_mon.sh


# 生成osd的密钥信息、生成osd的密钥信息
docker exec -it mon ceph auth get client.bootstrap-osd -o /var/lib/ceph/bootstrap-osd/ceph.keyring
docker exec mon ceph auth get client.bootstrap-rgw -o /var/lib/ceph/bootstrap-rgw/ceph.keyring

sh /data/ceph/admin/start_osd.sh
sh /data/ceph/admin/start_mgr.sh
sh /data/ceph/admin/start_rgw.sh
sh /data/ceph/admin/start_mds.sh

# 检查状态
docker exec mon ceph -s 
docker exec mon ceph health detail

# 处理：mons are allowing insecure global_id reclaim
docker exec mon ceph config set mon auth_allow_insecure_global_id_reclaim false

安装 Dashboard(主节点执行)

# 开启dashboard功能、创建证书、设置用户、 配置外部访问端口、 配置外部访问端口、关闭https、 重启Mgr DashBoard服务、 重启Mgr DashBoard服务
docker exec mgr ceph mgr module enable dashboard
docker exec mgr ceph config set mgr mgr/dashboard/ssl false
docker exec mgr ceph config set mgr mgr/dashboard/server_port 18080
docker exec mgr ceph config set mgr mgr/dashboard/server_addr 172.19.0.17

docker exec -it mgr  /bin/sh -c "echo admin > passwd"
docker exec mgr ceph dashboard set-login-credentials admin -i passwd


docker restart mgr
docker exec mgr ceph mgr services

访问: http://公网IP:18080
admin
admin

CephFs 部署(主节点执行)

# 主节点创建Data Pool、创建Metadata Pool、创建CephFS、查看FS信息
docker exec osd ceph osd pool create cephfs_data 128 128
docker exec osd ceph osd pool create cephfs_metadata 64 64
docker exec osd ceph fs new cephfs cephfs_metadata cephfs_data
docker exec osd ceph fs ls