# Docker-ceph集群
### 环境
Centos7.9
### 所有节点Host
```bash
cat >> /etc/hosts << EOF
172.19.0.17 ceph-node1
172.19.0.14 ceph-node2
172.19.0.3 ceph-node3
EOF
```
### 所有节点 Hostname
```bash
hostnamectl set-hostname ceph-node1 # 节点一
hostnamectl set-hostname ceph-node2 # 节点二
hostnamectl set-hostname ceph-node3 # 节点三
```
### 所有节点安装 Docker
```bash
mkdir -p /data/ceph/{admin,data,etc,lib,logs}
curl -fsSL https://get.docker.com | bash -s docker --mirror Aliyun
systemctl daemon-reload
systemctl restart docker
systemctl enable docker
```
### 所有节点挂载云盘
## chmod +x \*.sh
```bash
lsblk |grep vdb
mkfs.xfs -f /dev/vdb
mkdir /data/ceph/data/osd && mount /dev/vdb /data/ceph/data/osd
df -Th
```
### 主节点执行
```bash
cd /data/ceph/admin
cat > start_mon.sh << EOF
#!/bin/bash
docker run -d --net=host \
--name=mon \
-v /etc/localtime:/etc/localtime \
-v /data/ceph/etc:/etc/ceph \
-v /data/ceph/lib:/var/lib/ceph \
-v /data/ceph/logs:/var/log/ceph \
-e MON_IP=172.19.0.17 \
-e CEPH_PUBLIC_NETWORK=172.19.0.0/16 \
ceph/daemon:latest-nautilus mon
EOF
cat > start_osd.sh << EOF
#!/bin/bash
docker run -d \
--name=osd \
--net=host \
--restart=always \
--privileged=true \
--pid=host \
-v /etc/localtime:/etc/localtime \
-v /data/ceph/etc:/etc/ceph \
-v /data/ceph/lib:/var/lib/ceph \
-v /data/ceph/logs:/var/log/ceph \
-v /data/ceph/data/osd:/var/lib/ceph/osd \
ceph/daemon:latest-nautilus osd_directory
EOF
cat > start_mgr.sh << EOF
#!/bin/bash
docker run -d --net=host \
--name=mgr \
-v /etc/localtime:/etc/localtime \
-v /data/ceph/etc:/etc/ceph \
-v /data/ceph/lib:/var/lib/ceph \
-v /data/ceph/logs:/var/log/ceph \
ceph/daemon:latest-nautilus mgr
EOF
cat > start_mds.sh << EOF
#!/bin/bash
docker run -d \
--net=host \
--name=mds \
--privileged=true \
-v /etc/localtime:/etc/localtime \
-v /data/ceph/etc:/etc/ceph \
-v /data/ceph/lib:/var/lib/ceph \
-v /data/ceph/logs:/var/log/ceph \
-e CEPHFS_CREATE=0 \
-e CEPHFS_METADATA_POOL_PG=512 \
-e CEPHFS_DATA_POOL_PG=512 \
ceph/daemon:latest-nautilus mds
EOF
cat > start_rgw.sh << EOF
#!/bin/bash
docker run \
-d --net=host \
--name=rgw \
-v /data/ceph/lib:/var/lib/ceph/ \
-v /data/ceph/etc:/etc/ceph \
-v /etc/localtime:/etc/localtime \
ceph/daemon:latest-nautilus rgw
EOF
```
### 主节点目录详情
```bash
[root@VM-200-17-centos ~]# chmod +x /data/ceph/admin/*.sh
[root@VM-200-17-centos ~]# yum install tree -y
[root@VM-200-17-centos ~]# tree /data/ceph
.
|-- admin
| |-- start_mds.sh
| |-- start_mgr.sh
| |-- start_mon.sh
| |-- start_rgw.sh
| `-- start_osd.sh
|-- data
| `-- osd
|-- etc
|-- lib
`-- logs
```
### 执行(主节点执行)
```bash
[root@VM-200-17-centos ~]# sh /data/ceph/admin/start_mon.sh
[root@VM-200-17-centos ~]# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
2d287f3c5f73 ceph/daemon:latest-nautilus "/opt/ceph-container…" 3 hours ago Up 3 hours mon
[root@VM-200-17-centos ~]# vi /data/ceph/etc/ceph.conf
[global]
fsid = 171912aa-2b67-42e9-a988-37615b91f3e2
mon initial members = ceph-node1
mon host = 172.19.0.17,172.19.0.14,172.19.0.3
public network = 172.19.0.0/16
cluster network = 172.19.0.0/16
osd journal size = 100
# 容忍更多的时钟误差
mon clock drift allowed = 2
mon clock drift warn backoff = 30
mon_max_pg_per_osd = 1000
# 推送到各节点：
# 允许删除pool
mon_allow_pool_delete = true
osd max object name len = 256
osd max object namespace len = 64
[mgr]
# 开启WEB仪表盘
mgr modules = dashboard
[client.rgw.ceph1]
# 设置rgw网关的web访问端口
rgw_frontends = "civetweb port=7480"
```
### 复制文件到其他节点（主节点执行）
> 注意修改ceph.conf： mon initial members 的名字 注意修改 start\_mon.sh 脚本 IP 地址
```bash
ssh-keygen
ssh-copy-id -i ceph-node2
ssh-copy-id -i ceph-node3
scp -r /data/ceph ceph-node2:/data
scp -r /data/ceph ceph-node3:/data
```
### 执行启动(所有节点)
```bash
# 启动 osd、mgr、rgw
sh /data/ceph/admin/start_mon.sh
# 生成osd的密钥信息、生成osd的密钥信息
docker exec -it mon ceph auth get client.bootstrap-osd -o /var/lib/ceph/bootstrap-osd/ceph.keyring
docker exec mon ceph auth get client.bootstrap-rgw -o /var/lib/ceph/bootstrap-rgw/ceph.keyring
sh /data/ceph/admin/start_osd.sh
sh /data/ceph/admin/start_mgr.sh
sh /data/ceph/admin/start_rgw.sh
sh /data/ceph/admin/start_mds.sh
# 检查状态
docker exec mon ceph -s
docker exec mon ceph health detail
# 处理：mons are allowing insecure global_id reclaim
docker exec mon ceph config set mon auth_allow_insecure_global_id_reclaim false
```
### 安装 Dashboard(主节点执行)
```bash
# 开启dashboard功能、创建证书、设置用户、 配置外部访问端口、 配置外部访问端口、关闭https、 重启Mgr DashBoard服务、 重启Mgr DashBoard服务
docker exec mgr ceph mgr module enable dashboard
docker exec mgr ceph config set mgr mgr/dashboard/ssl false
docker exec mgr ceph config set mgr mgr/dashboard/server_port 18080
docker exec mgr ceph config set mgr mgr/dashboard/server_addr 172.19.0.17
docker exec -it mgr /bin/sh -c "echo admin > passwd"
docker exec mgr ceph dashboard set-login-credentials admin -i passwd
docker restart mgr
docker exec mgr ceph mgr services
```
```bahs
访问: http://公网IP:18080
admin
admin
```
### CephFs 部署(主节点执行)
```bash
# 主节点创建Data Pool、创建Metadata Pool、创建CephFS、查看FS信息
docker exec osd ceph osd pool create cephfs_data 128 128
docker exec osd ceph osd pool create cephfs_metadata 64 64
docker exec osd ceph fs new cephfs cephfs_metadata cephfs_data
docker exec osd ceph fs ls
```
---
# Agent Instructions: Querying This Documentation
If you need additional information that is not directly available in this page, you can query the documentation dynamically by asking a question.
Perform an HTTP GET request on the current page URL with the `ask` query parameter:
```
GET https://close.gitbook.io/yun-wei-bi-ji/kubernetes/ceph/dockerceph-ji-qun.md?ask=
```
The question should be specific, self-contained, and written in natural language.
The response will contain a direct answer to the question and relevant excerpts and sources from the documentation.
Use this mechanism when the answer is not explicitly present in the current page, you need clarification or additional context, or you want to retrieve related documentation sections.