一、服务器准备

二、环境装备设置主机名

1.每个节点修改主机名

hostnamectl set-hostname ceph-node1 # 节点一
hostnamectl set-hostname ceph-node2 # 节点二
hostnamectl set-hostname ceph-node3 # 节点三
hostnamectl set-hostname ceph-node4 # 节点四

2.设置hosts文件

在4台机器上都执行以下命令：

echo "10.0.30.11 ceph-node1" >> /etc/hosts
echo "10.0.30.12 ceph-node2" >> /etc/hosts
echo "10.0.30.13 ceph-node3" >> /etc/hosts
echo "10.0.30.14 ceph-node4" >> /etc/hosts

3.关闭防火墙和selinux

systemctl stop firewalld
systemctl disable firewalld
setenforce 0
sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config

4.安装和配置NTP(官方推荐集群所有节点全部安装NTP，需要保证各节点时间一致)

yum install ntp ntpdate ntp-doc -y
systemctl restart ntpd
systemctl status ntpd

4.创建用户(每台机器都执行)

useradd -d /home/ceph -m ceph
echo "123456" | passwd ceph --stdin
# sudo权限
echo "ceph ALL = (root) NOPASSWD:ALL" | tee /etc/sudoers.d/ceph
chmod 0440 /etc/sudoers.d/ceph

5.设置免密登录(因为ceph-deploy安装在ceph-node1上，所以只需要在ceph-node1上执行即可)

su - ceph
ssh-keygen -t rsa
ssh-copy-id ceph@ceph-node1
ssh-copy-id ceph@ceph-node2
ssh-copy-id ceph@ceph-node3
ssh-copy-id ceph@ceph-node4

6.设置ceph镜像源（我这里用的是清华源）

cat > /etc/yum.repos.d/ceph.repo<<'EOF'
[Ceph]
name=Ceph packages for $basearch
baseurl=https://mirror.tuna.tsinghua.edu.cn/ceph/rpm-mimic/el7/$basearch
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=https://mirror.tuna.tsinghua.edu.cn/ceph/keys/release.asc
priority=1

[Ceph-noarch]
name=Ceph noarch packages
baseurl=https://mirror.tuna.tsinghua.edu.cn/ceph/rpm-mimic/el7/noarch
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=https://mirror.tuna.tsinghua.edu.cn/ceph/keys/release.asc
priority=1

[ceph-source]
name=Ceph source packages
baseurl=https://mirror.tuna.tsinghua.edu.cn/ceph/rpm-mimic/el7/SRPMS
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=https://mirror.tuna.tsinghua.edu.cn/ceph/keys/release.asc
priority=1
EOF

三、安装ceph-deploy(之后的操作都在ceph-node1节点上执行)

yum update -y && yum install ceph-deploy -y

四、初始化集群

su - ceph # 切换到ceph用户
mkdir config
cd config

ceph-deploy new --cluster-network 10.0.0.0/16 --public-network 10.0.0.0/16 ceph-node{1,2,3}
# 此操作会报错
# Traceback (most recent call last):
#  File "/bin/ceph-deploy", line 18, in <module>
#    from ceph_deploy.cli import main
#  File "/usr/lib/python2.7/site-packages/ceph_deploy/cli.py", line 1, in <module>
#    import pkg_resources
# ImportError: No module named pkg_resources
# 以上报错是因为没有pip，需要安装pip
yum install epel-release
yum install python-pip

# 重新执行初始化
ceph-deploy new --cluster-network 10.0.0.0/16 --public-network 10.0.0.0/16 ceph-node{1,2,3}
ls -l
# -rw-rw-r--  1 ceph ceph    361 10月 25 17:24 ceph.conf
# -rw-rw-r--. 1 ceph ceph 504752 10月 25 17:24 ceph-deploy-ceph.log
# -rw-------. 1 ceph ceph     73 10月 25 17:24 ceph.mon.keyring

# 修改副本数为2
echo "osd_pool_default_size = 2" | tee -a ceph.conf

# 安装ceph相关的包(这一步可能会有点久)
ceph-deploy install ceph-node{1,2,3} 

五、初始化mon节点

ceph-deploy mon create-initial

# 将配置文件和admin密钥拷贝到管理节点
ceph-deploy admin ceph-node1 ceph-node2 ceph-node3

此时，集群可以通过ceph -s查看状态了

sudo ceph -s
#  cluster:
#    id:     b558acbf-fbdb-41ff-b609-f9bd4b543cb4
#    health: HEALTH_OK
#
#  services:
#    mon: 3 daemons, quorum ceph-node1,ceph-node2,ceph-node3
#    mgr: no daemons active
#    osd: 0 osds: 0 up, 0 in
#
#  data:
#    pools:   0 pools, 0 pgs
#    objects: 0  objects, 0 B
#    usage:   0 B used, 0 B / 0 B avail
#    pgs:     

六、部署管理节点

ceph-deploy mgr create ceph-node1 ceph-node2 ceph-node3

管理节点部署完，执行ceph -s即可看到mgr相关信息：

#    mon: 3 daemons, quorum ceph-node1,ceph-node2,ceph-node3
#    mgr: ceph-node1(active), standbys: ceph-node2, ceph-node3
#    osd: 0 osds: 0 up, 0 in

七、初始化磁盘并创建OSD

ceph-deploy osd create ceph-node1 --data /dev/sdb
ceph-deploy osd create ceph-node2 --data /dev/sdb
ceph-deploy osd create ceph-node3 --data /dev/sdb
ceph-deploy osd create ceph-node4 --data /dev/sdb

如果磁盘中有分区信息，可以用一下命令清除

ceph-volume lvm zap /dev/sdb --destroy

此时一个3mon节点，4 osd节点的ceph集群已搭建完毕

ceph -s
#  cluster:
#    id:     b558acbf-fbdb-41ff-b609-f9bd4b543cb4
#    health: HEALTH_OK
#
#  services:
#    mon: 3 daemons, quorum ceph-node1,ceph-node2,ceph-node3
#    mgr: ceph-node1(active), standbys: ceph-node2, ceph-node3
#    osd: 4 osds: 4 up, 4 in
#
#  data:
#    pools:   0 pools, 0 pgs
#    objects: 0  objects, 0 B
#    usage:   4.0 GiB used, 796 GiB / 800 GiB avail
#    pgs:     

八、安装ceph-mgr-dashboard

在ceph-node1上安装

yum install -y ceph-mgr-dashboard
ceph mgr module enable dashboard --force
ceph config-key set mgr/dashboard/server_port 8080 # 设置端口为8080
ceph config-key set mgr/dashboard/server_addr 10.0.30.11 # 设置绑定ip
ceph config set mgr mgr/dashboard/ssl false # 因为是内网使用，所以关闭ssl
# 重启一下dashboard
ceph mgr module disable dashboard
ceph mgr module enable dashboard
ceph dashboard set-login-credentials admin admin@123 # 设置用户名密码

至此，可以访问http://10.0.30.11:8080查看dashboard

参考链接

https://www.cnblogs.com/huchong/p/12435957.html

https://www.daxuxu.info/blog/post/centos7-an-zhuang-ceph-14-2-x-nautilusban-ben-ji-q/