Installing Ceph Cluster on CentOS 7.9

Hardware Value
CPU 2C
Memory 4G
Disk(OS) 50G *1
Data(HDD) 100G *2
Network(Pub) 1G *1
Network(Cls) 10G *1
Hostname Pub IP Cls IP Disk OS
ceph01 192.168.3.131 192.168.10.131 sda, sdb, sdc CentOS 7.9.2009
ceph02 192.168.3.132 192.168.10.132 sda, sdb, sdc CentOS 7.9.2009
ceph03 192.168.3.133 192.168.10.133 sda, sdb, sdc CentOS 7.9.2009

System Settings

  1. Resolve hostname.
~]# systemctl stop firewalld
~]# systemctl disable firewalld
~]# sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config
~]# setenforce 0
~]# sed -i '/swap/s/^[^#]/#&/' /etc/fstab

~]# cat >>/etc/hosts <<EOF

# ceph
192.168.3.131 ceph01
192.168.3.132 ceph02
192.168.3.133 ceph03
EOF
  1. SSH authentication
~]# ssh-keygen -t rsa
~]# ssh-copy-id ceph01
~]# ssh-copy-id ceph02
~]# ssh-copy-id ceph03
  1. YUM repository
mkdir /etc/yum.repos.d/bak
mv /etc/yum.repos.d/*.repo /etc/yum.repos.d/bak

cat >/etc/yum.repos.d/media.repo <<EOF
[InstallMedia]
name=CentOS 7
baseurl=/assets/tmp/mnt
gpgcheck=0
enabled=1
EOF

curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo
curl -o /etc/yum.repos.d/epel-7.repo http://mirrors.aliyun.com/repo/epel-7.repo
sed -i -e '/mirrors.cloud.aliyuncs.com/d' -e '/mirrors.aliyuncs.com/d' /etc/yum.repos.d/CentOS-Base.repo

yum clean all && yum makecache
  1. NTP
~]# yum install -y chrony
~]# sed -i 's/^server/#server/g' /etc/chrony.conf
~]# echo 'server ntp1.aliyun.com iburst' >>/etc/chrony.conf
~]# echo 'server ntp2.aliyun.com iburst' >>/etc/chrony.conf
~]# echo 'server ntp3.aliyun.com iburst' >>/etc/chrony.conf
~]# echo 'allow 192.168.3.0/24' >>/etc/chrony.conf
~]# systemctl restart chronyd
~]# systemctl enable chronyd
~]# chronyc sources -v

Install Ceph

  1. Ceph YUM repository.
~]# cat >>/etc/yum.repos.d/ceph.repo <<EOF
[ceph]
name=ceph
baseurl=http://mirrors.aliyun.com/ceph/rpm-mimic/el7/x86_64/
enabled=1
gpgcheck=0
priority=1

[ceph-noarch]
name=cephnoarch
baseurl=http://mirrors.aliyun.com/ceph/rpm-mimic/el7/noarch/
enabled=1
gpgcheck=0
priority=1

[ceph-source]
name=Ceph source packages
baseurl=http://mirrors.aliyun.com/ceph/rpm-mimic/el7/SRPMS
enabled=1
gpgcheck=0
priority=1
EOF
  1. Install ceph-deploy package.
~]# yum -y install ceph-deploy python-setuptools python2-subprocess32
  1. Initial cluster.
~]# mkdir /etc/ceph && cd /etc/ceph

[root@ceph01 ceph]# ceph-deploy new ceph0{1..3} --public-network=192.168.3.0/24 --cluster-network=192.168.10.0/24

[root@ceph01 ceph]# ll
total 16
-rw-r--r-- 1 root root 307 Jan 17 19:23 ceph.conf
-rw-r--r-- 1 root root 5055 Jan 17 19:23 ceph-deploy-ceph.log
-rw------- 1 root root 73 Jan 17 19:23 ceph.mon.keyring

[root@ceph01 ceph]# cat ceph.conf
[global]
fsid = 9f62601d-8131-447f-8fbb-6dd5327cd96f
public_network = 192.168.3.0/24
cluster_network = 192.168.10.0/24
mon_initial_members = ceph01, ceph02, ceph03
mon_host = 192.168.3.131,192.168.3.132,192.168.3.133
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
  1. Deploy cluster.
[root@ceph01 ceph]# ceph-deploy install ceph01 ceph02 ceph03

[root@ceph01 ceph]# ceph-deploy mon create-initial

[root@ceph01 ceph]# ceph -s
cluster:
id: 9f62601d-8131-447f-8fbb-6dd5327cd96f
health: HEALTH_OK

services:
mon: 3 daemons, quorum ceph01,ceph02,ceph03
mgr: no daemons active
osd: 0 osds: 0 up, 0 in

data:
pools: 0 pools, 0 pgs
objects: 0 objects, 0 B
usage: 0 B used, 0 B / 0 B avail
pgs:

[root@ceph01 ceph]# ceph mon stat
[root@ceph01 ceph]# ceph quorum_status -f json-pretty
[root@ceph01 ceph]# ceph mon dump
[root@ceph01 ceph]# ceph daemon mon.ceph01 mon_status

[root@ceph01 ceph]# ceph-deploy --overwrite-conf admin ceph01 ceph02 ceph03
[root@ceph01 ceph]# ceph-deploy mgr create ceph01 ceph02 ceph03
[root@ceph01 ceph]# ceph mgr services

[root@ceph01 ceph]# ceph-deploy disk list ceph01 ceph02 ceph03
[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf
[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy disk list ceph01 ceph02 ceph03
[ceph_deploy.cli][INFO ] ceph-deploy options:
[ceph_deploy.cli][INFO ] username : None
[ceph_deploy.cli][INFO ] verbose : False
[ceph_deploy.cli][INFO ] debug : False
[ceph_deploy.cli][INFO ] overwrite_conf : False
[ceph_deploy.cli][INFO ] subcommand : list
[ceph_deploy.cli][INFO ] quiet : False
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7f6a1f86d4d0>
[ceph_deploy.cli][INFO ] cluster : ceph
[ceph_deploy.cli][INFO ] host : ['ceph01', 'ceph02', 'ceph03']
[ceph_deploy.cli][INFO ] func : <function disk at 0x7f6a1fab8938>
[ceph_deploy.cli][INFO ] ceph_conf : None
[ceph_deploy.cli][INFO ] default_release : False
[ceph01][DEBUG ] connected to host: ceph01
[ceph01][DEBUG ] detect platform information from remote host
[ceph01][DEBUG ] detect machine type
[ceph01][DEBUG ] find the location of an executable
[ceph01][INFO ] Running command: fdisk -l
[ceph01][INFO ] Disk /dev/sda: 53.7 GB, 53687091200 bytes, 104857600 sectors
[ceph01][INFO ] Disk /dev/mapper/centos-root: 48.4 GB, 48444211200 bytes, 94617600 sectors
[ceph01][INFO ] Disk /dev/mapper/centos-swap: 4160 MB, 4160749568 bytes, 8126464 sectors
[ceph01][INFO ] Disk /dev/sdb: 21.5 GB, 21474836480 bytes, 41943040 sectors
[ceph01][INFO ] Disk /dev/sdc: 21.5 GB, 21474836480 bytes, 41943040 sectors
[ceph02][DEBUG ] connected to host: ceph02
[ceph02][DEBUG ] detect platform information from remote host
[ceph02][DEBUG ] detect machine type
[ceph02][DEBUG ] find the location of an executable
[ceph02][INFO ] Running command: fdisk -l
[ceph02][INFO ] Disk /dev/sda: 53.7 GB, 53687091200 bytes, 104857600 sectors
[ceph02][INFO ] Disk /dev/mapper/centos-root: 48.4 GB, 48444211200 bytes, 94617600 sectors
[ceph02][INFO ] Disk /dev/mapper/centos-swap: 4160 MB, 4160749568 bytes, 8126464 sectors
[ceph02][INFO ] Disk /dev/sdb: 21.5 GB, 21474836480 bytes, 41943040 sectors
[ceph02][INFO ] Disk /dev/sdc: 21.5 GB, 21474836480 bytes, 41943040 sectors
[ceph03][DEBUG ] connected to host: ceph03
[ceph03][DEBUG ] detect platform information from remote host
[ceph03][DEBUG ] detect machine type
[ceph03][DEBUG ] find the location of an executable
[ceph03][INFO ] Running command: fdisk -l
[ceph03][INFO ] Disk /dev/sda: 53.7 GB, 53687091200 bytes, 104857600 sectors
[ceph03][INFO ] Disk /dev/mapper/centos-root: 48.4 GB, 48444211200 bytes, 94617600 sectors
[ceph03][INFO ] Disk /dev/mapper/centos-swap: 4160 MB, 4160749568 bytes, 8126464 sectors
[ceph03][INFO ] Disk /dev/sdb: 21.5 GB, 21474836480 bytes, 41943040 sectors
[ceph03][INFO ] Disk /dev/sdc: 21.5 GB, 21474836480 bytes, 41943040 sectors

[root@ceph01 ceph]# ceph-deploy disk zap ceph01 /dev/sdb /dev/sdc
[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf
[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy disk zap ceph01 /dev/sdb /dev/sdc
[ceph_deploy.cli][INFO ] ceph-deploy options:
[ceph_deploy.cli][INFO ] username : None
[ceph_deploy.cli][INFO ] verbose : False
[ceph_deploy.cli][INFO ] debug : False
[ceph_deploy.cli][INFO ] overwrite_conf : False
[ceph_deploy.cli][INFO ] subcommand : zap
[ceph_deploy.cli][INFO ] quiet : False
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7f5c908204d0>
[ceph_deploy.cli][INFO ] cluster : ceph
[ceph_deploy.cli][INFO ] host : ceph01
[ceph_deploy.cli][INFO ] func : <function disk at 0x7f5c90a6b938>
[ceph_deploy.cli][INFO ] ceph_conf : None
[ceph_deploy.cli][INFO ] default_release : False
[ceph_deploy.cli][INFO ] disk : ['/dev/sdb', '/dev/sdc']
[ceph_deploy.osd][DEBUG ] zapping /dev/sdb on ceph01
[ceph01][DEBUG ] connected to host: ceph01
[ceph01][DEBUG ] detect platform information from remote host
[ceph01][DEBUG ] detect machine type
[ceph01][DEBUG ] find the location of an executable
[ceph_deploy.osd][INFO ] Distro info: CentOS Linux 7.9.2009 Core
[ceph01][DEBUG ] zeroing last few blocks of device
[ceph01][DEBUG ] find the location of an executable
[ceph01][INFO ] Running command: /usr/sbin/ceph-volume lvm zap /dev/sdb
[ceph01][WARNIN] --> Zapping: /dev/sdb
[ceph01][WARNIN] --> --destroy was not specified, but zapping a whole device will remove the partition table
[ceph01][WARNIN] Running command: /usr/bin/dd if=/dev/zero of=/dev/sdb bs=1M count=10 conv=fsync
[ceph01][WARNIN] stderr: 10+0 records in
[ceph01][WARNIN] 10+0 records out
[ceph01][WARNIN] 10485760 bytes (10 MB) copied
[ceph01][WARNIN] stderr: , 0.872308 s, 12.0 MB/s
[ceph01][WARNIN] --> Zapping successful for: <Raw Device: /dev/sdb>
[ceph_deploy.osd][DEBUG ] zapping /dev/sdc on ceph01
[ceph01][DEBUG ] connected to host: ceph01
[ceph01][DEBUG ] detect platform information from remote host
[ceph01][DEBUG ] detect machine type
[ceph01][DEBUG ] find the location of an executable
[ceph_deploy.osd][INFO ] Distro info: CentOS Linux 7.9.2009 Core
[ceph01][DEBUG ] zeroing last few blocks of device
[ceph01][DEBUG ] find the location of an executable
[ceph01][INFO ] Running command: /usr/sbin/ceph-volume lvm zap /dev/sdc
[ceph01][WARNIN] --> Zapping: /dev/sdc
[ceph01][WARNIN] --> --destroy was not specified, but zapping a whole device will remove the partition table
[ceph01][WARNIN] Running command: /usr/bin/dd if=/dev/zero of=/dev/sdc bs=1M count=10 conv=fsync
[ceph01][WARNIN] stderr: 10+0 records in
[ceph01][WARNIN] 10+0 records out
[ceph01][WARNIN] 10485760 bytes (10 MB) copied
[ceph01][WARNIN] stderr: , 0.635255 s, 16.5 MB/s
[ceph01][WARNIN] --> Zapping successful for: <Raw Device: /dev/sdc>

[root@ceph01 ceph]# ceph-deploy disk zap ceph02 /dev/sdb /dev/sdc
[root@ceph01 ceph]# ceph-deploy disk zap ceph03 /dev/sdb /dev/sdc

[root@ceph01 ceph]# ceph-deploy osd create ceph01 --data /dev/sdb
[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf
[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy osd create ceph01 --data /dev/sdb
[ceph_deploy.cli][INFO ] ceph-deploy options:
[ceph_deploy.cli][INFO ] verbose : False
[ceph_deploy.cli][INFO ] bluestore : None
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7fceadec85f0>
[ceph_deploy.cli][INFO ] cluster : ceph
[ceph_deploy.cli][INFO ] fs_type : xfs
[ceph_deploy.cli][INFO ] block_wal : None
[ceph_deploy.cli][INFO ] default_release : False
[ceph_deploy.cli][INFO ] username : None
[ceph_deploy.cli][INFO ] journal : None
[ceph_deploy.cli][INFO ] subcommand : create
[ceph_deploy.cli][INFO ] host : ceph01
[ceph_deploy.cli][INFO ] filestore : None
[ceph_deploy.cli][INFO ] func : <function osd at 0x7fceae10d8c0>
[ceph_deploy.cli][INFO ] ceph_conf : None
[ceph_deploy.cli][INFO ] zap_disk : False
[ceph_deploy.cli][INFO ] data : /dev/sdb
[ceph_deploy.cli][INFO ] block_db : None
[ceph_deploy.cli][INFO ] dmcrypt : False
[ceph_deploy.cli][INFO ] overwrite_conf : False
[ceph_deploy.cli][INFO ] dmcrypt_key_dir : /etc/ceph/dmcrypt-keys
[ceph_deploy.cli][INFO ] quiet : False
[ceph_deploy.cli][INFO ] debug : False
[ceph_deploy.osd][DEBUG ] Creating OSD on cluster ceph with data device /dev/sdb
[ceph01][DEBUG ] connected to host: ceph01
[ceph01][DEBUG ] detect platform information from remote host
[ceph01][DEBUG ] detect machine type
[ceph01][DEBUG ] find the location of an executable
[ceph_deploy.osd][INFO ] Distro info: CentOS Linux 7.9.2009 Core
[ceph_deploy.osd][DEBUG ] Deploying osd to ceph01
[ceph01][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
[ceph01][WARNIN] osd keyring does not exist yet, creating one
[ceph01][DEBUG ] create a keyring file
[ceph01][DEBUG ] find the location of an executable
[ceph01][INFO ] Running command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sdb
[ceph01][WARNIN] Running command: /usr/bin/ceph-authtool --gen-print-key
[ceph01][WARNIN] Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new 9bbd32ca-95ba-4879-a334-359bad1431b0
[ceph01][WARNIN] Running command: /usr/sbin/vgcreate --force --yes ceph-81f6b1cc-5b47-4e6e-a669-e0868678e93d /dev/sdb
[ceph01][WARNIN] stdout: Physical volume "/dev/sdb" successfully created.
[ceph01][WARNIN] stdout: Volume group "ceph-81f6b1cc-5b47-4e6e-a669-e0868678e93d" successfully created
[ceph01][WARNIN] Running command: /usr/sbin/lvcreate --yes -l 100%FREE -n osd-block-9bbd32ca-95ba-4879-a334-359bad1431b0 ceph-81f6b1cc-5b47-4e6e-a669-e0868678e93d
[ceph01][WARNIN] stdout: Logical volume "osd-block-9bbd32ca-95ba-4879-a334-359bad1431b0" created.
[ceph01][WARNIN] Running command: /usr/bin/ceph-authtool --gen-print-key
[ceph01][WARNIN] Running command: /usr/bin/mount -t tmpfs tmpfs /var/lib/ceph/osd/ceph-0
[ceph01][WARNIN] Running command: /usr/bin/chown -h ceph:ceph /dev/ceph-81f6b1cc-5b47-4e6e-a669-e0868678e93d/osd-block-9bbd32ca-95ba-4879-a334-359bad1431b0
[ceph01][WARNIN] Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2
[ceph01][WARNIN] Running command: /usr/bin/ln -s /dev/ceph-81f6b1cc-5b47-4e6e-a669-e0868678e93d/osd-block-9bbd32ca-95ba-4879-a334-359bad1431b0 /var/lib/ceph/osd/ceph-0/block
[ceph01][WARNIN] Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring mon getmap -o /var/lib/ceph/osd/ceph-0/activate.monmap
[ceph01][WARNIN] stderr: got monmap epoch 1
[ceph01][WARNIN] Running command: /usr/bin/ceph-authtool /var/lib/ceph/osd/ceph-0/keyring --create-keyring --name osd.0 --add-key AQArmm1p0oRkLRAAewuwDznjoQjxcQjkv3YSDg==
[ceph01][WARNIN] stdout: creating /var/lib/ceph/osd/ceph-0/keyring
[ceph01][WARNIN] stdout: added entity osd.0 auth auth(auid = 18446744073709551615 key=AQArmm1p0oRkLRAAewuwDznjoQjxcQjkv3YSDg== with 0 caps)
[ceph01][WARNIN] Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0/keyring
[ceph01][WARNIN] Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0/
[ceph01][WARNIN] Running command: /usr/bin/ceph-osd --cluster ceph --osd-objectstore bluestore --mkfs -i 0 --monmap /var/lib/ceph/osd/ceph-0/activate.monmap --keyfile - --osd-data /var/lib/ceph/osd/ceph-0/ --osd-uuid 9bbd32ca-95ba-4879-a334-359bad1431b0 --setuser ceph --setgroup ceph
[ceph01][WARNIN] --> ceph-volume lvm prepare successful for: /dev/sdb
[ceph01][WARNIN] Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0
[ceph01][WARNIN] Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/ceph-81f6b1cc-5b47-4e6e-a669-e0868678e93d/osd-block-9bbd32ca-95ba-4879-a334-359bad1431b0 --path /var/lib/ceph/osd/ceph-0 --no-mon-config
[ceph01][WARNIN] Running command: /usr/bin/ln -snf /dev/ceph-81f6b1cc-5b47-4e6e-a669-e0868678e93d/osd-block-9bbd32ca-95ba-4879-a334-359bad1431b0 /var/lib/ceph/osd/ceph-0/block
[ceph01][WARNIN] Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block
[ceph01][WARNIN] Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2
[ceph01][WARNIN] Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0
[ceph01][WARNIN] Running command: /usr/bin/systemctl enable ceph-volume@lvm-0-9bbd32ca-95ba-4879-a334-359bad1431b0
[ceph01][WARNIN] stderr: Created symlink from /etc/systemd/system/multi-user.target.wants/ceph-volume@lvm-0-9bbd32ca-95ba-4879-a334-359bad1431b0.service to /usr/lib/systemd/system/ceph-volume@.service.
[ceph01][WARNIN] Running command: /usr/bin/systemctl enable --runtime ceph-osd@0
[ceph01][WARNIN] stderr: Created symlink from /run/systemd/system/ceph-osd.target.wants/ceph-osd@0.service to /usr/lib/systemd/system/ceph-osd@.service.
[ceph01][WARNIN] Running command: /usr/bin/systemctl start ceph-osd@0
[ceph01][WARNIN] --> ceph-volume lvm activate successful for osd ID: 0
[ceph01][WARNIN] --> ceph-volume lvm create successful for: /dev/sdb
[ceph01][INFO ] checking OSD status...
[ceph01][DEBUG ] find the location of an executable
[ceph01][INFO ] Running command: /bin/ceph --cluster=ceph osd stat --format=json
[ceph_deploy.osd][DEBUG ] Host ceph01 is now ready for osd use.

[root@ceph01 ceph]# ceph-deploy osd create ceph01 --data /dev/sdc
[root@ceph01 ceph]# ceph-deploy osd create ceph02 --data /dev/sdb
[root@ceph01 ceph]# ceph-deploy osd create ceph02 --data /dev/sdc
[root@ceph01 ceph]# ceph-deploy osd create ceph03 --data /dev/sdb
[root@ceph01 ceph]# ceph-deploy osd create ceph03 --data /dev/sdc

[root@ceph01 ceph]# for i in {1..3}; do ceph-deploy osd create ceph0$i --data /dev/sdb; ceph-deploy osd create ceph0$i --data /dev/sdc; done

[root@ceph01 ceph]# ceph osd status
+----+--------+-------+-------+--------+---------+--------+---------+-----------+
| id | host | used | avail | wr ops | wr data | rd ops | rd data | state |
+----+--------+-------+-------+--------+---------+--------+---------+-----------+
| 0 | ceph01 | 1027M | 18.9G | 0 | 0 | 0 | 0 | exists,up |
| 1 | ceph01 | 1027M | 18.9G | 0 | 0 | 0 | 0 | exists,up |
| 2 | ceph02 | 1027M | 18.9G | 0 | 0 | 0 | 0 | exists,up |
| 3 | ceph02 | 1027M | 18.9G | 0 | 0 | 0 | 0 | exists,up |
| 4 | ceph03 | 1027M | 18.9G | 0 | 0 | 0 | 0 | exists,up |
| 5 | ceph03 | 1027M | 18.9G | 0 | 0 | 0 | 0 | exists,up |
+----+--------+-------+-------+--------+---------+--------+---------+-----------+

RBD

Create rbd pool.

[root@ceph01 ceph]# ceph-deploy --overwrite-conf admin ceph0{1..3}
[root@ceph01 ceph]# ceph osd pool create rbd_pool 128 128
[root@ceph01 ceph]# ceph auth get-or-create client.rbd mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=rbd_pool'
[root@ceph01 ceph]# ceph auth get client.rbd
[root@ceph01 ceph]# ceph auth get client.rbd -o /etc/ceph/ceph.client.rbd.keyring
[root@ceph01 ceph]# cat /etc/ceph/ceph.client.rbd.keyring
[root@ceph01 ceph]# scp /etc/ceph/ceph.client.rbd.keyring pve01:/etc/ceph/
[root@ceph01 ceph]# ceph osd pool application enable rbd_pool rbd_pool

PVE mount rbd image.

root@pve01:~# apt install ceph-common
root@pve01:~# ceph -s --id rbd
root@pve01:~# cp /etc/ceph/ceph.client.rbd.keyring /etc/pve/priv/ceph/ceph-rbd.keyring
root@pve01:~# cat /etc/pve/priv/ceph/ceph-rbd.keyring
root@pve01:~# pvesm add rbd ceph-rbd --pool rbd_pool --monhost "192.168.3.131 192.168.3.132 192.168.3.133" --username admin --content images --keyring /etc/pve/priv/ceph/ceph-rbd.keyring

General mount rbd image.

[root@ceph-client ~]# yum install ceph-common
[root@ceph-client ~]# ceph -s --id rbd
[root@ceph-client ~]# rbd create rbd -p rbd_pool --size 10G --id rbd
[root@ceph-client ~]# rbd ls -l rbd_pool --id rbd
[root@ceph-client ~]# rbd info -p rbd_pool rbd --id rbd
[root@ceph-client ~]# rbd feature disable -p rbd_pool rbd exclusive-lock, object-map, fast-diff, deep-flatten --id rbd
[root@ceph-client ~]# rbd info -p rbd_pool rbd --id rbd
[root@ceph-client ~]# rbd map -p rbd_pool rbd --id rbd
[root@ceph-client ~]# lsblk
[root@ceph-client ~]# mkfs.xfs /dev/rbd0
[root@ceph-client ~]# mkdir /mnt/rdb
[root@ceph-client ~]# mount /dev/rbd0 /mnt/rdb/
[root@ceph-client ~]# df -h
[root@ceph-client ~]# echo "/dev/rbd0 /mnt/rdb xfs defaults,_netdev 0 0" >>/etc/fstab

Umount and remove rbd image.

[root@ceph-client ~]# umount /mnt/rdb
[root@ceph-client ~]# rbd showmapped --id rbd
[root@ceph-client ~]# rbd unmap -p rbd_pool rbd --id rbd
[root@ceph-client ~]# rbd showmapped --id rbd
[root@ceph-client ~]# rbd ls -l rbd_pool --id rbd
[root@ceph-client ~]# rbd rm -p rbd_pool rbd --id rbd
[root@ceph-client ~]# sed -i '/rbd0/d' /etc/fstab

Remove rbd pool and client keyring.

[root@ceph01 ceph]# ceph osd pool ls
[root@ceph01 ceph]# ceph config set mon mon_allow_pool_delete true
[root@ceph01 ceph]# ceph osd pool rm rbd_pool rbd_pool --yes-i-really-really-mean-it
[root@ceph01 ceph]# ceph config set mon mon_allow_pool_delete false
[root@ceph01 ceph]# ceph auth ls
[root@ceph01 ceph]# ceph auth rm client.rbd

Remove osd disk.

[root@ceph01 ceph]# for i in {0..11}; do ceph osd out osd.$i; ceph osd down osd.$i; ceph osd rm osd.$i; ceph osd crush rm osd.$i; ceph auth del osd.$i; done
~]# df -h | grep ceph | awk '{print $6}' | xargs umount
~]# for i in /dev/mapper/ceph* ; do lvremove $i; done
~]# vgs | grep ceph | awk '{print $1}' | xargs vgremove
~]# for i in b c d e; do pvremove /dev/sd$i; done

Troubleshooting

Issue:

~]# ceph-deploy install ceph01 ceph02 ceph03
...
[ceph01][DEBUG ] Downloading packages:
[ceph01][WARNIN] No data was received after 300 seconds, disconnecting...
[ceph01][INFO ] Running command: ceph --version
[ceph01][ERROR ] Traceback (most recent call last):
[ceph01][ERROR ] File "/usr/lib/python2.7/site-packages/ceph_deploy/lib/vendor/remoto/process.py", line 119, in run
[ceph01][ERROR ] reporting(conn, result, timeout)
[ceph01][ERROR ] File "/usr/lib/python2.7/site-packages/ceph_deploy/lib/vendor/remoto/log.py", line 13, in reporting
[ceph01][ERROR ] received = result.receive(timeout)
[ceph01][ERROR ] File "/usr/lib/python2.7/site-packages/ceph_deploy/lib/vendor/remoto/lib/vendor/execnet/gateway_base.py", line 704, in receive
[ceph01][ERROR ] raise self._getremoteerror() or EOFError()
[ceph01][ERROR ] RemoteError: Traceback (most recent call last):
[ceph01][ERROR ] File "<string>", line 1036, in executetask
[ceph01][ERROR ] File "<remote exec>", line 12, in _remote_run
[ceph01][ERROR ] File "/usr/lib64/python2.7/subprocess.py", line 711, in __init__
[ceph01][ERROR ] errread, errwrite)
[ceph01][ERROR ] File "/usr/lib64/python2.7/subprocess.py", line 1327, in _execute_child
[ceph01][ERROR ] raise child_exception
[ceph01][ERROR ] OSError: [Errno 2] No such file or directory
[ceph01][ERROR ]
[ceph01][ERROR ]
[ceph_deploy][ERROR ] RuntimeError: Failed to execute command: ceph --version

Issue: VM 100 qmp command 'set_password' failed - unable to connect to VM 100 qmp socket - timeout after 51 retries TASK ERROR: Failed to run vncproxy.

TASK ERROR: start failed: command '/usr/bin/kvm -id 100 -name 'vm1,debug-threads=on' -no-shutdown -chardev 'socket,id=qmp,path=/var/run/qemu-server/100.qmp,server=on,wait=off' -mon 'chardev=qmp,mode=control' -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' -mon 'chardev=qmp-event,mode=control' -pidfile /var/run/qemu-server/100.pid -daemonize -smbios 'type=1,uuid=b8f34476-014d-4269-bb4a-2f87fedfd04f' -smp '1,sockets=1,cores=1,maxcpus=1' -nodefaults -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' -vnc 'unix:/var/run/qemu-server/100.vnc,password=on' -cpu qemu64,+aes,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+pni,+popcnt,+sse4.1,+sse4.2,+ssse3 -m 2048 -object '{"id":"throttle-drive-ide2","limits":{},"qom-type":"throttle-group"}' -object 'iothread,id=iothread-virtioscsi0' -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' -global 'PIIX4_PM.disable_s3=1' -global 'PIIX4_PM.disable_s4=1' -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' -device 'pci-bridge,id=pci.3,chassis_nr=3,bus=pci.0,addr=0x5' -device 'vmgenid,guid=7a16d85b-7a56-4df7-bc6c-5c5fbe4619d7' -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' -device 'VGA,id=vga,bus=pci.0,addr=0x2' -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' -iscsi 'initiator-name=iqn.1993-08.org.debian:01:9d2082fd7f15' -blockdev '{"driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"driver":"raw","file":{"aio":"io_uring","cache":{"direct":false,"no-flush":false},"driver":"file","filename":"/var/lib/vz/template/iso/CentOS-7-x86_64-Minimal-2009.iso","node-name":"e45ca1075b0e69a611a74d2ea44a118","read-only":true},"node-name":"f45ca1075b0e69a611a74d2ea44a118","read-only":true},"node-name":"drive-ide2","read-only":true,"throttle-group":"throttle-drive-ide2"}' -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=101' -device 'virtio-scsi-pci,id=virtioscsi0,bus=pci.3,addr=0x1,iothread=iothread-virtioscsi0' -blockdev '{"detect-zeroes":"on","discard":"ignore","driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"raw","file":{"auth-client-required":["cephx"],"cache":{"direct":true,"no-flush":false},"conf":"/etc/pve/priv/ceph/ceph-rbd.conf","detect-zeroes":"on","discard":"ignore","driver":"rbd","image":"vm-100-disk-0","node-name":"e6de88efbc5038c7107fe97b819db6c","pool":"rbd_pool","read-only":false,"server":[{"host":"192.168.3.131","port":"3300"},{"host":"192.168.3.132","port":"3300"},{"host":"192.168.3.133","port":"3300"}],"user":"admin"},"node-name":"f6de88efbc5038c7107fe97b819db6c","read-only":false},"node-name":"drive-scsi0","read-only":false,"throttle-group":"throttle-drive-scsi0"}' -device 'scsi-hd,bus=virtioscsi0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,device_id=drive-scsi0,bootindex=100,write-cache=on' -netdev 'type=tap,id=net0,ifname=tap100i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' -device 'virtio-net-pci,mac=BC:24:11:DE:11:0C,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=102,host_mtu=1500' -machine 'hpet=off,type=pc+pve0'' failed: got timeout

Issue: High write latency.

[root@ceph01 ceph]# rados bench -p rbd_pool 10 write --no-cleanup
hints = 1
Maintaining 16 concurrent writes of 4194304 bytes to objects of size 4194304 for up to 10 seconds or 0 objects
Object prefix: benchmark_data_ceph01_32031
sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s)
0 0 0 0 0 0 - 0
1 16 16 0 0 0 - 0
2 16 16 0 0 0 - 0
3 16 16 0 0 0 - 0
4 16 17 1 0.999873 1 3.9026 3.9026
5 16 20 4 3.19959 12 4.24611 4.08181
6 16 22 6 3.99948 8 5.80697 4.55603
7 16 26 10 5.71356 16 6.90788 5.3588
8 16 33 17 8.49887 28 3.73309 5.8086
9 16 36 20 8.88768 12 4.52645 5.79464
10 16 36 20 7.99891 0 - 5.79464
11 16 37 21 7.63531 2 3.50707 5.68571
12 16 37 21 6.99904 0 - 5.68571
13 16 37 21 6.46066 0 - 5.68571
14 15 37 22 6.28486 1.33333 7.02547 5.7466
Total time run: 14.274
Total writes made: 37
Write size: 4194304
Object size: 4194304
Bandwidth (MB/sec): 10.3685
Stddev Bandwidth: 8.49352
Max bandwidth (MB/sec): 28
Min bandwidth (MB/sec): 0
Average IOPS: 2
Stddev IOPS: 2
Max IOPS: 7
Min IOPS: 0
Average Latency(s): 6.1557
Stddev Latency(s): 1.57095
Max latency(s): 9.04431
Min latency(s): 3.29049

[root@ceph01 ceph]# rados bench -p rbd_pool 10 seq
hints = 1
sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s)
0 0 0 0 0 0 - 0
Total time run: 0.919018
Total reads made: 37
Read size: 4194304
Object size: 4194304
Bandwidth (MB/sec): 161.042
Average IOPS: 40
Stddev IOPS: 0
Max IOPS: 36
Min IOPS: 36
Average Latency(s): 0.37721
Max latency(s): 0.917139
Min latency(s): 0.0524276

[root@ceph01 ceph]# rados bench -p rbd_pool 10 rand
hints = 1
sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s)
0 0 0 0 0 0 - 0
1 15 317 302 1207.7 1208 0.208717 0.0487666
2 16 620 604 1207.09 1208 0.0397822 0.0505016
3 16 921 905 1205.86 1204 0.121147 0.0509149
4 16 1213 1197 1196.32 1168 0.0416905 0.0514738
5 15 1536 1521 1216.18 1296 0.0358462 0.0508281
6 15 1831 1816 1209.75 1180 0.0199542 0.0511784
7 15 2135 2120 1210.62 1216 0.010359 0.0512213
8 16 2448 2432 1215.21 1248 0.131977 0.0509898
9 16 2742 2726 1210.73 1176 0.0444173 0.0513644
10 14 3059 3045 1217.24 1276 0.0183521 0.0511602
Total time run: 10.0626
Total reads made: 3059
Read size: 4194304
Object size: 4194304
Bandwidth (MB/sec): 1215.99
Average IOPS: 303
Stddev IOPS: 10
Max IOPS: 324
Min IOPS: 292
Average Latency(s): 0.0513217
Max latency(s): 0.276891
Min latency(s): 0.00234561

Workaround:

Install package manually:

yum -y install ceph ceph-radosgw

Reference

https://docs.ceph.com/en/latest/
https://www.hikunpeng.com/document/detail/zh/kunpengsdss/ecosystemEnable/Ceph/kunpengcephblock_05_0013.html
https://www.linuxcenter.cn/ceph/introducing-ceph-architecture.html