Installing Ceph Cluster on Ubuntu 18.04

admin.ai.com admin
mon01.ai.com mon01
mon02.ai.com mon02
mon03.ai.com mon03
mgr01.ai.com mgr01
mgr02.ai.com mgr02
stor01.ai.com stor01
stor02.ai.com stor02
stor03.ai.com stor03

1. Create cephadm user.

cat > /tmp/create_cephadm.sh << EOF
#!/bin/bash
useradd -m -s /bin/bash cephadm
echo cephadm:welcome | chpasswd
echo "cephadm ALL=(ALL) NOPASSWORD:ALL" > /etc/sudoers.d/cephadm
chmod 0440 /etc/sudoers.d/cephadm
EOF

for i in {131..133}; do ssh root@192.168.3.$i bash < /tmp/create_cephadm.sh; done

2. SSH auth.

su - cephadm
ssh-keygen -t rsa -P "" -f ~/.ssh/id_rsa

PASS=welcome
for i in {131..133}; do
    sshpass -p $PASS ssh-copy-id -o StrictHostKeyChecking=no cephadm@192.168.3.$i
done

3. YUM repository.

cat > /tmp/ceph_repo.sh << EOF
#!/bin/bash
echo "deb http://mirrors.aliyun.com/ceph/debian-pacific/ $(lsb_release -sc) main" > /etc/apt/sources.list.d/ceph.list
wget -q -O- 'https://download.ceph.com/keys/release.asc' | apt-key add -
apt update
EOF

for i in {131..133}; do ssh root@192.168.3.$i bash < /tmp/ceph_repo.sh; done

4. Install ceph-deploy package.

su - cephadm
sudo apt-cache madison ceph-deploy
sudo apt -y install ceph-deploy

ceph-deploy --version

5. Cluster initial on admin node.

mkdir ceph-cluster && cd ceph-cluster
ceph-deploy new --public-network 10.0.0.0/24 --cluster-network 192.168.10.0/24 mon01:mon01.ai.com

Deploy Mon Node

1. Install ceph-mon on monitor node.

ceph-deploy install --no-adjust-repos --nogpgcheck --mon mon01
apt install ceph-mon
dpkg -l | grep ceph
getent passwd ceph

2. Create mon initial config on admin node.

cd ceph-cluster
ceph-deploy --overwrite-conf mon create-initial

3. Install ceph-common on admin node.

sudo apt install ceph-common acl
ceph-deploy admin admin
ls /etc/ceph
sudo setfacl -m u:cephadm:rw /etc/ceph/ceph.client.admin.keyring
ceph -s
ceph config set mon auth_allow_insecure_global_id_reclaim false

Deploy Mgr Node

1. Install ceph-mgr package.

ceph-deploy install --mgr mgr01
apt install ceph-mgr
ceph-deploy mgr create mgr01
ceph -s

Deploy OSD Node

lsblk
apt -y autoremove --purge snapd

1. Install ceph-osd package.

ceph-deploy install --release pacific --osd stor01 stor02 stor03
apt install ceph-osd
ceph-deploy disk list stor01 stor02 stor03
for i in {1..3}; do ceph-deploy disk zap stor0$i /dev/sdb /dev/sdc; done 
for i in {1..3}; do 
  ceph-deploy --overwrite-conf osd create stor0$i --data /dev/sdb
  ceph-deploy --overwrite-conf osd create stor0$i --data /dev/sdc
done

ceph -s
ceph osd ls
ceph osd df
ceph osd status

HA

1. Mon HA.

ceph-deploy install -mon mon02 mon03
apt install ceph-mon
ceph-deploy mon add mon02 mon03
ceph -s

vim ceph.conf
mon_host = 10.0.0.101,10.0.0.102,10.0.0.103
ceph-deploy --overwrite-conf config push admin mon0{1,2,3} mgr0{1,2} stor0{1,2,3}

2. Mgr HA.

ceph-deploy install --mgr mgr02
apt install ceph-mgr
ceph-deploy mgr create mgr02
ceph -s