Installing QEMU-KVM Pacemaker Corosync Cluster on RHEL 9

RHEL 9.7
pacemaker-2.1.7-5.2.el8_10.x86_64
corosync-3.1.8-1.el8.x86_64

Object Value
Node cls01, cls02
Node Public IP 192.168.3.91, 192.168.3.92
Node Private IP 100.100.100.91, 100.100.100.92
Cluster Name vmcluster
BusType iSCSI, 192.168.3.112
Shared Disk 1 20GB, data
Shared Disk 2 5GB, data
Shared Disk 3 1GB, fence_scsi

Setting Media ISO YUM Repository

・ Configuring Yum Repositories

Installing Cluster Software

# online
sudo subscription-manager register
sudo subscription-manager repos --enable=rhel-9-for-x86_64-highavailability-rpms
sudo dnf -y install pcs pacemaker fence-agents-all pcp-zeroconf
sudo subscription-manager unregister

# offline
wget -P ~/soft https://miss.lostyu.qzz.io/assets/tmp/rhel9.7-pacemaker-rpms.tar.gz
tar -zxf ~/soft/rhel9.7-pacemaker-rpms.tar.gz -C ~/soft
sudo dnf -y localinstall ~/soft/*.rpm

Setting firewall.

sudo firewall-cmd --permanent --add-service=high-availability
sudo firewall-cmd --add-service=high-availability

Setting hacluster password.

echo hacluster | sudo passwd hacluster --stdin

Starting pcsd.

sudo systemctl enable --now pcsd.service

Performance Co-Pilot (PCP)

sudo dnf -y install pcp-zeroconf

pminfo -a /var/log/pcp/pmlogger/cls01/20260901.22.13.0

pmval -t 2sec -f 3 filesys.free -S @"2026-09-01 22:25" -T @"2026-09-01 22:26" -a /var/log/pcp/pmlogger/cls01/20260901.22.13.0
pmval -t 2sec -f 3 mem.util.available -S @"2026-09-01 22:25" -T @"2026-09-01 22:26" -a /var/log/pcp/pmlogger/cls01/20260901.22.13.0
pmval -t 2sec -f 3 kernel.cpu.util.user -S @"2026-09-01 22:25" -T @"2026-09-01 22:26" -a /var/log/pcp/pmlogger/cls01/20260901.22.13.0
pmval -t 2sec -f 3 kernel.cpu.util.sys -S @"2026-09-01 22:25" -T @"2026-09-01 22:26" -a /var/log/pcp/pmlogger/cls01/20260901.22.13.0

Configuring Network

・ Configuring Network Bonding on RHEL 7,8,9,10

sudo nmcli con add type bond con-name bond-hb ifname bond-hb bond.options "mode=active-backup,miimon=1000"
sudo nmcli con add type ethernet slave-type bond con-name bond-hb-ens224 ifname ens224 master bond-hb
sudo nmcli con add type ethernet slave-type bond con-name bond-hb-ens256 ifname ens256 master bond-hb
sudo nmcli con modify bond-hb ipv4.addresses 100.100.100.91/24 ipv4.method manual autoconnect yes connection.autoconnect-slaves 1
sudo nmcli con up bond-hb
cat /proc/net/bonding/bond-hb

Configuring hosts resolve.

sudo bash -c "cat >>/etc/hosts" <<EOF
# Cluster
192.168.3.91    cls01
192.168.3.92    cls02
EOF

Configuring ISCSI

・ Configuring iSCSI Share on Linux

Configuring startup order for resource dependencies not managed by pacemaker:

sudo mkdir /etc/systemd/system/resource-agents-deps.target.d
sudo bash -c "cat >/etc/systemd/system/resource-agents-deps.target.d/blk-availability.conf" <<EOF
[Unit]
Requires=blk-availability.service
After=blk-availability.service
EOF

sudo systemctl daemon-reload

Creating a High Availability Cluster

sudo pcs host auth cls01 cls02 -u hacluster
sudo pcs cluster setup vmcluster --start cls01 cls02
sudo pcs cluster enable --all
sudo pcs cluster status
sudo pcs cluster config show
sudo pcs cluster link add cls01=100.100.100.91 cls02=100.100.100.92

sudo pcs cluster stop --all
sudo pcs cluster link update 1 cls01=100.100.100.91 cls02=100.100.100.92 options link_priority=0
sudo pcs cluster link remove 0
sudo pcs cluster start --all

Configuring STONITH

・ Using SCSI Persistent Reservation Fencing (fence_scsi) with pacemaker in a Red Hat High Availability cluster

pcs stonith describe fence_scsi
cat /proc/partitions

ll /dev/disk/by-id/ | grep sdc    # get the /dev/sdc wwn

sudo pcs stonith create scsi-shooter fence_scsi \
  pcmk_host_list="cls01 cls02" \
  devices="/dev/disk/by-id/wwn-0x60014053a118daeb0374b63a156409c3" \
  pcmk_reboot_action="off" meta provides=unfencing --force

sudo pcs stonith config scsi-shooter

Test scsi fencing.

# Check the reservation/registration status
sudo sg_persist -n -i -r -d /dev/sdc
sudo sg_persist -n -i -k -d /dev/sdc

# Simulate a node failure, such as by panicking the system with SysRq+C
sudo bash -c "echo c > /proc/sysrq-trigger"

Configuring Cluster Resources

Configuring a GFS2 File System

Install GFS2 software.

sudo subscription-manager repos --enable=rhel-9-for-x86_64-resilientstorage-rpms
sudo dnf -y install lvm2-lockd gfs2-utils dlm

Configuring LVM and GFS2 parameter.

# set the use_lvmlockd = 1
sudo sed -i '/^global/a\\tuse_lvmlockd = 1' /etc/lvm/lvm.conf

sudo pcs property set no-quorum-policy=freeze

sudo pcs resource create dlm --group locking ocf:pacemaker:controld \
  op monitor interval=30s on-fail=fence
sudo pcs resource clone locking interleave=true
sudo pcs resource create lvmlockd --group locking ocf:heartbeat:lvmlockd \
  op monitor interval=30s on-fail=fence

Create shared VG.

# On one node of the cluster
sudo vgcreate --shared shared_vg1 /dev/sdb
sudo vgcreate --shared shared_vg2 /dev/sdc

# On the second node in the cluster
sudo lvmdevices --adddev /dev/sdb
sudo lvmdevices --adddev /dev/sdc
sudo vgchange --lockstart shared_vg1
sudo vgchange --lockstart shared_vg2

Create LV and filesystem.

# On one node of the cluster
sudo lvcreate --activate sy -L10G -n shared_lv1 shared_vg1
sudo lvcreate --activate sy -l +100%FREE -n shared_lv2 shared_vg2
sudo mkfs.gfs2 -j2 -p lock_dlm -t vmcluster:gfs2-fs1 /dev/shared_vg1/shared_lv1
sudo mkfs.gfs2 -j2 -p lock_dlm -t vmcluster:gfs2-fs2 /dev/shared_vg2/shared_lv2

Create LVM-activate resource.

sudo pcs resource create sharedlv1 --group shared_vg1 ocf:heartbeat:LVM-activate \
lvname=shared_lv1 vgname=shared_vg1 activation_mode=shared vg_access_mode=lvmlockd

sudo pcs resource create sharedlv2 --group shared_vg2 ocf:heartbeat:LVM-activate \
lvname=shared_lv2 vgname=shared_vg2 activation_mode=shared vg_access_mode=lvmlockd

sudo pcs resource clone shared_vg1 interleave=true
sudo pcs resource clone shared_vg2 interleave=true

sudo pcs constraint order start locking-clone then shared_vg1-clone
sudo pcs constraint order start locking-clone then shared_vg2-clone

sudo pcs constraint colocation add shared_vg1-clone with locking-clone
sudo pcs constraint colocation add shared_vg2-clone with locking-clone

sudo pcs constraint config --full

Create Filesystem resource.

sudo mkdir -p /vm/data /vm/config

sudo pcs resource create sharedfs1 --group shared_vg1 ocf:heartbeat:Filesystem \
device="/dev/shared_vg1/shared_lv1" directory="/vm/data" fstype="gfs2" \
options=noatime op monitor interval=10s on-fail=fence

sudo pcs resource create sharedfs2 --group shared_vg2 ocf:heartbeat:Filesystem \
device="/dev/shared_vg2/shared_lv2" directory="/vm/config" fstype="gfs2" \
options=noatime op monitor interval=10s on-fail=fence

Verification.

~]$ mount | grep gfs2
/dev/mapper/shared_vg1-shared_lv1 on /vm/data type gfs2 (rw,noatime,seclabel,rgrplvb)
/dev/mapper/shared_vg2-shared_lv2 on /vm/config type gfs2 (rw,noatime,seclabel,rgrplvb)

~]$ sudo pcs status --full
Cluster name: vmcluster
Cluster Summary:
  * Stack: corosync (Pacemaker is running)
  * Current DC: cls02 (2) (version 2.1.10-3.el9_8-5693eaeee) - partition with quorum
  * Last updated: Mon Sep 21 14:47:48 2026 on cls01
  * Last change:  Mon Sep 21 14:44:02 2026 by root via root on cls01
  * 2 nodes configured
  * 13 resource instances configured

Node List:
  * Node cls01 (1): online, feature set 3.19.6
  * Node cls02 (2): online, feature set 3.19.6

Full List of Resources:
  * scsi-shooter        (stonith:fence_scsi):    Started cls01
  * Clone Set: locking-clone [locking]:
    * Resource Group: locking:0:
      * dlm     (ocf:pacemaker:controld):        Started cls02
      * lvmlockd        (ocf:heartbeat:lvmlockd):        Started cls02
    * Resource Group: locking:1:
      * dlm     (ocf:pacemaker:controld):        Started cls01
      * lvmlockd        (ocf:heartbeat:lvmlockd):        Started cls01
  * Clone Set: shared_vg1-clone [shared_vg1]:
    * Resource Group: shared_vg1:0:
      * sharedlv1       (ocf:heartbeat:LVM-activate):    Started cls02
      * sharedfs1       (ocf:heartbeat:Filesystem):      Started cls02
    * Resource Group: shared_vg1:1:
      * sharedlv1       (ocf:heartbeat:LVM-activate):    Started cls01
      * sharedfs1       (ocf:heartbeat:Filesystem):      Started cls01
  * Clone Set: shared_vg2-clone [shared_vg2]:
    * Resource Group: shared_vg2:0:
      * sharedlv2       (ocf:heartbeat:LVM-activate):    Started cls02
      * sharedfs2       (ocf:heartbeat:Filesystem):      Started cls02
    * Resource Group: shared_vg2:1:
      * sharedlv2       (ocf:heartbeat:LVM-activate):    Started cls01
      * sharedfs2       (ocf:heartbeat:Filesystem):      Started cls01

Migration Summary:

Fencing History:
  * unfencing of cls02 successful: delegate=cls02, client=pacemaker-fenced.1461, origin=cls01, completed='2026-09-21 09:54:08.032570 +08:00'
  * unfencing of cls01 successful: delegate=cls01, client=pacemaker-controld.1487, origin=cls02, completed='2026-09-21 09:54:08.026570 +08:00'

Tickets:

PCSD Status:
  cls01: Online
  cls02: Online

Daemon Status:
  corosync: active/enabled
  pacemaker: active/enabled
  pcsd: active/enabled

QEMU-KVM resources

・ Installing QEMU-KVM on Linux

Create a VM.

sudo virt-install \
  --virt-type kvm \
  --name vm1 \
  --memory 2048 \
  --vcpus 1 \
  --os-variant centos7.0 \
  --network bridge=br0 \
  --disk /vm/data/vm1.qcow2,size=10,bus=virtio,format=qcow2 \
  --location http://192.168.3.112/assets/tmp/mnt -x console=ttyS0 \
  --graphics none

sudo virsh list
sudo bash -c 'virsh dumpxml vm1 > /vm/config/vm1.xml'
sudo virsh shutdown vm1
sudo virsh undefine vm1

Manual test migration.

sudo virsh define /vm/config/vm1.xml
sudo virsh start vm1

sudo virsh migrate --domain vm1 \
  --desturi qemu+ssh://cls02/system \
  --persistent --undefinesource \
  --unsafe --live

sudo virsh migrate --domain vm1 \
  --desturi qemu+ssh://cls01/system \
  --persistent --undefinesource \
  --unsafe --live

sudo virsh shutdown vm1
sudo virsh undefine vm1

Create VirtualDomain resources.

~]$ sudo pcs resource describe virtualdomain

sudo pcs resource create vm1 VirtualDomain \
  hypervisor="qemu:///system" \
  config="/vm/config/vm1.xml" \
  migration_transport=ssh \
  meta allow-migrate="true"

sudo pcs constraint order start shared_vg1-clone then vm1
sudo pcs constraint order start shared_vg2-clone then vm1

Constraints

sudo pcs resource create --future ping ocf:pacemaker:ping multiplier=1000 host_list=192.168.3.1 clone
sudo pcs constraint location vm1 rule score=-INFINITY pingd lt 1 or not_defined pingd

Troubleshooting

The requested volume capacity will exceed the available pool space when the volume is fully allocated

~]$ sudo virt-install \
  --virt-type kvm \
  --name vm1 \
  --memory 2048 \
  --vcpus 1 \
  --os-type linux \
  --os-variant centos7.0 \
  --network bridge=br0 \
  --disk /vm/data/vm1.qcow2,size=10,bus=virtio,format=qcow2 \
  --location http://192.168.3.112/assets/tmp/mnt -x console=ttyS0 \
  --graphics none
WARNING  --os-type is deprecated and does nothing. Please stop using it.
WARNING  The requested volume capacity will exceed the available pool space when the volume is fully allocated. (10240 M requested capacity > 10172 M available)
ERROR    The requested volume capacity will exceed the available pool space when the volume is fully allocated. (10240 M requested capacity > 10172 M available) (Use --check disk_size=off or --check all=off to override)

Solution:

sudo lvextend -L+5G shared_vg1/shared_lv1
sudo lvchange --refresh shared_vg1/shared_lv1  # on others node.
sudo gfs2_grow /vm/data/

Migration fails permission denied with pacemaker

sudo tail -flink
2026-10-03 10:26:58.090+0000: initiating migration
2026-10-03T10:26:58.618176Z qemu-kvm: migration_block_inactivate: bdrv_inactivate_all() failed: -1
2026-10-03T10:26:58.618654Z qemu-kvm: Could not reopen qcow2 layer: Could not read qcow2 header: Permission denied
2026-10-03T10:26:58.699506Z qemu-kvm: Error in migration completion: Bad address
2026-10-03 10:27:09.011+0000: IO error device='virtio-disk0' node-name='libvirt-1-format' reason='No medium found'
sudo tail -flink
Oct  3 18:28:25 cls02 pacemaker-controld[1488]: notice: Requesting local execution of start operation for vm1 on cls02
Oct  3 18:28:25 cls02 VirtualDomain(vm1)[38974]: ERROR: Failed to start virtual domain vm1.
Oct  3 18:28:25 cls02 pacemaker-controld[1488]: notice: Result of start operation for vm1 on cls02: error (Failed to start virtual domain vm1.)
Oct  3 18:28:25 cls02 pacemaker-controld[1488]: notice: vm1_start_0@cls02 output [ error: Failed to open file '/vm/config/vm1.xml': Permission denied ]
Oct  3 18:28:25 cls02 pacemaker-controld[1488]: notice: vm1_start_0@cls02 output [ ocf-exit-reason:Failed to start virtual domain vm1. ]
Oct  3 18:28:25 cls02 pacemaker-attrd[1486]: notice: Setting last-failure-vm1#start_0[cls02] in instance_attributes: (unset) -> 1791023305

Workaround:

Disable SElinux.

sudo setenforce 0

migration failed: Invalid argument

~]$ sudo virsh migrate --domain vm1 \
  --desturi qemu+ssh://cls02/system \
  --persistent --undefinesource \
  --unsafe --live
error: operation failed: migration failed. Message from the source host: operation failed: job 'migration out' failed: Error in migration completion: Bad address. Message from the destination host: operation failed: job 'migration in' failed: load of migration failed: Invalid argument
sudo tail -flink
>> cls02
Oct  3 22:52:26 cls02 sssd_kcm[483704]: Starting up
Oct  3 22:52:27 cls02 virtqemud[218689]: operation failed: job 'migration out' failed: Error in migration completion: Bad address

>> cls01
Oct  3 22:52:26 cls01 systemd-machined[1023]: New machine qemu-1-vm1.
Oct  3 22:52:26 cls01 systemd[1]: Started Virtual Machine qemu-1-vm1.
Oct  3 22:52:26 cls01 virtqemud[433517]: 2026-10-03 14:52:26.913+0000: 433517: info : libvirt version: 11.10.0, package: 12.3.el9_8 (Red Hat, Inc. <http://bugzilla.redhat.com/bugzilla>, 2026-06-04-11:12:18, )
Oct  3 22:52:26 cls01 virtqemud[433517]: 2026-10-03 14:52:26.913+0000: 433517: info : hostname: cls01
Oct  3 22:52:26 cls01 virtqemud[433517]: 2026-10-03 14:52:26.913+0000: 433517: warning : virSecurityValidateTimestamp:205 : Invalid XATTR timestamp detected on /vm/data/vm1.qcow2 secdriver=selinux
Oct  3 22:52:26 cls01 virtqemud[433518]: 2026-10-03 14:52:26.961+0000: 433518: info : libvirt version: 11.10.0, package: 12.3.el9_8 (Red Hat, Inc. <http://bugzilla.redhat.com/bugzilla>, 2026-06-04-11:12:18, )
Oct  3 22:52:26 cls01 virtqemud[433518]: 2026-10-03 14:52:26.961+0000: 433518: info : hostname: cls01
Oct  3 22:52:26 cls01 virtqemud[433518]: 2026-10-03 14:52:26.961+0000: 433518: warning : virSecurityValidateTimestamp:205 : Invalid XATTR timestamp detected on /vm/data/vm1.qcow2 secdriver=dac
Oct  3 22:52:27 cls01 virtqemud[433463]: operation failed: job 'migration in' failed: load of migration failed: Invalid argument
sudo tail -flink
2026-10-03T14:52:27.677437Z qemu-kvm: check_section_footer: Read section footer failed: -5
2026-10-03 14:52:27.685+0000: shutting down, reason=failed
2026-10-03T14:52:27.687333Z qemu-kvm: terminating on signal 15 from pid 433463 (/usr/sbin/virtqemud)

Cause:

The SELinux prevent it.

~]$ sudo ausearch -m AVC -ts recent
type=AVC msg=audit(1791040849.953:2200): avc:  denied  { lock } for  pid=525572 comm="mig/src/main" path="/vm/data/vm1.qcow2" dev="dm-3" ino=81573 scontext=system_u:system_r:svirt_t:s0:c52,c386 tcontext=system_u:object_r:svirt_image_t:s0:c60,c398 tclass=file permissive=1

~]$ sudo dnf install setroubleshoot-server
~]$ sudo sealert -a /var/log/audit/audit.log
100% done
found 1 alerts in /var/log/audit/audit.log
--------------------------------------------------------------------------------

SELinux is preventing /usr/libexec/qemu-kvm from read access on the file /vm/data/vm1.qcow2.

*****  Plugin qemu_file_image (98.8 confidence) suggests   *******************

If vm1.qcow2 is a virtualization target
Then you need to change the label on vm1.qcow2'
Do
# semanage fcontext -a -t virt_image_t '/vm/data/vm1.qcow2'
# restorecon -v '/vm/data/vm1.qcow2'

*****  Plugin catchall (2.13 confidence) suggests   **************************

If you believe that qemu-kvm should be allowed read access on the vm1.qcow2 file by default.
Then you should report this as a bug.
You can generate a local policy module to allow this access.
Do
allow this access for now by executing:
# ausearch -c 'mig/src/main' --raw | audit2allow -M my-migsrcmain
# semodule -X 300 -i my-migsrcmain.pp


Additional Information:
Source Context                system_u:system_r:svirt_t:s0:c262,c621
Target Context                system_u:object_r:svirt_image_t:s0:c117,c538
Target Objects                /vm/data/vm1.qcow2 [ file ]
Source                        mig/src/main
Source Path                   /usr/libexec/qemu-kvm
Port                          <Unknown>
Host                          <Unknown>
Source RPM Packages           qemu-kvm-core-10.1.0-17.el9_8.5.x86_64
Target RPM Packages
SELinux Policy RPM            selinux-policy-targeted-38.1.65-1.el9.noarch
Local Policy RPM              selinux-policy-targeted-38.1.65-1.el9.noarch
Selinux Enabled               True
Policy Type                   targeted
Enforcing Mode                Enforcing
Host Name                     cls01
Platform                      Linux cls01 5.14.0-611.5.1.el9_7.x86_64 #1 SMP
                              PREEMPT_DYNAMIC Fri Oct 17 14:16:35 EDT 2025
                              x86_64 x86_64
Alert Count                   2
First Seen                    2026-10-07 18:36:53 CST
Last Seen                     2026-10-07 18:36:53 CST
Local ID                      23ba612a-59fa-4059-bcfa-4104455b1838

Raw Audit Messages
type=AVC msg=audit(1791369413.57:3119): avc:  denied  { read } for  pid=3831373 comm="worker" path="/vm/data/vm1.qcow2" dev="dm-4" ino=81573 scontext=system_u:system_r:svirt_t:s0:c262,c621 tcontext=system_u:object_r:svirt_image_t:s0:c117,c538 tclass=file permissive=0


type=SYSCALL msg=audit(1791369413.57:3119): arch=x86_64 syscall=pread success=no exit=EACCES a0=d a1=7f14f64ebf58 a2=70 a3=0 items=0 ppid=1 pid=3831373 auid=4294967295 uid=107 gid=107 euid=107 suid=107 fsuid=107 egid=107 sgid=107 fsgid=107 tty=(none) ses=4294967295 comm=worker exe=/usr/libexec/qemu-kvm subj=system_u:system_r:svirt_t:s0:c262,c621 key=(null)ARCH=x86_64 SYSCALL=pread AUID=unset UID=qemu GID=qemu EUID=qemu SUID=qemu FSUID=qemu EGID=qemu SGID=qemu FSGID=qemu

Hash: mig/src/main,svirt_t,svirt_image_t,file,read

Workaround:

Disable SElinux.

~]$ sudo semanage fcontext -a -t svirt_image_t "/vm/data(/.*)?"    # this not work.
~]$ sudo restorecon -Rv /vm/data

~]$ sudo setenforce 0
~]$ sudo sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config

error: internal error: unable to execute QEMU command

~]$ sudo virsh domblkinfo vm1 --all
error: internal error: unable to execute QEMU command 'query-named-block-nodes': Block device libvirt-1-format is ejected

Workaround:

The vm1 is shutting down, but hung. Kill it.

~]$ ps -ef | grep "[g]uest=vm1" | awk '{print $2}' | xargs sudo kill -15