How to troubleshoot disk I/O performance issues on Linux

dd if=/dev/zero of=/tmp/testfile bs=1M oflag=append,direct
~]$ iostat -x 1

avg-cpu: %user %nice %system %iowait %steal %idle
0.75 0.00 2.00 12.00 0.00 85.25

Device r/s rkB/s rrqm/s %rrqm r_await rareq-sz w/s wkB/s wrqm/s %wrqm w_await wareq-sz d/s dkB/s drqm/s %drqm d_await dareq-sz f/s f_await aqu-sz %util
dm-0 0.00 0.00 0.00 0.00 0.00 0.00 67.00 68608.00 0.00 0.00 14.88 1024.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 1.00 98.60
dm-1 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
dm-2 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
dm-3 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
dm-4 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
sda 0.00 0.00 0.00 0.00 0.00 0.00 133.00 68096.00 0.00 0.00 11.89 512.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 1.58 98.70
sdb 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
sdc 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
sdd 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
sr0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00

# find the process pid
~]$ sudo iotop -oP

Total DISK READ : 0.00 B/s | Total DISK WRITE : 22.81 M/s
Actual DISK READ: 0.00 B/s | Actual DISK WRITE: 22.81 M/s
PID PRIO USER DISK READ DISK WRITE> COMMAND
301914 be/4 jove 0.00 B/s 22.81 M/s dd if=/dev/zero of=/tmp/testfile bs=1M oflag=append,direct


# find open files by pid, w(write)
~]$ lsof -p 301914
COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME
dd 301914 jove cwd DIR 253,2 8192 145 /home/jove/soft/sysbench-master
dd 301914 jove rtd DIR 253,0 245 128 /
dd 301914 jove txt REG 253,0 69768 101276821 /usr/bin/dd
dd 301914 jove mem REG 253,0 2586930 93158 /usr/lib/locale/en_US.utf8/LC_COLLATE
dd 301914 jove mem REG 253,0 2549256 93250 /usr/lib64/libc.so.6
dd 301914 jove mem REG 253,0 346132 100702652 /usr/lib/locale/C.utf8/LC_CTYPE
dd 301914 jove mem REG 253,0 54 93161 /usr/lib/locale/en_US.utf8/LC_NUMERIC
dd 301914 jove mem REG 253,0 3284 33561178 /usr/lib/locale/en_US.utf8/LC_TIME
dd 301914 jove mem REG 253,0 286 33561176 /usr/lib/locale/en_US.utf8/LC_MONETARY
dd 301914 jove mem REG 253,0 57 33561189 /usr/lib/locale/en_US.utf8/LC_MESSAGES/SYS_LC_MESSAGES
dd 301914 jove mem REG 253,0 34 100702626 /usr/lib/locale/en_US.utf8/LC_PAPER
dd 301914 jove mem REG 253,0 77 93160 /usr/lib/locale/en_US.utf8/LC_NAME
dd 301914 jove mem REG 253,0 167 33561172 /usr/lib/locale/en_US.utf8/LC_ADDRESS
dd 301914 jove mem REG 253,0 59 33561177 /usr/lib/locale/en_US.utf8/LC_TELEPHONE
dd 301914 jove mem REG 253,0 23 33561175 /usr/lib/locale/en_US.utf8/LC_MEASUREMENT
dd 301914 jove mem REG 253,0 26988 67255870 /usr/lib64/gconv/gconv-modules.cache
dd 301914 jove mem REG 253,0 369 33561174 /usr/lib/locale/en_US.utf8/LC_IDENTIFICATION
dd 301914 jove mem REG 253,0 932656 93182 /usr/lib64/ld-linux-x86-64.so.2
dd 301914 jove 0r CHR 1,5 0t0 6 /dev/zero
dd 301914 jove 1w REG 253,0 4424990720 67471041 /tmp/testfile
dd 301914 jove 2u CHR 136,0 0t0 3 /dev/pts/0

~]$ strace -p 301914 -y -e trace=write
strace: Process 301914 attached
write(1</tmp/testfile>, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 1048576) = 1048576
write(1</tmp/testfile>, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 1048576) = 1048576
write(1</tmp/testfile>, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 1048576) = 1048576
write(1</tmp/testfile>, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 1048576) = 1048576
write(1</tmp/testfile>, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 1048576) = 1048576
...