My test steps:


4.18.0-222.el8.x86_64
qemu-kvm-core-4.2.0-29.module+el8.3.0+7212+401047e6.x86_64
qemu-kvm-common-4.2.0-29.module+el8.3.0+7212+401047e6.x86_64


1.host FC env
root@hp-dl385pg8-03 /home/rworkdir/vbugs/feature/passthrough # multipath -l
mpatha (360050763008084e6e00000000000019d) dm-3 IBM,2145
size=130G features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=0 status=enabled
| `- 2:0:0:0 sdb 8:16 active undef running
`-+- policy='service-time 0' prio=0 status=enabled
  `- 2:0:1:0 sdc 8:32 active undef running

There are 2 paths 

3.boot vm with passthrough path,set rerror=stop,werror=stop on the device

/usr/libexec/qemu-kvm \
  -name src_vm1 \
  -machine q35 \
  -nodefaults \
  -vga qxl \
  -device pcie-root-port,id=pcie.0-root-port-2,slot=2,bus=pcie.0,multifunction=on \
  -device pcie-root-port,id=pcie.0-root-port-2-1,chassis=3,bus=pcie.0,addr=0x2.0x1 \
  -device pcie-root-port,id=pcie.0-root-port-2-2,chassis=4,bus=pcie.0,addr=0x2.0x2 \
  -device pcie-root-port,id=pcie.0-root-port-3,slot=3,bus=pcie.0 \
  -device pcie-root-port,id=pcie.0-root-port-4,slot=4,bus=pcie.0 \
  -device pcie-root-port,id=pcie.0-root-port-5,slot=5,bus=pcie.0 \
  -device pcie-root-port,id=pcie.0-root-port-7,slot=7,bus=pcie.0 \
  -device pcie-root-port,id=pcie.0-root-port-8,slot=8,bus=pcie.0 \
  -device pcie-root-port,id=pcie.0-root-port-9,slot=9,bus=pcie.0 \
  -device qemu-xhci,id=usb1,bus=pcie.0-root-port-2-1,addr=0x0 \
  -device usb-tablet,id=usb-tablet1,bus=usb1.0,port=1 \
  -object iothread,id=iothread0 \
  -device virtio-scsi-pci,id=scsi0,bus=pcie.0-root-port-2-2,addr=0x0,iothread=iothread0 \
  \
  -blockdev driver=qcow2,file.driver=file,cache.direct=off,cache.no-flush=on,file.filename=/home/kvm_autotest_root/images/rhel830-64-virtio-scsi.qcow2,node-name=drive_image1 \
  -device scsi-hd,id=os1,drive=drive_image1,bootindex=0 \
  \
  -device virtio-scsi-pci,id=scsi1,bus=pcie.0-root-port-8,addr=0x0 \
  -blockdev node-name=host_device_stg0,driver=host_device,filename=/dev/mapper/mpatha \
  -blockdev node-name=drive_stg0,driver=raw,file=host_device_stg0 \
  -device scsi-block,id=stg0,rerror=stop,werror=stop,drive=drive_stg0,bus=scsi1.0 \
  -vnc :5 \
  -qmp tcp:0:5955,server,nowait \
  -monitor stdio \
  -m 8192 \
  -smp 8 \
  -device virtio-net-pci,mac=9a:b5:b6:b1:b2:b5,id=idMmq1jH,vectors=4,netdev=idxgXAlm,bus=pcie.0-root-port-5,addr=0x0 \
  -netdev tap,id=idxgXAlm \

4.access guest then do io 
root@bootp-73-131-253 ~ # lsblk
NAME                       MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
sda                          8:0    0   20G  0 disk 
\u251c\u2500sda1                       8:1    0    1G  0 part /boot
\u2514\u2500sda2                       8:2    0   19G  0 part 
  \u251c\u2500rhel_vm--115--131-root 253:0    0   17G  0 lvm  /
  \u2514\u2500rhel_vm--115--131-swap 253:1    0    2G  0 lvm  [SWAP]
sdb                          8:16   0  130G  0 disk 
root@bootp-73-131-253 ~ # dd if=/dev/zero of=/dev/sdb bs=1M count=30000


5. check status 
(qemu) info status
VM status: running


6.Alternately close a path every 10 seconds on host

root@hp-dl385pg8-03 /home/rworkdir/vbugs/feature/passthrough # ./offline.sh 
offline ===== sdb
| `- 2:0:0:0 sdb 8:16 failed faulty offline
online ===== sdb
| `- 2:0:0:0 sdb 8:16 failed undef running
offline ===== sdc
  `- 2:0:1:0 sdc 8:32 failed faulty offline
online ===== sdc
  `- 2:0:1:0 sdc 8:32 failed undef running
mpatha (360050763008084e6e00000000000019d) dm-3 IBM,2145
size=130G features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=0 status=enabled
| `- 2:0:0:0 sdb 8:16 active undef running
`-+- policy='service-time 0' prio=0 status=enabled
  `- 2:0:1:0 sdc 8:32 active undef running
offline ===== sdb
| `- 2:0:0:0 sdb 8:16 active faulty offline
online ===== sdb
| `- 2:0:0:0 sdb 8:16 failed undef running
offline ===== sdc
  `- 2:0:1:0 sdc 8:32 failed faulty offline
online ===== sdc
  `- 2:0:1:0 sdc 8:32 active undef running
mpatha (360050763008084e6e00000000000019d) dm-3 IBM,2145
size=130G features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=0 status=enabled
| `- 2:0:0:0 sdb 8:16 active undef running
`-+- policy='service-time 0' prio=0 status=enabled
  `- 2:0:1:0 sdc 8:32 active undef running
offline ===== sdb
| `- 2:0:0:0 sdb 8:16 failed faulty offline
online ===== sdb
| `- 2:0:0:0 sdb 8:16 failed undef running
offline ===== sdc
  `- 2:0:1:0 sdc 8:32 failed faulty offline



7 check guest status
(qemu) info status
VM status: paused (io-error)

8.resume the guest and stop offline script
(qemu) c
(qemu) info status
VM status: running

9.guest may finish job
root@bootp-73-131-253 ~ # dd if=/dev/zero of=/dev/sdb bs=1M count=30000
30000+0 records in
30000+0 records out
31457280000 bytes (31 GB, 29 GiB) copied, 973.289 s, 32.3 MB/s


So in my testing i think it encount io-error is expected if multipath switch path.
My point is the guest may resume if path is ready.

If i boot vm without rerror=stop,werror=stop on the device,the guest will always keep running status
and dd operatoin will finish finally. find message in /var/log/message:
Jul  8 17:25:59 bootp-73-131-253 kernel: blk_update_request: I/O error, dev sdb, sector 59475968 op 0x1:(WRITE) flags 0x4800 phys_seg 12 prio class 0
Jul  8 17:25:59 bootp-73-131-253 kernel: sd 1:0:0:0: [sdb] tag#93 CDB: Write(10) 2a 00 03 8b 84 38 00 03 c8 00
Jul  8 17:25:59 bootp-73-131-253 kernel: sd 1:0:0:0: [sdb] tag#75 FAILED Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE cmd_age=0s
Jul  8 17:25:59 bootp-73-131-253 kernel: blk_update_request: I/O error, dev sdb, sector 59475000 op 0x1:(WRITE) flags 0x100000 phys_seg 9 prio class 0
Jul  8 17:25:59 bootp-73-131-253 kernel: sd 1:0:0:0: [sdb] tag#75 Sense Key : Aborted Command [current] 
Jul  8 17:25:59 bootp-73-131-253 kernel: sd 1:0:0:0: [sdb] tag#75 Add. Sense: I/O process terminated
Jul  8 17:25:59 bootp-73-131-253 kernel: sd 1:0:0:0: [sdb] tag#75 CDB: Write(10) 2a 00 03 8b 8c 00 00 04 00 00
Jul  8 17:25:59 bootp-73-131-253 kernel: sd 1:0:0:0: [sdb] tag#87 FAILED Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE cmd_age=0s
Jul  8 17:25:59 bootp-73-131-253 kernel: blk_update_request: I/O error, dev sdb, sector 59476992 op 0x1:(WRITE) flags 0x4800 phys_seg 14 prio class 0
Jul  8 17:25:59 bootp-73-131-253 kernel: sd 1:0:0:0: [sdb] tag#87 Sense Key : Aborted Command [current] 
Jul  8 17:25:59 bootp-73-131-253 kernel: sd 1:0:0:0: [sdb] tag#87 Add. Sense: I/O process terminated
Jul  8 17:25:59 bootp-73-131-253 kernel: sd 1:0:0:0: [sdb] tag#87 CDB: Write(10) 2a 00 03 8b 95 c0 00 03 90 00
Jul  8 17:25:59 bootp-73-131-253 kernel: sd 1:0:0:0: [sdb] tag#74 FAILED Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE cmd_age=0s
Jul  8 17:25:59 bootp-73-131-253 kernel: blk_update_request: I/O error, dev sdb, sector 59479488 op 0x1:(WRITE) flags 0x100000 phys_seg 8 prio class 0
Jul  8 17:25:59 bootp-73-131-253 kernel: sd 1:0:0:0: [sdb] tag#86 FAILED Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE cmd_age=0s
Jul  8 17:25:59 bootp-73-131-253 kernel: sd 1:0:0:0: [sdb] tag#74 Sense Key : Aborted Command [current] 
Jul  8 17:25:59 bootp-73-131-253 kernel: sd 1:0:0:0: [sdb] tag#86 Sense Key : Aborted Command [current] 
Jul  8 17:25:59 bootp-73-131-253 kernel: sd 1:0:0:0: [sdb] tag#74 Add. Sense: I/O process terminated
Jul  8 17:25:59 bootp-73-131-253 kernel: sd 1:0:0:0: [sdb] tag#74 CDB: Write(10) 2a 00 03 8b 91 d0 00 03 f0 00
Jul  8 17:25:59 bootp-73-131-253 kernel: blk_update_request: I/O error, dev sdb, sector 59478480 op 0x1:(WRITE) flags 0x4800 phys_seg 9 prio class 0
Jul  8 17:25:59 bootp-73-131-253 kernel: sd 1:0:0:0: [sdb] tag#86 Add. Sense: I/O process terminated
Jul  8 17:25:59 bootp-73-131-253 kernel: sd 1:0:0:0: [sdb] tag#86 CDB: Write(10) 2a 00 03 8b 9c f8 00 03 f0 00
Jul  8 17:25:59 bootp-73-131-253 kernel: blk_update_request: I/O error, dev sdb, sector 59481336 op 0x1:(WRITE) flags 0x104000 phys_seg 40 prio class 0


if guest in pause status please confirm there are error_policy on disk like as following

<disk type='block' device='lun' sgio='filtered' snapshot='no'>
      <driver name='qemu' type='raw' cache='none' error_policy='stop' io='native'/>
      <source dev='/dev/mapper/360014059b31089df5114cbd830320c2b'/>
      <backingStore/>
      <target dev='sdb' bus='scsi'/>
      <alias name='scsi0-0-0-1'/>
      <address type='drive' controller='0' bus='0' target='0' unit='1'/>
    </disk>

In the cases I looked at the Guest saw the Aborted Command / I/O process terminated
which caused the I/O tests to fail.  The Guest did not pause, although I believe there
are other BZs that have reported a paused guest that did not see I/O errors, so this
may have been the cause.

The I/O errors on write will cause a guest's file system on that device to go read-only.

At the design level, the main issue is that SG_IO returns sense data and a normal errno, not the bio-level BLK_STS_* errors that blk_path_error() uses to decide whether to retry a request.

The proposed solution is to implement SG_IO-specific logic in dm-mpath, or directly in dm.c, so that it can issue REQ_OP_SCSI_IN and REQ_OP_SCSI_OUT bios directly to the paths.

Currently, DM's ioctl path really doesn't have anything SCSI specific---it doesn't even pass the ioctl command to the dm target (see dm_prepare_ioctl in drivers/md/dm.c).  It is simply about passing ioctl down to underlying device(s), so the target selects a bdev and dm_blk_ioctl sends it down.

I'm adding CondNAK Upstream, but this is an important issue for which the only other solution would be redoing multipath in userspace (clearly suboptimal, to say the least).

Yes, it seems like what we need is a some kernel interface that provides the
semantics needed.  Reimplementing multipath in userspace would be a lot of work
and would require that all of the storage vendor partners test that as a whole
other mechanism, plus we could get different failover/recovery behavior which
seems like it would be a huge support problem.

The other alternative I see in the interim is to perhaps pass through all of the
underlying device paths to the Guest and have multipath run in the Guest on those
paths.  I don't know if that can be configured, though, and whether that approach
has been tested/validated.

We have done this before and it does work.
The device has to pass through as a direct device with inquiry and then multipath on guest can be enabled.

> The other alternative I see in the interim is to perhaps pass through all of the
> underlying device paths to the Guest and have multipath run in the Guest on those
> paths.

This would work but I think RHEV very much prefers to have multipath configured on
the host.  Configuring multipath on the guest would remove a lot of the appeal of
storage virtualization; it would complicate the configuration of the VM (i.e. have
to add all paths one by one) and also in the VM (having to configure dm-mpath
on all VMs, possibly with both Linux and Windows guest OSes, instead of just in the
hosts).

Also, path fallback would not work for device handlers that need REQ_FAILFAST_*,
because there's no way to pass them down from the guest to the host through SG_IO.

*** Bug 1776758 has been marked as a duplicate of this bug. ***

*** Bug 1783161 has been marked as a duplicate of this bug. ***

*** Bug 1803716 has been marked as a duplicate of this bug. ***

------- Comment From shlyi.com 2020-03-10 02:17 EDT-------
(In reply to comment #9)
> Only for ASC 10h, which is reported for T10 DIF errors
>
> 10 00 D ID CRC OR ECC ERROR
> 10 01 D LOGICAL BLOCK GUARD CHECK FAILED
> 10 02 D LOGICAL BLOCK APPLICATION TAG CHECK FAILED
> 10 03 D LOGICAL BLOCK REFERENCE TAG CHECK FAILED
> 10 04   LOGICAL BLOCK PROTECTION ERROR ON RECOVER BUFFERED DATA
> 10 05   LOGICAL BLOCK PROTECTION METHOD ERROR
>
> case ABORTED_COMMAND:
> action = ACTION_FAIL;
> if (sshdr.asc == 0x10) /* DIF */
> error = -EILSEQ;
> break;
>
> The errors we see here are ASC/ASCQ 00h / 06h:
>
> 00 06 D I/O PROCESS TERMINATED
>
> It doesn't look to me like this would set the codes to make
> blk_path_error() return false

Hi emilne,

As I know, Redhat 7.8 has already released several snapshot versions after snapshot2, can you help me answer below two questions?

1, Does redhat has release notes for Redhat 7.8 snapshot version, such as Redhat 7.8 snapshot 5, I've just seen the Redhat 7.8 Beta version release note at https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/7-beta/

2, Could you help check if the version snapshot 5 have some updates to solve this issue?

Many thanks to you.

------- Comment From shlyi.com 2020-03-10 22:51 EDT-------
(In reply to comment #12)
> I replied in the email thread, but I'll reply here too to keep everything in
> one place.
>
> We don't have release notes for snapshots, those are just for public
> releases like the beta and GA
>
> We released the RC of RHEL 7.8 today. If you're not being notified, please
> let me know and I can get you added to the mailing list so you receive
> schedules, pre-release announcements, etc.

Thank you all for your updates. As you haven't replied to me for my question 2, I assume the new versions such as Rhel 7.8 RC don't contain updates for this issue.

Besides, do you have a document to explain the upgrade path, for example, from snapshot version to another snapshot version, from snapshot version to RC version, from RC version to GA version, etc?

Based on my experience, my understanding are as below:

1, we cannot upgrade from snapshot version to another snapshot version, or from snapshot version to RC/GA version, since I encountered some conflict when I tried to do that last time.
2, the RC version and GA version may be the same, so I can do normal update/upgrade based on a RC version.

Appreciate for your time.

------- Comment From shlyi.com 2020-03-17 01:19 EDT-------
Hello,
Could you take a look at the comments provided by a redhat focal below? Based on my understanding, his experiment has just illustrated that this issue is caused by the multipath function. Looking forward to hear you back. Thanks.

below comments are coming from issue: https://bugzilla.linux.ibm.com/show_bug.cgi?id=183977

I did some investigations and this is what I found:

I setup the following test enviroment (much more modest that you have :-) ):

a host (my laptop) running iscsi target from a 1 GB file.

Then I connected to that target twice using qemu's iscsi driver, with blkdebug instance running on top of each

I hacked the blkdebug qemu driver such as it passes through the SG_IO and sometimes fails them, such
as the two instances of blkdebug don't fail at the same time.

Thus I have a guest with two paths to same disk and each of paths fails sometimes.

In that guest I aggregated the two paths using the multipath device mapper and I passed
the resulting multipath node to a nested guest running in it.
Inside the nested guest I had run a fio job on the passed through disk.

The results:

Sadly the nested guest sees IO errors.

I tried both direct and buffered IO in the guest, both reads
and writes, and I tried pass the multipath node to the nested guest both as a raw LUN and
as a regular block device.
In all cases I see errors in the nested guest, however the multipath
code does recover eventually and switches the paths.
If I allow the fio to continue on IO errors,
I see that multipath code switches patches in very timely manner with a bunch of IO errors in between.

I even tried to fail only one path (the first one) while keeping the second one always online,
and even then I have seen IO errors in the guest, suggesting
that kernel's multipath code doesn't retry the IO requests on remaining paths when it encounters
a failing path.

I think that this means that in-kernel multipath code is just designed this way, that is geared towards
once in a while disk failure and assumes that the client will retry failing requests
(which is true for linux filesystem code from my experience with nvme-mdev driver I once developed).

One thing for sure, that this bug is not related to virtualization in any way.
in fact running fio in the outer guest (which just simulates the real hardware) produces the same
results.

------- Comment From shlyi.com 2020-03-17 21:51 EDT-------
(In reply to comment #17)
> Created attachment 141606 [details]
> systemtap script to print the error code on failed requests
>
>
> ------- Comment on attachment From bmarzins 2020-03-17 16:55:44
> EDT-------
>
>
> The kernel multipath code will retry requests down different paths, as long
> as the error code on the failing request appears to be transport related.
> Would you be able to run this systemtap script, and reproduce the error, and
> post the script output and /var/log/messages from when the issue occured?
>
> Can you run the script with
>
> # stap mpath_error.stp
>
> Wait for it to log "starting", and then reproduce the issue. When you're
> done, you can kill the script with ctrl-c
>
> To run the script, you need the systemtap package, and the kernel devel and
> debuginfo packages.

Hello,

I failed to start the script you provided as you can see below. I've installed "systemtap" and "kernel-devel" pkg, but it seems that there isn't a "debuginfo" package. Looking forward to hear you back.

[root@ilx24s ~]# yum list |grep debuginfo
[root@ilx24s ~]# yum list |grep kernel-devel
kernel-devel.x86_64                     3.10.0-1127.el7            @anaconda/7.8
[root@ilx24s ~]# yum install debuginfo
Loaded plugins: langpacks, product-id, search-disabled-repos, subscription-manager

This system is not registered with an entitlement server. You can use subscription-manager to register.

No package debuginfo available.
Error: Nothing to do
[root@ilx24s ~]# yum install systemtap
Loaded plugins: langpacks, product-id, search-disabled-repos, subscription-manager

This system is not registered with an entitlement server. You can use subscription-manager to register.

Package systemtap-4.0-11.el7.x86_64 already installed and latest version
Nothing to do
[root@ilx24s ~]# yum install kernel-devel
Loaded plugins: langpacks, product-id, search-disabled-repos, subscription-manager

This system is not registered with an entitlement server. You can use subscription-manager to register.

Package kernel-devel-3.10.0-1127.el7.x86_64 already installed and latest version
Nothing to do

[root@ilx24s ~]# stap mpath_error.stp
semantic error: while resolving probe point: identifier 'module' at mpath_error.stp:9:7
source: probe module("dm_multipath").function("do_end_io"),
^

semantic error: no match (similar functions: multipath_end_io, assign_bit, bypass_pg, init_module, pg_init_done)

Pass 2: analysis failed.  [man error::pass2]
Number of similar error messages suppressed: 1.
Rerun with -v to see them.
[root@ilx24s ~]# stap mpath_error.stp -v
Pass 1: parsed user script and 479 library scripts using 279848virt/77084res/3492shr/73724data kb, in 600usr/30sys/640real ms.
semantic error: while resolving probe point: identifier 'module' at mpath_error.stp:9:7
source: probe module("dm_multipath").function("do_end_io"),
^

semantic error: no match (similar functions: multipath_end_io, assign_bit, bypass_pg, init_module, pg_init_done)

semantic error: while resolving probe point: identifier 'module' at :10:7
source:       module("dm_multipath").function("do_end_io_bio") {
^

semantic error: no match (similar functions: multipath_end_io_bio, assign_bit, fail_path, init_module, multipath_end_io)

Pass 2: analyzed script: 2 probes, 9 functions, 1 embed, 0 globals using 282248virt/80364res/4368shr/76124data kb, in 30usr/60sys/86real ms.
Pass 2: analysis failed.  [man error::pass2]

[root@ilx24s ~]# yum repolist
Loaded plugins: langpacks, product-id, search-disabled-repos, subscription-manager

This system is not registered with an entitlement server. You can use subscription-manager to register.

repo id                                       repo name                                                                   status
HighAvailability                              Red Hat Enterprise Linux 7.8 HighAvailability                                  52
InstallMedia                                  Red Hat Enterprise Linux 7.8                                                5,231
InstallMediasup                               sup Red Hat Enterprise Linux 7.8                                               18
ResilientStorage                              Red Hat Enterprise Linux 7.8 ResilientStorage                                  57
repolist: 5,358

------- Comment From shlyi.com 2020-04-07 05:46 EDT-------
(In reply to comment #17)
> Created attachment 141606 [details]
> systemtap script to print the error code on failed requests
>
>
> ------- Comment on attachment From bmarzins 2020-03-17 16:55:44
> EDT-------
>
>
> The kernel multipath code will retry requests down different paths, as long
> as the error code on the failing request appears to be transport related.
> Would you be able to run this systemtap script, and reproduce the error, and
> post the script output and /var/log/messages from when the issue occured?
>
> Can you run the script with
>
> # stap mpath_error.stp
>
> Wait for it to log "starting", and then reproduce the issue. When you're
> done, you can kill the script with ctrl-c
>
> To run the script, you need the systemtap package, and the kernel devel and
> debuginfo packages.

Hello,

Here is the log you requested:
[root@ilx24s ~]# stap mpath_error.stp
Thu Apr  2 20:00:33 2020 MST: starting

Thu Apr  2 21:57:51 2020 MST: 0: multipath_end_io called with error -5
Thu Apr  2 21:57:51 2020 MST: 6999: multipath_end_io called with error -5
Thu Apr  2 21:57:51 2020 MST: 61: multipath_end_io called with error -5
Fri Apr  3 02:10:12 2020 MST: 14: multipath_end_io called with error -5
Fri Apr  3 02:11:27 2020 MST: 0: multipath_end_io called with error -5
Fri Apr  3 02:11:27 2020 MST: 45: multipath_end_io called with error -5
Fri Apr  3 02:11:27 2020 MST: 45: multipath_end_io called with error -5
Fri Apr  3 05:07:01 2020 MST: 20: multipath_end_io called with error -5
Fri Apr  3 06:29:32 2020 MST: 14: multipath_end_io called with error -5
Fri Apr  3 06:29:32 2020 MST: 14: multipath_end_io called with error -5
Fri Apr  3 06:30:38 2020 MST: 0: multipath_end_io called with error -5
Fri Apr  3 09:21:36 2020 MST: 0: multipath_end_io called with error -5
Fri Apr  3 09:21:36 2020 MST: 20: multipath_end_io called with error -5
Fri Apr  3 09:21:36 2020 MST: 20: multipath_end_io called with error -5
^CTue Apr  7 02:30:16 2020 MST: ending
You have new mail in /var/spool/mail/root
[root@ilx24s ~]#

[root@ilx25s ~]# screen -ls
There is a screen on:
26305.multipath (Detached)
1 Socket in /var/run/screen/S-root.

[root@ilx25s ~]# screen -r 26305.multipath
Thu Apr  2 20:12:35 2020 MST: starting

Fri Apr  3 00:48:40 2020 MST: 0: multipath_end_io called with error -5
Fri Apr  3 00:48:40 2020 MST: 0: multipath_end_io called with error -5
Fri Apr  3 00:48:40 2020 MST: 0: multipath_end_io called with error -5
Fri Apr  3 00:48:40 2020 MST: 0: multipath_end_io called with error -5
Fri Apr  3 00:48:40 2020 MST: 0: multipath_end_io called with error -5
Fri Apr  3 00:48:40 2020 MST: 0: multipath_end_io called with error -5
Fri Apr  3 00:48:45 2020 MST: 50: multipath_end_io called with error -5
Fri Apr  3 02:11:19 2020 MST: 50: multipath_end_io called with error -5
Fri Apr  3 02:11:19 2020 MST: 50: multipath_end_io called with error -5
Fri Apr  3 06:29:33 2020 MST: 6: multipath_end_io called with error -5
Fri Apr  3 06:29:33 2020 MST: 6: multipath_end_io called with error -5
Fri Apr  3 06:29:33 2020 MST: 6: multipath_end_io called with error -5
Fri Apr  3 06:29:33 2020 MST: 6: multipath_end_io called with error -5
Fri Apr  3 06:30:39 2020 MST: 14: multipath_end_io called with error -5
Fri Apr  3 06:30:39 2020 MST: 14: multipath_end_io called with error -5
Fri Apr  3 06:30:39 2020 MST: 14: multipath_end_io called with error -5
Fri Apr  3 06:30:39 2020 MST: 14: multipath_end_io called with error -5
Fri Apr  3 06:30:39 2020 MST: 14: multipath_end_io called with error -5
Fri Apr  3 06:30:39 2020 MST: 14: multipath_end_io called with error -5
Fri Apr  3 06:30:39 2020 MST: 14: multipath_end_io called with error -5
Fri Apr  3 09:21:37 2020 MST: 25: multipath_end_io called with error -5
Fri Apr  3 09:21:37 2020 MST: 25: multipath_end_io called with error -5
Fri Apr  3 09:21:37 2020 MST: 25: multipath_end_io called with error -5
Fri Apr  3 09:21:37 2020 MST: 25: multipath_end_io called with error -5
Fri Apr  3 13:15:25 2020 MST: 0: multipath_end_io called with error -5

^CTue Apr  7 02:30:58 2020 MST: ending
You have new mail in /var/spool/mail/root
[root@ilx25s ~]#

/var/log/messages will be uploaded soon.
messages-ilx24s-20200405
messages-ilx25s-20200405

------- Comment From shlyi.com 2020-04-07 22:00 EDT-------
(In reply to comment #26)
> I can't find any indication that your filesystem went read-only in the
> messages. Do you have the disktest logs as well?

Yes, I've uploaded one disktest log of the guest of ilx24s:
Below is partial content, the error message occurred at the time perfectly matched with your script output.

*************** disktest_16K_FS starting Thu Apr  2 21:30:49 MST 2020 ***************
| 2020/04/02-21:30:49 | START | 7240 | v1.4.2.1 | /fc/dev/sda/test | Start args: -K1 -t 0:1:120 -N 8856661 -PXT -h0 -pl -rw -I f -z -ma -E0 -T 1800 -B 16K (-p u) (-o 0)
| 2020/04/02-21:30:49 | INFO  | 7240 | v1.4.2.1 | /fc/dev/sda/test | Starting pass
| 2020/04/02-21:57:51 | ERROR | 7240 | v1.4.2.1 | /fc/dev/sda/test | Thread 0: Write failed: seek 1216442, lba 1749744 (0x1AB2F0), got = -1, asked for = 8192, errno 30
| 2020/04/02-21:57:51 | ERROR | 7240 | v1.4.2.1 | /fc/dev/sda/test | Thread 0: fsync error = 30
| 2020/04/02-21:59:51 | WARN  | 7240 | v1.4.2.1 | /fc/dev/sda/test | Possible IO hang condition, IO timeout reached, 120 seconds

------- Comment From shlyi.com 2020-04-08 22:28 EDT-------
(In reply to comment #30)
> Thanks, but I still don't see any indication in the logs that a multipath
> device returned an IO error, or lost all of it's paths.  So, just to be
> sure, the test output says "/fc/dev/sda/test". These tests aren't actually
> running directly on top of /dev/sda, are they? If so, then they aren't using
> multipath at all.

"/fc/dev/sda/test" is seen from the guest(ilx24s1r) side, and the guest think it's a raw device without multipath. But in the dependent host's(ilx24s) view, the mapped lun has multiple paths;

------- Comment From shlyi.com 2020-04-26 22:55 EDT-------
We run the EI at weekend, so the earlier error messages flashed out of the screen, below are the error message in the end of the test:

[root@ilx24s ~]# screen -r 152142.stap
Sun Apr 26 15:47:22 2020 MST: 2373: failed path 69:96, nr_active 2
Sun Apr 26 15:47:24 2020 MST: 2373: failed path 68:208, nr_active 3
Sun Apr 26 15:47:24 2020 MST: 2373: failed path 68:224, nr_active 2
Sun Apr 26 15:47:24 2020 MST: 2373: failed path 69:176, nr_active 2
Sun Apr 26 15:57:49 2020 MST: 2373: reinstated path 67:48, nr_active 3
Sun Apr 26 15:57:49 2020 MST: 2373: reinstated path 67:64, nr_active 3
Sun Apr 26 15:57:49 2020 MST: 2373: reinstated path 67:80, nr_active 3
Sun Apr 26 15:57:50 2020 MST: 2373: reinstated path 66:16, nr_active 3
Sun Apr 26 15:57:51 2020 MST: 2373: reinstated path 65:192, nr_active 4
Sun Apr 26 15:57:51 2020 MST: 2373: reinstated path 65:208, nr_active 4
Sun Apr 26 15:57:51 2020 MST: 2373: reinstated path 65:224, nr_active 3
Sun Apr 26 15:57:51 2020 MST: 2373: reinstated path 65:240, nr_active 3
Sun Apr 26 15:57:51 2020 MST: 2373: reinstated path 66:0, nr_active 3
Sun Apr 26 15:57:51 2020 MST: 2373: reinstated path 66:32, nr_active 3
Sun Apr 26 15:57:51 2020 MST: 2373: reinstated path 66:48, nr_active 3
Sun Apr 26 15:57:51 2020 MST: 2373: reinstated path 66:64, nr_active 3
Sun Apr 26 15:57:51 2020 MST: 2373: reinstated path 66:80, nr_active 4
Sun Apr 26 15:57:51 2020 MST: 2373: reinstated path 66:96, nr_active 3
Sun Apr 26 15:57:51 2020 MST: 2373: reinstated path 66:112, nr_active 3
Sun Apr 26 15:57:51 2020 MST: 2373: reinstated path 66:128, nr_active 3
Sun Apr 26 15:57:51 2020 MST: 2373: reinstated path 66:144, nr_active 3
Sun Apr 26 15:57:51 2020 MST: 2373: reinstated path 66:192, nr_active 4
Sun Apr 26 15:57:51 2020 MST: 2373: reinstated path 67:0, nr_active 4
Sun Apr 26 15:57:51 2020 MST: 2373: reinstated path 67:16, nr_active 4
Sun Apr 26 15:57:51 2020 MST: 2373: reinstated path 67:96, nr_active 4
Sun Apr 26 15:57:51 2020 MST: 2373: reinstated path 67:112, nr_active 4
Sun Apr 26 15:57:52 2020 MST: 2373: reinstated path 66:160, nr_active 4
Sun Apr 26 15:57:52 2020 MST: 2373: reinstated path 66:176, nr_active 4
Sun Apr 26 15:57:52 2020 MST: 2373: reinstated path 66:208, nr_active 4
Sun Apr 26 15:57:52 2020 MST: 2373: reinstated path 66:224, nr_active 4
Sun Apr 26 15:57:52 2020 MST: 2373: reinstated path 66:240, nr_active 4
Sun Apr 26 15:57:52 2020 MST: 2373: reinstated path 67:32, nr_active 4
Sun Apr 26 16:00:03 2020 MST: 2373: reinstated path 69:32, nr_active 3
Sun Apr 26 16:00:03 2020 MST: 2373: reinstated path 69:64, nr_active 3
Sun Apr 26 16:00:04 2020 MST: 2373: reinstated path 68:240, nr_active 3
Sun Apr 26 16:00:04 2020 MST: 2373: reinstated path 69:96, nr_active 4
Sun Apr 26 16:00:04 2020 MST: 2373: reinstated path 69:128, nr_active 4
Sun Apr 26 16:00:06 2020 MST: 2373: reinstated path 68:192, nr_active 3
Sun Apr 26 16:00:06 2020 MST: 2373: reinstated path 68:208, nr_active 3
Sun Apr 26 16:00:06 2020 MST: 2373: reinstated path 68:224, nr_active 3
Sun Apr 26 16:00:06 2020 MST: 2373: reinstated path 69:0, nr_active 3
Sun Apr 26 16:00:06 2020 MST: 2373: reinstated path 69:16, nr_active 3
Sun Apr 26 16:00:06 2020 MST: 2373: reinstated path 69:48, nr_active 3
Sun Apr 26 16:00:06 2020 MST: 2373: reinstated path 69:80, nr_active 3
Sun Apr 26 16:00:06 2020 MST: 2373: reinstated path 69:112, nr_active 4
Sun Apr 26 16:00:06 2020 MST: 2373: reinstated path 69:144, nr_active 4
Sun Apr 26 16:00:06 2020 MST: 2373: reinstated path 69:160, nr_active 4
Sun Apr 26 16:00:06 2020 MST: 2373: reinstated path 69:176, nr_active 4
Sun Apr 26 16:00:06 2020 MST: 2373: reinstated path 69:192, nr_active 4
Sun Apr 26 16:00:06 2020 MST: 2373: reinstated path 69:208, nr_active 4
Sun Apr 26 16:00:06 2020 MST: 2373: reinstated path 69:224, nr_active 4
Sun Apr 26 16:00:06 2020 MST: 2373: reinstated path 69:240, nr_active 4

^CSun Apr 26 19:44:14 2020 MST: ending
You have new mail in /var/spool/mail/root

Corresponding log files you asked have been uploaded as "disktest, /var/log/messages of ilx24s and ilx24s1r", since ilx24s1r and ilx25s1s has the similar issue, I only uploaded the related logs of ilx24s1r.

------- Comment From shlyi.com 2020-06-28 00:20 EDT-------
Hello,

Thanks for the guide.

Before you told us these suggested modifications. we've tried three of your suggested modifications, which made a huge difference for the behavior of migrating a guest running io.
modifications we've made:
cache='none' io='native' device='disk'

We haven't run any auto ei case after making the above change yet, we'll update you with the result once we get it.

Besides, both "bus='virtio'" and "bus='scsi'" would be good for our manual test result by now.

*** Bug 1817087 has been marked as a duplicate of this bug. ***

*** Bug 1779758 has been marked as a duplicate of this bug. ***

------- Comment From shlyi.com 2019-12-23 20:59 EDT-------
(In reply to comment #13)
> Hi, i have no your mentioned storage ENV, but i want to clear following
> questions:
> 1. How to install  RHEL 8.1 cluster env ? Do you mean there are 4 paths in
> one host which point to same lun, each path pass-through to respective
> guest? what software installed in the guest?
> 2. What qemu command line of your guest?
> 3. Could you please share tool "StorageSideCablePull", how to test ?
> 4. Could you test two set two paths to eliminate physical storage issue .
> For example, one cluster is path A and B, the other is path C and D.

Hello, I tried to answer your questions as below:

For question 1: DMMP is used in host, guest use virtio for boot lun, guest use raw scsi map for the data lun;

For question 2: I am not sure what's your meaning, I used the rhel cluster to manage the guest, the resource type is:  "ocf:heartbeat:VirtualDomain". It "Manages virtual domains through the libvirt virtualization framework".

For question 3: 4 Paths are as below, our script for case "StorageSideCablePull" will break and recover below four paths sequentially, we anticipate the failed path could recover after the path recovered.

Host port1 ---------------- switchA port1 ----------Path1----------storage port1
|
switch? port2 -----------Path2---------storage port2

switch? port1 -----------Path3----------storage port3
|
Host port2 ---------------- switchB port2 ----------Path4----------storage port4

For question 4, we only have one cluster, so I don't understand your meaning. Can my above explanation answer your question?

------- Comment From shlyi.com 2019-12-23 21:25 EDT-------
Part of guest config file "ilx21r1r.xml":

<domain type='kvm'>
<name>ilx21r1r</name>
<uuid>f37748d4-3b13-4a95-906e-24606a619736</uuid>
<metadata>
<libosinfo:libosinfo xmlns:libosinfo="http://libosinfo.org/xmlns/libvirt/domain/1.0">
<libosinfo:os id="http://redhat.com/rhel/8.0"/>
</libosinfo:libosinfo>
</metadata>

------- Comment From shlyi.com 2019-12-23 22:06 EDT-------
> (In reply to IBM Bug Proxy from comment #10)
> > (In reply to comment #13)
> > > Hi, i have no your mentioned storage ENV, but i want to clear following
> > > questions:
> > >
> > > 1. How to install  RHEL 8.1 cluster env ? Do you mean there are 4 paths in
> > > one host which point to same lun, each path pass-through to respective
> > > guest? what software installed in the guest?
> > > 2. What qemu command line of your guest?
> > > 3. Could you please share tool "StorageSideCablePull", how to test ?
> > > 4. Could you test two set two paths to eliminate physical storage issue .
> > > For example, one cluster is path A and B, the other is path C and D.
> >
> > Hello, I tried to answer your questions as below:
> >
> > For question 1: DMMP is used in host, guest use virtio for boot lun, guest
> > use raw scsi map for the data lun;
> >
> > For question 2: I am not sure what's your meaning, I used the rhel cluster
> > to manage the guest, the resource type is:  "ocf:heartbeat:VirtualDomain".
> > It "Manages virtual domains through the libvirt virtualization framework".
> >
> > For question 3: 4 Paths are as below, our script for case
> > "StorageSideCablePull" will break and recover below four paths sequentially,
> > we anticipate the failed path could recover after the path recovered.
> >
> > Host port1 ---------------- switchA port1 ----------Path1----------storage
> > port1
> > |
> > switch? port2 -----------Path2---------storage port2
> >
> > switch? port1 -----------Path3----------storage port3
> > |
> > Host port2 ---------------- switchB port2 ----------Path4----------storage
> > port4
> >
> > For question 4, we only have one cluster, so I don't understand your
> > meaning. Can my above explanation answer your question?
> I do not familiar with your software and hardware ENV.
> for the question4 , i mean you may build 2 separated clusters, one using
> path A and B,
> the other using path C and D to verify all path may work well.
> I suggest using targetcli to simulate 4 paths then check it in your ENV

Hello, as I know "targetcli" is used in iscsi env, but my env is in FCP env.

------- Comment From shlyi.com 2019-12-26 01:18 EDT-------
(In reply to comment #20)
> 1.You have 4 path in host, you may run " multipath -l " to check the result.
> 2.I do not know the usage of "disktest", it come from where?  what the usage
> of tool?
> 3.This tool can run on host?
> 4.Could you get the qemu command line of the guest, like as "ps -ax|grep
> qemu"
> 5. if the testing related to switch path, transient io error is expected?

1, Yes, we use that command, and it shows the paths are good after the ei
2, "disktest" is a io tool used by IBM, it can generate and verify the io.
3, it can run on host, but in rhel 8.1 cluster env, we run io on its guest;
4, here is the output of the guest:
[root@ilx21r1r ~]# ps -ax|grep qemu
1398 ?        Ss     0:00 /usr/bin/qemu-ga --method=virtio-serial --path=/dev/virtio-ports/org.qemu.guest_agent.0 --blacklist=guest-file-open,guest-file-close,guest-file-read,guest-file-write,guest-file-seek,guest-file-flush,guest-exec,guest-exec-status -F/etc/qemu-ga/fsfreeze-hook
8323 pts/0    S+     0:00 grep --color=auto qemu
5, since the host 4 paths for each lun running io, so we expect there is no error in the io log for each lun, which would break only half of the paths when running ei.

------- Comment From shlyi.com 2019-12-26 01:33 EDT-------
(In reply to comment #22)
> More questions:
> 1 What the guest behavior when it hit io-error, will it change to "pause"
> status?
> 2 You may set error_policy='stop' for the guest to see what happen ?
> 3 In my understanding, it should be expect result have io error when active
> path is broken,
> am i right?  the point is it may back to if other path may work.
> 4 if you may set error_policy='stop' , the guest will change to pause status
> when hit io error, if
> this issue is fixed. the guest can be back to running after we ask for
> continue running. that is expected result.
> For example:
> <disk type='block' device='lun' sgio='filtered' snapshot='no'>
> <driver name='qemu' type='raw' cache='none' error_policy='stop' io='native'/>
> <source dev='/dev/mapper/360014059b31089df5114cbd830320c2b'/>
> <backingStore/>
> <target dev='sdb' bus='scsi'/>
> <alias name='scsi0-0-0-1'/>
> <address type='drive' controller='0' bus='0' target='0' unit='1'/>
> </disk>

Please see my last comments for its expecting behavior, "we expect there is no error in the io log for each lun, which would break only half of the paths when running ei".
While below is the log when io hit the error, which we think the guest mishandled the issue:

/var/log/message of ilx21r:
Dec  3 01:01:07 ilx21r kernel: sd 3:0:4:0: rejecting I/O to offline device

/var/log/message of ilx21r1r:
Dec  3 01:01:07 ilx21r1r kernel: print_req_error: I/O error, dev sde, sector 135448 flags 801

I don't understand why we should " set error_policy='stop' ", and how to do it.

------- Comment From shlyi.com 2019-12-26 23:02 EDT-------
(In reply to comment #25)
> (In reply to IBM Bug Proxy from comment #19)
> > (In reply to comment #20)
> > > 1.You have 4 path in host, you may run " multipath -l " to check the result.
> > > 2.I do not know the usage of "disktest", it come from where?  what the usage
> > > of tool?
> > > 3.This tool can run on host?
> > > 4.Could you get the qemu command line of the guest, like as "ps -ax|grep
> > > qemu"
> > > 5. if the testing related to switch path, transient io error is expected?
> > 1, Yes, we use that command, and it shows the paths are good after the ei
> > 2, "disktest" is a io tool used by IBM, it can generate and verify the io.
> > 3, it can run on host, but in rhel 8.1 cluster env, we run io on its guest;
> > 4, here is the output of the guest:
> > [root@ilx21r1r ~]# ps -ax|grep qemu
> > 1398 ?        Ss     0:00 /usr/bin/qemu-ga --method=virtio-serial
> > --path=/dev/virtio-ports/org.qemu.guest_agent.0
> > --blacklist=guest-file-open,guest-file-close,guest-file-read,guest-file-
> > write,guest-file-seek,guest-file-flush,guest-exec,guest-exec-status
> > -F/etc/qemu-ga/fsfreeze-hook
> > 8323 pts/0    S+     0:00 grep --color=auto qemu
> > 5, since the host 4 paths for each lun running io, so we expect there is no
> > error in the io log for each lun, which would break only half of the paths
> > when running ei.
> 1.Please share the result/ouput of " multipath -l"
> 2.What result if your do disktest on host during StorageSideCablePull
> running, there is no any ioerror?
> 3.In my understanding, your 4 paths for fail-over,right?
> so only one path in active , if active path is broken, other path will take
> place. if have io-error is accepted in the switch time.
1, I'll put the out put in the bottom;
2, yes, we expect no io error;
3, I think the io shouldn't break during the switch time, at least, this io error didn't happen in previous rhel version.

[root@ilx21r ~]# multipath -l
ilx21r22r_8G_d_0 (36005076b19bcb49b180000004600005c) dm-28 IBM,FlashSystem-9840
size=5.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
|- 3:0:4:0 sdcr 69:240 active undef running
|- 3:0:3:0 sdcl 69:144 active undef running
|- 4:0:1:0 sddh 70:240 active undef running
`- 4:0:4:0 sddt 71:176 active undef running
FC16G_1_D0 (36005076119909fc118000000bf0001d0) dm-25 IBM,FlashSystem-9840
size=3.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
|- 3:0:1:3 sdcc 69:0   active undef running
|- 3:0:2:3 sdci 69:96  active undef running
|- 4:0:0:3 sdde 70:192 active undef running
`- 4:0:2:3 sddq 71:128 active undef running
ilx21r22r_16G_d_5 (36005076b19bcb49b180000005900006f) dm-17 IBM,FlashSystem-9840
size=5.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
|- 1:0:4:3 sdaa 65:160 active undef running
|- 1:0:5:3 sdag 66:0   active undef running
|- 2:0:3:3 sdba 67:64  active undef running
`- 2:0:6:3 sdbr 68:80  active undef running
FC8G_D5 (36005076119909fc118000000730001cb) dm-11 IBM,FlashSystem-9840
size=3.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
|- 1:0:2:3 sdo  8:224  active undef running
|- 1:0:3:3 sdu  65:64  active undef running
|- 2:0:2:3 sdau 66:224 active undef running
`- 2:0:4:3 sdbg 67:160 active undef running
9840Q_8G_d_1 (3600507639281b4a3900000006c0000ab) dm-21 IBM,FlashSystem-9840
size=4.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
|- 1:0:8:1 sdak 66:64  active undef running
|- 1:0:9:1 sdam 66:96  active undef running
|- 2:0:0:1 sdao 66:128 active undef running
`- 2:0:1:1 sdaq 66:160 active undef running
ilx21r_22r_kvmcfg (360050762198c1fc21800000041000109) dm-3 IBM,FlashSystem-9840
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
|- 1:0:0:3 sde  8:64   active undef running
|- 1:0:1:3 sdj  8:144  active undef running
|- 2:0:5:3 sdbm 68:0   active undef running
`- 2:0:7:3 sdbx 68:176 active undef running
9840Q_8G_d_0 (3600507639281b4a3900000006b0000aa) dm-20 IBM,FlashSystem-9840
size=4.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
|- 1:0:8:0 sdaj 66:48  active undef running
|- 1:0:9:0 sdal 66:80  active undef running
|- 2:0:0:0 sdan 66:112 active undef running
`- 2:0:1:0 sdap 66:144 active undef running
ilx21r22r_16G_d_2 (36005076b19bcb49b180000004d000063) dm-16 IBM,FlashSystem-9840
size=5.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
|- 1:0:4:2 sdz  65:144 active undef running
|- 1:0:5:2 sdaf 65:240 active undef running
|- 2:0:3:2 sdaz 67:48  active undef running
`- 2:0:6:2 sdbq 68:64  active undef running
ilx21r1r_b0 (360050762198c1fc21800000032000107) dm-2 IBM,FlashSystem-9840
size=65G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
|- 1:0:0:2 sdd  8:48   active undef running
|- 1:0:1:2 sdi  8:128  active undef running
|- 2:0:5:2 sdbl 67:240 active undef running
`- 2:0:7:2 sdbw 68:160 active undef running
FC8G_D2 (36005076119909fc1180000003a0001c8) dm-10 IBM,FlashSystem-9840
size=3.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
|- 1:0:2:2 sdn  8:208  active undef running
|- 1:0:3:2 sdt  65:48  active undef running
|- 2:0:2:2 sdat 66:208 active undef running
`- 2:0:4:2 sdbf 67:144 active undef running
mpathag (3600605b00dee874024d75438b8e61213) dm-39 Lenovo,RAID 530-8i
size=223G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
`- 0:2:0:0 sda  8:0    active undef running
mpatha (360050762198c1fc218000000520000ff) dm-0 IBM,FlashSystem-9840
size=120G features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=0 status=active
|- 1:0:0:0 sdb  8:16   active undef running
|- 2:0:5:0 sdbj 67:208 active undef running
|- 1:0:1:0 sdg  8:96   active undef running
`- 2:0:7:0 sdbu 68:128 active undef running
ilx21r22r_16G_d_1 (36005076b19bcb49b180000004c000062) dm-15 IBM,FlashSystem-9840
size=5.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
|- 1:0:4:1 sdy  65:128 active undef running
|- 1:0:5:1 sdae 65:224 active undef running
|- 2:0:3:1 sday 67:32  active undef running
`- 2:0:6:1 sdbp 68:48  active undef running
ilx22r1s_b0 (360050762198c1fc21800000030000108) dm-1 IBM,FlashSystem-9840
size=65G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
|- 1:0:0:1 sdc  8:32   active undef running
|- 1:0:1:1 sdh  8:112  active undef running
|- 2:0:5:1 sdbk 67:224 active undef running
`- 2:0:7:1 sdbv 68:144 active undef running
FC8G_D1 (36005076119909fc118000000390001c7) dm-9 IBM,FlashSystem-9840
size=3.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
|- 1:0:2:1 sdm  8:192  active undef running
|- 1:0:3:1 sds  65:32  active undef running
|- 2:0:2:1 sdas 66:192 active undef running
`- 2:0:4:1 sdbe 67:128 active undef running
ilx21r22r_16G_d_0 (36005076b19bcb49b180000004b000061) dm-14 IBM,FlashSystem-9840
size=5.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
|- 1:0:4:0 sdx  65:112 active undef running
|- 1:0:5:0 sdad 65:208 active undef running
|- 2:0:3:0 sdax 67:16  active undef running
`- 2:0:6:0 sdbo 68:32  active undef running
ilx21r22r_8G_d_7 (36005076b19bcb49b1800000052000068) dm-33 IBM,FlashSystem-9840
size=5.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
|- 3:0:4:5 sdcw 70:64  active undef running
|- 3:0:3:5 sdcq 69:224 active undef running
|- 4:0:1:5 sddm 71:64  active undef running
`- 4:0:4:5 sddy 128:0  active undef running
FC8G_D0 (36005076119909fc118000000330001c6) dm-8 IBM,FlashSystem-9840
size=3.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
|- 1:0:2:0 sdl  8:176  active undef running
|- 1:0:3:0 sdr  65:16  active undef running
|- 2:0:2:0 sdar 66:176 active undef running
`- 2:0:4:0 sdbd 67:112 active undef running
ilx21r22r_8G_d_6 (36005076b19bcb49b1800000051000067) dm-32 IBM,FlashSystem-9840
size=5.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
|- 3:0:4:4 sdcv 70:48  active undef running
|- 3:0:3:4 sdcp 69:208 active undef running
|- 4:0:1:4 sddl 71:48  active undef running
`- 4:0:4:4 sddx 71:240 active undef running
9840Q_16G_d_1 (3600507639281b4a3900000006e0000ad) dm-35 IBM,FlashSystem-9840
size=4.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
|- 3:0:8:1 sdcy 70:96  active undef running
|- 3:0:9:1 sdda 70:128 active undef running
|- 4:0:6:1 sdea 128:32 active undef running
`- 4:0:7:1 sdec 128:64 active undef running
ilx21r22r_8G_d_5 (36005076b19bcb49b1800000050000066) dm-31 IBM,FlashSystem-9840
size=5.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
|- 3:0:4:3 sdcu 70:32  active undef running
|- 3:0:3:3 sdco 69:192 active undef running
|- 4:0:1:3 sddk 71:32  active undef running
`- 4:0:4:3 sddw 71:224 active undef running
FC16G_2_D2 (36005076119909fc118000000ff0001d7) dm-24 IBM,FlashSystem-9840
size=3.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
|- 3:0:1:2 sdcb 68:240 active undef running
|- 3:0:2:2 sdch 69:80  active undef running
|- 4:0:0:2 sddd 70:176 active undef running
`- 4:0:2:2 sddp 71:112 active undef running
9840Q_16G_d_0 (3600507639281b4a3900000006d0000ac) dm-34 IBM,FlashSystem-9840
size=4.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
|- 3:0:8:0 sdcx 70:80  active undef running
|- 3:0:9:0 sdcz 70:112 active undef running
|- 4:0:6:0 sddz 128:16 active undef running
`- 4:0:7:0 sdeb 128:48 active undef running
FC16G_2_D1 (36005076119909fc118000000fe0001d6) dm-23 IBM,FlashSystem-9840
size=3.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
|- 3:0:1:1 sdca 68:224 active undef running
|- 3:0:2:1 sdcg 69:64  active undef running
|- 4:0:0:1 sddc 70:160 active undef running
`- 4:0:2:1 sddo 71:96  active undef running
FC16G_2_D0 (36005076119909fc118000000fd0001d5) dm-22 IBM,FlashSystem-9840
size=3.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
|- 3:0:1:0 sdbz 68:208 active undef running
|- 3:0:2:0 sdcf 69:48  active undef running
|- 4:0:0:0 sddb 70:144 active undef running
`- 4:0:2:0 sddn 71:80  active undef running
ilx21r22r_8G_d_2 (36005076b19bcb49b180000004800005e) dm-30 IBM,FlashSystem-9840
size=5.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
|- 3:0:4:2 sdct 70:16  active undef running
|- 3:0:3:2 sdcn 69:176 active undef running
|- 4:0:1:2 sddj 71:16  active undef running
`- 4:0:4:2 sddv 71:208 active undef running
FC16G_1_D2 (36005076119909fc118000000fa0001d2) dm-27 IBM,FlashSystem-9840
size=3.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
|- 3:0:1:5 sdce 69:32  active undef running
|- 3:0:2:5 sdck 69:128 active undef running
|- 4:0:0:5 sddg 70:224 active undef running
`- 4:0:2:5 sdds 71:160 active undef running
ilx21r22r_16G_d_7 (36005076b19bcb49b180000005b000071) dm-19 IBM,FlashSystem-9840
size=5.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
|- 1:0:4:5 sdac 65:192 active undef running
|- 1:0:5:5 sdai 66:32  active undef running
|- 2:0:3:5 sdbc 67:96  active undef running
`- 2:0:6:5 sdbt 68:112 active undef running
FC8G_D7 (36005076119909fc118000000a80001cd) dm-12 IBM,FlashSystem-9840
size=3.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
|- 1:0:2:5 sdp  8:240  active undef running
|- 1:0:3:5 sdv  65:80  active undef running
|- 2:0:2:5 sdav 66:240 active undef running
`- 2:0:4:5 sdbh 67:176 active undef running
ilx21r22r_8G_d_1 (36005076b19bcb49b180000004700005d) dm-29 IBM,FlashSystem-9840
size=5.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
|- 3:0:4:1 sdcs 70:0   active undef running
|- 3:0:3:1 sdcm 69:160 active undef running
|- 4:0:1:1 sddi 71:0   active undef running
`- 4:0:4:1 sddu 71:192 active undef running
FC16G_1_D1 (36005076119909fc118000000f90001d1) dm-26 IBM,FlashSystem-9840
size=3.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
|- 3:0:1:4 sdcd 69:16  active undef running
|- 3:0:2:4 sdcj 69:112 active undef running
|- 4:0:0:4 sddf 70:208 active undef running
`- 4:0:2:4 sddr 71:144 active undef running
ilx21r22r_16G_d_6 (36005076b19bcb49b180000005a000070) dm-18 IBM,FlashSystem-9840
size=5.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
|- 1:0:4:4 sdab 65:176 active undef running
|- 1:0:5:4 sdah 66:16  active undef running
|- 2:0:3:4 sdbb 67:80  active undef running
`- 2:0:6:4 sdbs 68:96  active undef running
ilx21r22r_fence (360050762198c1fc2180000005f00012d) dm-4 IBM,FlashSystem-9840
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
|- 1:0:0:4 sdf  8:80   active undef running
|- 1:0:1:4 sdk  8:160  active undef running
|- 2:0:5:4 sdbn 68:16  active undef running
`- 2:0:7:4 sdby 68:192 active undef running
FC8G_D6 (36005076119909fc118000000780002fe) dm-13 IBM,FlashSystem-9840
size=3.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
|- 1:0:2:6 sdq  65:0   active undef running
|- 1:0:3:6 sdw  65:96  active undef running
|- 2:0:2:6 sdaw 67:0   active undef running
`- 2:0:4:6 sdbi 67:192 active undef running

------- Comment From shlyi.com 2019-12-30 02:33 EDT-------
(In reply to comment #27)
> 1.What result same test on host only ? please execute it some times.
> I think the host should have same issue. it maybe have lower frequency.
> 2.Does it can recover when hit io-error on guest?
> 3.What do your mean previous rhel? do you mean it is regression issue?
> What is the previous revision of kernel and qemu?

2, io-error reported in this issue won't break the guest
3, when the host and the guest are in os  rhel 7.7, we don't have this kind of issue.
host ilx21r:  rhel 7.7, 3.10.0-1060.el7.x86_64
guest ilx21r1r: rhel 7.7, 3.10.0-1062.el7.x86_64

------- Comment From shlyi.com 2019-12-30 03:38 EDT-------
(In reply to comment #29)
> (In reply to IBM Bug Proxy from comment #24)
> > (In reply to comment #27)
> > > 1.What result same test on host only ? please execute it some times.
> > > I think the host should have same issue. it maybe have lower frequency.
> > > 2.Does it can recover when hit io-error on guest?
> > >
> > > 3.What do your mean previous rhel? do you mean it is regression issue?
> > > What is the previous revision of kernel and qemu?
> > 2, io-error reported in this issue won't break the guest
> If this issue can be recovered finally, I think it is expected result.
> It maybe related to performance.
> Please refer to https://access.redhat.com/solutions/137073
> Could you please collect /var/log/messages and dmesg in host and guest when
> error happened ?
> > 3, when the host and the guest are in os  rhel 7.7, we don't have this kind
> > of issue.
> > host ilx21r:  rhel 7.7, 3.10.0-1060.el7.x86_64
> > guest ilx21r1r: rhel 7.7, 3.10.0-1062.el7.x86_64

I've modified the /etc/mulpath.conf file as that link suggested, let's see if the issue can be solved by this way.

/var/log/messages has already uploaded to the attachment, and I've also uploaded the "sosreport", can you have a check to see if it can cover the config info you need for "dmesg"?

------- Comment From shlyi.com 2019-12-30 21:00 EDT-------
(In reply to comment #31)
> The attachment of message looks like html file, and how to extract your xz
> file ?
> root@QW ~/Downloads # unxz sosreport-ilx21r-2019-12-03-mksahnv.tar.xz
> unxz: sosreport-ilx21r-2019-12-03-mksahnv.tar.xz: File format not recognized

Hello, you can extract it as below:
# tar -xvJf sosreport-ilx21r-2019-12-03-mksahnv.tar.xz

------- Comment From shlyi.com 2019-12-30 21:49 EDT-------
(In reply to comment #35)
> (In reply to IBM Bug Proxy from comment #28)
> > (In reply to comment #31)
> > > The attachment of message looks like html file, and how to extract your xz
> > > file ?
> > > root@QW ~/Downloads # unxz sosreport-ilx21r-2019-12-03-mksahnv.tar.xz
> > > unxz: sosreport-ilx21r-2019-12-03-mksahnv.tar.xz: File format not recognized
> > Hello, you can extract it as below:
> > # tar -xvJf sosreport-ilx21r-2019-12-03-mksahnv.tar.xz
> root@QW ~/Downloads # tar -xvJf sosreport-ilx21r-2019-12-03-mksahnv.tar.xz
> xz: (stdin): File format not recognized
> tar: Child returned status 1
> tar: Error is not recoverable: exiting now
> root@QW ~/Downloads # file sosreport-ilx21r-2019-12-03-mksahnv.tar.xz
> sosreport-ilx21r-2019-12-03-mksahnv.tar.xz: HTML document, ASCII text, with
> very long lines
> Please double check the message and sos file you uploaded..

Sorry for the confusion. I've uploaded it again, this should be good, I've verified it.

------- Comment From shlyi.com 2019-12-31 02:03 EDT-------
(In reply to comment #40)
> I suggest to do same testing on host only several times to check have same
> issue?
> if it exist, the issue not related to qemu.

I'll try it when I have the resource to do it.

------- Comment From shlyi.com 2019-12-31 03:11 EDT-------
(In reply to comment #41)
> (In reply to comment #40)
> > I suggest to do same testing on host only several times to check have same
> > issue?
> > if it exist, the issue not related to qemu.
> I'll try it when I have the resource to do it.

Actually, we do have a standalone host running rhel 8.1 and running io on one of  the storage ilx21r also being using, and from the io log I've checked, it did not encounter the issue.

[root@ilx37s archive]# uname -a
Linux ilx37s.tuc.stglabs.ibm.com 4.18.0-147.el8.x86_64 #1 SMP Thu Sep 26 15:52:44 UTC 2019 x86_64 x86_64 x86_64 GNU/Linux
[root@ilx37s archive]# cat /etc/redhat-release
Red Hat Enterprise Linux release 8.1 (Ootpa)

------- Comment From shlyi.com 2020-01-02 00:44 EDT-------
(In reply to comment #43)
> (In reply to IBM Bug Proxy from comment #20)
> > (In reply to comment #22)
> > > More questions:
> > > 1 What the guest behavior when it hit io-error, will it change to "pause"
> > > status?
> > > 2 You may set error_policy='stop' for the guest to see what happen ?
> > > 3 In my understanding, it should be expect result have io error when active
> > > path is broken,
> > > am i right?  the point is it may back to if other path may work.
> > > 4 if you may set error_policy='stop' , the guest will change to pause status
> > > when hit io error, if
> > > this issue is fixed. the guest can be back to running after we ask for
> > > continue running. that is expected result.
> > > For example:
> > > <disk type='block' device='lun' sgio='filtered' snapshot='no'>
> > > <driver name='qemu' type='raw' cache='none' error_policy='stop' io='native'/>
> > > <source dev='/dev/mapper/360014059b31089df5114cbd830320c2b'/>
> > > <backingStore/>
> > > <target dev='sdb' bus='scsi'/>
> > > <alias name='scsi0-0-0-1'/>
> > > <address type='drive' controller='0' bus='0' target='0' unit='1'/>
> > > </disk>
> > Hello,
> > Please see my last comments for its expecting behavior, "we expect there is
> > no error in the io log for each lun, which would break only half of the
> > paths when running ei".
> > While below is the log when io hit the error, which we think the guest
> > mishandled the issue:
> > /var/log/message of ilx21r:
> > Dec  3 01:01:07 ilx21r kernel: sd 3:0:4:0: rejecting I/O to offline device
> > /var/log/message of ilx21r1r:
> > Dec  3 01:01:07 ilx21r1r kernel: print_req_error: I/O error, dev sde, sector
> > 135448 flags 801
> >
> > I don't understand why we should " set error_policy='stop' ", and how to do
> > it.
> 1. What the behavior of your script "StorageSideCablePull" ? is it dedicated
> to your ENV?
> 2. It looks like the host detect the io error, so the guest report io-error.
> That should be expected ,right?

1, Yes, it dedicate to our env, it will simulate the cable pull behavior on the storage side by breaking/recovering the cable connection between the switch and the storage.

2, as you can see from the /var/log/message, the host won't report io error, just report part of paths of the lun to be offline.

------- Comment From shlyi.com 2020-01-02 21:02 EDT-------
(In reply to comment #45)
> (In reply to IBM Bug Proxy from comment #24)
> > (In reply to comment #27)
> > > 1.What result same test on host only ? please execute it some times.
> > > I think the host should have same issue. it maybe have lower frequency.
> > > 2.Does it can recover when hit io-error on guest?
> > >
> > > 3.What do your mean previous rhel? do you mean it is regression issue?
> > > What is the previous revision of kernel and qemu?
> > 2, io-error reported in this issue won't break the guest
> > 3, when the host and the guest are in os  rhel 7.7, we don't have this kind
> > of issue.
> > host ilx21r:  rhel 7.7, 3.10.0-1060.el7.x86_64
> > guest ilx21r1r: rhel 7.7, 3.10.0-1062.el7.x86_64
>
> 1. I suggest using mentioned guest(7.7) do testing:
>
> If issue exist, it may prove this issue does not related to the guest
> version.
> host 8.1 + guest 7.7
> host 7.7 + guest 8.1
>
> 2. There are many disks in guest, is it possible just attach one data disk
> in your guest to check whether issue exist or not,
> it may simplify test scenario to help to address issue ?

1, Could we focus on the env which having the issue, I am afraid we don't have resource and time to test all the config combination which may not have the issue;

2, I can try this one, but we still need to wait to another turn to run this case. Before that, could you explain what the result could tell for "having/ not having issue when only one data lun runing io"?

Besides, I've already modified the parameter according to article "https://access.redhat.com/solutions/137073", I'd like to verify if this workaround could work firstly before trying anything else.

------- Comment From shlyi.com 2020-01-03 01:11 EDT-------
(In reply to comment #47)
> (In reply to IBM Bug Proxy from comment #41)
> > (In reply to comment #45)
> > > (In reply to IBM Bug Proxy from comment #24)
> > > > (In reply to comment #27)
> > > > > 1.What result same test on host only ? please execute it some times.
> > > > > I think the host should have same issue. it maybe have lower frequency.
> > > > > 2.Does it can recover when hit io-error on guest?
> > > > >
> > > > > 3.What do your mean previous rhel? do you mean it is regression issue?
> > > > > What is the previous revision of kernel and qemu?
> > > > 2, io-error reported in this issue won't break the guest
> > > > 3, when the host and the guest are in os  rhel 7.7, we don't have this kind
> > > > of issue.
> > > > host ilx21r:  rhel 7.7, 3.10.0-1060.el7.x86_64
> > > > guest ilx21r1r: rhel 7.7, 3.10.0-1062.el7.x86_64
> > > 1. I suggest using mentioned guest(7.7) do testing:
> > > If issue exist, it may prove this issue does not related to the guest
> > > version.
> > > host 8.1 + guest 7.7
> > > host 7.7 + guest 8.1
> > > 2. There are many disks in guest, is it possible just attach one data disk
> > > in your guest to check whether issue exist or not,
> > > it may simplify test scenario to help to address issue ?
> > Hello,
> > 1, Could we focus on the env which having the issue, I am afraid we don't
> > have resource and time to test all the config combination which may not have
> > the issue;
>
> From your log, it display all disks have same issue, so i want to confirm
> using simple configuration to reproduce this issue.
>
> [root@ilx21r1r ~]# cat /opt/ilab/msdi/log/disktest/disktest.sd* |grep error
> | 2019/12/03-00:51:54 | ERROR | 26388 | v1.4.2.1 | /fc/dev/sda/test | Thread
> 0: fsync error = 5
> | 2019/12/03-04:21:56 | ERROR | 1668 | v1.4.2.1 | /fc/dev/sda/test | Thread
> 0: fsync error = 5
> | 2019/12/03-07:21:58 | ERROR | 9815 | v1.4.2.1 | /fc/dev/sda/test | Thread
> 0: fsync error = 5
> | 2019/12/03-08:21:59 | ERROR | 13233 | v1.4.2.1 | /fc/dev/sda/test | Thread
> 0: fsync error = 5
> | 2019/12/03-00:39:48 | ERROR | 26358 | v1.4.2.1 | /fc/dev/sdb/test | Thread
> 0: fsync error = 30
> | 2019/12/03-00:39:49 | ERROR | 26352 | v1.4.2.1 | /fc/dev/sdc/test | Thread
> 0: fsync error = 30
> | 2019/12/03-00:51:52 | ERROR | 26336 | v1.4.2.1 | /fc/dev/sdd/test | Thread
> 0: fsync error = 5
> | 2019/12/03-04:21:55 | ERROR | 1632 | v1.4.2.1 | /fc/dev/sdd/test | Thread
> 0: fsync error = 5
> | 2019/12/03-07:21:57 | ERROR | 9790 | v1.4.2.1 | /fc/dev/sdd/test | Thread
> 0: fsync error = 5
> | 2019/12/03-08:21:58 | ERROR | 13203 | v1.4.2.1 | /fc/dev/sdd/test | Thread
> 0: fsync error = 5
> | 2019/12/03-01:01:08 | ERROR | 26989 | v1.4.2.1 | /fc/dev/sde/test | Thread
> 0: fsync error = 30
> | 2019/12/03-00:39:48 | ERROR | 26342 | v1.4.2.1 | /fc/dev/sdg/test | Thread
> 0: fsync error = 30
> | 2019/12/03-03:10:55 | ERROR | 31641 | v1.4.2.1 | /fc/dev/sdh/test | Thread
> 0: fsync error = 30
> | 2019/12/03-03:10:55 | ERROR | 31647 | v1.4.2.1 | /fc/dev/sdi/test | Thread
> 0: fsync error = 30
> | 2019/12/03-03:10:55 | ERROR | 31659 | v1.4.2.1 | /fc/dev/sdj/test | Thread
> 0: fsync error = 30
> | 2019/12/03-03:10:55 | ERROR | 31635 | v1.4.2.1 | /fc/dev/sdk/test | Thread
> 0: fsync error = 30
> | 2019/12/03-03:10:55 | ERROR | 31653 | v1.4.2.1 | /fc/dev/sdl/test | Thread
> 0: fsync error = 30
> | 2019/12/03-00:39:48 | ERROR | 26370 | v1.4.2.1 | /fc/dev/sdm/test | Thread
> 0: fsync error = 30
> | 2019/12/03-03:10:55 | ERROR | 31617 | v1.4.2.1 | /fc/dev/sdn/test | Thread
> 0: fsync error = 30
> > 2, I can try this one, but we still need to wait to another turn to run this
> > case. Before that, could you explain what the result could tell for "having/
> > not having issue when only one data lun runing io"?
> > Besides, I've already modified the parameter according to article
> > "https://access.redhat.com/solutions/137073", I'd like to verify if this
> > workaround could work firstly before trying anything else.

1? /fc/dev/sdf didn't show up the error info;

------- Comment From shlyi.com 2020-03-31 05:58 EDT-------
FYI, suggestion in article "https://access.redhat.com/solutions/137073" couldn't solve this issue.

Rhel 8.2 has the similar issue, please look at issue "https://bugzilla.linux.ibm.com/show_bug.cgi?id=184579" for the detail info.

------- Comment From shlyi.com 2020-04-16 04:32 EDT-------
(In reply to comment #51)
> Hanns, bug 1817087 and 1779758 seems related to this one. Are they sharing
> the same environment and related use-case?
>
> We're having trouble reproducing this because of the specific environment
> and test-case. Do you have a way to reproduce it using standard open-source
> tools or manual steps?
>
> The best analysis seems to be from
> https://bugzilla.redhat.com/show_bug.cgi?id=1779758#c21, which I quote:
>
> "so my suspicion would be a memory corruption induced crash involving the
> srb_t in the
> qla2xxx driver, given the number of other bug reports we have had, but
> without a Call Trace
> or a crash dump we cannot be sure.  In addition, note that IBM says this
> persists in 8.2,
> although we do not have log files from that."
>
> Do you think it applies here as well? Thanks.

For reproducing this defect, you can refer to comment 14 of defect: https://bugzilla.linux.ibm.com/show_bug.cgi?id=183977, it may help you reproduce this issue;
I don't quite understand analysis, what's "srb_t", I cannot be sure it can apply. But I can told you that Rhel 7.8, Rhel 8.0, Rhel 8.1 and Rhel 8.2 are having similar issue based on our test result.

------- Comment From shlyi.com 2020-04-20 22:11 EDT-------
(In reply to comment #57)
> (In reply to IBM Bug Proxy from comment #4)
> > Created attachment 1644651 [details]
> > /var/log/messages of ilx21r
>
> This attachment does not contain /var/log/messages, it appears to contain
> Bugzilla XML.

Hello,

Please look at attachment " /var/log/messages of ilx21r_Dec3 ". Thanks.

------- Comment From shlyi.com 2020-06-28 00:16 EDT-------
(In reply to comment #67)
> We are investigating and have a possible reproduction of this issue on RHEL8.
>
> At the present time, we believe it may be related to the use of SCSI
> passthrough
> in QEMU with an underlying multipath device on the hypervisor.
>
> If possible, reconfigure the guest VM to avoid the use of SCSI passthrough.
> i.e. instead of XML that uses device="lun":
>
> <disk type="block" device="lun">
> <driver name="qemu" type="raw"/>
> <source dev="/dev/mapper/9840Q_8G_d_0"/>
> <target dev="sdb" bus="scsi"/>
> <address type="drive" controller="0" bus="0" target="0" unit="1"/>
> </disk>
>
> use device='disk'  (The exact parameters of the underlying device and the
> PCI info would depend upon your configuration, this is only what we tested
> here, so you would probably need to do this with virt-manager):
>
> <disk type='block' device='disk'>
> <driver name='qemu' type='raw' cache='none' io='native'/>
> <source dev='/dev/mapper/9840Q_8G_d_0'/>
> <backingStore/>
> <target dev='vda' bus='virtio'/>
> <alias name='virtio-disk0'/>
> <address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/>
> </disk>
>
> Unfortunately, due to authentication issues with our VPN, I cannot run the
> graphical virt-manager software on our lab machines so cannot give more
> specific
> instructions.

Hello,

------- Comment From shlyi.com 2019-12-17 21:12 EDT-------
(In reply to comment #14)
> There does not seem to be any useful information in the sosreport log files.
> Is it possible to obtain a crash dump of the guest OS ?  It looks like it
> just stopped.
> There is an incidental finding, the allocation failures while scanning the
> MegaRAID
> devices is a known issue which has been fixed.  I don't see how that caused
> your guest
> to crash though.
> Dec  2 02:17:32 ilx21r kernel: scsi host0: Avago SAS based MegaRAID driver
> Dec  2 02:17:32 ilx21r kernel: scsi_alloc_sdev: Allocation failure during
> SCSI scanning, some SCSI devices might not be configured

I've enabled "ulimit -c unlimited", but no file generated in /var/crash directory after the guest crashed again.

I've also done below config according to https://access.redhat.com/solutions/56021
# abrt-install-ccpp-hook install
# abrt-install-ccpp-hook is-installed; echo $?;
The second command should return 0 (hooks installed)
Ensure that this service is up and the ccpp hook to capture core dumps is enabled:
# service abrtd start
# service abrt-ccpp start

Please confirm what else should I configure to get the dump file, thanks.

------- Comment From shlyi.com 2019-12-18 21:39 EDT-------
(In reply to comment #18)
> Hello
> The crash here, are you saying the guest O/S is panicking.
> You are not talking about Qemu userspace cores right .
> We will need a sosreport to check your kdump configuration and then we can
> guide you to test
> and validate kdump.
> ftp dropbox.redhat.com
> anonymous ENTER
> cd incoming
> binary
> put 1779758.sosreport.xxxx
> Let us know when its there, we will correct any kdump issues, test kdump and
> then we should get a vmcore
> next time the guest panics.
> Regards
> Laurence

File "sosreport-ilx21r-2019-12-18-fdrvtfi.tar.xz" uploaded:

root@iltuc10-nfs:/home/eason/errlog/rhel8.1_guest_crash# ftp dropbox.redhat.com
Connected to origin-dropbox.redhat.com.
220-+---------------------------------------------------------------+
220-| ???????Welcome to Red Hat Global Support Services FTP! ???????|
220-| ??????????????????????????????????????????????????????????????|
220-| ?????Now you can upload files up to 250GB in size directly ???|
220-| ??to the Customer Portal. With this new capability, you can ??|
220-| ????manage everything in one place. You can start taking ?????|
220-| ???????advantage of this improved functionality now. ?????????|
220-| ??????????????????????????????????????????????????????????????|
220-| ???Access to dropbox.redhat.com and dropbox-ssl.redhat.com ???|
220-| ??will no longer be available on March 1, 2019. Please visit ?|
220-| ????https://access.redhat.com/solutions/2112 for details. ????|
220-+---------------------------------------------------------------+
220
Name (dropbox.redhat.com:root): anonymous
331 Please specify the password.
Password:
230 Login successful.
Remote system type is UNIX.
Using binary mode to transfer files.
ftp> binary
200 Switching to Binary mode.
ftp> put sosreport-ilx21r-2019-12-18-fdrvtfi.tar.xz
local: sosreport-ilx21r-2019-12-18-fdrvtfi.tar.xz remote: sosreport-ilx21r-2019-12-18-fdrvtfi.tar.xz
200 PORT command successful. Consider using PASV.
553 Could not create file.
ftp> ls
200 PORT command successful. Consider using PASV.
550 Permission denied.

------- Comment From shlyi.com 2019-12-23 03:00 EDT-------
Done.

root@iltuc10-nfs:/home/eason/errlog/rhel8.1_guest_crash# ftp dropbox.redhat.com
Connected to origin-dropbox.redhat.com.
220-+---------------------------------------------------------------+
220-| ???????Welcome to Red Hat Global Support Services FTP! ???????|
220-| ??????????????????????????????????????????????????????????????|
220-| ?????Now you can upload files up to 250GB in size directly ???|
220-| ??to the Customer Portal. With this new capability, you can ??|
220-| ????manage everything in one place. You can start taking ?????|
220-| ???????advantage of this improved functionality now. ?????????|
220-| ??????????????????????????????????????????????????????????????|
220-| ???Access to dropbox.redhat.com and dropbox-ssl.redhat.com ???|
220-| ??will no longer be available on March 1, 2019. Please visit ?|
220-| ????https://access.redhat.com/solutions/2112 for details. ????|
220-+---------------------------------------------------------------+
220
Name (dropbox.redhat.com:root): anonymous
331 Please specify the password.
Password:
230 Login successful.
Remote system type is UNIX.
Using binary mode to transfer files.
ftp> cd incoming
250 Directory successfully changed.
ftp> binary
200 Switching to Binary mode.
ftp> put sosreport-ilx21r-2019-12-18-fdrvtfi.tar.xz
local: sosreport-ilx21r-2019-12-18-fdrvtfi.tar.xz remote: sosreport-ilx21r-2019-12-18-fdrvtfi.tar.xz
200 PORT command successful. Consider using PASV.
150 Ok to send data.
226 File receive OK.
17823688 bytes sent in 5.59 secs (3.0399 MB/s)
> Hello
> You forgot to cd incoming, so file was not uploaded
> ftp dropbox.redhat.com
> anonymous ENTER
> cd incoming
> binary
> put 1779758.sosreport.xxxx
> 553 Could not create file.
> ftp> ls
> 200 PORT command successful. Consider using PASV.
> 550 Permission denied.
> Regards
> Laurence

Done.

root@iltuc10-nfs:/home/eason/errlog/rhel8.1_guest_crash# ftp dropbox.redhat.com
Connected to origin-dropbox.redhat.com.
220-+---------------------------------------------------------------+
220-| ???????Welcome to Red Hat Global Support Services FTP! ???????|
220-| ??????????????????????????????????????????????????????????????|
220-| ?????Now you can upload files up to 250GB in size directly ???|
220-| ??to the Customer Portal. With this new capability, you can ??|
220-| ????manage everything in one place. You can start taking ?????|
220-| ???????advantage of this improved functionality now. ?????????|
220-| ??????????????????????????????????????????????????????????????|
220-| ???Access to dropbox.redhat.com and dropbox-ssl.redhat.com ???|
220-| ??will no longer be available on March 1, 2019. Please visit ?|
220-| ????https://access.redhat.com/solutions/2112 for details. ????|
220-+---------------------------------------------------------------+
220
Name (dropbox.redhat.com:root): anonymous
331 Please specify the password.
Password:
230 Login successful.
Remote system type is UNIX.
Using binary mode to transfer files.
ftp> cd incoming
250 Directory successfully changed.
ftp> binary
200 Switching to Binary mode.
ftp> put sosreport-ilx21r-2019-12-18-fdrvtfi.tar.xz
local: sosreport-ilx21r-2019-12-18-fdrvtfi.tar.xz remote: sosreport-ilx21r-2019-12-18-fdrvtfi.tar.xz
200 PORT command successful. Consider using PASV.
150 Ok to send data.
226 File receive OK.
17823688 bytes sent in 5.59 secs (3.0399 MB/s)

------- Comment From shlyi.com 2020-01-13 22:03 EDT-------
(In reply to comment #24)
> Hello
> Can we schedule a screen share so I can assist with getting the vmcore.
> Anytime after Wednesday morning should be good.
> I work USA Eastern from 9AM to 6PM.
> Regards
> Laurence Oberman

The time you provided do not work for me, I work in China time, so we have 11 hours time gap.
Would you please provide the steps so I can do it by myself?

------- Comment From shlyi.com 2020-03-31 06:01 EDT-------

------- Comment From chavez.com 2020-03-31 10:38 EDT-------
The mentioned LTC bug 184579 corresponds to Red Hat Bugzilla bug 1817087

------- Comment From shlyi.com 2020-06-04 06:53 EDT-------
> Any progress of obtaining a crash dump of the guest ?

Hello emilne,
1, Since nobody replied me after my comments at "2020-01-13 21:03:17 CST", I still am not able to collect the core dump for the guest.
2, We are testing Rhel 8.2, instead of Rhel 8.1, right now. Below is the current output for the command, I don't know if it meet your need. But if you look at my attachment named of "sosreport of ilx21r", after uncompressing it, you can find the multipath.conf at "sosreport-ilx21r-2019-12-18-fdrvtfi/etc/multipath.conf"
[root@ilx21r ~]# multipath -ll
FC16G_1_D0 (36005076119909fc118000000bf0001d0) dm-33 IBM,FlashSystem-9840
`-+- policy='service-time 0' prio=1 status=active
|- 3:0:5:3 sdco 69:192 active ready running
|- 3:0:7:3 sdcw 70:64  active ready running
|- 4:0:1:3 sdde 70:192 active ready running
`- 4:0:6:3 sdds 71:160 active ready running
ilx21r22r_8G_d_0 (36005076b19bcb49b180000004600005c) dm-24 IBM,FlashSystem-9840
`-+- policy='service-time 0' prio=1 status=active
|- 3:0:1:0 sdcf 69:48  active ready running
|- 3:0:0:0 sdbz 68:208 active ready running
|- 4:0:3:0 sddj 71:16  active ready running
`- 4:0:8:0 sddv 71:208 active ready running
mpathe (360050762198c1fc2180000005f00012d) dm-7 IBM,FlashSystem-9840
`-+- policy='service-time 0' prio=1 status=active
|- 1:0:0:4 sdf  8:80   active ready running
|- 1:0:9:4 sdam 66:96  active ready running
|- 2:0:0:4 sdar 66:176 active ready running
`- 2:0:5:4 sdbq 68:64  active ready running
ilx21r22r_16G_d_5 (36005076b19bcb49b180000005900006f) dm-13 IBM,FlashSystem-9840
`-+- policy='service-time 0' prio=1 status=active
|- 1:0:1:3 sdj  8:144  active ready running
|- 1:0:3:3 sdp  8:240  active ready running
|- 2:0:3:3 sdbd 67:112 active ready running
`- 2:0:8:3 sdbu 68:128 active ready running
FC8G_D5 (36005076119909fc118000000730001cb) dm-19 IBM,FlashSystem-9840
`-+- policy='service-time 0' prio=1 status=active
|- 1:0:5:3 sdv  65:80  active ready running
|- 1:0:8:3 sdaf 65:240 active ready running
|- 2:0:1:3 sdav 66:240 active ready running
`- 2:0:4:3 sdbj 67:208 active ready running
9840Q_8G_d_1 (3600507639281b4a3900000006c0000ab) dm-23 IBM,FlashSystem-9840
`-+- policy='service-time 0' prio=1 status=active
|- 1:0:6:1 sdz  65:144 active ready running
|- 1:0:7:1 sdab 65:176 active ready running
|- 2:0:2:1 sdaz 67:48  active ready running
`- 2:0:9:1 sdby 68:192 active ready running
ilx21r_22r_kvmcfg (360050762198c1fc21800000041000109) dm-6 IBM,FlashSystem-9840
`-+- policy='service-time 0' prio=1 status=active
|- 1:0:0:3 sde  8:64   active ready running
|- 1:0:9:3 sdal 66:80  active ready running
|- 2:0:0:3 sdaq 66:160 active ready running
`- 2:0:5:3 sdbp 68:48  active ready running
9840Q_8G_d_0 (3600507639281b4a3900000006b0000aa) dm-22 IBM,FlashSystem-9840
`-+- policy='service-time 0' prio=1 status=active
|- 1:0:6:0 sdy  65:128 active ready running
|- 1:0:7:0 sdaa 65:160 active ready running
|- 2:0:2:0 sday 67:32  active ready running
`- 2:0:9:0 sdbx 68:176 active ready running
mpathah (3600605b00dee874024d75438b8e61213) dm-38 Lenovo,RAID 530-8i
`-+- policy='service-time 0' prio=1 status=active
`- 0:2:0:0 sda  8:0    active ready running
ilx21r22r_16G_d_2 (36005076b19bcb49b180000004d000063) dm-12 IBM,FlashSystem-9840
`-+- policy='service-time 0' prio=1 status=active
|- 1:0:1:2 sdi  8:128  active ready running
|- 1:0:3:2 sdo  8:224  active ready running
|- 2:0:3:2 sdbc 67:96  active ready running
`- 2:0:8:2 sdbt 68:112 active ready running
`-+- policy='service-time 0' prio=1 status=active
|- 1:0:0:2 sdd  8:48   active ready running
|- 1:0:9:2 sdak 66:64  active ready running
|- 2:0:0:2 sdap 66:144 active ready running
`- 2:0:5:2 sdbo 68:32  active ready running
FC8G_D2 (36005076119909fc1180000003a0001c8) dm-18 IBM,FlashSystem-9840
`-+- policy='service-time 0' prio=1 status=active
|- 1:0:5:2 sdu  65:64  active ready running
|- 1:0:8:2 sdae 65:224 active ready running
|- 2:0:1:2 sdau 66:224 active ready running
`- 2:0:4:2 sdbi 67:192 active ready running
`-+- policy='service-time 0' prio=1 status=active
|- 1:0:0:0 sdb  8:16   active ready running
|- 1:0:9:0 sdai 66:32  active ready running
|- 2:0:0:0 sdan 66:112 active ready running
`- 2:0:5:0 sdbm 68:0   active ready running
ilx21r22r_16G_d_1 (36005076b19bcb49b180000004c000062) dm-10 IBM,FlashSystem-9840
`-+- policy='service-time 0' prio=1 status=active
|- 1:0:1:1 sdh  8:112  active ready running
|- 1:0:3:1 sdn  8:208  active ready running
|- 2:0:3:1 sdbb 67:80  active ready running
`- 2:0:8:1 sdbs 68:96  active ready running
`-+- policy='service-time 0' prio=1 status=active
|- 1:0:0:1 sdc  8:32   active ready running
|- 1:0:9:1 sdaj 66:48  active ready running
|- 2:0:0:1 sdao 66:128 active ready running
`- 2:0:5:1 sdbn 68:16  active ready running
FC8G_D1 (36005076119909fc118000000390001c7) dm-17 IBM,FlashSystem-9840
`-+- policy='service-time 0' prio=1 status=active
|- 1:0:5:1 sdt  65:48  active ready running
|- 1:0:8:1 sdad 65:208 active ready running
|- 2:0:1:1 sdat 66:208 active ready running
`- 2:0:4:1 sdbh 67:176 active ready running
ilx21r22r_16G_d_0 (36005076b19bcb49b180000004b000061) dm-9 IBM,FlashSystem-9840
`-+- policy='service-time 0' prio=1 status=active
|- 1:0:1:0 sdg  8:96   active ready running
|- 1:0:3:0 sdm  8:192  active ready running
|- 2:0:3:0 sdba 67:64  active ready running
`- 2:0:8:0 sdbr 68:80  active ready running
ilx21r22r_8G_d_7 (36005076b19bcb49b1800000052000068) dm-29 IBM,FlashSystem-9840
`-+- policy='service-time 0' prio=1 status=active
|- 3:0:1:5 sdck 69:128 active ready running
|- 3:0:0:5 sdce 69:32  active ready running
|- 4:0:3:5 sddo 71:96  active ready running
`- 4:0:8:5 sdea 128:32 active ready running
FC8G_D0 (36005076119909fc118000000330001c6) dm-16 IBM,FlashSystem-9840
`-+- policy='service-time 0' prio=1 status=active
|- 1:0:5:0 sds  65:32  active ready running
|- 1:0:8:0 sdac 65:192 active ready running
|- 2:0:1:0 sdas 66:192 active ready running
`- 2:0:4:0 sdbg 67:160 active ready running
ilx21r22r_8G_d_6 (36005076b19bcb49b1800000051000067) dm-28 IBM,FlashSystem-9840
`-+- policy='service-time 0' prio=1 status=active
|- 3:0:1:4 sdcj 69:112 active ready running
|- 3:0:0:4 sdcd 69:16  active ready running
|- 4:0:3:4 sddn 71:80  active ready running
`- 4:0:8:4 sddz 128:16 active ready running
9840Q_16G_d_1 (3600507639281b4a3900000006e0000ad) dm-37 IBM,FlashSystem-9840
`-+- policy='service-time 0' prio=1 status=active
|- 3:0:6:1 sdcs 70:0   active ready running
|- 3:0:9:1 sdda 70:128 active ready running
|- 4:0:2:1 sddi 71:0   active ready running
`- 4:0:9:1 sdec 128:64 active ready running
FC16G_2_D2 (36005076119909fc118000000ff0001d7) dm-32 IBM,FlashSystem-9840
`-+- policy='service-time 0' prio=1 status=active
|- 3:0:5:2 sdcn 69:176 active ready running
|- 3:0:7:2 sdcv 70:48  active ready running
|- 4:0:1:2 sddd 70:176 active ready running
`- 4:0:6:2 sddr 71:144 active ready running
ilx21r22r_8G_d_5 (36005076b19bcb49b1800000050000066) dm-27 IBM,FlashSystem-9840
`-+- policy='service-time 0' prio=1 status=active
|- 3:0:1:3 sdci 69:96  active ready running
|- 3:0:0:3 sdcc 69:0   active ready running
|- 4:0:3:3 sddm 71:64  active ready running
`- 4:0:8:3 sddy 128:0  active ready running
9840Q_16G_d_0 (3600507639281b4a3900000006d0000ac) dm-36 IBM,FlashSystem-9840
`-+- policy='service-time 0' prio=1 status=active
|- 3:0:6:0 sdcr 69:240 active ready running
|- 3:0:9:0 sdcz 70:112 active ready running
|- 4:0:2:0 sddh 70:240 active ready running
`- 4:0:9:0 sdeb 128:48 active ready running
FC16G_2_D1 (36005076119909fc118000000fe0001d6) dm-31 IBM,FlashSystem-9840
`-+- policy='service-time 0' prio=1 status=active
|- 3:0:5:1 sdcm 69:160 active ready running
|- 3:0:7:1 sdcu 70:32  active ready running
|- 4:0:1:1 sddc 70:160 active ready running
`- 4:0:6:1 sddq 71:128 active ready running
FC16G_2_D0 (36005076119909fc118000000fd0001d5) dm-30 IBM,FlashSystem-9840
`-+- policy='service-time 0' prio=1 status=active
|- 3:0:5:0 sdcl 69:144 active ready running
|- 3:0:7:0 sdct 70:16  active ready running
|- 4:0:1:0 sddb 70:144 active ready running
`- 4:0:6:0 sddp 71:112 active ready running
FC16G_1_D2 (36005076119909fc118000000fa0001d2) dm-35 IBM,FlashSystem-9840
`-+- policy='service-time 0' prio=1 status=active
|- 3:0:5:5 sdcq 69:224 active ready running
|- 3:0:7:5 sdcy 70:96  active ready running
|- 4:0:1:5 sddg 70:224 active ready running
`- 4:0:6:5 sddu 71:192 active ready running
ilx21r22r_8G_d_2 (36005076b19bcb49b180000004800005e) dm-26 IBM,FlashSystem-9840
`-+- policy='service-time 0' prio=1 status=active
|- 3:0:1:2 sdch 69:80  active ready running
|- 3:0:0:2 sdcb 68:240 active ready running
|- 4:0:3:2 sddl 71:48  active ready running
`- 4:0:8:2 sddx 71:240 active ready running
ilx21r22r_16G_d_7 (36005076b19bcb49b180000005b000071) dm-15 IBM,FlashSystem-9840
`-+- policy='service-time 0' prio=1 status=active
|- 1:0:1:5 sdl  8:176  active ready running
|- 1:0:3:5 sdr  65:16  active ready running
|- 2:0:3:5 sdbf 67:144 active ready running
`- 2:0:8:5 sdbw 68:160 active ready running
FC8G_D7 (36005076119909fc118000000a80001cd) dm-20 IBM,FlashSystem-9840
`-+- policy='service-time 0' prio=1 status=active
|- 1:0:5:5 sdw  65:96  active ready running
|- 1:0:8:5 sdag 66:0   active ready running
|- 2:0:1:5 sdaw 67:0   active ready running
`- 2:0:4:5 sdbk 67:224 active ready running
FC16G_1_D1 (36005076119909fc118000000f90001d1) dm-34 IBM,FlashSystem-9840
`-+- policy='service-time 0' prio=1 status=active
|- 3:0:5:4 sdcp 69:208 active ready running
|- 3:0:7:4 sdcx 70:80  active ready running
|- 4:0:1:4 sddf 70:208 active ready running
`- 4:0:6:4 sddt 71:176 active ready running
ilx21r22r_8G_d_1 (36005076b19bcb49b180000004700005d) dm-25 IBM,FlashSystem-9840
`-+- policy='service-time 0' prio=1 status=active
|- 3:0:1:1 sdcg 69:64  active ready running
|- 3:0:0:1 sdca 68:224 active ready running
|- 4:0:3:1 sddk 71:32  active ready running
`- 4:0:8:1 sddw 71:224 active ready running
ilx21r22r_16G_d_6 (36005076b19bcb49b180000005a000070) dm-14 IBM,FlashSystem-9840
`-+- policy='service-time 0' prio=1 status=active
|- 1:0:1:4 sdk  8:160  active ready running
|- 1:0:3:4 sdq  65:0   active ready running
|- 2:0:3:4 sdbe 67:128 active ready running
`- 2:0:8:4 sdbv 68:144 active ready running
FC8G_D6 (36005076119909fc118000000780002fe) dm-21 IBM,FlashSystem-9840
`-+- policy='service-time 0' prio=1 status=active
|- 1:0:5:6 sdx  65:112 active ready running
|- 1:0:8:6 sdah 66:16  active ready running
|- 2:0:1:6 sdax 67:16  active ready running
`- 2:0:4:6 sdbl 67:240 active ready running

------- Comment From shlyi.com 2020-06-04 06:58 EDT-------
Besides, please notice that we don't have env for Rhel 8.1 cluster right now. Your future need for this issue would better based on Rhel 8.2, and would better to update at "Bug 184579 - RH1817087".

------- Comment From shlyi.com 2020-06-04 20:51 EDT-------
> And, one more question.  How was the guest VM for this test created, exactly?
> >Test details
> >1, Set up RHEL 8.1 cluster env, install RHEL 8.1 guest on them, ...
> >Host names: ilx21r ilx22r
> >Guest name: ilx21r1r

I used virt-manager with a rhel iso file to create the guest, and the os is installed on a sanboot lun.

------- Comment From shlyi.com 2020-06-04 20:52 EDT-------
(In reply to comment #36)
> The guest
> System Information
> Manufacturer: Red Hat
> Product Name: KVM
> Version: RHEL-7.6.0 PC (Q35 + ICH9, 2009)
> Serial Number: Not Specified
> UUID: f37748d4-3b13-4a95-906e-24606a619736
> Wake-up Type: Power Switch
> SKU Number: Not Specified
> Family: Red Hat Enterprise Linux
> I assume IBM used virt-manager, created the guest and then assigned
> passthrough LUNS.
> I know its passthrough because I dont see virtio scsi or virtio block. the
> devices shows up as
> Attached devices:
> Host: scsi0 Channel: 00 Id: 00 Lun: 06
> Vendor: IBM      Model: FlashSystem-9840 Rev: 1611
> Type:   Direct-Access                    ANSI  SCSI revision: 05
> We have some known issues with KVM and passthrough so I need to try
> reproduce here in our lab.
> I should be able to work on this next week.
> One thing I need to know, if I reboot storage controllers and toggle switch
> ports, is that similar to your error injection
> Thanks
> Laurence

Yes, I think so.

*** Bug 1804850 has been marked as a duplicate of this bug. ***

------- Comment From shlyi.com 2020-02-23 19:45 EDT-------
(In reply to comment #7)
> Moving to rhel-7.9.0 since rhel-7.8.0 is essentially closed for changes.
> Victor, I'm unclear why this ends up being a Virtualization problem. Hard to
> get that from the problem report, but from my read I wonder about the fabric
> and host interactions.
> The ilx24s part1 and part2 host logs seem to have quite a few ominous (to
> me) looking messages regarding the storage...  the part2 logs include some
> tracebacks.  Whether that's normal as part of what the "disktest" and
> "SwitchReboot" do is really unclear to me.
> Could just be that the guest is a victim here.
> IBM: Need some more information regarding the host env - specifically what
> version of qemu (and libvirt) is installed?  Makes it easier to route the
> bug that way.

The libvirt version is 4.5.0-31.el7.x86_64 as you can see below; disktest is a IBM io tool used by our team, it keeps running io with a specific block size on a raw lun mapped from IBM storage; SwitchReboot is just reboot the switch which the host/storage connected to, it would break partial lun paths, we need to see the broken paths recovered after the switch recovered without affecting the guest status or its io status.

[root@ilx24s ~]# rpm -qa |grep libvirt
libvirt-glib-1.0.0-1.el7.x86_64
libvirt-daemon-4.5.0-31.el7.x86_64
libvirt-daemon-driver-qemu-4.5.0-31.el7.x86_64
libvirt-daemon-driver-nodedev-4.5.0-31.el7.x86_64
libvirt-daemon-driver-storage-rbd-4.5.0-31.el7.x86_64
libvirt-daemon-driver-nwfilter-4.5.0-31.el7.x86_64
libvirt-daemon-driver-lxc-4.5.0-31.el7.x86_64
libvirt-daemon-driver-storage-scsi-4.5.0-31.el7.x86_64
libvirt-4.5.0-31.el7.x86_64
libvirt-libs-4.5.0-31.el7.x86_64
libvirt-client-4.5.0-31.el7.x86_64
libvirt-bash-completion-4.5.0-31.el7.x86_64
libvirt-daemon-driver-storage-disk-4.5.0-31.el7.x86_64
libvirt-daemon-kvm-4.5.0-31.el7.x86_64
libvirt-daemon-driver-secret-4.5.0-31.el7.x86_64
libvirt-gobject-1.0.0-1.el7.x86_64
libvirt-daemon-driver-storage-iscsi-4.5.0-31.el7.x86_64
libvirt-daemon-driver-network-4.5.0-31.el7.x86_64
libvirt-daemon-config-nwfilter-4.5.0-31.el7.x86_64
libvirt-gconfig-1.0.0-1.el7.x86_64
libvirt-daemon-driver-storage-core-4.5.0-31.el7.x86_64
libvirt-daemon-driver-storage-mpath-4.5.0-31.el7.x86_64
libvirt-daemon-driver-interface-4.5.0-31.el7.x86_64
libvirt-daemon-driver-storage-logical-4.5.0-31.el7.x86_64
libvirt-daemon-config-network-4.5.0-31.el7.x86_64
libvirt-daemon-driver-storage-gluster-4.5.0-31.el7.x86_64
libvirt-python-4.5.0-1.el7.x86_64
libvirt-daemon-driver-storage-4.5.0-31.el7.x86_64

------- Comment From shlyi.com 2020-03-12 22:26 EDT-------
(In reply to comment #10)
> I need some info
> As I understand it you have a LUN which is connected by 4 redundant patches
> to the host.
> How do you pass these paths to the guest? Each separately or you pass
> /dev/mpathN node to the guest?
> Could you provide the xml (and if possible qemu command line) that was used?

Hello, the guest's os disk is managed by virtio as you can see below, and the guest cannot see the multipath the host has. I think the correct behavior of the host should be that it should not let the guest be affected if only partial of its lun paths lost.

<disk type='file' device='disk'>
<driver name='qemu' type='qcow2'/>
<source file='/mnt/gfs1/rhel7.7.qcow2'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/>

besides, the guest was managed by the redhat cluster(pcs), so I don't need to use qemu command line to manage the guest.

Below is the command line I use to create the guest resource:
# pcs resource create ilx24s1r_guest ocf:heartbeat:VirtualDomain hypervisor="qemu:///system" config="/mnt/kvmconf/rhel7.7.xml" migration_transport=ssh op start timeout="120s" op stop timeout="120s" op monitor  timeout="30" interval="10"  meta allow-migrate="true" priority="100" op migrate_from interval="0" timeout="120s" op migrate_to interval="0" timeout="120"

Based on my analysis, the real problem is lying on the host's cpu usage, the usage is very high at Feb 16 00:59:14 and some other time, and our check logs showed that the host hung during that time, then recovered after a while. But the guest may not be so lucky, it failed to be recovered.

Below is the "xml" file you asked:

[root@ilx24s ~]# cat /mnt/kvmconf/rhel7.7.xml
<!--
WARNING: THIS IS AN AUTO-GENERATED FILE. CHANGES TO IT ARE LIKELY TO BE
OVERWRITTEN AND LOST. Changes to this xml configuration should be made using:
virsh edit rhel7.7
or other application using the libvirt API.
-->

<name>rhel7.7</name>
<uuid>99e9771e-9dd3-4f83-8fa8-6ca530c96513</uuid>
<memory unit='KiB'>8388608</memory>
<currentMemory unit='KiB'>8388608</currentMemory>
<vcpu placement='static'>8</vcpu>
<os>
<type arch='x86_64' machine='pc-i440fx-rhel7.0.0'>hvm</type>
<boot dev='hd'/>
</os>
<features>
<acpi/>
<apic/>
</features>
<cpu mode='custom' match='exact' check='partial'>
<model fallback='allow'>Skylake-Server-IBRS</model>
<feature policy='require' name='md-clear'/>
<feature policy='require' name='spec-ctrl'/>
<feature policy='require' name='ssbd'/>
</cpu>
<clock offset='utc'>
<timer name='rtc' tickpolicy='catchup'/>
<timer name='pit' tickpolicy='delay'/>
<timer name='hpet' present='no'/>
</clock>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>destroy</on_crash>
<pm>
<suspend-to-mem enabled='no'/>
<suspend-to-disk enabled='no'/>
</pm>
<devices>
<emulator>/usr/libexec/qemu-kvm</emulator>
<disk type='file' device='disk'>
<driver name='qemu' type='qcow2'/>
<source file='/mnt/gfs1/rhel7.7.qcow2'/>
<target dev='vda' bus='virtio'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/>
<disk type='file' device='cdrom'>
<driver name='qemu' type='raw'/>
<target dev='hda' bus='ide'/>
<readonly/>
<driver name='qemu' type='raw'/>
<source dev='/dev/mapper/ilx24s25s_9840l_q16_d0'/>
<address type='drive' controller='0' bus='0' target='0' unit='0'/>
</disk>
<disk type='block' device='lun'>
<driver name='qemu' type='raw'/>
<source dev='/dev/mapper/ilx24s25s_9840l_q32_d0'/>
</disk>
<disk type='block' device='lun'>
<driver name='qemu' type='raw'/>
<source dev='/dev/mapper/ilx24s25s_9840o_q16_d0'/>
<target dev='sdc' bus='scsi'/>
<address type='drive' controller='0' bus='0' target='0' unit='2'/>
</disk>
<disk type='block' device='lun'>
<driver name='qemu' type='raw'/>
<source dev='/dev/mapper/ilx24s25s_9840o_q16_d1'/>
<target dev='sdd' bus='scsi'/>
<address type='drive' controller='0' bus='0' target='0' unit='3'/>
</disk>
<disk type='block' device='lun'>
<driver name='qemu' type='raw'/>
<source dev='/dev/mapper/ilx24s25s_9840o_q32_d0'/>
<target dev='sde' bus='scsi'/>
<address type='drive' controller='0' bus='0' target='0' unit='4'/>
</disk>
<disk type='block' device='lun'>
<driver name='qemu' type='raw'/>
<source dev='/dev/mapper/ilx24s25s_9840o_q32_d1'/>
<target dev='sdf' bus='scsi'/>
<address type='drive' controller='0' bus='0' target='0' unit='5'/>
</disk>
<disk type='block' device='lun'>
<driver name='qemu' type='raw'/>
<source dev='/dev/mapper/ilx24s25s_9840q_q16_d0'/>
<target dev='sdg' bus='scsi'/>
<address type='drive' controller='0' bus='0' target='0' unit='6'/>
</disk>
<disk type='block' device='lun'>
<driver name='qemu' type='raw'/>
<source dev='/dev/mapper/ilx24s25s_9840q_q16_d1'/>
<target dev='sdh' bus='scsi'/>
<address type='drive' controller='0' bus='0' target='0' unit='7'/>
</disk>
<disk type='block' device='lun'>
<driver name='qemu' type='raw'/>
<source dev='/dev/mapper/ilx24s25s_9840q_q32_d0'/>
<target dev='sdi' bus='scsi'/>
<address type='drive' controller='0' bus='0' target='0' unit='8'/>
</disk>
<disk type='block' device='lun'>
<driver name='qemu' type='raw'/>
<source dev='/dev/mapper/ilx24s25s_9840q_q32_d1'/>
<target dev='sdl' bus='scsi'/>
<address type='drive' controller='0' bus='0' target='0' unit='9'/>
</disk>
<controller type='usb' index='0' model='ich9-ehci1'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x7'/>
</controller>
<controller type='usb' index='0' model='ich9-uhci1'>
<master startport='0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0' multifunction='on'/>
</controller>
<controller type='usb' index='0' model='ich9-uhci2'>
<master startport='2'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x1'/>
</controller>
<controller type='usb' index='0' model='ich9-uhci3'>
<master startport='4'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x2'/>
</controller>
<controller type='pci' index='0' model='pci-root'/>
<controller type='ide' index='0'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x1'/>
</controller>
<controller type='virtio-serial' index='0'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x0'/>
</controller>
<interface type='bridge'>
<mac address='52:54:00:73:98:1a'/>
<source bridge='br1'/>
<model type='virtio'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
</interface>
<serial type='pty'>
<target type='isa-serial' port='0'>
<model name='isa-serial'/>
</target>
</serial>
<console type='pty'>
<target type='serial' port='0'/>
</console>
<channel type='unix'>
<target type='virtio' name='org.qemu.guest_agent.0'/>
<address type='virtio-serial' controller='0' bus='0' port='1'/>
</channel>
<channel type='spicevmc'>
<target type='virtio' name='com.redhat.spice.0'/>
<address type='virtio-serial' controller='0' bus='0' port='2'/>
</channel>
<input type='tablet' bus='usb'>
<address type='usb' bus='0' port='1'/>
</input>
<input type='mouse' bus='ps2'/>
<input type='keyboard' bus='ps2'/>
<graphics type='spice' autoport='yes'>
<listen type='address'/>
<image compression='off'/>
</graphics>
<sound model='ich6'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
</sound>
<video>
<model type='qxl' ram='65536' vram='65536' vgamem='16384' heads='1' primary='yes'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
</video>
<redirdev bus='usb' type='spicevmc'>
<address type='usb' bus='0' port='2'/>
</redirdev>
<redirdev bus='usb' type='spicevmc'>
<address type='usb' bus='0' port='3'/>
</redirdev>
<memballoon model='virtio'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x08' function='0x0'/>
</memballoon>
<rng model='virtio'>
<backend model='random'>/dev/urandom</backend>
<address type='pci' domain='0x0000' bus='0x00' slot='0x09' function='0x0'/>
</rng>
</devices>
</domain>

------- Comment From shlyi.com 2020-03-12 22:33 EDT-------
(In reply to comment #11)
> (In reply to comment #10)
> > I need some info
> > As I understand it you have a LUN which is connected by 4 redundant patches
> > to the host.
> > How do you pass these paths to the guest? Each separately or you pass
> > /dev/mpathN node to the guest?
> > Could you provide the xml (and if possible qemu command line) that was used?
> Hello, the guest's os disk is managed by virtio as you can see below, and
> the guest cannot see the multipath the host has. I think the correct
> behavior of the host should be that it should not let the guest be affected
> if only partial of its lun paths lost.
>     <disk type='file' device='disk'>
>       <driver name='qemu' type='qcow2'/>
>       <source file='/mnt/gfs1/rhel7.7.qcow2'/>
>       <target dev='vda' bus='virtio'/>
>       <address type='pci' domain='0x0000' bus='0x00' slot='0x07'
> function='0x0'/>
>     </disk>
> besides, the guest was managed by the redhat cluster(pcs), so I don't need
> to use qemu command line to manage the guest.
> Below is the command line I use to create the guest resource:
> # pcs resource create ilx24s1r_guest ocf:heartbeat:VirtualDomain
> hypervisor="qemu:///system" config="/mnt/kvmconf/rhel7.7.xml"
> migration_transport=ssh op start timeout="120s" op stop timeout="120s" op
> monitor  timeout="30" interval="10"  meta allow-migrate="true"
> priority="100" op migrate_from interval="0" timeout="120s" op migrate_to
> interval="0" timeout="120"
> Based on my analysis, the real problem is lying on the host's cpu usage, the
> usage is very high at Feb 16 00:59:14 and some other time, and our check
> logs showed that the host hung during that time, then recovered after a
> while. But the guest may not be so lucky, it failed to be recovered.
> Below is the "xml" file you asked:
> [root@ilx24s ~]# cat /mnt/kvmconf/rhel7.7.xml
> <!--
> WARNING: THIS IS AN AUTO-GENERATED FILE. CHANGES TO IT ARE LIKELY TO BE
> OVERWRITTEN AND LOST. Changes to this xml configuration should be made using:
>   virsh edit rhel7.7
> or other application using the libvirt API.
> -->
> <domain type='kvm'>
>   <name>rhel7.7</name>
>   <uuid>99e9771e-9dd3-4f83-8fa8-6ca530c96513</uuid>
>   <memory unit='KiB'>8388608</memory>
>   <currentMemory unit='KiB'>8388608</currentMemory>
>   <vcpu placement='static'>8</vcpu>
>   <os>
>     <type arch='x86_64' machine='pc-i440fx-rhel7.0.0'>hvm</type>
>     <boot dev='hd'/>
>   </os>
>   <features>
>     <acpi/>
>     <apic/>
>   </features>
>   <cpu mode='custom' match='exact' check='partial'>
>     <model fallback='allow'>Skylake-Server-IBRS</model>
>     <feature policy='require' name='md-clear'/>
>     <feature policy='require' name='spec-ctrl'/>
>     <feature policy='require' name='ssbd'/>
>   </cpu>
>   <clock offset='utc'>
>     <timer name='rtc' tickpolicy='catchup'/>
>     <timer name='pit' tickpolicy='delay'/>
>     <timer name='hpet' present='no'/>
>   </clock>
>   <on_poweroff>destroy</on_poweroff>
>   <on_reboot>restart</on_reboot>
>   <on_crash>destroy</on_crash>
>   <pm>
>     <suspend-to-mem enabled='no'/>
>     <suspend-to-disk enabled='no'/>
>   </pm>
>   <devices>
>     <emulator>/usr/libexec/qemu-kvm</emulator>
>     <disk type='file' device='disk'>
>       <driver name='qemu' type='qcow2'/>
>       <source file='/mnt/gfs1/rhel7.7.qcow2'/>
>       <target dev='vda' bus='virtio'/>
>       <address type='pci' domain='0x0000' bus='0x00' slot='0x07'
> function='0x0'/>
>     </disk>
>     <disk type='file' device='cdrom'>
>       <driver name='qemu' type='raw'/>
>       <target dev='hda' bus='ide'/>
>       <readonly/>
>       <address type='drive' controller='0' bus='0' target='0' unit='0'/>
>     </disk>
>     <disk type='block' device='lun'>
>       <driver name='qemu' type='raw'/>
>       <source dev='/dev/mapper/ilx24s25s_9840l_q16_d0'/>
>       <target dev='sda' bus='scsi'/>
>       <address type='drive' controller='0' bus='0' target='0' unit='0'/>
>     </disk>
>     <disk type='block' device='lun'>
>       <driver name='qemu' type='raw'/>
>       <source dev='/dev/mapper/ilx24s25s_9840l_q32_d0'/>
>       <target dev='sdb' bus='scsi'/>
>       <address type='drive' controller='0' bus='0' target='0' unit='1'/>
>     </disk>
>     <disk type='block' device='lun'>
>       <driver name='qemu' type='raw'/>
>       <source dev='/dev/mapper/ilx24s25s_9840o_q16_d0'/>
>       <target dev='sdc' bus='scsi'/>
>       <address type='drive' controller='0' bus='0' target='0' unit='2'/>
>     </disk>
>     <disk type='block' device='lun'>
>       <driver name='qemu' type='raw'/>
>       <source dev='/dev/mapper/ilx24s25s_9840o_q16_d1'/>
>       <target dev='sdd' bus='scsi'/>
>       <address type='drive' controller='0' bus='0' target='0' unit='3'/>
>     </disk>
>     <disk type='block' device='lun'>
>       <driver name='qemu' type='raw'/>
>       <source dev='/dev/mapper/ilx24s25s_9840o_q32_d0'/>
>       <target dev='sde' bus='scsi'/>
>       <address type='drive' controller='0' bus='0' target='0' unit='4'/>
>     </disk>
>     <disk type='block' device='lun'>
>       <driver name='qemu' type='raw'/>
>       <source dev='/dev/mapper/ilx24s25s_9840o_q32_d1'/>
>       <target dev='sdf' bus='scsi'/>
>       <address type='drive' controller='0' bus='0' target='0' unit='5'/>
>     </disk>
>     <disk type='block' device='lun'>
>       <driver name='qemu' type='raw'/>
>       <source dev='/dev/mapper/ilx24s25s_9840q_q16_d0'/>
>       <target dev='sdg' bus='scsi'/>
>       <address type='drive' controller='0' bus='0' target='0' unit='6'/>
>     </disk>
>     <disk type='block' device='lun'>
>       <driver name='qemu' type='raw'/>
>       <source dev='/dev/mapper/ilx24s25s_9840q_q16_d1'/>
>       <target dev='sdh' bus='scsi'/>
>       <address type='drive' controller='0' bus='0' target='0' unit='7'/>
>     </disk>
>     <disk type='block' device='lun'>
>       <driver name='qemu' type='raw'/>
>       <source dev='/dev/mapper/ilx24s25s_9840q_q32_d0'/>
>       <target dev='sdi' bus='scsi'/>
>       <address type='drive' controller='0' bus='0' target='0' unit='8'/>
>     </disk>
>     <disk type='block' device='lun'>
>       <driver name='qemu' type='raw'/>
>       <source dev='/dev/mapper/ilx24s25s_9840q_q32_d1'/>
>       <target dev='sdl' bus='scsi'/>
>       <address type='drive' controller='0' bus='0' target='0' unit='9'/>
>     </disk>
>     <controller type='usb' index='0' model='ich9-ehci1'>
>       <address type='pci' domain='0x0000' bus='0x00' slot='0x05'
> function='0x7'/>
>     </controller>
>     <controller type='usb' index='0' model='ich9-uhci1'>
>       <master startport='0'/>
>       <address type='pci' domain='0x0000' bus='0x00' slot='0x05'
> function='0x0' multifunction='on'/>
>     </controller>
>     <controller type='usb' index='0' model='ich9-uhci2'>
>       <master startport='2'/>
>       <address type='pci' domain='0x0000' bus='0x00' slot='0x05'
> function='0x1'/>
>     </controller>
>     <controller type='usb' index='0' model='ich9-uhci3'>
>       <master startport='4'/>
>       <address type='pci' domain='0x0000' bus='0x00' slot='0x05'
> function='0x2'/>
>     </controller>
>     <controller type='pci' index='0' model='pci-root'/>
>     <controller type='ide' index='0'>
>       <address type='pci' domain='0x0000' bus='0x00' slot='0x01'
> function='0x1'/>
>     </controller>
>     <controller type='virtio-serial' index='0'>
>       <address type='pci' domain='0x0000' bus='0x00' slot='0x06'
> function='0x0'/>
>     </controller>
>     <interface type='bridge'>
>       <mac address='52:54:00:73:98:1a'/>
>       <source bridge='br1'/>
>       <model type='virtio'/>
>       <address type='pci' domain='0x0000' bus='0x00' slot='0x03'
> function='0x0'/>
>     </interface>
>     <serial type='pty'>
>       <target type='isa-serial' port='0'>
>         <model name='isa-serial'/>
>       </target>
>     </serial>
>     <console type='pty'>
>       <target type='serial' port='0'/>
>     </console>
>     <channel type='unix'>
>       <target type='virtio' name='org.qemu.guest_agent.0'/>
>       <address type='virtio-serial' controller='0' bus='0' port='1'/>
>     </channel>
>     <channel type='spicevmc'>
>       <target type='virtio' name='com.redhat.spice.0'/>
>       <address type='virtio-serial' controller='0' bus='0' port='2'/>
>     </channel>
>     <input type='tablet' bus='usb'>
>       <address type='usb' bus='0' port='1'/>
>     </input>
>     <input type='mouse' bus='ps2'/>
>     <input type='keyboard' bus='ps2'/>
>     <graphics type='spice' autoport='yes'>
>       <listen type='address'/>
>       <image compression='off'/>
>     </graphics>
>     <sound model='ich6'>
>       <address type='pci' domain='0x0000' bus='0x00' slot='0x04'
> function='0x0'/>
>     </sound>
>     <video>
>       <model type='qxl' ram='65536' vram='65536' vgamem='16384' heads='1'
> primary='yes'/>
>       <address type='pci' domain='0x0000' bus='0x00' slot='0x02'
> function='0x0'/>
>     </video>
>     <redirdev bus='usb' type='spicevmc'>
>       <address type='usb' bus='0' port='2'/>
>     </redirdev>
>     <redirdev bus='usb' type='spicevmc'>
>       <address type='usb' bus='0' port='3'/>
>     </redirdev>
>     <memballoon model='virtio'>
>       <address type='pci' domain='0x0000' bus='0x00' slot='0x08'
> function='0x0'/>
>     </memballoon>
>     <rng model='virtio'>
>       <backend model='random'>/dev/urandom</backend>
>       <address type='pci' domain='0x0000' bus='0x00' slot='0x09'
> function='0x0'/>
>     </rng>
>   </devices>
> </domain>

Correct some words of below sentence:
Before:
Based on my analysis, the real problem is lying on the host's cpu usage, the usage is very high at Feb 16 00:59:14 and some other time, and our check logs showed that the host hung during that time, then recovered after a while. But the guest may not be so lucky, it failed to be recovered.
Corrected:
Based on my analysis, the real problem should be caused by the host's cpu usage, the usage was very high at Feb 16 00:59:14 and some other time, and our check logs showed that the host hung during that time, then recovered after a while. But the guest may not be so lucky, it failed to be recovered.

------- Comment From shlyi.com 2020-03-16 21:50 EDT-------
> I did some investigations and this is what I found:
> I setup the following test enviroment (much more modest that you have :-) ):
> a host (my laptop) running iscsi target from a 1 GB file.
> Then I connected to that target twice using qemu's iscsi driver, with
> blkdebug instance running on top of each
> I hacked the blkdebug qemu driver such as it passes through the SG_IO and
> sometimes fails them, such
> as the two instances of blkdebug don't fail at the same time.
> Thus I have a guest with two paths to same disk and each of paths fails
> sometimes.
> In that guest I aggregated the two paths using the multipath device mapper
> and I passed
> the resulting multipath node to a nested guest running in it.
> Inside the nested guest I had run a fio job on the passed through disk.
> The results:
> Sadly the nested guest sees IO errors.
> I tried both direct and buffered IO in the guest, both reads
> and writes, and I tried pass the multipath node to the nested guest both as
> a raw LUN and
> as a regular block device.
> In all cases I see errors in the nested guest, however the multipath
> code does recover eventually and switches the paths.
> If I allow the fio to continue on IO errors,
> I see that multipath code switches patches in very timely manner with a
> bunch of IO errors in between.
> I even tried to fail only one path (the first one) while keeping the second
> one always online,
> and even then I have seen IO errors in the guest, suggesting
> that kernel's multipath code doesn't retry the IO requests on remaining
> paths when it encounters
> a failing path.
> I think that this means that in-kernel multipath code is just designed this
> way, that is geared towards
> once in a while disk failure and assumes that the client will retry failing
> requests
> (which is true for linux filesystem code from my experience with nvme-mdev
> driver I once developed).
> One thing for sure, that this bug is not related to virtualization in any
> way.
> in fact running fio in the outer guest (which just simulates the real
> hardware) produces the same
> results.

Thanks for the updates. I think your experiment had just reproduced my another defect: https://bugzilla.linux.ibm.com/show_bug.cgi?id=183930

The key difference between this issue and that issue is the behavior, rebooting the switch, which somehow caused the host hung, then caused the guest hung. And this "key difference" is what I suggest we should focus on.

Besides, could you give me some introduction for the "fio job", is it refer to blocking one of the two paths your iscsi lun has?

------- Comment From shlyi.com 2020-03-17 23:56 EDT-------
> fio just means is just running fio random read/write benchmark on the disk -
> anything else doing IO would work just fine, like dd for instance.
> OK so the issue here is that we know that guest is getting *lots* of IO
> errors but these shouldn't make it hung like that.
> I can in theory try to reproduce this but to be honest this could be caused
> by lots of different problems in the guest kernel or even guest userspace,
> and I don't yet have enough experience to suggest how you should debug this
> on your system since this issue most likely won't be reproducible
> (or at least reproducible in the same way) on anything else but your system.
> I guess you can try to install qemu-kvm's debug symbols, then reproduce this
> and then attach gdb to qemu process and try to capture where it is
> stuck, but most likely this won't produce much meaningful info.
> Qemu also supports built-in gdbstub which allows to debug the guest kernel,
> something I used for few times.
> The abrt packet you attached is unfortunately just from systemd-journald
> crashing so this doesn't really help.

I tried as you said, I am not sure if my config is correct/would work or not, it reported "Missing separate debuginfos, use: debuginfo-install qemu-kvm-1.5.3-173.el7.x86_64", but the rhel 7.8 repo seems not to contain "debuginfos" package. Please give me some comments for my trying. Thanks.

[root@ilx24s ~]# gdb program 7598
GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-119.el7
Copyright (C) 2013 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-redhat-linux-gnu".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
program: No such file or directory.
Attaching to process 7598
Reading symbols from /usr/libexec/qemu-kvm...Reading symbols from /usr/libexec/qemu-kvm...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Reading symbols from /lib64/libz.so.1...Reading symbols from /lib64/libz.so.1...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libz.so.1
Reading symbols from /lib64/libaio.so.1...Reading symbols from /lib64/libaio.so.1...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libaio.so.1
Reading symbols from /usr/lib64/iscsi/libiscsi.so.2...Reading symbols from /usr/lib64/iscsi/libiscsi.so.2...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/iscsi/libiscsi.so.2
Reading symbols from /lib64/libcurl.so.4...Reading symbols from /lib64/libcurl.so.4...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libcurl.so.4
Reading symbols from /lib64/libgfapi.so.0...Reading symbols from /lib64/libgfapi.so.0...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libgfapi.so.0
Reading symbols from /lib64/libgfrpc.so.0...Reading symbols from /lib64/libgfrpc.so.0...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libgfrpc.so.0
Reading symbols from /lib64/libgfxdr.so.0...Reading symbols from /lib64/libgfxdr.so.0...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libgfxdr.so.0
Reading symbols from /lib64/libssh2.so.1...Reading symbols from /lib64/libssh2.so.1...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libssh2.so.1
Reading symbols from /lib64/librt.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/librt.so.1
Reading symbols from /lib64/libtcmalloc.so.4...Reading symbols from /lib64/libtcmalloc.so.4...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libtcmalloc.so.4
Reading symbols from /lib64/libgthread-2.0.so.0...Reading symbols from /lib64/libgthread-2.0.so.0...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libgthread-2.0.so.0
Reading symbols from /lib64/libglib-2.0.so.0...Reading symbols from /lib64/libglib-2.0.so.0...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libglib-2.0.so.0
Reading symbols from /lib64/libssl3.so...Reading symbols from /lib64/libssl3.so...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libssl3.so
Reading symbols from /lib64/libsmime3.so...Reading symbols from /lib64/libsmime3.so...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libsmime3.so
Reading symbols from /lib64/libnss3.so...Reading symbols from /lib64/libnss3.so...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libnss3.so
Reading symbols from /lib64/libnssutil3.so...Reading symbols from /lib64/libnssutil3.so...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libnssutil3.so
Reading symbols from /lib64/libplds4.so...Reading symbols from /lib64/libplds4.so...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libplds4.so
Reading symbols from /lib64/libplc4.so...Reading symbols from /lib64/libplc4.so...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libplc4.so
Reading symbols from /lib64/libnspr4.so...Reading symbols from /lib64/libnspr4.so...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libnspr4.so
Reading symbols from /lib64/libpthread.so.0...(no debugging symbols found)...done.
[New LWP 240551]
[New LWP 240550]
[New LWP 240549]
[New LWP 240548]
[New LWP 240547]
[New LWP 240546]
[New LWP 240545]
[New LWP 240544]
[New LWP 240543]
[New LWP 240542]
[New LWP 240541]
[New LWP 240540]
[New LWP 240539]
[New LWP 240538]
[New LWP 240537]
[New LWP 240536]
[New LWP 240535]
[New LWP 240534]
[New LWP 240533]
[New LWP 240532]
[New LWP 240531]
[New LWP 202053]
[New LWP 202050]
[New LWP 202048]
[New LWP 200709]
[New LWP 200707]
[New LWP 200706]
[New LWP 200705]
[New LWP 200704]
[New LWP 200703]
[New LWP 200702]
[New LWP 200699]
[New LWP 200698]
[New LWP 200697]
[New LWP 200694]
[New LWP 200693]
[New LWP 199809]
[New LWP 199807]
[New LWP 199806]
[New LWP 199805]
[New LWP 199800]
[New LWP 199796]
[New LWP 199795]
[New LWP 199794]
[New LWP 199793]
[New LWP 199791]
[New LWP 197317]
[New LWP 197315]
[New LWP 197312]
[New LWP 197310]
[New LWP 196327]
[New LWP 196325]
[New LWP 196321]
[New LWP 196307]
[New LWP 195116]
[New LWP 195110]
[New LWP 194521]
[New LWP 194516]
[New LWP 194514]
[New LWP 194503]
[New LWP 194047]
[New LWP 194029]
[New LWP 191956]
[New LWP 180103]
[New LWP 7635]
[New LWP 7634]
[New LWP 7633]
[New LWP 7632]
[New LWP 7631]
[New LWP 7630]
[New LWP 7629]
[New LWP 7628]
[New LWP 7627]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
Loaded symbols for /lib64/libpthread.so.0
Reading symbols from /lib64/libdl.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libdl.so.2
Reading symbols from /lib64/libutil.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libutil.so.1
Reading symbols from /lib64/librbd.so.1...Reading symbols from /lib64/librbd.so.1...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/librbd.so.1
Reading symbols from /lib64/librados.so.2...Reading symbols from /lib64/librados.so.2...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/librados.so.2
Reading symbols from /lib64/libasound.so.2...Reading symbols from /lib64/libasound.so.2...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libasound.so.2
Reading symbols from /lib64/libpulse.so.0...Reading symbols from /lib64/libpulse.so.0...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libpulse.so.0
Reading symbols from /lib64/libuuid.so.1...Reading symbols from /lib64/libuuid.so.1...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libuuid.so.1
Reading symbols from /lib64/libpng15.so.15...Reading symbols from /lib64/libpng15.so.15...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libpng15.so.15
Reading symbols from /lib64/libsasl2.so.3...Reading symbols from /lib64/libsasl2.so.3...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libsasl2.so.3
Reading symbols from /lib64/libgnutls.so.28...Reading symbols from /lib64/libgnutls.so.28...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libgnutls.so.28
Reading symbols from /lib64/liblzo2.so.2...Reading symbols from /lib64/liblzo2.so.2...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/liblzo2.so.2
Reading symbols from /lib64/libsnappy.so.1...Reading symbols from /lib64/libsnappy.so.1...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libsnappy.so.1
Reading symbols from /lib64/libseccomp.so.2...Reading symbols from /lib64/libseccomp.so.2...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libseccomp.so.2
Reading symbols from /lib64/librdmacm.so.1...Reading symbols from /lib64/librdmacm.so.1...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/librdmacm.so.1
Reading symbols from /lib64/libibverbs.so.1...Reading symbols from /lib64/libibverbs.so.1...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libibverbs.so.1
Reading symbols from /lib64/libspice-server.so.1...Reading symbols from /lib64/libspice-server.so.1...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libspice-server.so.1
Reading symbols from /lib64/libusb-1.0.so.0...Reading symbols from /lib64/libusb-1.0.so.0...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libusb-1.0.so.0
Reading symbols from /lib64/libusbredirparser.so.1...Reading symbols from /lib64/libusbredirparser.so.1...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libusbredirparser.so.1
Reading symbols from /lib64/libpixman-1.so.0...Reading symbols from /lib64/libpixman-1.so.0...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libpixman-1.so.0
Reading symbols from /lib64/libm.so.6...(no debugging symbols found)...done.
Loaded symbols for /lib64/libm.so.6
Reading symbols from /lib64/libc.so.6...(no debugging symbols found)...done.
Loaded symbols for /lib64/libc.so.6
Reading symbols from /lib64/libgcrypt.so.11...Reading symbols from /lib64/libgcrypt.so.11...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libgcrypt.so.11
Reading symbols from /lib64/libidn.so.11...Reading symbols from /lib64/libidn.so.11...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libidn.so.11
Reading symbols from /lib64/libgssapi_krb5.so.2...Reading symbols from /lib64/libgssapi_krb5.so.2...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libgssapi_krb5.so.2
Reading symbols from /lib64/libkrb5.so.3...Reading symbols from /lib64/libkrb5.so.3...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libkrb5.so.3
Reading symbols from /lib64/libk5crypto.so.3...Reading symbols from /lib64/libk5crypto.so.3...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libk5crypto.so.3
Reading symbols from /lib64/libcom_err.so.2...Reading symbols from /lib64/libcom_err.so.2...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libcom_err.so.2
Reading symbols from /lib64/liblber-2.4.so.2...Reading symbols from /lib64/liblber-2.4.so.2...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/liblber-2.4.so.2
Reading symbols from /lib64/libldap-2.4.so.2...Reading symbols from /lib64/libldap-2.4.so.2...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libldap-2.4.so.2
Reading symbols from /lib64/libacl.so.1...Reading symbols from /lib64/libacl.so.1...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libacl.so.1
Reading symbols from /lib64/libglusterfs.so.0...Reading symbols from /lib64/libglusterfs.so.0...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libglusterfs.so.0
Reading symbols from /lib64/libcrypto.so.10...Reading symbols from /lib64/libcrypto.so.10...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libcrypto.so.10
Reading symbols from /lib64/libssl.so.10...Reading symbols from /lib64/libssl.so.10...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libssl.so.10
Reading symbols from /lib64/libstdc++.so.6...(no debugging symbols found)...done.
Loaded symbols for /lib64/libstdc++.so.6
Reading symbols from /lib64/ld-linux-x86-64.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/ld-linux-x86-64.so.2
Reading symbols from /lib64/libgcc_s.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libgcc_s.so.1
Reading symbols from /lib64/libpcre.so.1...Reading symbols from /lib64/libpcre.so.1...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libpcre.so.1
Reading symbols from /lib64/libboost_thread-mt.so.1.53.0...Reading symbols from /lib64/libboost_thread-mt.so.1.53.0...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libboost_thread-mt.so.1.53.0
Reading symbols from /lib64/libboost_system-mt.so.1.53.0...Reading symbols from /lib64/libboost_system-mt.so.1.53.0...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libboost_system-mt.so.1.53.0
Reading symbols from /lib64/libboost_random-mt.so.1.53.0...Reading symbols from /lib64/libboost_random-mt.so.1.53.0...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libboost_random-mt.so.1.53.0
Reading symbols from /lib64/libblkid.so.1...Reading symbols from /lib64/libblkid.so.1...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libblkid.so.1
Reading symbols from /lib64/libboost_iostreams-mt.so.1.53.0...Reading symbols from /lib64/libboost_iostreams-mt.so.1.53.0...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libboost_iostreams-mt.so.1.53.0
Reading symbols from /usr/lib64/pulseaudio/libpulsecommon-10.0.so...Reading symbols from /usr/lib64/pulseaudio/libpulsecommon-10.0.so...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/pulseaudio/libpulsecommon-10.0.so
Reading symbols from /lib64/libdbus-1.so.3...Reading symbols from /lib64/libdbus-1.so.3...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libdbus-1.so.3
Reading symbols from /lib64/libcap.so.2...Reading symbols from /lib64/libcap.so.2...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libcap.so.2
Reading symbols from /lib64/libresolv.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libresolv.so.2
Reading symbols from /lib64/libcrypt.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libcrypt.so.1
Reading symbols from /lib64/libkrb5support.so.0...Reading symbols from /lib64/libkrb5support.so.0...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libkrb5support.so.0
Reading symbols from /lib64/libp11-kit.so.0...Reading symbols from /lib64/libp11-kit.so.0...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libp11-kit.so.0
Reading symbols from /lib64/libtasn1.so.6...Reading symbols from /lib64/libtasn1.so.6...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libtasn1.so.6
Reading symbols from /lib64/libnettle.so.4...Reading symbols from /lib64/libnettle.so.4...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libnettle.so.4
Reading symbols from /lib64/libhogweed.so.2...Reading symbols from /lib64/libhogweed.so.2...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libhogweed.so.2
Reading symbols from /lib64/libgmp.so.10...Reading symbols from /lib64/libgmp.so.10...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libgmp.so.10
Reading symbols from /lib64/libnl-route-3.so.200...Reading symbols from /lib64/libnl-route-3.so.200...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libnl-route-3.so.200
Reading symbols from /lib64/libnl-3.so.200...Reading symbols from /lib64/libnl-3.so.200...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libnl-3.so.200
Reading symbols from /lib64/libcelt051.so.0...Reading symbols from /lib64/libcelt051.so.0...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libcelt051.so.0
Reading symbols from /lib64/libopus.so.0...Reading symbols from /lib64/libopus.so.0...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libopus.so.0
Reading symbols from /lib64/libgio-2.0.so.0...Reading symbols from /lib64/libgio-2.0.so.0...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libgio-2.0.so.0
Reading symbols from /lib64/libgobject-2.0.so.0...Reading symbols from /lib64/libgobject-2.0.so.0...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libgobject-2.0.so.0
Reading symbols from /lib64/libjpeg.so.62...Reading symbols from /lib64/libjpeg.so.62...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libjpeg.so.62
Reading symbols from /lib64/liblz4.so.1...Reading symbols from /lib64/liblz4.so.1...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/liblz4.so.1
Reading symbols from /lib64/libudev.so.1...Reading symbols from /lib64/libudev.so.1...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libudev.so.1
Reading symbols from /lib64/libgpg-error.so.0...Reading symbols from /lib64/libgpg-error.so.0...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libgpg-error.so.0
Reading symbols from /lib64/libkeyutils.so.1...Reading symbols from /lib64/libkeyutils.so.1...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libkeyutils.so.1
Reading symbols from /lib64/libattr.so.1...Reading symbols from /lib64/libattr.so.1...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libattr.so.1
Reading symbols from /lib64/libbz2.so.1...Reading symbols from /lib64/libbz2.so.1...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libbz2.so.1
Reading symbols from /lib64/libX11-xcb.so.1...Reading symbols from /lib64/libX11-xcb.so.1...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libX11-xcb.so.1
Reading symbols from /lib64/libX11.so.6...Reading symbols from /lib64/libX11.so.6...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libX11.so.6
Reading symbols from /lib64/libxcb.so.1...Reading symbols from /lib64/libxcb.so.1...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libxcb.so.1
Reading symbols from /lib64/libICE.so.6...Reading symbols from /lib64/libICE.so.6...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libICE.so.6
Reading symbols from /lib64/libSM.so.6...Reading symbols from /lib64/libSM.so.6...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libSM.so.6
Reading symbols from /lib64/libXtst.so.6...Reading symbols from /lib64/libXtst.so.6...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libXtst.so.6
Reading symbols from /lib64/libsystemd.so.0...Reading symbols from /lib64/libsystemd.so.0...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libsystemd.so.0
Reading symbols from /lib64/libwrap.so.0...Reading symbols from /lib64/libwrap.so.0...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libwrap.so.0
Reading symbols from /lib64/libsndfile.so.1...Reading symbols from /lib64/libsndfile.so.1...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libsndfile.so.1
Reading symbols from /lib64/libasyncns.so.0...Reading symbols from /lib64/libasyncns.so.0...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libasyncns.so.0
Reading symbols from /lib64/libfreebl3.so...Reading symbols from /lib64/libfreebl3.so...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libfreebl3.so
Reading symbols from /lib64/libselinux.so.1...Reading symbols from /lib64/libselinux.so.1...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libselinux.so.1
Reading symbols from /lib64/libffi.so.6...Reading symbols from /lib64/libffi.so.6...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libffi.so.6
Reading symbols from /lib64/libgmodule-2.0.so.0...Reading symbols from /lib64/libgmodule-2.0.so.0...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libgmodule-2.0.so.0
Reading symbols from /lib64/libmount.so.1...Reading symbols from /lib64/libmount.so.1...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libmount.so.1
Reading symbols from /lib64/libdw.so.1...Reading symbols from /lib64/libdw.so.1...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libdw.so.1
Reading symbols from /lib64/libXau.so.6...Reading symbols from /lib64/libXau.so.6...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libXau.so.6
Reading symbols from /lib64/libXext.so.6...Reading symbols from /lib64/libXext.so.6...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libXext.so.6
Reading symbols from /lib64/libXi.so.6...Reading symbols from /lib64/libXi.so.6...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libXi.so.6
Reading symbols from /lib64/liblzma.so.5...Reading symbols from /lib64/liblzma.so.5...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/liblzma.so.5
Reading symbols from /lib64/libnsl.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libnsl.so.1
Reading symbols from /lib64/libgsm.so.1...Reading symbols from /lib64/libgsm.so.1...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libgsm.so.1
Reading symbols from /lib64/libFLAC.so.8...Reading symbols from /lib64/libFLAC.so.8...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libFLAC.so.8
Reading symbols from /lib64/libvorbisenc.so.2...Reading symbols from /lib64/libvorbisenc.so.2...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libvorbisenc.so.2
Reading symbols from /lib64/libvorbis.so.0...Reading symbols from /lib64/libvorbis.so.0...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libvorbis.so.0
Reading symbols from /lib64/libogg.so.0...Reading symbols from /lib64/libogg.so.0...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libogg.so.0
Reading symbols from /lib64/libelf.so.1...Reading symbols from /lib64/libelf.so.1...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libelf.so.1
Reading symbols from /lib64/libnss_files.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libnss_files.so.2
Reading symbols from /usr/lib64/sasl2/libanonymous.so...Reading symbols from /usr/lib64/sasl2/libanonymous.so...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/sasl2/libanonymous.so
Reading symbols from /usr/lib64/sasl2/libsasldb.so...Reading symbols from /usr/lib64/sasl2/libsasldb.so...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/sasl2/libsasldb.so
Reading symbols from /lib64/libdb-5.3.so...Reading symbols from /lib64/libdb-5.3.so...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /lib64/libdb-5.3.so
Reading symbols from /usr/lib64/sasl2/libgssapiv2.so...Reading symbols from /usr/lib64/sasl2/libgssapiv2.so...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/sasl2/libgssapiv2.so
Reading symbols from /usr/lib64/sasl2/libscram.so...Reading symbols from /usr/lib64/sasl2/libscram.so...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/sasl2/libscram.so
Reading symbols from /usr/lib64/sasl2/liblogin.so...Reading symbols from /usr/lib64/sasl2/liblogin.so...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/sasl2/liblogin.so
Reading symbols from /usr/lib64/sasl2/libplain.so...Reading symbols from /usr/lib64/sasl2/libplain.so...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/sasl2/libplain.so
Reading symbols from /usr/lib64/sasl2/libcrammd5.so...Reading symbols from /usr/lib64/sasl2/libcrammd5.so...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/sasl2/libcrammd5.so
Reading symbols from /usr/lib64/sasl2/libdigestmd5.so...Reading symbols from /usr/lib64/sasl2/libdigestmd5.so...(no debugging symbols found)...done.
(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/sasl2/libdigestmd5.so
0x00007f03ea075c3d in poll () from /lib64/libc.so.6
Missing separate debuginfos, use: debuginfo-install qemu-kvm-1.5.3-173.el7.x86_64
(gdb) debuginfo-install qemu-kvm-1.5.3-173.el7.x86_64
Undefined command: "debuginfo-install".  Try "help".
(gdb) quit
A debugging session is active.

Inferior 1 [process 7598] will be detached.

Quit anyway? (y or n) y
Detaching from program: /usr/libexec/qemu-kvm, process 7598
[Inferior 1 (process 7598) detached]
[root@ilx24s ~]# debuginfo-install qemu-kvm-1.5.3-173.el7.x86_64
Loaded plugins: langpacks, product-id, subscription-manager

Could not find debuginfo for main pkg: 10:qemu-kvm-1.5.3-173.el7.x86_64
Could not find debuginfo pkg for dependency package libaio-0.3.109-13.el7.x86_64
Could not find debuginfo pkg for dependency package alsa-lib-1.1.8-1.el7.x86_64
Could not find debuginfo pkg for dependency package glibc-2.17-307.el7.1.x86_64
Could not find debuginfo pkg for dependency package libcurl-7.29.0-57.el7.x86_64
Could not find debuginfo pkg for dependency package glusterfs-api-6.0-29.el7.x86_64
Could not find debuginfo pkg for dependency package glusterfs-libs-6.0-29.el7.x86_64
Could not find debuginfo pkg for dependency package glib2-2.56.1-5.el7.x86_64
Could not find debuginfo pkg for dependency package gnutls-3.3.29-9.el7_6.x86_64
Could not find debuginfo pkg for dependency package libibverbs-22.4-1.el7.x86_64
Could not find debuginfo pkg for dependency package libiscsi-1.9.0-7.el7.x86_64
Could not find debuginfo pkg for dependency package lzo-2.06-8.el7.x86_64
Could not find debuginfo pkg for dependency package nspr-4.21.0-1.el7.x86_64
Could not find debuginfo pkg for dependency package nss-3.44.0-7.el7_7.x86_64
Could not find debuginfo pkg for dependency package nss-util-3.44.0-4.el7_7.x86_64
Could not find debuginfo pkg for dependency package pixman-0.34.0-1.el7.x86_64
Could not find debuginfo pkg for dependency package 2:libpng-1.5.13-7.el7_2.x86_64
Could not find debuginfo pkg for dependency package pulseaudio-libs-10.0-5.el7.x86_64
Could not find debuginfo pkg for dependency package 1:librados2-10.2.5-4.el7.x86_64
Could not find debuginfo pkg for dependency package 1:librbd1-10.2.5-4.el7.x86_64
Could not find debuginfo pkg for dependency package librdmacm-22.4-1.el7.x86_64
Could not find debuginfo pkg for dependency package cyrus-sasl-lib-2.1.26-23.el7.x86_64
Could not find debuginfo pkg for dependency package libseccomp-2.3.1-4.el7.x86_64
Could not find debuginfo pkg for dependency package snappy-1.1.0-3.el7.x86_64
Could not find debuginfo pkg for dependency package spice-server-0.14.0-9.el7.x86_64
Could not find debuginfo pkg for dependency package libssh2-1.8.0-3.el7.x86_64
Could not find debuginfo pkg for dependency package gperftools-libs-2.6.1-1.el7.x86_64
Could not find debuginfo pkg for dependency package libusbx-1.0.21-1.el7.x86_64
Could not find debuginfo pkg for dependency package usbredir-0.7.1-3.el7.x86_64
Could not find debuginfo pkg for dependency package libuuid-2.23.2-63.el7.x86_64
Could not find debuginfo pkg for dependency package zlib-1.2.7-18.el7.x86_64
No debuginfo packages available to install
[root@ilx24s ~]# yum install debuginfo-install qemu-kvm-1.5.3-173.el7.x86_64

No package debuginfo-install available.
Package 10:qemu-kvm-1.5.3-173.el7.x86_64 already installed and latest version
[root@ilx24s ~]# yum install kernel-debuginfo

No package kernel-debuginfo available.
[root@ilx24s ~]# yum list |grep kernel-debuginfo

------- Comment From shlyi.com 2020-03-17 23:57 EDT-------

[root@ilx24s ~]#

------- Comment From shlyi.com 2020-03-19 04:59 EDT-------
> Yep, the missing debuginfo messages is a very good way to install them.
> Its different in RHEL8/Fedora, here I use 'dnf debuginfo-install'
> Turns out that in RHEL7 you should just run this as is, like
> debuginfo-install qemu-kvm-1.5.3-173.el7.x86_64
> I don't hold high hopes though that a backtrace from the qemu would reveal
> anything interesting, however,
> does the issue reproduce (that is did the guest hung again when you had the
> gdb as you had shown)?
> If it does reproduce more or less consistently, I can try too, to setup
> something, maybe even connect to your disks
> using iscsi or something.
> Best regards,
> Maxim Levitsky

Hello, I still failed to use command "debuginfo-install", do you know which pkg should I use?

(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/sasl2/libdigestmd5.so
0x00007f03ea075c3d in poll () from /lib64/libc.so.6
Missing separate debuginfos, use: debuginfo-install qemu-kvm-1.5.3-173.el7.x86_64
(gdb) debuginfo-install qemu-kvm-1.5.3-173.el7.x86_64
Undefined command: "debuginfo-install".  Try "help".

[root@ilx24s yum.repos.d]# debuginfo-install qemu-kvm-1.5.3-173.el7.x86_64
Loaded plugins: langpacks, product-id, subscription-manager

This system is not registered with an entitlement server. You can use subscription-manager to register.

Could not find debuginfo for main pkg: 10:qemu-kvm-1.5.3-173.el7.x86_64
Could not find debuginfo pkg for dependency package libaio-0.3.109-13.el7.x86_64
Could not find debuginfo pkg for dependency package alsa-lib-1.1.8-1.el7.x86_64
Could not find debuginfo pkg for dependency package glibc-2.17-307.el7.1.x86_64
Could not find debuginfo pkg for dependency package libcurl-7.29.0-57.el7.x86_64
Could not find debuginfo pkg for dependency package glusterfs-api-6.0-29.el7.x86_64
Could not find debuginfo pkg for dependency package glusterfs-libs-6.0-29.el7.x86_64
Could not find debuginfo pkg for dependency package glib2-2.56.1-5.el7.x86_64
Could not find debuginfo pkg for dependency package gnutls-3.3.29-9.el7_6.x86_64
Could not find debuginfo pkg for dependency package libibverbs-22.4-1.el7.x86_64
Could not find debuginfo pkg for dependency package libiscsi-1.9.0-7.el7.x86_64
Could not find debuginfo pkg for dependency package lzo-2.06-8.el7.x86_64
Could not find debuginfo pkg for dependency package nspr-4.21.0-1.el7.x86_64
Could not find debuginfo pkg for dependency package nss-3.44.0-7.el7_7.x86_64
Could not find debuginfo pkg for dependency package nss-util-3.44.0-4.el7_7.x86_64
Could not find debuginfo pkg for dependency package pixman-0.34.0-1.el7.x86_64
Could not find debuginfo pkg for dependency package 2:libpng-1.5.13-7.el7_2.x86_64
Could not find debuginfo pkg for dependency package pulseaudio-libs-10.0-5.el7.x86_64
Could not find debuginfo pkg for dependency package 1:librados2-10.2.5-4.el7.x86_64
Could not find debuginfo pkg for dependency package 1:librbd1-10.2.5-4.el7.x86_64
Could not find debuginfo pkg for dependency package librdmacm-22.4-1.el7.x86_64
Could not find debuginfo pkg for dependency package cyrus-sasl-lib-2.1.26-23.el7.x86_64
Could not find debuginfo pkg for dependency package libseccomp-2.3.1-4.el7.x86_64
Could not find debuginfo pkg for dependency package snappy-1.1.0-3.el7.x86_64
Could not find debuginfo pkg for dependency package spice-server-0.14.0-9.el7.x86_64
Could not find debuginfo pkg for dependency package libssh2-1.8.0-3.el7.x86_64
Could not find debuginfo pkg for dependency package gperftools-libs-2.6.1-1.el7.x86_64
Could not find debuginfo pkg for dependency package libusbx-1.0.21-1.el7.x86_64
Could not find debuginfo pkg for dependency package usbredir-0.7.1-3.el7.x86_64
Could not find debuginfo pkg for dependency package libuuid-2.23.2-63.el7.x86_64
Could not find debuginfo pkg for dependency package zlib-1.2.7-18.el7.x86_64
No debuginfo packages available to install
[root@ilx24s yum.repos.d]# yum install qemu-kvm-1.5.3-173.el7.x86_64

This system is not registered with an entitlement server. You can use subscription-manager to register.

Package 10:qemu-kvm-1.5.3-173.el7.x86_64 already installed and latest version
[root@ilx24s yum.repos.d]#

------- Comment From shlyi.com 2020-04-17 05:16 EDT-------
> Can I assume from responses shown in
> https://bugzilla.redhat.com/show_bug.cgi?id=1803716#c17 that perhaps the
> debuginfo pkgs have been successfully installed?
> There is also concern about the time spent chasing a needle in the haystack
> type problem on older software....
> It's not that RHEL7 problems aren't important, but there are a lot of
> variables to consider.
> Do you know whether the same problem exists using RHEL8?

Yes, similar problem do exist on RHEL8, but I have no access to view https://bugzilla.redhat.com/show_bug.cgi?id=1803716#c17

Created attachment 1723438 [details]
sosreport-ilx21r


------- Comment on attachment From shlyi.com 2019-12-30 21:45 EDT-------


re-upload sosreport-ilx21r

Created attachment 1723439 [details]
Connection TOPO of the host ilx21r


------- Comment on attachment From shlyi.com 2019-12-23 21:30 EDT-------


Connection TOPO of the host ilx21r

Created attachment 1723440 [details]
sosreport of ilx21r1r


------- Comment on attachment From shlyi.com 2019-12-31 01:53 EDT-------


re-upload sosreport of ilx21r1r

Created attachment 1723441 [details]
guest config file


------- Comment on attachment From shlyi.com 2019-12-24 02:04 EDT-------


Here is the guest config file used by domain in libvirt

Created attachment 1723442 [details]
/var/log/messages of ilx21r_Dec3

Created attachment 1723443 [details]
io logs containing error info

Created attachment 1723444 [details]
/var/log/messages of ilx21r

Created attachment 1723445 [details]
/var/log/messages of ilx21r

Created attachment 1723446 [details]
sosreport-ilx21r1r

Created attachment 1723447 [details]
updated system tap script with more output

Created attachment 1723448 [details]
messages-ilx24s-20200405

Created attachment 1723449 [details]
disktest, /var/log/messages of ilx24s and ilx24s1r

Created attachment 1723450 [details]
ilx24s1r.disktest.sda.20200402200029, guest lun of ilx24s

Created attachment 1723451 [details]
messages-ilx25s-20200405

Created attachment 1723452 [details]
io logs and system logs

Created attachment 1723453 [details]
systemtap script to print the error code on failed requests

------- Comment From shlyi.com 2020-04-08 04:25 EDT-------
(In reply to comment #15)
> Why is the storage target returning Aborted Command / IO Process Terminated
> errors ?
> It appears as if you are expecting that multipath will retry those errors,
> but this was not a path error, it was an error returned by the storage
> device.
> bug 1779758 refers to this bug, but 1779758 would be investigated as a
> kernel crash.
> Does the host or the guest OS crash here?

Yes, There was guest OS crash here, below is the comments from my original statements:
Actual results
For the guest ilx21r1r, there was io error on it starting from time 2020/03/23-15:01:32
For the guest ilx22r1s, it was rebooted by the host at 2020-03-23T14:39:53.

Could you be more specific for question "Why is the storage target returning Aborted Command / IO Process Terminated errors ?" For example, could you provide us the timeline of the "Aborted Command / IO Process Terminated errors".

------- Comment From shlyi.com 2020-04-29 23:41 EDT-------
> Please refer to the log files that were attached to this BZ to see
> when the Aborted Command / IO Process Terminated errors were received
> from the storage.

I think you refer to below logs when you were saying "Aborted Command / IO Process Terminated errors"
/var/log/messages of ilx21r1r:
Mar 23 14:38:05 ilx21r1r kernel: sd 1:0:0:6: [sdh] tag#21 Add. Sense: I/O process terminated

And I've checked with our storage dev, asked if the storage aborted any write request during that timeslot, and what storage had done during that time which may related to this issue.

Below is what he told me, in briefly speaking, the storage didn't abort any write request during that time.
Could you take a look at the mechanism of communication between the guest and its dependent host to see if the host could mislead the guest to that error by mishandling the partial failing path? Thanks a lot.

"The only commands I'm aware of that we are aborting are the ones I note above with the illegal unmap size, which is not a write command. There are plenty of other commands we aborth because they are optional and not supported, but again, not write commands. A quick look at the logs and I can find no reason we would abort the write commands. Furthermore, from the Linux logs, I have to presume the Sense Code is 0xB (for "Aborted Command") and the Additional Sense Code/Additional Sense Code Qualifier of 0x00/0x06 (for "I/O PROCESS TERMINATED"), as that seems to match closest to the report (since they don't supply the raw values). Well, just looked through our code, and we NEVER return that ASC/ASCQ value under any circumstance, and only issue an SC of Abort under a very specific overlapping command condition (which we would see in our logs. I can only presume that the "hypervisor" is generating this status to the guest OS, as I can say with near certainty that it's not the flash enclosure."

------- Comment From shlyi.com 2020-05-19 22:29 EDT-------
Hello emilne,

Do you have any updates for this issue?

We ran storage ei "Texan_ccl" yesterday, this case will cause half of the ports of the storage break and recover cyclically? and I get some new clue to share with you:

Summary: guest had io error and rebooted during the test. Below are the detailed trace:
EXT4-fs error detected on the guest during the test:, qemu-kvm crashed after path coming back on the host;

on the host ilx21r, /var/log/messages:
May 18 23:27:56 ilx21r abrt-hook-ccpp[390001]: Process 10020 (qemu-kvm) of user 107 killed by SIGABRT - dumping core

on the guest ilx21r1r, /var/log/messages:
May 18 21:28:52 ilx21r1r kernel: print_req_error: 459 callbacks suppressed
May 18 21:28:52 ilx21r1r kernel: print_req_error: I/O error, dev sdf, sector 53560 flags 801
May 18 21:28:52 ilx21r1r kernel: Aborting journal on device sdf-8.
May 18 21:28:52 ilx21r1r kernel: EXT4-fs error (device sdf) in ext4_reserve_inode_write:5894: Journal has aborted

I'll attach whole logs later.

------- Comment From mcneals.com 2020-06-03 10:41 EDT-------
This is a blocking defect and will prevent us from publishing support.

Created attachment 1723457 [details]
IO errors, /var/log/messages, and XML for RHEL8.2 guest

Created attachment 1723458 [details]
sosreport of ilx22r - part 2/2

Created attachment 1723459 [details]
abrt log of ilx24s1r

Created attachment 1723460 [details]
message logs and io log

Created attachment 1723468 [details]
collectlogs of ilx21r

Created attachment 1723469 [details]
sosreport of ilx21r

Created attachment 1723470 [details]
sosreport of ilx21r

Created attachment 1723471 [details]
sosreport of ilx21r1r

Created attachment 1723472 [details]
sosreport of ilx21r1r

Created attachment 1723476 [details]
sosreport of ilx21r  - part 2/2

Created attachment 1723477 [details]
sosreport of ilx22r - part 1/2

Created attachment 1723478 [details]
sosreport of ilx21r  - part 1/2

Created attachment 1723479 [details]
messages-ilx21r1r-May18

Created attachment 1723480 [details]
messages-ilx21r-May18

------- Comment From hannsj_uhl.com 2020-11-09 10:11 EDT-------
..

This issue depends on the kernel/DM work being tracked in Bug 1855868.

Hanns-Joachim Uhl: I'll put you in contact with the IBM Partner Manager (via email) who can coordinate the escalation of this issue.

(In reply to Ademar Reis from comment #56)
> This issue depends on the kernel/DM work being tracked in Bug 1855868.
> 
> Hanns-Joachim Uhl: I'll put you in contact with the IBM Partner Manager (via
> email) who can coordinate the escalation of this issue.
.
ah ok ... I just have seen that this bugzilla is tagged as 'FutureFeature' ...
and therefore I am resetting the severity to 'high' ... 
... let's see whether this bugzilla will make RHEL-AV 8.4.0 ...

(In reply to Hanns-Joachim Uhl from comment #57)
> (In reply to Ademar Reis from comment #56)
> > This issue depends on the kernel/DM work being tracked in Bug 1855868.
> > 
> > Hanns-Joachim Uhl: I'll put you in contact with the IBM Partner Manager (via
> > email) who can coordinate the escalation of this issue.
> .
> ah ok ... I just have seen that this bugzilla is tagged as 'FutureFeature'
> ...
> and therefore I am resetting the severity to 'high' ... 
> ... let's see whether this bugzilla will make RHEL-AV 8.4.0 ...

I can't speak on behalf of the kernel/dm team, but given the complexity of what's involved and the activity in the BZ, I would say it's unlikely to be fixed in 8.4.0. If this is important to IBM, then I recommend escalating it with the IBM Partner Manager for it to receive proper attention and prioritization. Thanks.

Guest hit io-error even if using virtio device

Red Hat Enterprise Linux release 8.2 (Ootpa)
4.18.0-193.el8.x86_64
qemu-kvm-common-4.2.0-29.module+el8.2.1+7297+a825794d.x86_64

1.Create multipath device

multipath -f /dev/mapper/mpathb;modprobe -r scsi_debug
modprobe scsi_debug dev_size_mb=5000 num_tgts=1 vpd_use_hostno=0 add_host=2 delay=1 max_luns=2 no_lun_0=1

root@dell-per440-10 /home/rworkdir/vbugs # multipath  -l
mpathb (333333330000007d1) dm-5 Linux,scsi_debug
size=4.9G features='0' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=0 status=active
| `- 18:0:0:1 sde 8:64 active undef running
`-+- policy='service-time 0' prio=0 status=enabled
  `- 17:0:0:1 sdd 8:48 active undef running

2.Boot vm with passthrough dm device, the disk attached as virtio-blk-pci

/usr/libexec/qemu-kvm \
  -name src_vm1 \
  -machine q35 \
  -nodefaults \
  -vga qxl \
  -device pcie-root-port,id=pcie.0-root-port-2,slot=2,bus=pcie.0,multifunction=on \
  -device pcie-root-port,id=pcie.0-root-port-2-1,chassis=3,bus=pcie.0,addr=0x2.0x1 \
  -device pcie-root-port,id=pcie.0-root-port-2-2,chassis=4,bus=pcie.0,addr=0x2.0x2 \
  -device pcie-root-port,id=pcie.0-root-port-3,slot=3,bus=pcie.0 \
  -device pcie-root-port,id=pcie.0-root-port-4,slot=4,bus=pcie.0 \
  -device pcie-root-port,id=pcie.0-root-port-5,slot=5,bus=pcie.0 \
  -device pcie-root-port,id=pcie.0-root-port-7,slot=7,bus=pcie.0 \
  -device pcie-root-port,id=pcie.0-root-port-8,slot=8,bus=pcie.0 \
  -device pcie-root-port,id=pcie.0-root-port-9,slot=9,bus=pcie.0 \
  -device qemu-xhci,id=usb1,bus=pcie.0-root-port-2-1,addr=0x0 \
  -device usb-tablet,id=usb-tablet1,bus=usb1.0,port=1 \
  -object iothread,id=iothread0 \
  -device virtio-scsi-pci,id=scsi0,bus=pcie.0-root-port-2-2,addr=0x0,iothread=iothread0 \
  \
  -blockdev driver=qcow2,file.driver=file,cache.direct=off,cache.no-flush=on,file.filename=/home/kvm_autotest_root/images/rhel830-64-virtio-scsi.qcow2,node-name=drive_image1 \
  -device scsi-hd,id=os1,drive=drive_image1,bootindex=0 \
  \
  -device virtio-scsi-pci,id=scsi1,bus=pcie.0-root-port-8,addr=0x0 \
  -blockdev node-name=host_device_stg0,driver=host_device,filename=/dev/mapper/mpathb \
  -blockdev node-name=drive_stg0,driver=raw,file=host_device_stg0 \
  -device virtio-blk-pci,scsi=off,bus=pcie.0-root-port-4,addr=0,drive=drive_stg0,id=stg0,write-cache=on,rerror=stop,werror=stop,bootindex=2 \
  \
  -vnc :5 \
  -qmp tcp:0:5955,server,nowait \
  -monitor stdio \
  -m 8192 \
  -smp 8 \
  -device virtio-net-pci,mac=9a:b5:b6:b1:b2:b5,id=idMmq1jH,vectors=4,netdev=idxgXAlm,bus=pcie.0-root-port-5,addr=0x0 \
  -netdev tap,id=idxgXAlm \

3. do io on the disk in guest
while true;do dd if=/dev/zero of=/dev/vda;done

4.Alternately close a path every 10 seconds on host
offline ===== sde
mpathb (333333330000007d1) dm-5 Linux,scsi_debug
size=4.9G features='0' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=0 status=active
| `- 18:0:0:1 sde 8:64 active faulty offline
`-+- policy='service-time 0' prio=0 status=enabled
  `- 17:0:0:1 sdd 8:48 active undef running
online ===== sde
mpathb (333333330000007d1) dm-5 Linux,scsi_debug
size=4.9G features='0' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=0 status=active
| `- 18:0:0:1 sde 8:64 failed undef running
`-+- policy='service-time 0' prio=0 status=enabled
  `- 17:0:0:1 sdd 8:48 active undef running

offline ===== sdd
mpathb (333333330000007d1) dm-5 Linux,scsi_debug
size=4.9G features='0' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=0 status=active
| `- 18:0:0:1 sde 8:64 active undef running
`-+- policy='service-time 0' prio=0 status=enabled
  `- 17:0:0:1 sdd 8:48 active faulty offline
online ===== sdd
mpathb (333333330000007d1) dm-5 Linux,scsi_debug
size=4.9G features='0' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=0 status=active
| `- 18:0:0:1 sde 8:64 active undef running
`-+- policy='service-time 0' prio=0 status=enabled
  `- 17:0:0:1 sdd 8:48 active undef running

offline ===== sde
....

5.check status for guest 1 minute latet

(qemu) info status
VM status: paused (io-error)

Is is not expected?

(In reply to qing.wang from comment #59)
> Guest hit io-error even if using virtio device
> 
> Red Hat Enterprise Linux release 8.2 (Ootpa)
> 4.18.0-193.el8.x86_64
> qemu-kvm-common-4.2.0-29.module+el8.2.1+7297+a825794d.x86_64
> 
> 1.Create multipath device
> 
> multipath -f /dev/mapper/mpathb;modprobe -r scsi_debug
> modprobe scsi_debug dev_size_mb=5000 num_tgts=1 vpd_use_hostno=0 add_host=2
> delay=1 max_luns=2 no_lun_0=1
> 
> root@dell-per440-10 /home/rworkdir/vbugs # multipath  -l
> mpathb (333333330000007d1) dm-5 Linux,scsi_debug
> size=4.9G features='0' hwhandler='1 alua' wp=rw
> |-+- policy='service-time 0' prio=0 status=active
> | `- 18:0:0:1 sde 8:64 active undef running
> `-+- policy='service-time 0' prio=0 status=enabled
>   `- 17:0:0:1 sdd 8:48 active undef running
> 
> 2.Boot vm with passthrough dm device, the disk attached as virtio-blk-pci
> 
> /usr/libexec/qemu-kvm \
>   -name src_vm1 \
>   -machine q35 \
>   -nodefaults \
>   -vga qxl \
>   -device
> pcie-root-port,id=pcie.0-root-port-2,slot=2,bus=pcie.0,multifunction=on \
>   -device
> pcie-root-port,id=pcie.0-root-port-2-1,chassis=3,bus=pcie.0,addr=0x2.0x1 \
>   -device
> pcie-root-port,id=pcie.0-root-port-2-2,chassis=4,bus=pcie.0,addr=0x2.0x2 \
>   -device pcie-root-port,id=pcie.0-root-port-3,slot=3,bus=pcie.0 \
>   -device pcie-root-port,id=pcie.0-root-port-4,slot=4,bus=pcie.0 \
>   -device pcie-root-port,id=pcie.0-root-port-5,slot=5,bus=pcie.0 \
>   -device pcie-root-port,id=pcie.0-root-port-7,slot=7,bus=pcie.0 \
>   -device pcie-root-port,id=pcie.0-root-port-8,slot=8,bus=pcie.0 \
>   -device pcie-root-port,id=pcie.0-root-port-9,slot=9,bus=pcie.0 \
>   -device qemu-xhci,id=usb1,bus=pcie.0-root-port-2-1,addr=0x0 \
>   -device usb-tablet,id=usb-tablet1,bus=usb1.0,port=1 \
>   -object iothread,id=iothread0 \
>   -device
> virtio-scsi-pci,id=scsi0,bus=pcie.0-root-port-2-2,addr=0x0,
> iothread=iothread0 \
>   \
>   -blockdev
> driver=qcow2,file.driver=file,cache.direct=off,cache.no-flush=on,file.
> filename=/home/kvm_autotest_root/images/rhel830-64-virtio-scsi.qcow2,node-
> name=drive_image1 \
>   -device scsi-hd,id=os1,drive=drive_image1,bootindex=0 \
>   \
>   -device virtio-scsi-pci,id=scsi1,bus=pcie.0-root-port-8,addr=0x0 \
>   -blockdev
> node-name=host_device_stg0,driver=host_device,filename=/dev/mapper/mpathb \
>   -blockdev node-name=drive_stg0,driver=raw,file=host_device_stg0 \
>   -device
> virtio-blk-pci,scsi=off,bus=pcie.0-root-port-4,addr=0,drive=drive_stg0,
> id=stg0,write-cache=on,rerror=stop,werror=stop,bootindex=2 \
>   \
>   -vnc :5 \
>   -qmp tcp:0:5955,server,nowait \
>   -monitor stdio \
>   -m 8192 \
>   -smp 8 \
>   -device
> virtio-net-pci,mac=9a:b5:b6:b1:b2:b5,id=idMmq1jH,vectors=4,netdev=idxgXAlm,
> bus=pcie.0-root-port-5,addr=0x0 \
>   -netdev tap,id=idxgXAlm \
> 
> 3. do io on the disk in guest
> while true;do dd if=/dev/zero of=/dev/vda;done
> 
> 4.Alternately close a path every 10 seconds on host
> offline ===== sde
> mpathb (333333330000007d1) dm-5 Linux,scsi_debug
> size=4.9G features='0' hwhandler='1 alua' wp=rw
> |-+- policy='service-time 0' prio=0 status=active
> | `- 18:0:0:1 sde 8:64 active faulty offline
> `-+- policy='service-time 0' prio=0 status=enabled
>   `- 17:0:0:1 sdd 8:48 active undef running
> online ===== sde
> mpathb (333333330000007d1) dm-5 Linux,scsi_debug
> size=4.9G features='0' hwhandler='1 alua' wp=rw
> |-+- policy='service-time 0' prio=0 status=active
> | `- 18:0:0:1 sde 8:64 failed undef running
> `-+- policy='service-time 0' prio=0 status=enabled
>   `- 17:0:0:1 sdd 8:48 active undef running
> 
> offline ===== sdd
> mpathb (333333330000007d1) dm-5 Linux,scsi_debug
> size=4.9G features='0' hwhandler='1 alua' wp=rw
> |-+- policy='service-time 0' prio=0 status=active
> | `- 18:0:0:1 sde 8:64 active undef running
> `-+- policy='service-time 0' prio=0 status=enabled
>   `- 17:0:0:1 sdd 8:48 active faulty offline
> online ===== sdd
> mpathb (333333330000007d1) dm-5 Linux,scsi_debug
> size=4.9G features='0' hwhandler='1 alua' wp=rw
> |-+- policy='service-time 0' prio=0 status=active
> | `- 18:0:0:1 sde 8:64 active undef running
> `-+- policy='service-time 0' prio=0 status=enabled
>   `- 17:0:0:1 sdd 8:48 active undef running
> 
> offline ===== sde
> ....
> 
> 5.check status for guest 1 minute latet
> 
> (qemu) info status
> VM status: paused (io-error)
> 
> Is is not expected?

Paolo?

Please help to check test step, does it meet test requirement? What is expected result on https://bugzilla.redhat.com/show_bug.cgi?id=1854659#c59, it should not hit io-error as long as any path is valid ?

> it should not hit io-error as long as any path is valid ?

It shouldn't.  In fact that scenario (or also scsi-hd) is not SCSI passthrough (SCSI passthrough is only with scsi-block/scsi-generic) so it seems to be a different bug?  Can you reproduce it just with "dd" on the host without involving virtualization at all?

I tried dd on the host only

1. create multipath device
modprobe scsi_debug dev_size_mb=5000 num_tgts=1 vpd_use_hostno=0 add_host=2 delay=1 max_luns=2 no_lun_0=1

mpatha (333333330000007d1) dm-3 Linux,scsi_debug
size=4.9G features='0' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=0 status=active
| `- 15:0:0:1 sdb 8:16 active undef running
`-+- policy='service-time 0' prio=0 status=enabled
  `- 16:0:0:1 sdc 8:32 active undef running

2.run dd on the DM-multipath device
 while true;do echo "dd..";dd if=/dev/urandom of=/dev/mapper/mpatha oflag=direct bs=4k count=100000;echo "dd end";sleep 1;done

3. Alternately close a path every 10 seconds on host
offline ===== sdc 02:16:44
mpatha (333333330000007d1) dm-3 Linux,scsi_debug
size=4.9G features='0' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=0 status=active
| `- 15:0:0:1 sdb 8:16 active undef running
`-+- policy='service-time 0' prio=0 status=enabled
  `- 16:0:0:1 sdc 8:32 failed faulty offline
online ====== sdc 02:16:54
mpatha (333333330000007d1) dm-3 Linux,scsi_debug
size=4.9G features='0' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=0 status=active
| `- 15:0:0:1 sdb 8:16 active undef running
`-+- policy='service-time 0' prio=0 status=enabled
  `- 16:0:0:1 sdc 8:32 failed undef running

offline ===== sdb 02:17:00
mpatha (333333330000007d1) dm-3 Linux,scsi_debug
size=4.9G features='0' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=0 status=enabled
| `- 15:0:0:1 sdb 8:16 failed faulty offline
`-+- policy='service-time 0' prio=0 status=active
  `- 16:0:0:1 sdc 8:32 active undef running
online ====== sdb 02:17:10
mpatha (333333330000007d1) dm-3 Linux,scsi_debug
size=4.9G features='0' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=0 status=enabled
| `- 15:0:0:1 sdb 8:16 failed undef running
`-+- policy='service-time 0' prio=0 status=active
  `- 16:0:0:1 sdc 8:32 active undef running

offline ===== sdc 02:17:15
mpatha (333333330000007d1) dm-3 Linux,scsi_debug
size=4.9G features='0' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=0 status=active
| `- 15:0:0:1 sdb 8:16 active undef running
`-+- policy='service-time 0' prio=0 status=enabled
  `- 16:0:0:1 sdc 8:32 failed faulty offline
online ====== sdc 02:17:25
mpatha (333333330000007d1) dm-3 Linux,scsi_debug
size=4.9G features='0' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=0 status=active
| `- 15:0:0:1 sdb 8:16 active undef running
`-+- policy='service-time 0' prio=0 status=enabled
  `- 16:0:0:1 sdc 8:32 failed undef running



Sometimes dd encounter io error
dd..
dd: error writing '/dev/mapper/mpatha': Input/output error
56305+0 records in
56304+0 records out
230621184 bytes (231 MB, 220 MiB) copied, 61.1572 s, 3.8 MB/s
dd end

/var/log/messages

Jan 12 02:23:02 dell-per440-10 kernel: sd 15:0:0:1: rejecting I/O to offline device
Jan 12 02:23:02 dell-per440-10 kernel: device-mapper: multipath: 253:3: Failing path 8:16.
Jan 12 02:23:02 dell-per440-10 multipathd[390972]: sdb: mark as failed
Jan 12 02:23:02 dell-per440-10 multipathd[390972]: mpatha: remaining active paths: 1
Jan 12 02:23:02 dell-per440-10 kernel: sd 16:0:0:1: alua: port group 1100 state N non-preferred supports tolUSNA
Jan 12 02:23:02 dell-per440-10 kernel: sd 16:0:0:1: alua: port group 1100 state N non-preferred supports tolUSNA
Jan 12 02:23:07 dell-per440-10 multipathd[390972]: mpatha: sdb - path offline
Jan 12 02:23:12 dell-per440-10 multipathd[390972]: mpatha: sdb - path offline
Jan 12 02:23:17 dell-per440-10 kernel: sd 16:0:0:1: rejecting I/O to offline device
Jan 12 02:23:17 dell-per440-10 kernel: device-mapper: multipath: 253:3: Failing path 8:32.
Jan 12 02:23:17 dell-per440-10 kernel: blk_update_request: I/O error, dev dm-3, sector 652864 op 0x1:(WRITE) flags 0x8800 phys_seg 1 prio class 0
Jan 12 02:23:17 dell-per440-10 multipathd[390972]: sdc: mark as failed
Jan 12 02:23:17 dell-per440-10 multipathd[390972]: mpatha: remaining active paths: 0
Jan 12 02:23:18 dell-per440-10 multipathd[390972]: mpatha: sdb - tur checker reports path is up
Jan 12 02:23:18 dell-per440-10 kernel: device-mapper: multipath: 253:3: Reinstating path 8:16.
Jan 12 02:23:18 dell-per440-10 multipathd[390972]: 8:16: reinstated
Jan 12 02:23:18 dell-per440-10 multipathd[390972]: mpatha: remaining active paths: 1
Jan 12 02:23:18 dell-per440-10 kernel: sd 15:0:0:1: alua: port group 1000 state A non-preferred supports tolUSNA
Jan 12 02:23:18 dell-per440-10 kernel: sd 15:0:0:1: alua: port group 1000 state A non-preferred supports tolUSNA
Jan 12 02:23:22 dell-per440-10 multipathd[390972]: mpatha: sdc - path offline
Jan 12 02:23:27 dell-per440-10 multipathd[390972]: mpatha: sdc - path offline
Jan 12 02:23:33 dell-per440-10 multipathd[390972]: mpatha: sdc - tur checker reports path is up
Jan 12 02:23:33 dell-per440-10 kernel: device-mapper: multipath: 253:3: Reinstating path 8:32.
Jan 12 02:23:33 dell-per440-10 multipathd[390972]: 8:32: reinstated
Jan 12 02:23:33 dell-per440-10 multipathd[390972]: mpatha: remaining active paths: 2
Jan 12 02:23:33 dell-per440-10 kernel: sd 15:0:0:1: rejecting I/O to offline device
Jan 12 02:23:33 dell-per440-10 kernel: device-mapper: multipath: 253:3: Failing path 8:16.
Jan 12 02:23:33 dell-per440-10 kernel: sd 16:0:0:1: alua: port group 1100 state N non-preferred supports tolUSNA
Jan 12 02:23:33 dell-per440-10 kernel: sd 16:0:0:1: alua: port group 1100 state N non-preferred supports tolUSNA
Jan 12 02:23:34 dell-per440-10 multipathd[390972]: mpatha: sdb - path offline
Jan 12 02:23:34 dell-per440-10 multipathd[390972]: checker failed path 8:16 in map mpatha
Jan 12 02:23:34 dell-per440-10 multipathd[390972]: mpatha: remaining active paths: 1
Jan 12 02:23:39 dell-per440-10 multipathd[390972]: mpatha: sdb - path offline
Jan 12 02:23:45 dell-per440-10 multipathd[390972]: mpatha: sdb - tur checker reports path is up
Jan 12 02:23:45 dell-per440-10 kernel: device-mapper: multipath: 253:3: Reinstating path 8:16.
Jan 12 02:23:45 dell-per440-10 multipathd[390972]: 8:16: reinstated
Jan 12 02:23:45 dell-per440-10 multipathd[390972]: mpatha: remaining active paths: 2
Jan 12 02:23:48 dell-per440-10 kernel: sd 16:0:0:1: rejecting I/O to offline device
Jan 12 02:23:48 dell-per440-10 kernel: device-mapper: multipath: 253:3: Failing path 8:32.
Jan 12 02:23:48 dell-per440-10 multipathd[390972]: sdc: mark as failed
Jan 12 02:23:48 dell-per440-10 multipathd[390972]: mpatha: remaining active paths: 1
Jan 12 02:23:48 dell-per440-10 kernel: sd 15:0:0:1: alua: port group 1000 state A non-preferred supports tolUSNA
Jan 12 02:23:48 dell-per440-10 kernel: sd 15:0:0:1: alua: port group 1000 state A non-preferred supports tolUSNA
Jan 12 02:23:49 dell-per440-10 multipathd[390972]: mpatha: sdc - path offline
Jan 12 02:23:54 dell-per440-10 multipathd[390972]: mpatha: sdc - path offline
Jan 12 02:23:59 dell-per440-10 multipathd[390972]: mpatha: sdc - tur checker reports path is up
Jan 12 02:23:59 dell-per440-10 kernel: device-mapper: multipath: 253:3: Reinstating path 8:32.
Jan 12 02:23:59 dell-per440-10 multipathd[390972]: 8:32: reinstated
Jan 12 02:23:59 dell-per440-10 multipathd[390972]: mpatha: remaining active paths: 2
Jan 12 02:24:04 dell-per440-10 kernel: sd 15:0:0:1: rejecting I/O to offline device
Jan 12 02:24:04 dell-per440-10 kernel: device-mapper: multipath: 253:3: Failing path 8:16.
Jan 12 02:24:04 dell-per440-10 multipathd[390972]: sdb: mark as failed
Jan 12 02:24:04 dell-per440-10 multipathd[390972]: mpatha: remaining active paths: 1
Jan 12 02:24:04 dell-per440-10 kernel: sd 16:0:0:1: alua: port group 1100 state N non-preferred supports tolUSNA
Jan 12 02:24:04 dell-per440-10 kernel: sd 16:0:0:1: alua: port group 1100 state N non-preferred supports tolUSNA
Jan 12 02:24:09 dell-per440-10 multipathd[390972]: mpatha: sdb - path offline
Jan 12 02:24:14 dell-per440-10 multipathd[390972]: mpatha: sdb - path offline
Jan 12 02:24:19 dell-per440-10 kernel: sd 16:0:0:1: rejecting I/O to offline device
Jan 12 02:24:19 dell-per440-10 kernel: device-mapper: multipath: 253:3: Failing path 8:32.
Jan 12 02:24:19 dell-per440-10 kernel: blk_update_request: I/O error, dev dm-3, sector 450432 op 0x1:(WRITE) flags 0x8800 phys_seg 1 prio class 0
Jan 12 02:24:19 dell-per440-10 multipathd[390972]: sdc: mark as failed
Jan 12 02:24:19 dell-per440-10 multipathd[390972]: mpatha: remaining active paths: 0
Jan 12 02:24:20 dell-per440-10 multipathd[390972]: mpatha: sdb - tur checker reports path is up
Jan 12 02:24:20 dell-per440-10 kernel: device-mapper: multipath: 253:3: Reinstating path 8:16.
Jan 12 02:24:20 dell-per440-10 multipathd[390972]: 8:16: reinstated
Jan 12 02:24:20 dell-per440-10 multipathd[390972]: mpatha: remaining active paths: 1
Jan 12 02:24:20 dell-per440-10 kernel: sd 15:0:0:1: alua: port group 1000 state A non-preferred supports tolUSNA
Jan 12 02:24:20 dell-per440-10 kernel: sd 15:0:0:1: alua: port group 1000 state A non-preferred supports tolUSNA
Jan 12 02:24:24 dell-per440-10 multipathd[390972]: mpatha: sdc - path offline
Jan 12 02:24:29 dell-per440-10 multipathd[390972]: mpatha: sdc - path offline
Jan 12 02:24:35 dell-per440-10 multipathd[390972]: mpatha: sdc - tur checker reports path is up
Jan 12 02:24:35 dell-per440-10 kernel: device-mapper: multipath: 253:3: Reinstating path 8:32.
Jan 12 02:24:35 dell-per440-10 multipathd[390972]: 8:32: reinstated
Jan 12 02:24:35 dell-per440-10 multipathd[390972]: mpatha: remaining active paths: 2
Jan 12 02:24:35 dell-per440-10 kernel: sd 15:0:0:1: rejecting I/O to offline device
Jan 12 02:24:35 dell-per440-10 kernel: device-mapper: multipath: 253:3: Failing path 8:16.
Jan 12 02:24:35 dell-per440-10 kernel: sd 16:0:0:1: alua: port group 1100 state N non-preferred supports tolUSNA
Jan 12 02:24:35 dell-per440-10 kernel: sd 16:0:0:1: alua: port group 1100 state N non-preferred supports tolUSNA
Jan 12 02:24:36 dell-per440-10 multipathd[390972]: sdb: mark as failed
Jan 12 02:24:36 dell-per440-10 multipathd[390972]: mpatha: remaining active paths: 1
Jan 12 02:24:41 dell-per440-10 multipathd[390972]: mpatha: sdb - path offline
Jan 12 02:24:47 dell-per440-10 multipathd[390972]: mpatha: sdb - tur checker reports path is up
Jan 12 02:24:47 dell-per440-10 kernel: device-mapper: multipath: 253:3: Reinstating path 8:16.
Jan 12 02:24:47 dell-per440-10 multipathd[390972]: 8:16: reinstated
Jan 12 02:24:47 dell-per440-10 multipathd[390972]: mpatha: remaining active paths: 2
Jan 12 02:24:50 dell-per440-10 kernel: sd 16:0:0:1: rejecting I/O to offline device
Jan 12 02:24:50 dell-per440-10 kernel: device-mapper: multipath: 253:3: Failing path 8:32.
Jan 12 02:24:50 dell-per440-10 multipathd[390972]: sdc: mark as failed
Jan 12 02:24:50 dell-per440-10 multipathd[390972]: mpatha: remaining active paths: 1
Jan 12 02:24:50 dell-per440-10 kernel: sd 15:0:0:1: alua: port group 1000 state A non-preferred supports tolUSNA
Jan 12 02:24:50 dell-per440-10 kernel: sd 15:0:0:1: alua: port group 1000 state A non-preferred supports tolUSNA
Jan 12 02:24:51 dell-per440-10 multipathd[390972]: mpatha: sdc - path offline

(In reply to Paolo Bonzini from comment #62)
> > it should not hit io-error as long as any path is valid ?
> 
> It shouldn't.  In fact that scenario (or also scsi-hd) is not SCSI
> passthrough (SCSI passthrough is only with scsi-block/scsi-generic) so it
> seems to be a different bug?  Can you reproduce it just with "dd" on the
> host without involving virtualization at all?

https://bugzilla.redhat.com/show_bug.cgi?id=1854659#c3 is the SCSI passthrough device(scsi-block), 
Do you think we need to open a bug to track 
 virtio-blk-pci device  or scsi-hd device based on the DM-multipath device? like as:

-blockdev node-name=host_device_stg0,driver=host_device,filename=/dev/mapper/mpathb \
-blockdev node-name=drive_stg0,driver=raw,file=host_device_stg0 \
-device  virtio-blk-pci,scsi=off,bus=pcie.0-root-port-4,addr=0,drive=drive_stg0,
 id=stg0,write-cache=on,rerror=stop,werror=stop,bootindex=2 \

(dd on host result please check https://bugzilla.redhat.com/show_bug.cgi?id=1854659#c63)

Based on the dd results this looks like a kernel bug that can be reproduced without virtualization, so I think it should be a separate bug.

Hi bmarzins,could you pleaes help to check https://bugzilla.redhat.com/show_bug.cgi?id=1854659#c63 . hitting io-error is expected result or bug during the path switching?

No, you should clearly not be hitting IO errors as long as there are valid paths.  However, there aren't valid paths when you are hitting IO errors.

Here are the IO error cases:

Jan 12 02:23:12 dell-per440-10 multipathd[390972]: mpatha: sdb - path offline
Jan 12 02:23:17 dell-per440-10 kernel: sd 16:0:0:1: rejecting I/O to offline device
Jan 12 02:23:17 dell-per440-10 kernel: device-mapper: multipath: 253:3: Failing path 8:32.
Jan 12 02:23:17 dell-per440-10 kernel: blk_update_request: I/O error, dev dm-3, sector 652864 op 0x1:(WRITE) flags 0x8800 phys_seg 1 prio class 0
Jan 12 02:23:17 dell-per440-10 multipathd[390972]: sdc: mark as failed
Jan 12 02:23:17 dell-per440-10 multipathd[390972]: mpatha: remaining active paths: 0


Jan 12 02:24:14 dell-per440-10 multipathd[390972]: mpatha: sdb - path offline
Jan 12 02:24:19 dell-per440-10 kernel: sd 16:0:0:1: rejecting I/O to offline device
Jan 12 02:24:19 dell-per440-10 kernel: device-mapper: multipath: 253:3: Failing path 8:32.
Jan 12 02:24:19 dell-per440-10 kernel: blk_update_request: I/O error, dev dm-3, sector 450432 op 0x1:(WRITE) flags 0x8800 phys_seg 1 prio class 0
Jan 12 02:24:19 dell-per440-10 multipathd[390972]: sdc: mark as failed
Jan 12 02:24:19 dell-per440-10 multipathd[390972]: mpatha: remaining active paths: 0

So, it appears that sdc is not accepting IO when sdb is down.  These are scsi debug devices and they aren't really two separate paths to the device.  If you remove multipath, and set path sdb to offline
and try your dd directly to sdc, does it work?

Actually, trying this myself, you don't even need to set one of the devices to offline for the other to fail IO:

[root@ask-08 ~]# modprobe scsi_debug dev_size_mb=100 num_tgts=1 vpd_use_hostno=0 add_host=2 delay=1 max_luns=2 no_lun_0=1
[root@ask-08 ~]# cat /proc/partitions 
major minor  #blocks  name

  11        0    1048575 sr0
   8        0   78125000 sda
   8        1    1048576 sda1
   8        2   77074432 sda2
 253        0   15728640 dm-0
 253        1    4169728 dm-1
   8       16  244059136 sdb
   8       32  244060160 sdc
   8       48     102400 sdd
   8       64     102400 sde
[root@ask-08 ~]# dd if=/dev/sdd of=/dev/null iflag=direct bs=1M count=1
dd: error reading '/dev/sdd': Input/output error
0+0 records in
0+0 records out
0 bytes copied, 0.00122501 s, 0.0 kB/s
[root@ask-08 ~]# dd if=/dev/sde of=/dev/null iflag=direct bs=1M count=1
1+0 records in
1+0 records out
1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00332771 s, 315 MB/s

Multipathd would mark the device sdd as valid because the Test Unit Ready checks succeed, even though IO to the device appears to fail. If you change the path_checker to directio (which issues diretio reads to test if the device is usable), you will see that the device is never really usable.  This is probably a bug, but it would be a bug in the scsi_debug device driver which should, I assume, create multiple devices capable of accepting IO.

Hi,bmarzins,thank for your quick reply, i still have questions on the issue.

This case come from customer test on FC multipath ,they  
do alternate switch port to check the path availbility.

The first question is how to trigger io error in your test.
In my test they truely located in multipath. if i offline the path of the device,
the io operation can not executed. 

root@dell-per440-10 ~ # multipath -l
mpatha (333333330000007d1) dm-3 Linux,scsi_debug
size=4.9G features='0' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=0 status=active
| `- 15:0:0:1 sdb 8:16 active undef running
`-+- policy='service-time 0' prio=0 status=enabled
  `- 16:0:0:1 sdc 8:32 active undef running

root@dell-per440-10 ~ # cat /proc/partitions 
major minor  #blocks  name

   8        0  585498624 sda
   8        1     614400 sda1
   8        2    1048576 sda2
   8        3  583833600 sda3
 253        0   73400320 dm-0
 253        1   32849920 dm-1
 253        2  477581312 dm-2
   8       16    5120000 sdb
   8       32    5120000 sdc
 253        3    5120000 dm-3

root@dell-per440-10 ~ # echo offline >/sys/block/sdb/device/state
root@dell-per440-10 ~ #  dd if=/dev/sdb of=/dev/null iflag=direct bs=1M count=1
dd: failed to open '/dev/sdb': No such device or address

root@dell-per440-10 ~ # echo running >/sys/block/sdb/device/state
root@dell-per440-10 ~ #  dd if=/dev/sdb of=/dev/null iflag=direct bs=1M count=1
1+0 records in
1+0 records out
1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00189668 s, 553 MB/s


==================================================================
The second question is it looks like the kernel need time to detect the path back to available.
if there is not enough time to detect path availbility, the io error is expexted?

I tested 2 cases:
both paths are available in advance.

case 1 offline path A -> sleep 10s ->online path A ->sleep 10s,then same operation on path B
case 2 offline path A -> sleep 10s ->online path A ->sleep 1s,then same operation on path B 
the difference is the sleep time after online path.
case 1 passed test , case 2 failed due to io error


case 1 please check timestamp from 02:06:36 to 02:08:22
it looks like kernel spend 15s to check the active paths: 1 to active paths: 2 

Jan 15 02:06:29 dell-per440-10 multipathd[1192]: mpatha: remaining active paths: 2
Jan 15 02:06:36 dell-per440-10 kernel: sd 15:0:0:1: rejecting I/O to offline device
Jan 15 02:06:36 dell-per440-10 kernel: device-mapper: multipath: 253:3: Failing path 8:16.
Jan 15 02:06:36 dell-per440-10 multipathd[1192]: sdb: mark as failed
Jan 15 02:06:36 dell-per440-10 multipathd[1192]: mpatha: remaining active paths: 1
Jan 15 02:06:36 dell-per440-10 kernel: sd 16:0:0:1: alua: port group 1100 state N non-preferred supports tolUSNA
Jan 15 02:06:36 dell-per440-10 kernel: sd 16:0:0:1: alua: port group 1100 state N non-preferred supports tolUSNA
Jan 15 02:06:40 dell-per440-10 multipathd[1192]: mpatha: sdb - path offline
Jan 15 02:06:45 dell-per440-10 multipathd[1192]: mpatha: sdb - path offline
Jan 15 02:06:50 dell-per440-10 multipathd[1192]: mpatha: sdb - tur checker reports path is up
Jan 15 02:06:50 dell-per440-10 kernel: device-mapper: multipath: 253:3: Reinstating path 8:16.
Jan 15 02:06:50 dell-per440-10 multipathd[1192]: 8:16: reinstated
Jan 15 02:06:50 dell-per440-10 multipathd[1192]: mpatha: remaining active paths: 2
Jan 15 02:06:56 dell-per440-10 kernel: sd 16:0:0:1: rejecting I/O to offline device
Jan 15 02:06:56 dell-per440-10 kernel: device-mapper: multipath: 253:3: Failing path 8:32.
Jan 15 02:06:56 dell-per440-10 multipathd[1192]: sdc: mark as failed
Jan 15 02:06:56 dell-per440-10 multipathd[1192]: mpatha: remaining active paths: 1
Jan 15 02:06:56 dell-per440-10 kernel: sd 15:0:0:1: alua: port group 1000 state A non-preferred supports tolUSNA
Jan 15 02:06:56 dell-per440-10 kernel: sd 15:0:0:1: alua: port group 1000 state A non-preferred supports tolUSNA
Jan 15 02:07:00 dell-per440-10 multipathd[1192]: mpatha: sdc - path offline
Jan 15 02:07:05 dell-per440-10 multipathd[1192]: mpatha: sdc - path offline
Jan 15 02:07:10 dell-per440-10 multipathd[1192]: mpatha: sdc - tur checker reports path is up
Jan 15 02:07:10 dell-per440-10 kernel: device-mapper: multipath: 253:3: Reinstating path 8:32.
Jan 15 02:07:10 dell-per440-10 multipathd[1192]: 8:32: reinstated
Jan 15 02:07:10 dell-per440-10 multipathd[1192]: mpatha: remaining active paths: 2
Jan 15 02:07:17 dell-per440-10 kernel: sd 15:0:0:1: rejecting I/O to offline device
Jan 15 02:07:17 dell-per440-10 kernel: device-mapper: multipath: 253:3: Failing path 8:16.
Jan 15 02:07:17 dell-per440-10 multipathd[1192]: sdb: mark as failed
Jan 15 02:07:17 dell-per440-10 multipathd[1192]: mpatha: remaining active paths: 1
Jan 15 02:07:17 dell-per440-10 kernel: sd 16:0:0:1: alua: port group 1100 state N non-preferred supports tolUSNA
Jan 15 02:07:17 dell-per440-10 kernel: sd 16:0:0:1: alua: port group 1100 state N non-preferred supports tolUSNA
Jan 15 02:07:21 dell-per440-10 multipathd[1192]: mpatha: sdb - path offline
Jan 15 02:07:26 dell-per440-10 multipathd[1192]: mpatha: sdb - path offline
Jan 15 02:07:31 dell-per440-10 multipathd[1192]: mpatha: sdb - tur checker reports path is up
Jan 15 02:07:31 dell-per440-10 kernel: device-mapper: multipath: 253:3: Reinstating path 8:16.
Jan 15 02:07:31 dell-per440-10 multipathd[1192]: 8:16: reinstated
Jan 15 02:07:31 dell-per440-10 multipathd[1192]: mpatha: remaining active paths: 2
Jan 15 02:07:37 dell-per440-10 kernel: sd 16:0:0:1: rejecting I/O to offline device
Jan 15 02:07:37 dell-per440-10 kernel: device-mapper: multipath: 253:3: Failing path 8:32.
Jan 15 02:07:37 dell-per440-10 multipathd[1192]: sdc: mark as failed
Jan 15 02:07:37 dell-per440-10 multipathd[1192]: mpatha: remaining active paths: 1
Jan 15 02:07:37 dell-per440-10 kernel: sd 15:0:0:1: alua: port group 1000 state A non-preferred supports tolUSNA
Jan 15 02:07:37 dell-per440-10 kernel: sd 15:0:0:1: alua: port group 1000 state A non-preferred supports tolUSNA
Jan 15 02:07:41 dell-per440-10 multipathd[1192]: mpatha: sdc - path offline
Jan 15 02:07:46 dell-per440-10 multipathd[1192]: mpatha: sdc - path offline
Jan 15 02:07:52 dell-per440-10 multipathd[1192]: mpatha: sdc - tur checker reports path is up
Jan 15 02:07:52 dell-per440-10 kernel: device-mapper: multipath: 253:3: Reinstating path 8:32.
Jan 15 02:07:52 dell-per440-10 multipathd[1192]: 8:32: reinstated
Jan 15 02:07:52 dell-per440-10 multipathd[1192]: mpatha: remaining active paths: 2
Jan 15 02:07:58 dell-per440-10 kernel: sd 15:0:0:1: rejecting I/O to offline device
Jan 15 02:07:58 dell-per440-10 kernel: device-mapper: multipath: 253:3: Failing path 8:16.
Jan 15 02:07:58 dell-per440-10 multipathd[1192]: sdb: mark as failed
Jan 15 02:07:58 dell-per440-10 multipathd[1192]: mpatha: remaining active paths: 1
Jan 15 02:07:58 dell-per440-10 kernel: sd 16:0:0:1: alua: port group 1100 state N non-preferred supports tolUSNA
Jan 15 02:07:58 dell-per440-10 kernel: sd 16:0:0:1: alua: port group 1100 state N non-preferred supports tolUSNA
Jan 15 02:08:03 dell-per440-10 multipathd[1192]: mpatha: sdb - path offline
Jan 15 02:08:08 dell-per440-10 multipathd[1192]: mpatha: sdb - path offline
Jan 15 02:08:13 dell-per440-10 multipathd[1192]: mpatha: sdb - tur checker reports path is up
Jan 15 02:08:13 dell-per440-10 kernel: device-mapper: multipath: 253:3: Reinstating path 8:16.
Jan 15 02:08:13 dell-per440-10 multipathd[1192]: 8:16: reinstated
Jan 15 02:08:13 dell-per440-10 multipathd[1192]: mpatha: remaining active paths: 2
Jan 15 02:08:18 dell-per440-10 kernel: sd 16:0:0:1: rejecting I/O to offline device
Jan 15 02:08:18 dell-per440-10 kernel: device-mapper: multipath: 253:3: Failing path 8:32.
Jan 15 02:08:18 dell-per440-10 multipathd[1192]: sdc: mark as failed
Jan 15 02:08:18 dell-per440-10 multipathd[1192]: mpatha: remaining active paths: 1
Jan 15 02:08:18 dell-per440-10 kernel: sd 15:0:0:1: alua: port group 1000 state A non-preferred supports tolUSNA
Jan 15 02:08:18 dell-per440-10 kernel: sd 15:0:0:1: alua: port group 1000 state A non-preferred supports tolUSNA
Jan 15 02:08:23 dell-per440-10 multipathd[1192]: mpatha: sdc - path offline
Jan 15 02:08:28 dell-per440-10 multipathd[1192]: mpatha: sdc - path offline
Jan 15 02:08:33 dell-per440-10 multipathd[1192]: mpatha: sdc - tur checker reports path is up
Jan 15 02:08:33 dell-per440-10 kernel: device-mapper: multipath: 253:3: Reinstating path 8:32.
Jan 15 02:08:33 dell-per440-10 multipathd[1192]: 8:32: reinstated
Jan 15 02:08:33 dell-per440-10 multipathd[1192]: mpatha: remaining active paths: 2


#do dd 
dd .. 02:06:33
100000+0 records in
100000+0 records out
409600000 bytes (410 MB, 391 MiB) copied, 108.698 s, 3.8 MB/s
dd end 02:08:22

#do switch path
Back to running status for all paths
offline ===== sdb 02:06:36
mpatha (333333330000007d1) dm-3 Linux,scsi_debug
size=4.9G features='0' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=0 status=enabled
| `- 15:0:0:1 sdb 8:16 failed faulty offline
`-+- policy='service-time 0' prio=0 status=active
  `- 16:0:0:1 sdc 8:32 active undef running
online ====== sdb 02:06:46
mpatha (333333330000007d1) dm-3 Linux,scsi_debug
size=4.9G features='0' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=0 status=enabled
| `- 15:0:0:1 sdb 8:16 failed undef running
`-+- policy='service-time 0' prio=0 status=active
  `- 16:0:0:1 sdc 8:32 active undef running

offline ===== sdc 02:06:56
mpatha (333333330000007d1) dm-3 Linux,scsi_debug
size=4.9G features='0' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=0 status=active
| `- 15:0:0:1 sdb 8:16 active undef running
`-+- policy='service-time 0' prio=0 status=enabled
  `- 16:0:0:1 sdc 8:32 failed faulty offline
online ====== sdc 02:07:06
mpatha (333333330000007d1) dm-3 Linux,scsi_debug
size=4.9G features='0' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=0 status=active
| `- 15:0:0:1 sdb 8:16 active undef running
`-+- policy='service-time 0' prio=0 status=enabled
  `- 16:0:0:1 sdc 8:32 failed undef running

offline ===== sdb 02:07:17
mpatha (333333330000007d1) dm-3 Linux,scsi_debug
size=4.9G features='0' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=0 status=enabled
| `- 15:0:0:1 sdb 8:16 failed faulty offline
`-+- policy='service-time 0' prio=0 status=active
  `- 16:0:0:1 sdc 8:32 active undef running
online ====== sdb 02:07:27
mpatha (333333330000007d1) dm-3 Linux,scsi_debug
size=4.9G features='0' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=0 status=enabled
| `- 15:0:0:1 sdb 8:16 failed undef running
`-+- policy='service-time 0' prio=0 status=active
  `- 16:0:0:1 sdc 8:32 active undef running

offline ===== sdc 02:07:37
mpatha (333333330000007d1) dm-3 Linux,scsi_debug
size=4.9G features='0' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=0 status=active
| `- 15:0:0:1 sdb 8:16 active undef running
`-+- policy='service-time 0' prio=0 status=enabled
  `- 16:0:0:1 sdc 8:32 failed faulty offline
online ====== sdc 02:07:47
mpatha (333333330000007d1) dm-3 Linux,scsi_debug
size=4.9G features='0' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=0 status=active
| `- 15:0:0:1 sdb 8:16 active undef running
`-+- policy='service-time 0' prio=0 status=enabled
  `- 16:0:0:1 sdc 8:32 failed undef running

offline ===== sdb 02:07:58
mpatha (333333330000007d1) dm-3 Linux,scsi_debug
size=4.9G features='0' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=0 status=enabled
| `- 15:0:0:1 sdb 8:16 failed faulty offline
`-+- policy='service-time 0' prio=0 status=active
  `- 16:0:0:1 sdc 8:32 active undef running
online ====== sdb 02:08:08
mpatha (333333330000007d1) dm-3 Linux,scsi_debug
size=4.9G features='0' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=0 status=enabled
| `- 15:0:0:1 sdb 8:16 failed undef running
`-+- policy='service-time 0' prio=0 status=active
  `- 16:0:0:1 sdc 8:32 active undef running

offline ===== sdc 02:08:18
mpatha (333333330000007d1) dm-3 Linux,scsi_debug
size=4.9G features='0' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=0 status=active
| `- 15:0:0:1 sdb 8:16 active undef running
`-+- policy='service-time 0' prio=0 status=enabled
  `- 16:0:0:1 sdc 8:32 failed faulty offline
online ====== sdc 02:08:28
mpatha (333333330000007d1) dm-3 Linux,scsi_debug
size=4.9G features='0' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=0 status=active
| `- 15:0:0:1 sdb 8:16 active undef running
`-+- policy='service-time 0' prio=0 status=enabled
  `- 16:0:0:1 sdc 8:32 failed undef running



case 2 ,please check the timestamp from 21:17:40  to 21:18:03

Jan 14 21:17:40 dell-per440-10 kernel: sd 15:0:0:1: rejecting I/O to offline device
Jan 14 21:17:40 dell-per440-10 kernel: device-mapper: multipath: 253:3: Failing path 8:16.
Jan 14 21:17:40 dell-per440-10 multipathd[1192]: sdb: mark as failed
Jan 14 21:17:40 dell-per440-10 multipathd[1192]: mpatha: remaining active paths: 1
Jan 14 21:17:40 dell-per440-10 kernel: sd 16:0:0:1: alua: port group 1100 state N non-preferred supports tolUSNA
Jan 14 21:17:40 dell-per440-10 kernel: sd 16:0:0:1: alua: port group 1100 state N non-preferred supports tolUSNA
Jan 14 21:17:40 dell-per440-10 multipathd[1192]: mpatha: sdb - path offline
Jan 14 21:17:45 dell-per440-10 multipathd[1192]: mpatha: sdb - path offline
Jan 14 21:17:50 dell-per440-10 multipathd[1192]: mpatha: sdb - tur checker reports path is up
Jan 14 21:17:50 dell-per440-10 kernel: device-mapper: multipath: 253:3: Reinstating path 8:16.
Jan 14 21:17:50 dell-per440-10 multipathd[1192]: 8:16: reinstated
Jan 14 21:17:50 dell-per440-10 multipathd[1192]: mpatha: remaining active paths: 2
Jan 14 21:17:51 dell-per440-10 kernel: sd 16:0:0:1: rejecting I/O to offline device
Jan 14 21:17:51 dell-per440-10 kernel: device-mapper: multipath: 253:3: Failing path 8:32.
（offline sdc）
Jan 14 21:17:51 dell-per440-10 kernel: sd 15:0:0:1: alua: port group 1000 state A non-preferred supports tolUSNA
Jan 14 21:17:51 dell-per440-10 kernel: sd 15:0:0:1: alua: port group 1000 state A non-preferred supports tolUSNA
Jan 14 21:17:51 dell-per440-10 multipathd[1192]: sdc: mark as failed
Jan 14 21:17:51 dell-per440-10 multipathd[1192]: mpatha: remaining active paths: 1
Jan 14 21:17:55 dell-per440-10 multipathd[1192]: mpatha: sdc - path offline
Jan 14 21:18:00 dell-per440-10 multipathd[1192]: mpatha: sdc - path offline
Jan 14 21:18:03 dell-per440-10 kernel: sd 15:0:0:1: rejecting I/O to offline device
Jan 14 21:18:03 dell-per440-10 kernel: device-mapper: multipath: 253:3: Failing path 8:16.
Jan 14 21:18:03 dell-per440-10 kernel: blk_update_request: I/O error, dev dm-3, sector 206600 op 0x1:(WRITE) flags 0x8800 phys_seg 1 prio class 0
Jan 14 21:18:03 dell-per440-10 multipathd[1192]: sdb: mark as failed
Jan 14 21:18:03 dell-per440-10 multipathd[1192]: mpatha: remaining active paths: 0
Jan 14 21:18:04 dell-per440-10 kernel: blk_update_request: I/O error, dev dm-3, sector 0 op 0x1:(WRITE) flags 0x8800 phys_seg 1 prio class 0
Jan 14 21:18:05 dell-per440-10 kernel: blk_update_request: I/O error, dev dm-3, sector 0 op 0x1:(WRITE) flags 0x8800 phys_seg 1 prio class 0
Jan 14 21:18:05 dell-per440-10 multipathd[1192]: mpatha: sdb - path offline
Jan 14 21:18:05 dell-per440-10 multipathd[1192]: mpatha: sdc - tur checker reports path is up
Jan 14 21:18:05 dell-per440-10 kernel: device-mapper: multipath: 253:3: Reinstating path 8:32.
Jan 14 21:18:05 dell-per440-10 multipathd[1192]: 8:32: reinstated
Jan 14 21:18:05 dell-per440-10 multipathd[1192]: mpatha: remaining active paths: 1
Jan 14 21:18:05 dell-per440-10 kernel: sd 16:0:0:1: alua: port group 1100 state N non-preferred supports tolUSNA
Jan 14 21:18:05 dell-per440-10 kernel: sd 16:0:0:1: alua: port group 1100 state N non-preferred supports tolUSNA
Jan 14 21:18:10 dell-per440-10 multipathd[1192]: mpatha: sdb - path offline


dd .. 21:17:35
dd: error writing '/dev/mapper/mpatha': Input/output error
25826+0 records in
25825+0 records out
105779200 bytes (106 MB, 101 MiB) copied, 28.0871 s, 3.8 MB/s
dd end 21:18:03



Back to running status for all paths
offline ===== sdb 21:17:40
mpatha (333333330000007d1) dm-3 Linux,scsi_debug
size=4.9G features='0' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=0 status=enabled
| `- 15:0:0:1 sdb 8:16 failed faulty offline
`-+- policy='service-time 0' prio=0 status=active
  `- 16:0:0:1 sdc 8:32 active undef running
online ====== sdb 21:17:50
mpatha (333333330000007d1) dm-3 Linux,scsi_debug
size=4.9G features='0' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=0 status=enabled
| `- 15:0:0:1 sdb 8:16 failed undef running
`-+- policy='service-time 0' prio=0 status=active
  `- 16:0:0:1 sdc 8:32 active undef running

offline ===== sdc 21:17:51
mpatha (333333330000007d1) dm-3 Linux,scsi_debug
size=4.9G features='0' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=0 status=active
| `- 15:0:0:1 sdb 8:16 active undef running
`-+- policy='service-time 0' prio=0 status=enabled
  `- 16:0:0:1 sdc 8:32 failed faulty offline
online ====== sdc 21:18:01
mpatha (333333330000007d1) dm-3 Linux,scsi_debug
size=4.9G features='0' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=0 status=active
| `- 15:0:0:1 sdb 8:16 active undef running
`-+- policy='service-time 0' prio=0 status=enabled
  `- 16:0:0:1 sdc 8:32 failed undef running

(In reply to qing.wang from comment #68)
> Hi,bmarzins,thank for your quick reply, i still have questions on the issue.
> 
> This case come from customer test on FC multipath ,they  
> do alternate switch port to check the path availbility.
> 
> The first question is how to trigger io error in your test.
> In my test they truely located in multipath. if i offline the path of the
> device,
> the io operation can not executed. 
> 
> root@dell-per440-10 ~ # multipath -l
> mpatha (333333330000007d1) dm-3 Linux,scsi_debug
> size=4.9G features='0' hwhandler='1 alua' wp=rw
> |-+- policy='service-time 0' prio=0 status=active
> | `- 15:0:0:1 sdb 8:16 active undef running
> `-+- policy='service-time 0' prio=0 status=enabled
>   `- 16:0:0:1 sdc 8:32 active undef running
> 
> root@dell-per440-10 ~ # cat /proc/partitions 
> major minor  #blocks  name
> 
>    8        0  585498624 sda
>    8        1     614400 sda1
>    8        2    1048576 sda2
>    8        3  583833600 sda3
>  253        0   73400320 dm-0
>  253        1   32849920 dm-1
>  253        2  477581312 dm-2
>    8       16    5120000 sdb
>    8       32    5120000 sdc
>  253        3    5120000 dm-3
> 
> root@dell-per440-10 ~ # echo offline >/sys/block/sdb/device/state
> root@dell-per440-10 ~ #  dd if=/dev/sdb of=/dev/null iflag=direct bs=1M
> count=1
> dd: failed to open '/dev/sdb': No such device or address
> 
> root@dell-per440-10 ~ # echo running >/sys/block/sdb/device/state
> root@dell-per440-10 ~ #  dd if=/dev/sdb of=/dev/null iflag=direct bs=1M
> count=1
> 1+0 records in
> 1+0 records out
> 1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00189668 s, 553 MB/s

You clearly aren't seeing what I did.  When I tried to reproduce your issue on a random machine (with kernel 4.19.8-200), one of the scsi_debug devices was always failing IO,
even if the other one was in the running state. I didn't need to do anything for it to fail.

> ==================================================================
> The second question is it looks like the kernel need time to detect the path
> back to available.
> if there is not enough time to detect path availbility, the io error is
> expexted?

Yes. The kernel will only attempt to use a device if multipathd has marked it as usable. When devices have been marked as failed, multipathd will check their status every 5 seconds. If you have two devices, and where device A is down and device B is up, and you bring device A up and then bring device B down before 5 seconds have passed, there is a chance that multipathd will not notice that device A is up before the kernel tries to do IO on device B and discovers it is down. In this case the kernel will see that there are no valid paths. If the multipath device is not configured to queue IO (it is not in your tests), it will fail the IO.  If you add "no_path_retry 1" to multipath configuration, that would solve this issue, since it means that multipathd will queue IO for one round of path checking after seeing that all the paths are down. If no paths come back after one round of path checking, it will disable queueing and let the pending IOs fail.  Obviously, increasing the sleep time in the tests to something greater than 5 seconds will fix this as well.

Passed host dd test https://bugzilla.redhat.com/show_bug.cgi?id=1854659#c68
and virtio-blk-pci device test https://bugzilla.redhat.com/show_bug.cgi?id=1854659#c59 
after add "no_path_retry  1" into /etc/multipath.conf

defaults {
	user_friendly_names yes
	find_multipaths yes
	enable_foreign "^$"
	no_path_retry 1  
}

Hi pbonzini,
Test failed on scsi-block device, it should be expected result so far.

-device virtio-scsi-pci,id=scsi1,bus=pcie.0-root-port-8,addr=0x0 \
  -blockdev node-name=host_device_stg0,driver=host_device,filename=/dev/mapper/mpathb \
  -blockdev node-name=drive_stg0,driver=raw,file=host_device_stg0 \

-device scsi-block,id=stg0,rerror=stop,werror=stop,drive=drive_stg0,bus=scsi1.0,bootindex=2 \


If The bug is fixed, does it need to set "no_path_retry 1" in host  /etc/multipath.conf like as https://bugzilla.redhat.com/show_bug.cgi?id=1854659#c71 ?

This feature need to cover device scsi-generic?

-device scsi-generic,id=stg0,drive=drive_stg0,bus=scsi1.0,bootindex=2 \

There are two problem on it:
1,device scsi-generic can not work with DM-multipath device.
The vm boot failed due to qemu-kvm: -device scsi-generic,id=stg0,drive=drive_stg0,bus=scsi1.0,bootindex=2: SG_GET_SCSI_ID ioctl failed
2,device scsi-generic not support rerror=stop,werror=stop option, so it can not step in pause even if encount io-error?

*** Bug 1970030 has been marked as a duplicate of this bug. ***

Hi,

scsi-block is enough for testing.  Patches have been posted upstream with subject "dm: dm_blk_ioctl(): implement failover for SG_IO on dm-multipath​".

Has anyone already a build to test the upstream patch?

(In reply to Nils Koenig from comment #75)
> Has anyone already a build to test the upstream patch?

Paolo, 

Can you share the upstream progress and is it possible to prepare a build?

Thanks.

Mike Snitzer has given detailed review comments, so the final patches probably won't look very much like the latest posted version.  Still, early testing is useful so I have started a build at https://brewweb.engineering.redhat.com/brew/taskinfo?taskID=37518510

@pbonzini

Thank you Paolo for providing the quick fix.
I installed that kernel on the hypervisor and after a reboot brought down a path.
VM still is being paused. Multipath config on the host is default as created by RHV:

blacklist {
    protocol "(scsi:adt|scsi:sbp)"
}

# Options defined here override device specific options embedded into
# multipathd.

overrides {
    # NOTE: see comments in default section how to compute this value.
    no_path_retry   16
}


dmesg 

[Thu Jun 17 12:07:42 2021] qla2xxx [0000:cf:00.1]-500b:16: LOOP DOWN detected (2 7 0 0).
[Thu Jun 17 12:07:47 2021] sd 16:0:0:11: rejecting I/O to offline device
[Thu Jun 17 12:07:47 2021] sd 16:0:1:14: rejecting I/O to offline device
[Thu Jun 17 12:07:47 2021] blk_update_request: I/O error, dev sdar, sector 264192 op 0x0:(READ) flags 0x4200 phys_seg 1 prio class 0
[Thu Jun 17 12:07:47 2021] sd 16:0:1:12: rejecting I/O to offline device
[Thu Jun 17 12:07:47 2021] sd 16:0:1:10: rejecting I/O to offline device
[Thu Jun 17 12:07:47 2021] device-mapper: multipath: 253:28: Failing path 66:176.
[Thu Jun 17 12:07:48 2021] device-mapper: multipath: 253:22: Failing path 65:224.
[Thu Jun 17 12:07:48 2021] device-mapper: multipath: 253:21: Failing path 65:208.
[Thu Jun 17 12:07:48 2021] device-mapper: multipath: 253:23: Failing path 65:240.
[Thu Jun 17 12:07:48 2021] device-mapper: multipath: 253:12: Failing path 65:192.
[Thu Jun 17 12:07:48 2021] device-mapper: multipath: 253:24: Failing path 66:0.
[Thu Jun 17 12:07:48 2021] device-mapper: multipath: 253:26: Failing path 66:32.
[Thu Jun 17 12:07:48 2021] device-mapper: multipath: 253:25: Failing path 66:16.
[Thu Jun 17 12:07:48 2021] device-mapper: multipath: 253:13: Failing path 66:48.
[Thu Jun 17 12:07:48 2021] device-mapper: multipath: 253:14: Failing path 66:64.
[Thu Jun 17 12:07:48 2021] device-mapper: multipath: 253:15: Failing path 66:80.
[Thu Jun 17 12:07:48 2021] device-mapper: multipath: 253:30: Failing path 66:112.
[Thu Jun 17 12:07:48 2021] device-mapper: multipath: 253:34: Failing path 66:128.
[Thu Jun 17 12:07:48 2021] device-mapper: multipath: 253:36: Failing path 66:160.
[Thu Jun 17 12:07:48 2021] device-mapper: multipath: 253:35: Failing path 66:144.
[Thu Jun 17 12:07:48 2021] device-mapper: multipath: 253:27: Failing path 66:96.
[Thu Jun 17 12:07:48 2021] device-mapper: multipath: 253:29: Failing path 66:192.
[Thu Jun 17 12:08:13 2021]  rport-16:0-3: blocked FC remote port time out: removing rport
[Thu Jun 17 12:08:22 2021] qla2xxx [0000:cf:00.1]-500a:16: LOOP UP detected (16 Gbps).
[Thu Jun 17 12:08:23 2021] device-mapper: multipath: 253:22: Reinstating path 65:224.
[Thu Jun 17 12:08:23 2021] device-mapper: multipath: 253:21: Reinstating path 65:208.
[Thu Jun 17 12:08:23 2021] device-mapper: multipath: 253:23: Reinstating path 65:240.
[Thu Jun 17 12:08:23 2021] device-mapper: multipath: 253:12: Reinstating path 65:192.
[Thu Jun 17 12:08:23 2021] device-mapper: multipath: 253:24: Reinstating path 66:0.
[Thu Jun 17 12:08:23 2021] device-mapper: multipath: 253:26: Reinstating path 66:32.
[Thu Jun 17 12:08:23 2021] device-mapper: multipath: 253:25: Reinstating path 66:16.
[Thu Jun 17 12:08:23 2021] device-mapper: multipath: 253:30: Reinstating path 66:112.
[Thu Jun 17 12:08:23 2021] device-mapper: multipath: 253:34: Reinstating path 66:128.
[Thu Jun 17 12:08:23 2021] device-mapper: multipath: 253:28: Reinstating path 66:176.
[Thu Jun 17 12:08:23 2021] device-mapper: multipath: 253:36: Reinstating path 66:160.
[Thu Jun 17 12:08:23 2021] device-mapper: multipath: 253:35: Reinstating path 66:144.
[Thu Jun 17 12:08:23 2021] device-mapper: multipath: 253:27: Reinstating path 66:96.
[Thu Jun 17 12:08:23 2021] device-mapper: multipath: 253:29: Reinstating path 66:192.
[Thu Jun 17 12:08:23 2021] sd 16:0:1:2: Power-on or device reset occurred
[Thu Jun 17 12:08:23 2021] sd 16:0:1:2: alua: rtpg retry
[Thu Jun 17 12:08:23 2021] sd 16:0:1:2: [alua] Sense Key : Unit Attention [current] 
[Thu Jun 17 12:08:23 2021] sd 16:0:1:2: [alua] Add. Sense: I_T nexus loss occurred
[Thu Jun 17 12:08:23 2021] sd 16:0:1:3: alua: port group f033 state A non-preferred supports toluSNA
[Thu Jun 17 12:08:23 2021] sd 16:0:1:8: alua: port group f033 state A non-preferred supports toluSNA
[Thu Jun 17 12:08:23 2021] sd 16:0:1:6: alua: port group f033 state A non-preferred supports toluSNA
[Thu Jun 17 12:08:23 2021] sd 16:0:1:4: alua: port group f033 state A non-preferred supports toluSNA
[Thu Jun 17 12:08:23 2021] sd 16:0:1:1: alua: port group f033 state A non-preferred supports toluSNA
[Thu Jun 17 12:08:23 2021] sd 16:0:0:11: Power-on or device reset occurred
[Thu Jun 17 12:08:23 2021] sd 16:0:0:11: alua: rtpg retry
[Thu Jun 17 12:08:23 2021] sd 16:0:0:11: [alua] Sense Key : Unit Attention [current] 
[Thu Jun 17 12:08:23 2021] sd 16:0:0:11: [alua] Add. Sense: I_T nexus loss occurred
[Thu Jun 17 12:08:23 2021] sd 16:0:0:4: alua: port group f032 state A non-preferred supports toluSNA
[Thu Jun 17 12:08:23 2021] sd 16:0:0:3: alua: port group f032 state A non-preferred supports toluSNA
[Thu Jun 17 12:08:23 2021] sd 16:0:1:7: alua: port group f033 state A non-preferred supports toluSNA
[Thu Jun 17 12:08:23 2021] sd 16:0:0:13: alua: port group f032 state A non-preferred supports toluSNA
[Thu Jun 17 12:08:23 2021] sd 16:0:0:1: alua: port group f032 state A non-preferred supports toluSNA
[Thu Jun 17 12:08:23 2021] sd 16:0:1:3: alua: port group f033 state A non-preferred supports toluSNA
[Thu Jun 17 12:08:23 2021] sd 16:0:0:9: alua: port group f032 state A non-preferred supports toluSNA
[Thu Jun 17 12:08:23 2021] sd 16:0:1:8: alua: port group f033 state A non-preferred supports toluSNA
[Thu Jun 17 12:08:23 2021] sd 16:0:1:4: alua: port group f033 state A non-preferred supports toluSNA
[Thu Jun 17 12:08:23 2021] sd 16:0:1:1: alua: port group f033 state A non-preferred supports toluSNA
[Thu Jun 17 12:08:23 2021] sd 16:0:0:5: alua: port group f032 state A non-preferred supports toluSNA
[Thu Jun 17 12:08:23 2021] sd 16:0:1:6: alua: port group f033 state A non-preferred supports toluSNA
[Thu Jun 17 12:08:23 2021] sd 16:0:1:7: alua: port group f033 state A non-preferred supports toluSNA
[Thu Jun 17 12:08:23 2021] sd 16:0:0:1: alua: port group f032 state A non-preferred supports toluSNA
[Thu Jun 17 12:08:23 2021] sd 16:0:0:13: alua: port group f032 state A non-preferred supports toluSNA
[Thu Jun 17 12:08:23 2021] sd 16:0:0:3: alua: port group f032 state A non-preferred supports toluSNA
[Thu Jun 17 12:08:23 2021] sd 16:0:0:9: alua: port group f032 state A non-preferred supports toluSNA
[Thu Jun 17 12:08:23 2021] sd 16:0:0:5: alua: port group f032 state A non-preferred supports toluSNA
[Thu Jun 17 12:08:23 2021] sd 16:0:0:4: alua: port group f032 state A non-preferred supports toluSNA
[Thu Jun 17 12:08:25 2021] sd 16:0:0:11: alua: port group f032 state A non-preferred supports toluSNA
[Thu Jun 17 12:08:25 2021] sd 16:0:1:2: alua: port group f033 state A non-preferred supports toluSNA
[Thu Jun 17 12:08:25 2021] sd 16:0:0:11: alua: port group f032 state A non-preferred supports toluSNA
[Thu Jun 17 12:08:25 2021] sd 16:0:1:2: alua: port group f033 state A non-preferred supports toluSNA
[Thu Jun 17 12:08:28 2021] device-mapper: multipath: 253:13: Reinstating path 66:48.
[Thu Jun 17 12:08:28 2021] device-mapper: multipath: 253:14: Reinstating path 66:64.
[Thu Jun 17 12:08:28 2021] device-mapper: multipath: 253:15: Reinstating path 66:80.
[Thu Jun 17 12:08:28 2021] sd 16:0:1:14: alua: port group f033 state A non-preferred supports toluSNA
[Thu Jun 17 12:08:28 2021] sd 16:0:1:12: alua: port group f033 state A non-preferred supports toluSNA
[Thu Jun 17 12:08:28 2021] sd 16:0:1:10: alua: port group f033 state A non-preferred supports toluSNA
[Thu Jun 17 12:08:28 2021] sd 16:0:1:14: alua: port group f033 state A non-preferred supports toluSNA
[Thu Jun 17 12:08:28 2021] sd 16:0:1:12: alua: port group f033 state A non-preferred supports toluSNA
[Thu Jun 17 12:08:28 2021] sd 16:0:1:10: alua: port group f033 state A non-preferred supports toluSNA


Did I oversee something and missed a setting?

The block device used is /dev/mapper/36000d31005771c000000000000000066

-blockdev {"driver":"host_device","filename":"/dev/mapper/36000d31005771c000000000000000066","aio":"native","node-name":"libvirt-1-storage","cache":{"direct":true,"no-flush":false},"auto-read-only":true,"discard":"unmap"} 
-blockdev {"node-name":"libvirt-1-format","read-only":false,"cache":{"direct":true,"no-flush":false},"driver":"raw","file":"libvirt-1-storage"} 
-device scsi-block,bus=ua-eff86ed8-d744-43d1-8f69-463c4974691f.0,channel=0,scsi-id=0,lun=0,drive=libvirt-1-format,id=ua-3686d9c0-d674-4ddd-b801-a69d7657ea47,bootindex=2,werror=stop,rerror=stop

I see werror=stop,rerror=stop, is this the culprit and should be werror=report,rerror=report?


Qemu command line

/usr/libexec/qemu-kvm -name guest=lu0565,debug-threads=on -S -object secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-1-lu0565/master-key.aes -machine pc-q35-rhel8.2.0,accel=kvm,usb=off,dump-guest-core=off -cpu Cascadelake-Server-noTSX,rdtscp=on,x2apic=on,hypervisor=on,mpx=off,pku=on,arch-capabilities=on,rdctl-no=on,ibrs-all=on,skip-l1dfl-vmentry=on,mds-no=on,tsc-frequency=2194843000,l3-cache=on -m 65536 -overcommit mem-lock=off -smp 1,maxcpus=16,sockets=16,dies=1,cores=1,threads=1 -object iothread,id=iothread1 -mem-prealloc -mem-path /dev/hugepages/libvirt/qemu/1-lu0565 -numa node,nodeid=0,cpus=0-15,mem=65536 -uuid 2f168ccf-3415-4cd4-8ad7-2a9f2ed2c9a0 -smbios type=1,manufacturer=Red Hat,product=RHEL,version=8.4-0.6.el8,serial=24254ba2-332e-11eb-a02b-0a94efbb050f,uuid=2f168ccf-3415-4cd4-8ad7-2a9f2ed2c9a0,sku=8.2.0,family=RHV -smbios type=2,manufacturer=Red Hat,product=RHEL-AV -no-user-config -nodefaults -device sga -chardev socket,id=charmonitor,fd=38,server,nowait -mon chardev=charmonitor,id=monitor,mode=control -rtc base=utc -no-shutdown -global ICH9-LPC.disable_s3=1 -global ICH9-LPC.disable_s4=1 -boot strict=on -device pcie-root-port,port=0x10,chassis=1,id=pci.1,bus=pcie.0,multifunction=on,addr=0x2 -device pcie-root-port,port=0x11,chassis=2,id=pci.2,bus=pcie.0,addr=0x2.0x1 -device pcie-root-port,port=0x12,chassis=3,id=pci.3,bus=pcie.0,addr=0x2.0x2 -device pcie-root-port,port=0x13,chassis=4,id=pci.4,bus=pcie.0,addr=0x2.0x3 -device pcie-root-port,port=0x14,chassis=5,id=pci.5,bus=pcie.0,addr=0x2.0x4 -device pcie-root-port,port=0x15,chassis=6,id=pci.6,bus=pcie.0,addr=0x2.0x5 -device pcie-root-port,port=0x16,chassis=7,id=pci.7,bus=pcie.0,addr=0x2.0x6 -device pcie-root-port,port=0x17,chassis=8,id=pci.8,bus=pcie.0,addr=0x2.0x7 -device pcie-root-port,port=0x18,chassis=9,id=pci.9,bus=pcie.0,multifunction=on,addr=0x3 -device pcie-root-port,port=0x19,chassis=10,id=pci.10,bus=pcie.0,addr=0x3.0x1 -device pcie-root-port,port=0x1a,chassis=11,id=pci.11,bus=pcie.0,addr=0x3.0x2 -device pcie-root-port,port=0x1b,chassis=12,id=pci.12,bus=pcie.0,addr=0x3.0x3 -device pcie-root-port,port=0x1c,chassis=13,id=pci.13,bus=pcie.0,addr=0x3.0x4 -device pcie-root-port,port=0x1d,chassis=14,id=pci.14,bus=pcie.0,addr=0x3.0x5 -device pcie-root-port,port=0x1e,chassis=15,id=pci.15,bus=pcie.0,addr=0x3.0x6 -device pcie-root-port,port=0x1f,chassis=16,id=pci.16,bus=pcie.0,addr=0x3.0x7 -device qemu-xhci,p2=8,p3=8,id=ua-eb8be61b-c280-41d4-ade9-cae87820dc26,bus=pci.4,addr=0x0 -device virtio-scsi-pci,iothread=iothread1,id=ua-eff86ed8-d744-43d1-8f69-463c4974691f,bus=pci.2,addr=0x0 -device virtio-serial-pci,id=ua-59640c37-9490-4a52-9d19-618aa046f370,max_ports=16,bus=pci.3,addr=0x0 -device ide-cd,bus=ide.2,id=ua-60932971-6d22-4724-96aa-39e2a933e735,werror=report,rerror=report -blockdev {"driver":"host_device","filename":"/dev/mapper/36000d31005771c000000000000000066","aio":"native","node-name":"libvirt-1-storage","cache":{"direct":true,"no-flush":false},"auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-1-format","read-only":false,"cache":{"direct":true,"no-flush":false},"driver":"raw","file":"libvirt-1-storage"} -device scsi-block,bus=ua-eff86ed8-d744-43d1-8f69-463c4974691f.0,channel=0,scsi-id=0,lun=0,drive=libvirt-1-format,id=ua-3686d9c0-d674-4ddd-b801-a69d7657ea47,bootindex=2,werror=stop,rerror=stop -netdev tap,fd=40,id=hostua-88832ca8-481e-41e5-8e0a-1b926be2b7d0,vhost=on,vhostfd=41 -device virtio-net-pci,host_mtu=1500,netdev=hostua-88832ca8-481e-41e5-8e0a-1b926be2b7d0,id=ua-88832ca8-481e-41e5-8e0a-1b926be2b7d0,mac=dc:10:00:00:00:65,bus=pci.1,addr=0x0,bootindex=1 -chardev socket,id=charserial0,fd=42,server,nowait -device isa-serial,chardev=charserial0,id=serial0 -chardev socket,id=charchannel0,fd=43,server,nowait -device virtserialport,bus=ua-59640c37-9490-4a52-9d19-618aa046f370.0,nr=1,chardev=charchannel0,id=channel0,name=ovirt-guest-agent.0 -chardev socket,id=charchannel1,fd=44,server,nowait -device virtserialport,bus=ua-59640c37-9490-4a52-9d19-618aa046f370.0,nr=2,chardev=charchannel1,id=channel1,name=org.qemu.guest_agent.0 -chardev spicevmc,id=charchannel2,name=vdagent -device virtserialport,bus=ua-59640c37-9490-4a52-9d19-618aa046f370.0,nr=3,chardev=charchannel2,id=channel2,name=com.redhat.spice.0 -spice port=5900,tls-port=5901,addr=10.76.35.60,x509-dir=/etc/pki/vdsm/libvirt-spice,tls-channel=main,tls-channel=display,tls-channel=inputs,tls-channel=cursor,tls-channel=playback,tls-channel=record,tls-channel=smartcard,tls-channel=usbredir,seamless-migration=on -device qxl-vga,id=ua-ad77dcb6-2cf0-42ad-be32-dc6eca1207f3,ram_size=67108864,vram_size=8388608,vram64_size_mb=0,vgamem_mb=16,max_outputs=1,bus=pcie.0,addr=0x1 -incoming defer -object rng-random,id=objua-ec8e1a1b-a035-4f75-8fb0-814a75d648fd,filename=/dev/urandom -device virtio-rng-pci,rng=objua-ec8e1a1b-a035-4f75-8fb0-814a75d648fd,id=ua-ec8e1a1b-a035-4f75-8fb0-814a75d648fd,bus=pci.5,addr=0x0 -sandbox off -msg timestamp=on

> I see werror=stop,rerror=stop, is this the culprit and should be werror=report,rerror=report?

No it should be stop because the I/O should be retried on a different path.

Paolo, is there a way to see why a VM is paused?

> Paolo, is there a way to see why a VM is paused?

It's because I/O failed, i.e. the patches didn't work as advertised.

Maybe you can try "sg_dd blk_sgio=1 if=/dev/mpath/... bs=4096 count=100 of=/dev/null" to remove QEMU from the equation.  You can use strace to check that it's really using SG_IO, and use blk_sgio=0 vs. 1 to test both the traditional read/write I/O and SCSI passthrough.

For what it's worth, I applied Martin Wilck's v3 patches to kernel-4.18.0-314.el8
and I do see a difference in behavior with the FC switch port toggle test of the
Initiator port (I am not able to test toggling the Target FC port at this time because
there are other tests running on this storage array we would have to quiesce first).

With -314.el8 I can readily reproduce I/O errors on the guest by disabling the FC
initiator port.  However, with the patches added, so far, I do not see errors.  So the
patches do seem to be doing something.  I will also review the code to see if it
looks like there are any cases that might not be covered.

I spoke with Mike and he and I have some concerns about how the functionality was
implemented, so this may need further refinement upstream.  But we may be getting somewhere.

Let me know how you are testing. I assume without Qemu.
I will test here as I can interrupt the ports on my switch. 
Regards
Laurence

See the Description in the BZ, I have a RHEL7 guest installed on a FC SAN LUN
that is multipathed on the hypervisor with the guest XML device configuration like:

    <disk type='block' device='lun'>
      <driver name='qemu' type='raw' cache='none' io='native'/>
      <source dev='/dev/mapper/mpathb'/>
      <backingStore/>
      <target dev='sda' bus='scsi'/>
      <alias name='scsi0-0-0-0'/>
      <address type='drive' controller='0' bus='0' target='0' unit='0'/>
    </disk>

Boot the guest, start I/O (e.g. writes) then disable/enable one of FC switch ports.
It may take several tries to induce the failure as it is timing dependent if the
path down is detected before the I/O is issued.  The basic problem is the lack of
path retry if the HBA returns the command.  If multipath *knows* the path is down
then it won't select it to begin with, but this isn't always the case e.g. if an
array controller is rebooted, or if a target side switch port is disabled and we
haven't seen the RSCN yet, etc... basically what we get with fault insertion testing.

With Martin Wilck's patches applied the behavior is different and the I/O errors
are not readily reproducible.  Note that Mike is working with Martin to further
refine the patch set so there will be more iterations of the code.

Hi guys,

I did test and compare two machines, one with a stock kernel and one with a Kernel that contains Martin's patch.
The tests were pursued on bare metal, so qemu-kvm was out of the equation.
To me, they both behaved the same.
Maybe I have a misunderstanding of the issue or my testcase is not sufficient.
On both machines, IO did not block if only one path goes down.
Or is the impact of Martin's patch more subtle or the IO would never be entirely blocked?
Or, the Dell SC5200 (rebranded Compellant) storage behaves different?


This is the machine with Martin's patch (host: lu0584)

[root@lu0584 ~]# cat /sys/class/fc_host/host??/port_state
Online
Online
[root@lu0584 ~]# multipath -ll /dev/mapper/36000d31005771c000000000000000065
36000d31005771c000000000000000065 dm-29 COMPELNT,Compellent Vol
size=50G features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
`-+- policy='service-time 0' prio=50 status=enabled
  |- 15:0:1:13 sdab 65:176 active ready running
  `- 16:0:0:13 sdas 66:192 active ready running

[root@lu0584 ~]# date
Fri Jun 25 07:42:18 CEST 2021

[root@lu0584 ~]# uname -a
Linux lu0584.wdf.sap.corp 4.18.0-314.el8.test.x86_64 #1 SMP Wed Jun 16 06:47:51 EDT 2021 x86_64 x86_64 x86_64 GNU/Linux


<------------------- now one path is disabled on FC switch --------------------------------------->

[root@lu0584 ~]# cat /sys/class/fc_host/host??/port_state
Linkdown
Online
[root@lu0584 ~]# multipath -ll /dev/mapper/36000d31005771c000000000000000065
36000d31005771c000000000000000065 dm-29 COMPELNT,Compellent Vol
size=50G features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
`-+- policy='service-time 0' prio=50 status=enabled
  |- 15:0:1:13 sdab 65:176 active faulty running
  `- 16:0:0:13 sdas 66:192 active ready running

[root@lu0584 ~]# time sg_dd blk_sgio=1 if=/dev/mapper/36000d31005771c000000000000000065 of=/dev/null count=10000000
Assume default 'bs' (block size) of 512 bytes
10000000+0 records in
10000000+0 records out

real    0m8.640s
user    0m0.046s
sys    0m1.043s

[root@lu0584 ~]# multipath -ll /dev/mapper/36000d31005771c000000000000000065
36000d31005771c000000000000000065 dm-29 COMPELNT,Compellent Vol
size=50G features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
`-+- policy='service-time 0' prio=50 status=active
  |- 15:0:1:13 sdab 65:176 failed faulty running
  `- 16:0:0:13 sdas 66:192 active ready running


<----------------------- second path is deactivated -------------------------------------------------------->

[root@lu0584 ~]# multipath -ll /dev/mapper/36000d31005771c000000000000000065
36000d31005771c000000000000000065 dm-29 COMPELNT,Compellent Vol
size=50G features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
`-+- policy='service-time 0' prio=0 status=enabled
  `- 16:0:0:13 sdas 66:192 active faulty running


---> IO is blocked


##################################################################################

This is the machine with stock kernel (host lu0551)

[root@lu0551 ~]# cat /sys/class/fc_host/host??/port_state
Online
Online
[root@lu0551 ~]# date
Fri Jun 25 07:48:59 CEST 2021
[root@lu0551 ~]# uname -a
Linux lu0551.wdf.sap.corp 4.18.0-305.3.1.el8_4.x86_64 #1 SMP Mon May 17 10:08:25 EDT 2021 x86_64 x86_64 x86_64 GNU/Linux
[root@lu0551 ~]#
[root@lu0551 ~]# date
Fri Jun 25 07:49:34 CEST 2021


<--------------------------------------- First path is disabled on switch ------------------------------------------------>


[root@lu0551 ~]# cat /sys/class/fc_host/host??/port_state
Linkdown
Online
[root@lu0551 ~]# multipath -ll /dev/mapper/36000d31005771c000000000000000065
36000d31005771c000000000000000065 dm-17 COMPELNT,Compellent Vol
size=50G features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
`-+- policy='service-time 0' prio=50 status=active
  |- 14:0:1:13 sdaj 66:48  active faulty running
  `- 15:0:1:13 sdai 66:32  active ready running

[root@lu0551 ~]# time sg_dd blk_sgio=1 if=/dev/mapper/36000d31005771c000000000000000065 of=/dev/null count=10000000
Assume default 'bs' (block size) of 512 bytes
10000000+0 records in
10000000+0 records out

real    0m6.912s
user    0m0.021s
sys    0m0.325s

<------------------------------------- IO works ------------------------------------------------->

[root@lu0551 ~]# date
Fri Jun 25 07:50:05 CEST 2021


[root@lu0551 ~]# multipath -ll /dev/mapper/36000d31005771c000000000000000065
36000d31005771c000000000000000065 dm-17 COMPELNT,Compellent Vol
size=50G features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
`-+- policy='service-time 0' prio=50 status=active
  |- 14:0:1:13 sdaj 66:48  failed faulty running
  `- 15:0:1:13 sdai 66:32  active ready running



<-------------------------------- second path is disabled -------------------------------------->

[root@lu0551 ~]# date
Fri Jun 25 07:50:30 CEST 2021
[root@lu0551 ~]# multipath -ll /dev/mapper/36000d31005771c000000000000000065
36000d31005771c000000000000000065 dm-17 COMPELNT,Compellent Vol
size=50G features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 14:0:1:13 sdaj 66:48  failed faulty running
  `- 15:0:1:13 sdai 66:32  failed faulty running
[root@lu0551 ~]# time sg_dd blk_sgio=1 if=/dev/mapper/36000d31005771c000000000000000065 of=/dev/null count=10000000
Assume default 'bs' (block size) of 512 bytes


INQUIRY failed on /dev/mapper/36000d31005771c000000000000000065

real    0m20.597s
user    0m0.000s
sys    0m0.014s
[root@lu0551 ~]#


<---------------------- IO was blocked until path(s) were reenabled ------------------------->


[root@lu0551 ~]# date
Fri Jun 25 07:51:01 CEST 2021






[root@lu0551 ~]# multipath -ll /dev/mapper/36000d31005771c000000000000000065
36000d31005771c000000000000000065 dm-17 COMPELNT,Compellent Vol
size=50G features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
`-+- policy='service-time 0' prio=50 status=active
  |- 15:0:1:13 sdai 66:32  active ready running
  `- 14:0:1:13 sdaj 66:48  active ready running

Bulk update: Move RHEL-AV bugs to RHEL8

------- Comment From chavez.com 2021-11-08 11:10 EDT-------
*** Bug 183977 has been marked as a duplicate of this bug. ***

------- Comment From mainamdar.com 2022-05-09 09:13 EDT-------
*** Bug 183930 has been marked as a duplicate of this bug. ***

------- Comment From mainamdar.com 2022-05-09 09:15 EDT-------
*** Bug 182703 has been marked as a duplicate of this bug. ***

Created attachment 1907169 [details]
sosreport of ilx22r

Created attachment 1907170 [details]
sosreport of ilx21r

------- Comment From sthoufee.com 2023-01-16 02:55 EDT-------
Any update on this bug?