Created attachment 1644307 [details]
detailed backtrace info about qemu core dump

Additonal info:
(1)
when booting the guest with PF attached (without multifunction=on) and then hot-unplugging the PF, qemu core dump will happen as well.
(2)
Using qemu-kvm-4.1.0-14.module+el8.2.0+4677+51176c2e.x86_64 , everything works well.

I feel like notifier_remove() is trying to access an invalid pointer.

Thread 1 (Thread 0x7f71413be700 (LWP 3626)):
#0  0x000055e076d55d2d in notifier_remove (notifier=notifier@entry=0x55e078e6b8b8) at util/notify.c:31
#1  0x000055e076a379e9 in kvm_irqchip_remove_change_notifier (n=n@entry=0x55e078e6b8b8) at /usr/src/debug/qemu-kvm-4.2.0-2.scrmod+el8.2.0+5137+ec04dc0c.x86_64/accel/kvm/kvm-all.c:1409
#2  0x000055e076a7ce98 in vfio_exitfn (pdev=<optimized out>) at /usr/src/debug/qemu-kvm-4.2.0-2.scrmod+el8.2.0+5137+ec04dc0c.x86_64/hw/vfio/pci.c:3103
        vdev = 0x55e078e6ac70
        __func__ = "vfio_exitfn"
#3  0x000055e076bc220b in pci_qdev_unrealize (dev=<optimized out>, errp=<optimized out>) at hw/pci/pci.c:1131
        pci_dev = 0x55e078e6ac70
        __func__ = "pci_qdev_unrealize"
        pc = 0x55e077b53e80
#4  0x000055e076b5e441 in device_set_realized (obj=<optimized out>, value=<optimized out>, errp=0x0) at hw/core/qdev.c:932
        local_errp = <optimized out>
        dev = 0x55e078e6ac70
        __func__ = "device_set_realized"
        dc = 0x55e077b53e80
        hotplug_ctrl = <optimized out>
        bus = 0x0
        local_err = 0x0
        unattached_parent = false
        unattached_count = 26
#5  0x000055e076c70c8b in property_set_bool (obj=0x55e078e6ac70, v=<optimized out>, name=<optimized out>, opaque=0x55e078d90fb0, errp=0x0) at qom/object.c:2078
        prop = 0x55e078d90fb0
        value = false
        local_err = 0x0
#6  0x000055e076c75153 in object_property_set_qobject (obj=0x55e078e6ac70, value=<optimized out>, name=0x55e076e2a8bd "realized", errp=0x0) at qom/qom-qobject.c:26
        v = 0x7f71300a4000
#7  0x000055e076c729b9 in object_property_set_bool (obj=0x55e078e6ac70, value=<optimized out>, name=0x55e076e2a8bd "realized", errp=0x0) at qom/object.c:1336
        qbool = 0x7f7130669ec0
#8  0x000055e076bc83b7 in pcie_unplug_device (bus=<optimized out>, dev=0x55e078e6ac70, opaque=<optimized out>) at hw/pci/pcie.c:463
        hotplug_ctrl = 0x55e078776760
        __func__ = "pcie_unplug_device"
#9  0x000055e076bc2551 in pci_for_each_device_under_bus (opaque=<optimized out>, fn=<optimized out>, bus=<optimized out>) at hw/pci/pci.c:1640
        d = <optimized out>
#10 0x000055e076bc2551 in pci_for_each_device (bus=bus@entry=0x55e078777040, bus_num=<optimized out>, fn=fn@entry=0x55e076bc8330 <pcie_unplug_device>, opaque=opaque@entry=0x0) at hw/pci/pci.c:1652
#11 0x000055e076bc9338 in pcie_cap_slot_write_config (dev=0x55e078776760, old_slt_ctl=<optimized out>, old_slt_sta=<optimized out>, addr=108, val=<optimized out>, len=<optimized out>) at hw/pci/pcie.c:668
        sec_bus = 0x55e078777040
        pos = 2021093440
        exp_cap = 0x55e0787793b4 "\020HB\001"
        sltsta = <optimized out>
        __func__ = "pcie_cap_slot_write_config"
#12 0x000055e076bbb1df in rp_write_config (d=0x55e078776760, address=108, val=2033, len=2) at hw/pci-bridge/pcie_root_port.c:41
        root_cmd = 7
        slt_ctl = 1777
        slt_sta = 64
#13 0x000055e076a29ef7 in memory_region_write_accessor (mr=<optimized out>, addr=<optimized out>, value=<optimized out>, size=<optimized out>, shift=<optimized out>, mask=<optimized out>, attrs=...) at /usr/src/debug/qemu-kvm-4.2.0-2.scrmod+el8.2.0+5137+ec04dc0c.x86_64/memory.c:483
        tmp = <optimized out>
#14 0x000055e076a2812e in access_with_adjusted_size (addr=addr@entry=0, value=value@entry=0x7f71413bd508, size=size@entry=2, access_size_min=<optimized out>, access_size_max=<optimized out>, access_fn=0x55e076a29e80 <memory_region_write_accessor>, mr=0x55e077db0b50, attrs=...) at /usr/src/debug/qemu-kvm-4.2.0-2.scrmod+el8.2.0+5137+ec04dc0c.x86_64/memory.c:544
        access_mask = 65535
        access_size = 2
        i = <optimized out>
        r = 0
#15 0x000055e076a2c00c in memory_region_dispatch_write (mr=0x55e077db0b50, addr=0, data=<optimized out>, op=<optimized out>, attrs=...) at /usr/src/debug/qemu-kvm-4.2.0-2.scrmod+el8.2.0+5137+ec04dc0c.x86_64/memory.c:1475
        size = 2
#16 0x000055e0769d9097 in flatview_write_continue (fv=0x7f7130809f10, addr=3324, attrs=..., buf=0x7f714fada000 <error: Cannot access memory at address 0x7f714fada000>, len=2, addr1=<optimized out>, l=<optimized out>, mr=0x55e077db0b50) at /usr/src/debug/qemu-kvm-4.2.0-2.scrmod+el8.2.0+5137+ec04dc0c.x86_64/include/qemu/host-utils.h:164
        ptr = <optimized out>
        val = <optimized out>
        result = 0
        release_lock = true
#17 0x000055e0769d92b6 in flatview_write (fv=0x7f7130809f10, addr=3324, attrs=..., buf=0x7f714fada000 <error: Cannot access memory at address 0x7f714fada000>, len=2) at /usr/src/debug/qemu-kvm-4.2.0-2.scrmod+el8.2.0+5137+ec04dc0c.x86_64/exec.c:3169
        l = 2
        addr1 = 0
        mr = <optimized out>
        result = 0
#18 0x000055e0769dd7cf in address_space_write () at /usr/src/debug/qemu-kvm-4.2.0-2.scrmod+el8.2.0+5137+ec04dc0c.x86_64/exec.c:3259
#19 0x000055e076a3af24 in kvm_cpu_exec (cpu=<optimized out>) at /usr/src/debug/qemu-kvm-4.2.0-2.scrmod+el8.2.0+5137+ec04dc0c.x86_64/accel/kvm/kvm-all.c:2116
        run = <optimized out>
        ret = <optimized out>
        run_ret = <optimized out>
#20 0x000055e076a1fd5e in qemu_kvm_cpu_thread_fn (arg=0x55e077c732f0) at /usr/src/debug/qemu-kvm-4.2.0-2.scrmod+el8.2.0+5137+ec04dc0c.x86_64/cpus.c:1318
        cpu = 0x55e077c732f0
        r = <optimized out>
#21 0x000055e076d48ed4 in qemu_thread_start (args=0x55e077c9c050) at util/qemu-thread-posix.c:519
        __clframe = {__cancel_routine = <optimized out>, __cancel_arg = 0x0, __do_it = 1, __cancel_type = <optimized out>}
        qemu_thread_args = 0x55e077c9c050
        start_routine = 0x55e076a1fca0 <qemu_kvm_cpu_thread_fn>
        arg = 0x55e077c732f0
        r = <optimized out>
#22 0x00007f714aa272de in start_thread () at /lib64/libpthread.so.0
#23 0x00007f714a758e83 in clone () at /lib64/libc.so.6

Also CC David Gibson for c5478fea27ac47ed3b57e0489a49b62f36024763.

Steps to test this bug in  qemu-kvm-4.2.0-4.el8.fix_unplug_vfio2.x86_64:
(1) start a RHEL82 guest with a XXV710 PF
/usr/libexec/qemu-kvm -name rhel8-2 -M q35 -enable-kvm \
-monitor stdio \
-nodefaults \
-m 4G \
-boot menu=on \
 -cpu Haswell-noTSX-IBRS \
-device pcie-root-port,id=root.1,chassis=1,addr=0x2.0,multifunction=on \
-device pcie-root-port,id=root.2,chassis=2,addr=0x2.1 \
-device pcie-root-port,id=root.3,chassis=3,addr=0x2.2 \
-device pcie-root-port,id=root.4,chassis=4,addr=0x2.3 \
-device pcie-root-port,id=root.5,chassis=5,addr=0x2.4 \
-device pcie-root-port,id=root.6,chassis=6,addr=0x2.5 \
-device pcie-root-port,id=root.7,chassis=7,addr=0x2.6 \
-device pcie-root-port,id=root.8,chassis=8,addr=0x2.7 \
-smp 2,sockets=1,cores=2,threads=2,maxcpus=4 \
-qmp tcp:0:6666,server,nowait \
-blockdev node-name=back_image,driver=file,cache.direct=on,cache.no-flush=off,filename=/home/images/rhel8.2_q35.qcow2,aio=threads \
-blockdev node-name=drive-virtio-disk0,driver=qcow2,cache.direct=on,cache.no-flush=off,file=back_image \
-device virtio-blk-pci,drive=drive-virtio-disk0,id=disk0,bus=root.1 \
-device VGA,id=video1,bus=root.2  \
-vnc :0 \
-device virtio-net-pci,netdev=nic1,id=vnet0,mac=54:43:00:1a:11:33,bus=root.3 \
-netdev tap,id=nic1,script=/etc/qemu-ifup,vhost=on \
-device vfio-pci,host=0000:83:00.0,bus=root.4,id=pf1 \

(2)hot unplug the XXV710 PF from RHEL82 guest and check the qmp output 
(QMP) {"execute":"device_del","arguments":{"id":"pf1"}}
output:
{"return": {}}
{"timestamp": {"seconds": 1577768700, "microseconds": 434135}, "event": "DEVICE_DELETED", "data": {"device": "pf1", "path": "/machine/peripheral/pf1"}}

(3)Check device status with "info pci" on host and check device status with "lspci" in guest 
The XXV710 PF has been hot unplug successfully.

(4)reboot the RHEL82 guest.
everything goes well.
===========================================================================================================================================
Steps to test this bug in  qemu-kvm-4.2.0-4.module+el8.2.0+5220+e82621dc.x86_64:
Using the same test steps as above, qemu core dump happens after hot unplug PF from the RHEL82 guest.

Thanks for the quick run, yanghliu.  Patch posted upstream:

https://lists.gnu.org/archive/html/qemu-devel/2019-12/msg05493.html

*** Bug 1785052 has been marked as a duplicate of this bug. ***

Verification:
host:
kernel:4.18.0-169.el8.x86_64
qemu-kvm-4.2.0-6.module+el8.2.0+5453+31b2b136.x86_64
guest:
kernel:4.18.0-169.el8.x86_64

Steps:
(1)start a vm with XXV710 PF
/usr/libexec/qemu-kvm -name rhel8-2 -M q35 -enable-kvm \
-monitor stdio \
-nodefaults \
-m 4G \
-boot menu=on \
-cpu Haswell-noTSX-IBRS \
-device pcie-root-port,id=root.1,chassis=1,addr=0x2.0,multifunction=on \
-device pcie-root-port,id=root.2,chassis=2,addr=0x2.1 \
-device pcie-root-port,id=root.3,chassis=3,addr=0x2.2 \
-device pcie-root-port,id=root.4,chassis=4,addr=0x2.3 \
-device pcie-root-port,id=root.5,chassis=5,addr=0x2.4 \
-device pcie-root-port,id=root.6,chassis=6,addr=0x2.5 \
-device pcie-root-port,id=root.7,chassis=7,addr=0x2.6 \
-device pcie-root-port,id=root.8,chassis=8,addr=0x2.7 \
-smp 2,sockets=1,cores=2,threads=2,maxcpus=4 \
-qmp tcp:0:5555,server,nowait \
-blockdev node-name=back_image,driver=file,cache.direct=on,cache.no-flush=off,filename=/home/images/rhel82.qcow2,aio=threads \
-blockdev node-name=drive-virtio-disk0,driver=qcow2,cache.direct=on,cache.no-flush=off,file=back_image \
-device virtio-blk-pci,drive=drive-virtio-disk0,id=disk0,bus=root.1 \
-device VGA,id=video1,bus=root.2  \
-vnc :0 \
-device vfio-pci,host=0000:83:00.1,bus=root.3,id=pf1 \

(2)hot unplug the XXV710 PF from RHEL82 guest and check the qmp output 
QMP: {"execute":"device_del","arguments":{"id":"pf1"}}

output:
{"return": {}}
{"timestamp": {"seconds": 1579153332, "microseconds": 645452}, "event": "DEVICE_DELETED", "data": {"device": "pf1", "path": "/machine/peripheral/pf1"}}

(3)Check device status with "info pci" on host and check device status with "lspci" in guest 
The XXV710 PF has been hot unplug successfully.

(4)reboot the RHEL82 guest.
everything goes well.

(5)repeat step 1-step 4 with the XL710,82599ES,NetXtreme BCM57810,Mellanox MT27800,82576
All NICs can be hot unplugged from vm successfully.


According to the test result, this problem has been fixed well.
Move the bug status to 'VERIFIED'.

*** Bug 1784676 has been marked as a duplicate of this bug. ***

QEMU has been recently split into sub-components and as a one-time operation to avoid breakage of tools, we are setting the QEMU sub-component of this BZ to "General". Please review and change the sub-component if necessary the next time you review this BZ. Thanks

Since the problem described in this bug report should be
resolved in a recent advisory, it has been closed with a
resolution of ERRATA.

For information on the advisory, and where to find the updated
files, follow the link below.

If the solution does not work for you, open a new bug report.

https://access.redhat.com/errata/RHBA-2020:2017

*** Bug 1786027 has been marked as a duplicate of this bug. ***