Description of problem:
Guest didn't boot up with vfio-pci device 

Version-Release number of selected component (if applicable):
qemu-img-rhev-2.5.0-3.el7.x86_64
kernel-3.10.0-369.el7.x86_64

How reproducible:
3/3

Steps to Reproduce:
1.unbind the nic device from the host and bind the nic device to the vfio
[root@localhost q35]# cat /boot/grub2/grub.cfg |grep iommu
	linux16 /vmlinuz-3.10.0-369.el7.x86_64 root=/dev/mapper/rhel-root ro crashkernel=auto rd.lvm.lv=rhel/root rd.lvm.lv=rhel/swap rhgb quiet intel_iommu=on 
[root@localhost Desktop]# modprobe vfio-pci
[root@localhost Desktop]# lsmod |grep vfio
vfio_iommu_type1       17632  0 
vfio_pci               36735  0 
vfio                   25291  2 vfio_iommu_type1,vfio_pci
[root@localhost Desktop]# echo "0000:00:19.0" >/sys/bus/pci/devices/0000\:00\:19.0/driver/unbind 
[root@localhost Desktop]# echo "8086 153a" >/sys/bus/pci/drivers/vfio-pci/new_id 
[root@localhost Desktop]# echo "8086 153a" >/sys/bus/pci/drivers/vfio-pci/remove_id 
[root@localhost Desktop]# ll /sys/bus/pci/devices/0000\:00\:19.0/driver
lrwxrwxrwx. 1 root root 0 Mar 29 15:57 /sys/bus/pci/devices/0000:00:19.0/driver -> ../../../bus/pci/drivers/vfio-pci

2. Boot the guest with following cli:

/usr/libexec/qemu-kvm \
-M pc \
-cpu SandyBridge \
-nodefaults -rtc base=utc \
-m 4G \
-smp 2,sockets=2,cores=1,threads=1 \
-enable-kvm \
-name rhel6 \
-uuid 990ea161-6b67-47b2-b803-19fb01d30d12 \
-smbios type=1,manufacturer='Red Hat',product='RHEV Hypervisor',version=el6,serial=koTUXQrb,uuid=feebc8fd-f8b0-4e75-abc3-e63fcdb67170 \
-k en-us \
-serial unix:/tmp/serial0,server,nowait \
-boot menu=on \
-bios /usr/share/seabios/bios.bin \
-vga std \
-vnc :0 \
-drive file=/home/rhel.img,if=none,id=drive-virtio-disk0,format=qcow2,cache=none,werror=stop,rerror=stop,aio=threads \
-device virtio-blk-pci,bus=pci.0,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=0 \
-chardev file,path=/home/seabios.log,id=seabios \
-device isa-debugcon,chardev=seabios,iobase=0x402 \
-device vfio-pci,host=00:19.0,id=assin_nic \
-monitor stdio \

3. Check the nic device with "info qtree" of hmp

(qemu) info qtree
bus: main-system-bus
  type System
  dev: kvm-ioapic, id ""
    gpio-in "" 24
    gsi_base = 0 (0x0)
    mmio 00000000fec00000/0000000000001000
  dev: i440FX-pcihost, id ""
    pci-hole64-size = 18446744073709551615 (16 EiB)
    short_root_bus = 0 (0x0)
    bus: pci.0
      type PCI
      dev: vfio-pci, id "assin_nic"
        host = "0000:00:19.0"
        x-intx-mmap-timeout-ms = 1100 (0x44c)
        x-vga = false
        x-req = true
        x-no-mmap = false
        x-no-kvm-intx = false
        x-no-kvm-msi = false
        x-no-kvm-msix = false
        x-pci-vendor-id = 32902 (0x8086)
        x-pci-device-id = 5434 (0x153a)
        x-pci-sub-vendor-id = 4294967295 (0xffffffff)
        x-pci-sub-device-id = 4294967295 (0xffffffff)
        addr = 04.0
        romfile = ""
        rombar = 1 (0x1)
        multifunction = false
        command_serr_enable = true
        class Ethernet controller, addr 00:04.0, pci id 8086:153a (sub 17aa:309f)
        bar 0: mem at 0xfebc0000 [0xfebdffff]
        bar 1: mem at 0xfebf2000 [0xfebf2fff]
        bar 2: i/o at 0xc040 [0xc05f]

4. Check the guest with "remote-viewer"



Actual results:

Guest didn't boot up and following is the seabios log

[root@localhost home]# cat seabios-bk.log 
SeaBIOS (version seabios-1.7.5-11.el7)
Running on QEMU (q35)
Running on KVM
RamSize: 0x80000000 [cmos]
Relocating init from 0x000e3f59 to 0x7ffb1cd0 (size 57960)
Found QEMU fw_cfg
RamBlock: addr 0x0000000000000000 len 0x0000000080000000 [e820]
RamBlock: addr 0x0000000100000000 len 0x0000000080000000 [e820]
Moving pm_base to 0x600
boot order:
1: /pci@i0cf8/scsi@2/disk@0,0
CPU Mhz=3396
=== PCI bus & bridge init ===
PCI: pci_bios_init_bus_rec bus = 0x0
=== PCI device probing ===
Found 7 PCI devices (max PCI bus is 00)
=== PCI new allocation pass #1 ===
PCI: check devices
=== PCI new allocation pass #2 ===
PCI: IO: c000 - c0bf
PCI: 32: 00000000c0000000 - 00000000fec00000
PCI: map device bdf=00:02.0  bar 0, addr 0000c000, size 00000040 [io]
PCI: map device bdf=00:1f.3  bar 4, addr 0000c040, size 00000040 [io]
PCI: map device bdf=00:03.0  bar 2, addr 0000c080, size 00000020 [io]
PCI: map device bdf=00:1f.2  bar 4, addr 0000c0a0, size 00000020 [io]
PCI: map device bdf=00:03.0  bar 0, addr febc0000, size 00020000 [mem]
PCI: map device bdf=00:01.0  bar 6, addr febe0000, size 00010000 [mem]
PCI: map device bdf=00:01.0  bar 2, addr febf0000, size 00001000 [mem]
PCI: map device bdf=00:02.0  bar 1, addr febf1000, size 00001000 [mem]
PCI: map device bdf=00:03.0  bar 1, addr febf2000, size 00001000 [mem]
PCI: map device bdf=00:1f.2  bar 5, addr febf3000, size 00001000 [mem]
PCI: map device bdf=00:01.0  bar 0, addr fd000000, size 01000000 [prefmem]
PCI: init bdf=00:00.0 id=8086:29c0
PCI: init bdf=00:01.0 id=1234:1111
PCI: init bdf=00:02.0 id=1af4:1001
PCI: init bdf=00:03.0 id=8086:153a
PCI: init bdf=00:1f.0 id=8086:2918
Q35 LPC init: elcr=00 0c
Using pmtimer, ioport 0x608
PCI: init bdf=00:1f.2 id=8086:2922
PCI: init bdf=00:1f.3 id=8086:2930
PCI: Using 00:01.0 for primary VGA
Found 2 cpu(s) max supported 2 cpu(s)
Copying PIR from 0x7ffbfcef to 0x000f2150
Copying MPTABLE from 0x00006e64/7ffa8b10 to 0x000f2050
Copying SMBIOS entry point from 0x00006e64 to 0x000f1df0
Scan for VGA option rom
Running option rom at c000:0003
Start SeaVGABIOS (version seabios-1.7.5-11.el7)
enter vga_post:
   a=00000008  b=0000ffff  c=00000000  d=0000ffff ds=0000 es=f000 ss=0000
  si=00000000 di=00007200 bp=00000000 sp=00006dc6 cs=f000 ip=d313  f=0000
VBE DISPI: bdf 00:01.0, bar 0
VBE DISPI: lfb_addr=fd000000, size 16 MB
Turning on vga text mode console
set VGA mode 3
SeaBIOS (version seabios-1.7.5-11.el7)
Machine UUID fdc8ebfe-b0f8-754e-abc3-e63fcdb67170
All threads complete.
Found 0 lpt ports
Found 1 serial ports
AHCI controller at 1f.2, iobase febf3000, irq 10
found virtio-blk at 0:2
Searching bootorder for: /pci@i0cf8/*@2
PS2 keyboard initialized
All threads complete.
Scan for option roms
Searching bootorder for: /rom@genroms/kvmvapic.bin

Press ESC for boot menu.




Expected results:

guest can boot up successfully and the nic device can passthrough to the guest successfully


Additional info:

1. We can reproduced the issue with q35 machine type

2. Guest can boot up successfully if we deleted the vfio-pci device

3. Didn't hit the issue with qemu-kvm-rhev-2.3.0-31.el7_2.10 and kernel-3.10.0-369.el7.x86_64