Bug 1185167

Check "info qtree" in src qemu-kvm:
(qemu) info qtree
bus: main-system-bus
  type System
  dev: i440FX-pcihost, id ""
    bus: pci.0
      type PCI
      dev: virtio-scsi-pci, id "scsi1"
        dev-prop: ioeventfd = on
        dev-prop: vectors = 4
        dev-prop: indirect_desc = on
        dev-prop: event_idx = on
        dev-prop: num_queues = 1
        dev-prop: max_sectors = 65535
        dev-prop: cmd_per_lun = 128
        dev-prop: hotplug = on
        dev-prop: param_change = on
        bus-prop: addr = 13.0
        bus-prop: romfile = <null>
        bus-prop: rombar = 1
        bus-prop: multifunction = off
        class SCSI controller, addr 00:13.0, pci id 1af4:1004 (sub 1af4:0008)
        bar 0: i/o at 0xffffffffffffffff [0x3e]
        bar 1: mem at 0xffffffffffffffff [0xffe]
        bus: scsi1.0
          type SCSI
          dev: scsi-hd, id "data-disk2"
            dev-prop: drive = drive-scsi-disk
            dev-prop: logical_block_size = 512
            dev-prop: physical_block_size = 512
            dev-prop: min_io_size = 0
            dev-prop: opt_io_size = 0
            dev-prop: bootindex = 0
            dev-prop: discard_granularity = 0
            dev-prop: ver = "0.12.1"
            dev-prop: serial = <null>
            dev-prop: wwn = 0x0
            dev-prop: removable = off
            bus-prop: channel = 0
            bus-prop: scsi-id = 0
            bus-prop: lun = 0
      dev: virtio-serial-pci, id "virtio-serial0"
        dev-prop: vectors = 32
        dev-prop: class = 0x780
        dev-prop: indirect_desc = on
        dev-prop: event_idx = on
        dev-prop: max_ports = 31
        dev-prop: flow_control = 1
        bus-prop: addr = 07.0
        bus-prop: romfile = <null>
        bus-prop: rombar = 1
        bus-prop: multifunction = off
        class Class 0780, addr 00:07.0, pci id 1af4:1003 (sub 1af4:0003)
        bar 0: i/o at 0xffffffffffffffff [0x1e]
        bar 1: mem at 0xffffffffffffffff [0xffe]
        bus: virtio-serial0.0
          type virtio-serial-bus
          dev: virtserialport, id "channel1"
            dev-prop: chardev = charchannel1
            bus-prop: nr = 2
            bus-prop: name = "org.qemu.guest_agent.0"
             dev-prop-int: id: 2
             dev-prop-int: guest_connected: 0
             dev-prop-int: host_connected: 0
             dev-prop-int: throttled: 0
          dev: virtserialport, id "channel0"
            dev-prop: chardev = charchannel0
            bus-prop: nr = 1
            bus-prop: name = "com.redhat.spice.0"
             dev-prop-int: id: 1
             dev-prop-int: guest_connected: 0
             dev-prop-int: host_connected: 0
             dev-prop-int: throttled: 0
      dev: virtio-scsi-pci, id "scsi0"
        dev-prop: ioeventfd = on
        dev-prop: vectors = 4
        dev-prop: indirect_desc = on
        dev-prop: event_idx = on
        dev-prop: num_queues = 1
        dev-prop: max_sectors = 65535
        dev-prop: cmd_per_lun = 234
        dev-prop: hotplug = on
        dev-prop: param_change = on
        bus-prop: addr = 08.0
        bus-prop: romfile = <null>
        bus-prop: rombar = 1
        bus-prop: multifunction = off
        class SCSI controller, addr 00:08.0, pci id 1af4:1004 (sub 1af4:0008)
        bar 0: i/o at 0xffffffffffffffff [0x3e]
        bar 1: mem at 0xffffffffffffffff [0xffe]
        bus: scsi0.0
          type SCSI
      dev: ich9-usb-uhci3, id ""
        dev-prop: masterbus = "usb0.0"
        dev-prop: firstport = 4
        bus-prop: addr = 05.2
        bus-prop: romfile = <null>
        bus-prop: rombar = 1
        bus-prop: multifunction = off
        class USB controller, addr 00:05.2, pci id 8086:2936 (sub 1af4:1100)
        bar 4: i/o at 0xffffffffffffffff [0x1e]
      dev: ich9-usb-uhci2, id ""
        dev-prop: masterbus = "usb0.0"
        dev-prop: firstport = 2
        bus-prop: addr = 05.1
        bus-prop: romfile = <null>
        bus-prop: rombar = 1
        bus-prop: multifunction = off
        class USB controller, addr 00:05.1, pci id 8086:2935 (sub 1af4:1100)
        bar 4: i/o at 0xffffffffffffffff [0x1e]
      dev: ich9-usb-uhci1, id ""
        dev-prop: masterbus = "usb0.0"
        dev-prop: firstport = 0
        bus-prop: addr = 05.0
        bus-prop: romfile = <null>
        bus-prop: rombar = 1
        bus-prop: multifunction = on
        class USB controller, addr 00:05.0, pci id 8086:2934 (sub 1af4:1100)
        bar 4: i/o at 0xffffffffffffffff [0x1e]
      dev: ich9-usb-ehci1, id "usb0"
        dev-prop: freq = 1000
        dev-prop: maxframes = 128
        bus-prop: addr = 05.7
        bus-prop: romfile = <null>
        bus-prop: rombar = 1
        bus-prop: multifunction = off
        class USB controller, addr 00:05.7, pci id 8086:293a (sub 1af4:1100)
        bar 0: mem at 0xffffffffffffffff [0xffe]
        bus: usb0.0
          type USB
          dev: usb-host, id "hostdev0"
            dev-prop: hostbus = 0
            dev-prop: hostaddr = 0
            dev-prop: hostport = <null>
            dev-prop: vendorid = 0x0
            dev-prop: productid = 0x0
            dev-prop: isobufs = 4
            bus-prop: port = <null>
            bus-prop: create_unique_serial = 1
            bus-prop: msos-desc = off
            addr 0.0, port 3, speed 1.5, name USB Host Device
          dev: usb-redir, id "redir0"
            dev-prop: chardev = charredir0
            dev-prop: debug = 0
            dev-prop: filter = <null>
            bus-prop: port = <null>
            bus-prop: create_unique_serial = 1
            bus-prop: msos-desc = off
            addr 0.0, port 2, speed 1.5, name USB Redirection Device
          dev: usb-tablet, id "input0"
            dev-prop: migrate = 1
            bus-prop: port = <null>
            bus-prop: create_unique_serial = 1
            bus-prop: msos-desc = off
            addr 0.0, port 1, speed 12, name QEMU USB Tablet, attached
      dev: PIIX4_PM, id ""
        dev-prop: smb_io_base = 45312
        dev-prop: disable_s3 = 1
        dev-prop: disable_s4 = 1
        dev-prop: s4_val = 2
        bus-prop: addr = 01.3
        bus-prop: romfile = <null>
        bus-prop: rombar = 1
        bus-prop: multifunction = off
        class Bridge, addr 00:01.3, pci id 8086:7113 (sub 1af4:1100)
      dev: piix3-ide, id ""
        bus-prop: addr = 01.1
        bus-prop: romfile = <null>
        bus-prop: rombar = 1
        bus-prop: multifunction = off
        class IDE controller, addr 00:01.1, pci id 8086:7010 (sub 1af4:1100)
        bar 4: i/o at 0xffffffffffffffff [0xe]
        bus: ide.1
          type IDE
        bus: ide.0
          type IDE
      dev: qxl-vga, id ""
        dev-prop: ram_size = 67108864
        dev-prop: vram_size = 16777216
        dev-prop: revision = 4
        dev-prop: debug = 0
        dev-prop: guestdebug = 0
        dev-prop: cmdlog = 0
        bus-prop: addr = 02.0
        bus-prop: romfile = "vgabios-qxl.bin"
        bus-prop: rombar = 1
        bus-prop: multifunction = off
        class VGA controller, addr 00:02.0, pci id 1b36:0100 (sub 1af4:1100)
        bar 0: mem at 0xffffffffffffffff [0x3fffffe]
        bar 1: mem at 0xffffffffffffffff [0xfffffe]
        bar 2: mem at 0xffffffffffffffff [0x1ffe]
        bar 3: i/o at 0xffffffffffffffff [0x1e]
        bar 6: mem at 0xffffffffffffffff [0xfffe]
      dev: PIIX3, id ""
        bus-prop: addr = 01.0
        bus-prop: romfile = <null>
        bus-prop: rombar = 1
        bus-prop: multifunction = on
        class ISA bridge, addr 00:01.0, pci id 8086:7000 (sub 1af4:1100)
        bus: isa.0
          type ISA
          dev: sga, id ""
          dev: isa-fdc, id ""
            dev-prop: driveA = <null>
            dev-prop: driveB = <null>
            dev-prop: bootindexA = -1
            dev-prop: bootindexB = -1
            dev-prop: migrate_dir = 1
            isa irq 6
          dev: i8042, id ""
            isa irqs 1,12
          dev: isa-serial, id ""
            dev-prop: index = 0
            dev-prop: iobase = 0x3f8
            dev-prop: irq = 4
            dev-prop: chardev = serial0
            dev-prop: wakeup = 0
            isa irq 4
          dev: mc146818rtc, id ""
            dev-prop: base_year = 2000
            isa irq 8
      dev: i440FX, id ""
        bus-prop: addr = 00.0
        bus-prop: romfile = <null>
        bus-prop: rombar = 1
        bus-prop: multifunction = off
        class Host bridge, addr 00:00.0, pci id 8086:1237 (sub 1af4:1100)
(qemu)

Created attachment 983185 [details]
check "info qtree" on dst qemu-kvm

I test qemu-kvm-rhev-1.5.3-60.el7_0.11.x86_64, also hit this this issue.

Gerd, any good ideas?
It looks like the usb device has changed id's from source to target?

> -spice port=5910,password=redhat-vga,disable-ticketing

Please add seamless-migration=on here.

> (qemu) migration -d tcp:10.55.106.67:5800

"client_migrate_info spice $host $port" is needed beforehand.

spice client (which has access to the redirected usb device) is involved here, so we have to make sure seamless migration is turned on and the spice client is notified correctly.  Please retest after fixing the testcase.

Q1: is a spice client connected while running the test?
Q2: if so, is a redirected usb device active while running the test?

Adding Hans for additional clues.

(In reply to Gerd Hoffmann from comment #8)
> > -spice port=5910,password=redhat-vga,disable-ticketing
> 
> Please add seamless-migration=on here.
> 
> > (qemu) migration -d tcp:10.55.106.67:5800
> 
> "client_migrate_info spice $host $port" is needed beforehand.
> 
> spice client (which has access to the redirected usb device) is involved
> here, so we have to make sure seamless migration is turned on and the spice
> client is notified correctly.  Please retest after fixing the testcase.
> 
> Q1: is a spice client connected while running the test?
> Q2: if so, is a redirected usb device active while running the test?
> 
> Adding Hans for additional clues.

Add seamless-migration=on and before migration, run "client_migrate_info spice $host $port", the results are as following:

1. When no spice client is conneted, the migation can be finished normally.

2. Before migration, a spice client is connected (remote-viewer spice://$src-host:$port), the usb device is not active, in fact, I do not plugin a usb device to the spice client. The migration is failed and qemu-kvm in dst host is quits as comment 0:
(qemu) main_channel_link: add main channel client
inputs_connect: inputs channel client create
red_dispatcher_set_cursor_peer: 
red_dispatcher_loadvm_commands: 
qemu-kvm: usbredirparser: error unserialize caps mismatch
qemu: warning: error while loading state for instance 0x0 of device '0000:00:05.7/2/usb-redir'
qemu-kvm: load of migration failed: Operation not permitted

Migration guest with source mode usbredir device and without spice client connection also has such issue:
 
1.Start usbredirserver:
# usbredirserver -p 4000 0951:1625

2.Prepare a running guest with source mode usbredir device.

qemu command line:
-chardev socket,id=charredir0,host=10.66.6.6,port=4000 -device usb-redir,chardev=charredir0,id=redir0

3.# virsh migrate rhel6.6-GUI qemu+ssh://10.66.4.141/system --live --verbose
root.4.141's password: 
Migration: [ 97 %]error: internal error: early end of file from monitor: possible problem:
RHEL-6 compat: ich9-usb-uhci1: irq_pin = 3
RHEL-6 compat: ich9-usb-uhci2: irq_pin = 3
RHEL-6 compat: ich9-usb-uhci3: irq_pin = 3
2015-08-13T10:07:20.549375Z qemu-kvm: usbredirparser: error unserialize caps mismatch
2015-08-13T10:07:20.549401Z qemu-kvm: error while loading state for instance 0x0 of device '0000:00:06.7/2/usb-redir'
2015-08-13T10:07:20.549476Z qemu-kvm: load of migration failed: Operation not permitted

Looking at current usbredir (0.7 - not the rhel version) it seems like the comparison is very tight:

usbredirparser/usbredirparser.c : usbredirparser_unserialize

    if (memcmp(parser->our_caps, orig_caps,
               USB_REDIR_CAPS_SIZE * sizeof(int32_t)) != 0) {
        ERROR("error unserialize caps mismatch");
        return -1;
    }

so any feature added is going to cause a migration failure.

equires changes to usbredir library, Dave worked on this issue.

The problem here is that qemu has asked for extra capabilities in newer versions; if we'd spotted this in 7.0 we could have fixed it in qemu by making sure that the init code made the capabilities machine-type dependent.  Unfortunately the 7.x boat has sailed, and thus we can't break the machine types that 7.0/7.1 have set.

The fix in usbredir is just to be more forgiving and allow an incoming stream with some capabilities that we don't have;  (If we actually depend on those caps that might get a bit hairy - but at least rhel6.x's don't seem too bad?)

so moving from qemu->usbredir and going to dupe the other qemu* version onto it as well.

*** Bug 1186095 has been marked as a duplicate of this bug. ***

Fix posted upstream:

http://lists.freedesktop.org/archives/spice-devel/2015-September/021959.html

Upstream patch applies to RHEL7 usbredir version. Moving to POST

reproducer and fix is clear - qa_ack+

I test migration from rhel6.8 -> rhel7.2.z using the command line of comment 0, before migration, use two spice clients to connect src and dst vm. Migration can be finished normally. The tested version is:

src host:
qemu-kvm-rhev-0.12.1.2-2.490.el6.x86_64
kernel-2.6.32-626.el6.x86_64
usbredir-0.5.1-3.el6.x86_64

dst host:
qemu-kvm-rhev-2.3.0-31.el7_2.10.x86_64
kernel-3.10.0-363.el7.x86_64
usbredir-0.6-8.el7.x86_64

Since the problem described in this bug report should be
resolved in a recent advisory, it has been closed with a
resolution of ERRATA.

For information on the advisory, and where to find the updated
files, follow the link below.

If the solution does not work for you, open a new bug report.

https://rhn.redhat.com/errata/RHBA-2016-2229.html