Created attachment 1756159 [details] vm xml Description of problem: Qemu crashed during startup with error: Unexpected error in qemu_ram_resize() at ../softmmu/physmem.c:1769: 2021-02-10T08:58:20.736861Z qemu-kvm: Size too large: /rom@etc/table-loader: 0x2000 > 0x1000: Invalid argument 2021-02-10 08:58:21.313+0000: shutting down, reason=crashed Version-Release number of selected component (if applicable): qemu-kvm-5.2.0-6.module+el8.4.0+9871+53903be9.x86_64 How reproducible: 100% Steps to Reproduce: 1.Start vm: /usr/libexec/qemu-kvm \ -name guest=rhel8.3.0,debug-threads=on \ -S \ -object secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-1-rhel8.3.0/master-key.aes \ -machine pc-q35-rhel8.4.0,accel=kvm,usb=off,vmport=off,smm=on,dump-guest-core=off,nvdimm=on,kernel_irqchip=split \ -global mch.extended-tseg-mbytes=48 \ -cpu Skylake-Server-IBRS,ss=on,vmx=on,pdcm=on,hypervisor=on,tsc-adjust=on,clflushopt=on,umip=on,pku=on,md-clear=on,stibp=on,arch-capabilities=on,ssbd=on,xsaves=on,ibpb=on,amd-stibp=on,amd-ssbd=on,skip-l1dfl-vmentry=on,pschange-mc-no=on,vme=on \ -bios /usr/share/seabios/bios.bin \ -m size=2097152k,slots=8,maxmem=4194304k \ -overcommit mem-lock=off \ -smp 8,maxcpus=16,sockets=2,dies=1,cores=4,threads=2 \ -object memory-backend-file,id=ram-node0,mem-path=/var/lib/libvirt/qemu/ram/1-rhel8.3.0/ram-node0,share=yes,size=1073741824 \ -numa node,nodeid=0,cpus=0,cpus=2,cpus=4,cpus=6,cpus=8,cpus=10,cpus=12,cpus=14,memdev=ram-node0 \ -object memory-backend-ram,id=ram-node1,size=1073741824 \ -numa node,nodeid=1,cpus=1,cpus=3,cpus=5,cpus=7,cpus=9,cpus=11,cpus=13,cpus=15,memdev=ram-node1 \ -numa dist,src=0,dst=0,val=10 \ -numa dist,src=0,dst=1,val=19 \ -numa dist,src=1,dst=0,val=19 \ -numa dist,src=1,dst=1,val=10 \ -object memory-backend-file,id=memnvdimm1,mem-path=/tmp/nvdimm,prealloc=yes,size=268435456 \ -device nvdimm,node=1,label-size=131072,memdev=memnvdimm1,id=nvdimm1,slot=1 \ -uuid df899f5c-db94-48b2-867a-e0c266b59b7a \ -device vmgenid,guid=001b2039-ca77-4352-ab4a-433521eabf48,id=vmgenid0 \ -no-user-config \ -nodefaults \ -device sga \ -chardev socket,id=charmonitor,fd=37,server,nowait \ -mon chardev=charmonitor,id=monitor,mode=control \ -rtc base=2021-02-10T08:58:19,driftfix=slew \ -global kvm-pit.lost_tick_policy=delay \ -no-hpet \ -no-shutdown \ -global ICH9-LPC.disable_s3=0 \ -global ICH9-LPC.disable_s4=1 \ -boot menu=on,reboot-timeout=0,splash-time=3000,strict=on \ -device intel-iommu,intremap=on,caching-mode=on,device-iotlb=on,aw-bits=48 \ -device pcie-root-port,port=0x10,chassis=1,id=pci.1,bus=pcie.0,multifunction=on,addr=0x2 \ -device pcie-root-port,port=0x11,chassis=2,id=pci.2,bus=pcie.0,addr=0x2.0x1 \ -device pcie-pci-bridge,id=pci.3,bus=pci.1,addr=0x0 \ -device pcie-root-port,port=0x12,chassis=4,id=pci.4,bus=pcie.0,addr=0x2.0x2 \ -device pcie-root-port,port=0x13,chassis=5,id=pci.5,bus=pcie.0,addr=0x2.0x3 \ -device pcie-root-port,port=0x14,chassis=6,id=pci.6,bus=pcie.0,addr=0x2.0x4 \ -device pcie-root-port,port=0x15,chassis=7,id=pci.7,bus=pcie.0,addr=0x2.0x5 \ -device pcie-root-port,port=0x16,chassis=8,id=pci.8,hotplug=on,bus=pcie.0,addr=0x2.0x6 \ -device pcie-root-port,port=0x17,chassis=9,id=pci.9,hotplug=off,bus=pcie.0,addr=0x2.0x7 \ -device pcie-root-port,port=0x18,chassis=10,id=pci.10,bus=pcie.0,multifunction=on,addr=0x3 \ -device pcie-root-port,port=0x19,chassis=11,id=pci.11,bus=pcie.0,addr=0x3.0x1 \ -device pcie-root-port,port=0x1a,chassis=12,id=pci.12,bus=pcie.0,addr=0x3.0x2 \ -device piix3-usb-uhci,id=usb,bus=pci.3,addr=0x4 \ -device ich9-usb-ehci1,id=usb1,bus=pcie.0,addr=0x1d.0x7 \ -device ich9-usb-uhci1,masterbus=usb1.0,firstport=0,bus=pcie.0,multifunction=on,addr=0x1d \ -device ich9-usb-uhci2,masterbus=usb1.0,firstport=2,bus=pcie.0,addr=0x1d.0x1 \ -device ich9-usb-uhci3,masterbus=usb1.0,firstport=4,bus=pcie.0,addr=0x1d.0x2 \ -device nec-usb-xhci,p2=4,p3=4,id=usb2,bus=pci.4,addr=0x0 \ -device qemu-xhci,p2=15,p3=15,id=usb3,bus=pci.5,addr=0x0 \ -device virtio-scsi-pci,id=scsi0,cmd_per_lun=5,max_sectors=10000,ioeventfd=on,bus=pci.6,addr=0x0 \ -device virtio-serial-pci,id=virtio-serial0,bus=pci.7,addr=0x0 \ -device usb-hub,id=hub0,bus=usb.0,port=1 \ -device usb-ccid,id=ccid0,bus=usb.0,port=1.1 \ -blockdev '{"driver":"file","filename":"/var/lib/libvirt/images/RHEL-8.4.0-20210209.n.0-x86_64.qcow2","aio":"threads","node-name":"libvirt-1-storage","cache":{"direct":true,"no-flush":false},"auto-read-only":true,"discard":"unmap"}' \ -blockdev '{"node-name":"libvirt-1-format","read-only":false,"discard":"unmap","cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":"libvirt-1-storage","backing":null}' \ -device virtio-blk-pci,bus=pci.8,addr=0x0,drive=libvirt-1-format,id=virtio-disk0,bootindex=1,write-cache=on,werror=stop,rerror=stop \ -netdev tap,fd=39,id=hostnet0 \ -device rtl8139,netdev=hostnet0,id=net0,mac=52:54:00:c8:f0:40,bus=pci.2,addr=0x0 \ -chardev spicevmc,id=charsmartcard0,name=smartcard \ -device ccid-card-passthru,chardev=charsmartcard0,id=smartcard0,bus=ccid0.0 \ -add-fd set=2,fd=41 \ -chardev pty,id=charserial0,logfile=/dev/fdset/2,logappend=on \ -device isa-serial,chardev=charserial0,id=serial0 \ -add-fd set=3,fd=43 \ -chardev file,id=charserial1,path=/dev/fdset/3,append=on \ -device isa-serial,chardev=charserial1,id=serial1 \ -chardev spicevmc,id=charchannel0,name=vdagent \ -device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=com.redhat.spice.0 \ -chardev socket,id=charchannel1,fd=42,server,nowait \ -device virtserialport,bus=virtio-serial0.0,nr=2,chardev=charchannel1,id=channel1,name=org.qemu.guest_agent.0 \ -chardev socket,id=charchannel2,fd=40,server,nowait \ -netdev user,guestfwd=tcp:10.0.2.1:4600-chardev:charchannel2,id=channel2 \ -device usb-tablet,id=input0,bus=usb.0,port=1.2 \ -device usb-kbd,id=input2,bus=usb.0,port=1.3 \ -spice port=5900,addr=127.0.0.1,agent-mouse=on,disable-ticketing,image-compression=auto_glz,streaming-video=filter,seamless-migration=on \ -vnc 127.0.0.1:1 \ -device qxl-vga,id=ua-cf99c3a1-ed5c-4d68-898b-0d45bdb6913d,ram_size=67108864,vram_size=8388608,vram64_size_mb=128,vgamem_mb=16,max_outputs=1,bus=pcie.0,addr=0x1 \ -device intel-hda,id=ua-4c8b8b34-d2a8-4482-ab03-aa389e77cca7,bus=pci.3,addr=0x3 \ -device hda-duplex,id=ua-4c8b8b34-d2a8-4482-ab03-aa389e77cca7-codec0,bus=ua-4c8b8b34-d2a8-4482-ab03-aa389e77cca7.0,cad=0 \ -chardev spicevmc,id=charredir0,name=usbredir \ -device 'usb-redir,chardev=charredir0,id=redir0,filter=0x08:0x1234:0xBEEF:0x0256:1|-1:-1:-1:-1:0,bus=usb.0,port=1.4' \ -device virtio-balloon-pci,id=balloon0,bus=pci.9,addr=0x0,deflate-on-oom=on \ -sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \ -msg timestamp=on Actual results: qemu-kvm crashed Expected results: Additional info:
Thread 1 (Thread 0x7f3dbe40d700 (LWP 48073)): #0 0x00007f3dcd72037f in raise () from /lib64/libc.so.6 #1 0x00007f3dcd70adb5 in abort () from /lib64/libc.so.6 #2 0x000055baa74dc6a8 in error_handle_fatal (errp=<optimized out>, err=0x7f3d18387730) at ../util/error.c:40 #3 0x000055baa787724d in error_setv (errp=0x55baa8102108 <error_abort>, src=0x55baa796ade5 "../softmmu/physmem.c", line=1769, func=0x55baa796bad0 <__func__.38895> "qemu_ram_resize", err_class=ERROR_CLASS_GENERIC_ERROR, fmt=<optimized out>, ap=0x7f3dbe40c180, suffix=0x7f3dcd87003a "Invalid argument") at ../util/error.c:73 #4 0x000055baa78774d4 in error_setg_errno_internal (errp=0x55baa8102108 <error_abort>, src=src@entry=0x55baa796ade5 "../softmmu/physmem.c", line=1769, func=0x55baa796bad0 <__func__.38895> "qemu_ram_resize", os_errno=<optimized out>, fmt=0x55baa796b460 "Size too large: %s: 0x%lx > 0x%lx") at ../util/error.c:109 #5 0x000055baa76f40a1 in qemu_ram_resize (block=<optimized out>, newsize=newsize@entry=8192, errp=<optimized out>) at ../softmmu/physmem.c:1766 #6 0x000055baa76d0242 in memory_region_ram_resize (mr=mr@entry=0x55baaa812d00, newsize=newsize@entry=8192, errp=<optimized out>) at ../softmmu/memory.c:2243 #7 0x000055baa7627f16 in acpi_ram_update (mr=0x55baaa812d00, data=0x55baaa7f0610) at ../hw/i386/acpi-build.c:2638 #8 0x000055baa762e7ba in acpi_build_update (build_opaque=0x55baab9db540) at ../hw/i386/acpi-build.c:2667 #9 acpi_build_update (build_opaque=0x55baab9db540) at ../hw/i386/acpi-build.c:2644 #10 0x000055baa75a8b10 in fw_cfg_select (s=s@entry=0x55baaa1e1760, key=key@entry=48) at ../hw/nvram/fw_cfg.c:296 --Type <RET> for more, q to quit, c to continue without paging--#11 0x000055baa75a961d in fw_cfg_dma_transfer (s=0x55baaa1e1760) at ../hw/nvram/fw_cfg.c:369 #12 0x000055baa76cbc18 in memory_region_write_accessor (mr=<optimized out>, addr=<optimized out>, value=<optimized out>, size=<optimized out>, shift=<optimized out>, mask=<optimized out>, attrs=...) at ../softmmu/memory.c:491 #13 0x000055baa76ca873 in access_with_adjusted_size (addr=addr@entry=4, value=value@entry=0x7f3dbe40c518, size=size@entry=4, access_size_min=<optimized out>, access_size_max=<optimized out>, access_fn=0x55baa76cbbc0 <memory_region_write_accessor>, mr=0x55baaa1e1ae0, attrs=...) at ../softmmu/memory.c:547 #14 0x000055baa76cdbb3 in memory_region_dispatch_write (mr=mr@entry=0x55baaa1e1ae0, addr=4, data=<optimized out>, op=<optimized out>, attrs=attrs@entry=...) at ../softmmu/memory.c:1501 #15 0x000055baa76f1d57 in flatview_write_continue (fv=fv@entry=0x7f3d18c70400, addr=addr@entry=1304, attrs=..., ptr=ptr@entry=0x7f3dd1500000, len=len@entry=4, addr1=<optimized out>, l=<optimized out>, mr=0x55baaa1e1ae0) at /usr/src/debug/qemu-kvm-5.2.0-6.module+el8.4.0+9871+53903be9.x86_64/include/qemu/host-utils.h:164 #16 0x000055baa76f1f76 in flatview_write (fv=0x7f3d18c70400, addr=1304, attrs=..., buf=0x7f3dd1500000, len=4) at ../softmmu/physmem.c:2799 #17 0x000055baa76f5b0f in address_space_write (as=<optimized out>, addr=<optimized out>, attrs=..., buf=<optimized out>, len=<optimized out>) at ../softmmu/physmem.c:2891 #18 0x000055baa77029b4 in kvm_handle_io (count=1, size=4, direction=<optimized out>, data=<optimized out>, attrs=..., port=1304) at ../accel/kvm/kvm-all.c:2297 #19 kvm_cpu_exec (cpu=cpu@entry=0x55baa9fa6990) at ../accel/kvm/kvm-all.c:2543 #20 0x000055baa76af1d5 in kvm_vcpu_thread_fn (arg=0x55baa9fa6990) at ../accel/kvm/kvm-cpus.c:49 #21 0x000055baa7889064 in qemu_thread_start (args=0x55baa9fce240) at ../util/qemu-thread-posix.c:521 #22 0x00007f3dcdab614a in start_thread () from /lib64/libpthread.so.0 #23 0x00007f3dcd7e5db3 in clone () from /lib64/libc.so.6
>> Version-Release number of selected component (if applicable): >> qemu-kvm-5.2.0-6.module+el8.4.0+9871+53903be9.x86_64 Indicates this is a RHEL-AV bug not a RHEL bug - so just making sure that was your expectation before changing this...
We have a resizable RAMblock that has a maximum size of 1 page, yet we try growing it to 2 pages. The relevant memory region is in ACPI code: build_state->linker_mr As we don't specify a maximum size when creating it via acpi_add_rom_blob(), the maximum size gets set to the initial blob size. The blob size is always aligned to ACPI_BUILD_ALIGN_SIZE, so multiples of single pages. We seem to exceed the initial size (1 page) after an update. I have no idea why this happens now (and why we don't specify a proper maximum size), but a simple solution would be doing it similar to build_state->table_mr and specifying a bigger maximum size.
Assigned to Amnon for initial triage per bz process and age of bug created or assigned to virt-maint without triage. Amnon - fwiw, I had asked David to take a quick look since I had seen from qemu git history that he had made changes in the qemu_ram_resize area at some point.
Here is the shortest qemu cli I can find to reproduce the issue. If remove any one of numa dist, nvdimm device, vmgenid device or intel-iommu device from below cli, the issue can't reproduce. # /usr/libexec/qemu-kvm \ -machine pc-q35-rhel8.4.0,nvdimm=on,kernel_irqchip=split \ -cpu host \ -m size=2097152k,slots=8,maxmem=4194304k \ -smp 8,maxcpus=16,sockets=2,dies=2,cores=4,threads=1 \ -object memory-backend-file,id=ram-node0,mem-path=/dev/hugepages,size=1073741824 \ -numa node,nodeid=0,memdev=ram-node0 \ -object memory-backend-ram,id=ram-node1,size=1073741824 \ -numa node,nodeid=1,memdev=ram-node1 \ -object memory-backend-file,id=memnvdimm1,mem-path=/tmp/nvdimm,size=268435456 \ -numa dist,src=0,dst=0,val=10 \ -numa dist,src=0,dst=1,val=19 \ -numa dist,src=1,dst=0,val=19 \ -numa dist,src=1,dst=1,val=10 \ -device nvdimm,node=1,label-size=131072,memdev=memnvdimm1,id=nvdimm1,slot=1 \ -no-user-config \ -nodefaults \ -device vmgenid,guid=001b2039-ca77-4352-ab4a-433521eabf48,id=vmgenid0 \ -device intel-iommu,intremap=on,caching-mode=on,device-iotlb=on,aw-bits=48 \ -device pcie-root-port,port=0x16,chassis=8,id=pci.8,bus=pcie.0,addr=0x2.0x6 \ -blockdev '{"driver":"file","filename":"/home/rhel84.qcow2.276","aio":"threads","node-name":"libvirt-1-storage","cache":{"direct":true,"no-flush":false},"auto-read-only":true,"discard":"unmap"}' \ -blockdev '{"node-name":"libvirt-1-format","read-only":false,"discard":"unmap","cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":"libvirt-1-storage","backing":null}' \ -device virtio-blk-pci,bus=pci.8,addr=0x0,drive=libvirt-1-format,id=virtio-disk0,bootindex=1,write-cache=on,werror=stop,rerror=stop \ -monitor stdio Packages: qemu-kvm-5.2.0-8.module+el8.4.0+10093+e085f1eb
Upstream is broken as well. Small reproducer: #! /bin/bash build/qemu-system-x86_64 --enable-kvm \ -machine q35,nvdimm=on,kernel_irqchip=split \ -smp 1 \ -cpu host \ -m size=2097152k,slots=8,maxmem=4194304k \ -object memory-backend-file,id=memnvdimm1,mem-path=/tmp/nvdimm,size=268435456 \ -device nvdimm,label-size=131072,memdev=memnvdimm1,id=nvdimm1,slot=1 \ -nodefaults \ -device vmgenid,guid=001b2039-ca77-4352-ab4a-433521eabf48,id=vmgenid0 \ -device intel-iommu,intremap=on,caching-mode=on,device-iotlb=on,aw-bits=48
Proposed an upstream patch: https://lkml.kernel.org/r/20210301104833.45580-1-david@redhat.com Let's see if we need any kind of compat handling (e.g., warning that migration might fail).
Fix is upstream and will be included in QEMU 6.0 commit 6c2b24d1d2b19cd330d971cdbc8e6b115dc97ca4 Author: David Hildenbrand <david> Date: Thu Mar 4 11:55:51 2021 +0100 acpi: Set proper maximum size for "etc/table-loader" blob
Changing to POST, since code is merged upstream.
Verify: qemu-kvm-6.0.0-16.module+el8.5.0+10848+2dccc46d kernel-4.18.0-305.2.el8.x86_64 Guest works well with the qemu cli in comment 5, no crash.
Since the problem described in this bug report should be resolved in a recent advisory, it has been closed with a resolution of ERRATA. For information on the advisory (virt:av bug fix and enhancement update), and where to find the updated files, follow the link below. If the solution does not work for you, open a new bug report. https://access.redhat.com/errata/RHBA-2021:4684