Attachment 888369 Details for Bug 1089873 – additional information about the test conditions

additional information about the test conditions

report.txt (text/plain), 10.45 KB, created by Francesco Romani on 2014-04-22 06:29:46 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Francesco Romani

Created: 2014-04-22 06:29:46 UTC

Size: 10.45 KB

patch

obsolete

>iptables output (see NFS ports blocked; 192.168.1.64 is the host running engine and hosting storage)
>
>Chain INPUT (policy ACCEPT 211K packets, 312M bytes)
> pkts bytes target     prot opt in     out     source               destination         
>
>Chain FORWARD (policy ACCEPT 0 packets, 0 bytes)
> pkts bytes target     prot opt in     out     source               destination         
>
>Chain OUTPUT (policy ACCEPT 111K packets, 6954K bytes)
> pkts bytes target     prot opt in     out     source               destination         
>   53 64964 REJECT     tcp  --  *      *       0.0.0.0/0            192.168.1.64        tcp dpt:2049 reject-with icmp-port-unreachable 
>    0     0 REJECT     udp  --  *      *       0.0.0.0/0            192.168.1.64        udp dpt:2049 reject-with icmp-port-unreachable 
>
>
>ps -faux once VDSM gets stuck (trimmed)
>USER       PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
>[...]
>root      2291  0.0  0.0 919604 14504 ?        Ssl  17:04   0:04 /usr/sbin/libvirtd --listen
>qemu      1132  0.0  0.0 919604  9528 ?        D    22:10   0:00  \_ /usr/sbin/libvirtd --listen
>root      2390  0.0  0.1  28956 22808 ?        S<Lsl 17:04   0:00 iscsiuio
>root      2398  0.0  0.0   4900   460 ?        Ss   17:04   0:00 iscsid
>root      2399  0.0  0.0   8148  4072 ?        S<Ls 17:04   0:01 iscsid
>root      2690  0.0  0.0   4064   536 tty1     Ss+  17:04   0:00 /sbin/mingetty /dev/tty1
>root      2692  0.0  0.0   4064   536 tty2     Ss+  17:04   0:00 /sbin/mingetty /dev/tty2
>root      2694  0.0  0.0   4064   536 tty3     Ss+  17:04   0:00 /sbin/mingetty /dev/tty3
>root      2696  0.0  0.0   4064   532 tty4     Ss+  17:04   0:00 /sbin/mingetty /dev/tty4
>root      2698  0.0  0.0   4064   532 tty5     Ss+  17:04   0:00 /sbin/mingetty /dev/tty5
>root      2700  0.0  0.0   4064   536 tty6     Ss+  17:04   0:00 /sbin/mingetty /dev/tty6
>sanlock  26579  0.0  0.2 791748 34952 ?        SLsl 17:37   0:10 sanlock daemon -U sanlock -G sanlock
>root     26580  0.0  0.0  23096   284 ?        S    17:37   0:00  \_ sanlock daemon -U sanlock -G sanlock
>root     26693  0.0  0.0  11340   692 pts/0    S<   17:37   0:00 /bin/bash -e /usr/share/vdsm/respawn --minlifetime 10 --daemon --masterpid /var/run/vdsm/supervdsm_respawn.pid /usr/share/vdsm/supervdsmServer --sockfile /var/run/vdsm/svdsm.sock --pidfile /var/run/vdsm/supervdsmd.pid
>root     26695  0.0  0.1 654168 28524 pts/0    S<l  17:37   0:00  \_ /usr/bin/python /usr/share/vdsm/supervdsmServer --sockfile /var/run/vdsm/svdsm.sock --pidfile /var/run/vdsm/supervdsmd.pid
>qemu     30046  0.0  0.0      0     0 pts/0    Z<   17:55   0:00      \_ [supervdsmServer] <defunct>
>vdsm       542  0.0  0.0   9236   680 ?        S<   22:08   0:00 /bin/bash -e /usr/share/vdsm/respawn --minlifetime 10 --daemon --masterpid /var/run/vdsm/respawn.pid /usr/share/vdsm/vdsm --pidfile /var/run/vdsm/vdsmd.pid
>vdsm       545  7.8  0.3 3471444 58288 ?       S<l  22:08   0:38  \_ /usr/bin/python /usr/share/vdsm/vdsm --pidfile /var/run/vdsm/vdsmd.pid
>vdsm       743  0.0  0.1 244904 17992 ?        D<   22:08   0:00      \_ /usr/bin/python /usr/share/vdsm/storage/remoteFileHandler.pyc 32 27
>vdsm       892  0.0  0.1 244900 17976 ?        S<   22:08   0:00      \_ /usr/bin/python /usr/share/vdsm/storage/remoteFileHandler.pyc 61 55
>vdsm      1840  0.1  0.1 244900 17936 ?        D<   22:15   0:00      \_ /usr/bin/python /usr/share/vdsm/storage/remoteFileHandler.pyc 32 27
>vdsm      1841  0.1  0.1 244900 17932 ?        D<   22:15   0:00      \_ /usr/bin/python /usr/share/vdsm/storage/remoteFileHandler.pyc 49 48
>vdsm      1844  0.1  0.1 244904 17940 ?        D<   22:15   0:00      \_ /usr/bin/python /usr/share/vdsm/storage/remoteFileHandler.pyc 48 44
>vdsm      1845  0.1  0.1 244904 17936 ?        D<   22:15   0:00      \_ /usr/bin/python /usr/share/vdsm/storage/remoteFileHandler.pyc 57 54
>
>output of /cat/1132/syscall
>21 0x7faff805b180 0x0 0x0 0x8 0xca 0x7fb019422280 0x7fb0194222f8 0x355f0db7e7
>
>output of /cat/1132/stack (clearly stuck in NFS)
>[<ffffffffa04eb1b2>] rpc_wait_bit_killable+0x42/0xa0 [sunrpc]
>[<ffffffffa04eb6f5>] __rpc_execute+0xf5/0x350 [sunrpc]
>[<ffffffffa04eb9b1>] rpc_execute+0x61/0xa0 [sunrpc]
>[<ffffffffa04e23a5>] rpc_run_task+0x75/0x90 [sunrpc]
>[<ffffffffa04e24c2>] rpc_call_sync+0x42/0x70 [sunrpc]
>[<ffffffffa077f73d>] nfs3_rpc_wrapper.clone.0+0x3d/0xd0 [nfs]
>[<ffffffffa077fb1c>] nfs3_proc_access+0xbc/0x180 [nfs]
>[<ffffffffa076758c>] nfs_do_access+0x19c/0x240 [nfs]
>[<ffffffffa07676d8>] nfs_permission+0xa8/0x1e0 [nfs]
>[<ffffffff81198d43>] __link_path_walk+0xb3/0xff0
>[<ffffffff81199f3a>] path_walk+0x6a/0xe0
>[<ffffffff8119a14b>] filename_lookup+0x6b/0xc0
>[<ffffffff8119b277>] user_path_at+0x57/0xa0
>[<ffffffff81186d11>] sys_faccessat+0xe1/0x230
>[<ffffffff81186e78>] sys_access+0x18/0x20
>[<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
>[<ffffffffffffffff>] 0xffffffffffffffff
>
>output of the 'mount' command
>/dev/mapper/vg_toji-lv_root on / type ext4 (rw)
>proc on /proc type proc (rw)
>sysfs on /sys type sysfs (rw)
>devpts on /dev/pts type devpts (rw,gid=5,mode=620)
>tmpfs on /dev/shm type tmpfs (rw,rootcontext="system_u:object_r:tmpfs_t:s0")
>/dev/sda3 on /boot type ext4 (rw)
>/dev/mapper/vg_toji-lv_home on /home type ext4 (rw)
>/dev/mapper/vg_toji-lv_srv on /srv type ext4 (rw)
>none on /proc/sys/fs/binfmt_misc type binfmt_misc (rw)
>sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw)
>nfsd on /proc/fs/nfsd type nfsd (rw)
>shinji:/srv/ovirt/iso on /rhev/data-center/mnt/shinji:_srv_ovirt_iso type nfs (rw,soft,nosharecache,timeo=600,retrans=6,nfsvers=3,addr=192.168.1.64)
>192.168.1.64:/srv/virtstore/nfs/deviscsi/data on /rhev/data-center/mnt/192.168.1.64:_srv_virtstore_nfs_deviscsi_data type nfs (rw,soft,nosharecache,timeo=600,retrans=6,nfsvers=3,addr=192.168.1.64)
>/dev/mapper/bc1e6c00--7a28--4d0f--860f--e620ac3bd24d-master on /rhev/data-center/mnt/blockSD/bc1e6c00-7a28-4d0f-860f-e620ac3bd24d/master type ext3 (rw)
>
>output of uname -a
>Linux toji.rokugan.lan 2.6.32-431.11.2.el6.x86_64 #1 SMP Mon Mar 3 13:32:45 EST 2014 x86_64 x86_64 x86_64 GNU/Linux
>
><?xml version="1.0" encoding="utf-8"?>
><domain type="kvm">
>        <name>Fedora2</name>
>        <uuid>3923c1d4-65e6-42c6-8768-501f0993ceb8</uuid>
>        <memory>1048576</memory>
>        <currentMemory>1048576</currentMemory>
>        <vcpu current="1">160</vcpu>
>        <memtune>
>                <min_guarantee>1048576</min_guarantee>
>        </memtune>
>        <devices>
>                <channel type="unix">
>                        <target name="com.redhat.rhevm.vdsm" type="virtio"/>
>                        <source mode="bind" path="/var/lib/libvirt/qemu/channels/3923c1d4-65e6-42c6-8768-501f0993ceb8.com.redhat.rhevm.vdsm"/>
>                </channel>
>                <channel type="unix">
>                        <target name="org.qemu.guest_agent.0" type="virtio"/>
>                        <source mode="bind" path="/var/lib/libvirt/qemu/channels/3923c1d4-65e6-42c6-8768-501f0993ceb8.org.qemu.guest_agent.0"/>
>                </channel>
>                <input bus="ps2" type="mouse"/>
>                <channel type="spicevmc">
>                        <target name="com.redhat.spice.0" type="virtio"/>
>                </channel>
>                <graphics autoport="yes" keymap="en-us" listen="0" passwd="*****" passwdValidTo="1970-01-01T00:00:01" port="-1" tlsPort="-1" type="spice"/>
>                <sound model="ich6">
>                        <address bus="0x00" domain="0x0000" function="0x0" slot="0x04" type="pci"/>
>                </sound>
>                <memballoon model="virtio"/>
>                <controller index="0" model="virtio-scsi" type="scsi">
>                        <address bus="0x00" domain="0x0000" function="0x0" slot="0x05" type="pci"/>
>                </controller>
>                <video>
>                        <address bus="0x00" domain="0x0000" function="0x0" slot="0x02" type="pci"/>
>                        <model heads="1" type="qxl" vram="32768"/>
>               </video>
>                <interface type="bridge">
>                        <address bus="0x00" domain="0x0000" function="0x0" slot="0x03" type="pci"/>
>                        <mac address="00:1a:4a:6d:33:19"/>
>                        <model type="virtio"/>
>                        <source bridge="ovirtmgmt"/>
>                        <filterref filter="vdsm-no-mac-spoofing"/>
>                        <link state="up"/>
>                        <boot order="3"/>
>                </interface>
>                <disk device="cdrom" snapshot="no" type="file">
>                        <source file="/rhev/data-center/mnt/shinji:_srv_ovirt_iso/029b397f-6a0a-432a-b5ac-03861cbc6b38/images/11111111-1111-1111-1111-111111111111/Fedora-Live-Desktop-x86_64-19-1.iso" startupPolicy="optional"/>
>                        <target bus="ide" dev="hdc"/>
>                        <readonly/>
>                        <serial/>
>                        <boot order="1"/>
>                </disk>
>                <disk device="disk" snapshot="no" type="block">
>                        <address bus="0x00" domain="0x0000" function="0x0" slot="0x07" type="pci"/>
>                        <source dev="/rhev/data-center/mnt/blockSD/bc1e6c00-7a28-4d0f-860f-e620ac3bd24d/images/e9d5398b-2f2c-4066-9f45-53ae6616899b/86226983-9b04-4b19-9b2b-85c87442d658"/>
>                        <target bus="virtio" dev="vda"/>
>                        <serial>e9d5398b-2f2c-4066-9f45-53ae6616899b</serial>
>                        <boot order="2"/>
>                        <driver cache="none" error_policy="stop" io="native" name="qemu" type="qcow2"/>
>                </disk>
>        </devices>
>        <os>
>                <type arch="x86_64" machine="rhel6.5.0">hvm</type>
>                <smbios mode="sysinfo"/>
>        </os>
>        <sysinfo type="smbios">
>                <system>
>                       <entry name="manufacturer">oVirt</entry>
>                        <entry name="product">oVirt Node</entry>
>                        <entry name="version">6Server-6.5.0.1.el6</entry>
>                        <entry name="serial">4C4C4544-0059-4410-8043-B6C04F4E5A31_10:fe:ed:02:02:e4</entry>
>                        <entry name="uuid">3923c1d4-65e6-42c6-8768-501f0993ceb8</entry>
>                </system>
>        </sysinfo>
>        <clock adjustment="0" offset="variable">
>                <timer name="rtc" tickpolicy="catchup"/>
>                <timer name="pit" tickpolicy="delay"/>
>                <timer name="hpet" present="no"/>
>        </clock>
>        <features>
>                <acpi/>
>        </features>
>        <cpu match="exact">
>                <model>SandyBridge</model>
>                <topology cores="1" sockets="160" threads="1"/>
>        </cpu>
></domain>
>

Actions: View

Attachments on bug 1089873: 888369 | 892814 | 892821