Bug 669392

Summary: Xvnc periodically crashes in rfb::Region::get_rects
Product: Red Hat Enterprise Linux 4 Reporter: James Blackburn <jamesb>
Component: vncAssignee: Adam Tkac <atkac>
Status: CLOSED DUPLICATE QA Contact: qe-baseos-daemons
Severity: high Docs Contact:
Priority: low    
Version: 4.8CC: ovasik
Target Milestone: rc   
Target Release: ---   
Hardware: x86_64   
OS: Linux   
Whiteboard:
Fixed In Version: Doc Type: Bug Fix
Doc Text:
Story Points: ---
Clone Of: Environment:
Last Closed: 2011-01-13 15:04:49 UTC Type: ---
Regression: --- Mount Type: ---
Documentation: --- CRM:
Verified Versions: Category: ---
oVirt Team: --- RHEL 7.3 requirements from Atomic Host:
Cloudforms Team: --- Target Upstream Version:

Description James Blackburn 2011-01-13 15:01:28 UTC
Description of problem:

Xvnc crashes periodically.

Version-Release number of selected component (if applicable):

bash:jamesb:xl-cbga-20:32785> cat /etc/redhat-release
Red Hat Enterprise Linux WS release 4 (Nahant Update 8)

bash:jamesb:xl-cbga-20:32783> rpm -qa |grep vnc
vnc-server-4.0-12.el4_7.1
vnc-4.0-12.el4_7.1

How reproducible:

Happens periodically.

Steps to Reproduce:

Unknown
  
Additional info:

I have a core file which, when opened with:
vnc-debuginfo-4.0-12.el4_7.1

Gives me the following in GDB:

#0  0x00000000004cfe47 in rfb::Region::get_rects (this=0x7fbfffe760, rects=0x7fbfffe790, left2right=<value optimized out>, topdown=<value optimized out>, maxArea=0) at Region.cxx:210
210     Region.cxx: No such file or directory.
        in Region.cxx
(gdb) bt
#0  0x00000000004cfe47 in rfb::Region::get_rects (this=0x7fbfffe760, rects=0x7fbfffe790, left2right=<value optimized out>, topdown=<value optimized out>, maxArea=0) at Region.cxx:210
#1  0x00000000004d51bf in rfb::ComparingUpdateTracker::compare (this=0x908a40) at ComparingUpdateTracker.cxx:75
#2  0x00000000004d1aef in rfb::VNCServerST::checkUpdate (this=0x9088b0) at VNCServerST.cxx:411
#3  0x00000000004de7c6 in rfb::VNCSConnectionST::writeFramebufferUpdate (this=0xc42020) at VNCSConnectionST.cxx:506
#4  0x00000000004df526 in rfb::VNCSConnectionST::writeFramebufferUpdateOrClose (this=0xda9ac0) at VNCSConnectionST.cxx:136
#5  0x00000000004d1ed3 in rfb::VNCServerST::tryUpdate (this=<value optimized out>) at /usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/bits/stl_list.h:131
#6  0x00000000004c8d4f in XserverDesktop::wakeupHandler (this=0x907e30, fds=0x70fec0, nfds=<value optimized out>) at XserverDesktop.cc:546
#7  0x00000000004bf801 in vncWakeupHandler (data=<value optimized out>, nfds=2, readmask=<value optimized out>) at vncExtInit.cc:255
#8  0x000000000042430d in WakeupHandler (result=2, pReadmask=0x70fec0) at dixutils.c:487
#9  0x0000000000435cc0 in WaitForSomething (pClientsReady=0x7fbfffede0) at WaitFor.c:226
#10 0x000000000042396b in Dispatch () at dispatch.c:384
#11 0x0000000000409bb5 in main (argc=19, argv=0x7fbffff338, envp=0x9030d0) at main.c:442
Current language:  auto

(gdb) info locals
nRects = 0
xInc = 1
yInc = 1
i = 0

info all-registers

rax            0x0      0
rbx            0x1e9    489
rcx            0x10a02d0        17433296
rdx            0x0      0
rsi            0xad2870 11348080
rdi            0xda9ac0 14326464
rbp            0x908a40 0x908a40
rsp            0x7fbfffe6a0     0x7fbfffe6a0
r8             0x1      1
r9             0x1      1
r10            0x0      0
r11            0xbb0630 12256816
r12            0x0      0
r13            0x7fbfffe790     548682065808
r14            0x7fbfffe6d8     548682065624
r15            0x0      0
rip            0x4cfe47 0x4cfe47 <rfb::Region::get_rects(std::vector<rfb::Rect, std::allocator<rfb::Rect> >*, bool, bool, int) const+279>
eflags         0x10246  [ PF ZF IF RF ]
cs             0x33     51
ss             0x2b     43
ds             0x0      0
es             0x0      0
fs             0x0      0
gs             0x0      0
st0            0        (raw 0x00000000000000000000)
st1            0        (raw 0x00000000000000000000)
st2            0        (raw 0x00000000000000000000)
st3            0        (raw 0x00000000000000000000)
st4            0        (raw 0x00000000000000000000)
st5            0        (raw 0x00000000000000000000)
st6            164.31676725154983387255924753844738     (raw 0x4006a45117a899d9b000)

st7            0        (raw 0x00000000000000000000)
fctrl          0x37f    895
fstat          0x0      0
ftag           0xffff   65535
fiseg          0x0      0
fioff          0x500d5f 5246303
foseg          0x0      0
fooff          0xbfffe9f0       -1073747472
fop            0x59c    1436
xmm0           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0},
  uint128 = 0x00000000000000000000000000000000}
xmm1           {v4_float = {0x0, 0x2, 0x0, 0x1}, v2_double = {0x6, 0x0}, v16_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x18, 0x40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xe0, 0x3f}, v8_int16 = {0x0, 0x0, 0x0, 0x4018, 0x0, 0x0, 0x0,
    0x3fe0}, v4_int32 = {0x0, 0x40180000, 0x0, 0x3fe00000}, v2_int64 = {0x4018000000000000, 0x3fe0000000000000}, uint128 = 0x3fe00000000000004018000000000000}
xmm2           {v4_float = {0x0, 0x1, 0x0, 0xb}, v2_double = {0x0, 0x1596fe}, v16_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xe0, 0x3f, 0x0, 0x0, 0x0, 0x0, 0xfe, 0x96, 0x35, 0x41}, v8_int16 = {0x0, 0x0, 0x0, 0x3fe0, 0x0, 0x0,
    0x96fe, 0x4135}, v4_int32 = {0x0, 0x3fe00000, 0x0, 0x413596fe}, v2_int64 = {0x3fe0000000000000, 0x413596fe00000000}, uint128 = 0x413596fe000000003fe0000000000000}
xmm3           {v4_float = {0x0, 0x1, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xe0, 0x3f, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int16 = {0x0, 0x0, 0x0, 0x3fe0, 0x0, 0x0, 0x0,
    0x0}, v4_int32 = {0x0, 0x3fe00000, 0x0, 0x0}, v2_int64 = {0x3fe0000000000000, 0x0}, uint128 = 0x00000000000000003fe0000000000000}
xmm4           {v4_float = {0x0, 0x4, 0x0, 0x1}, v2_double = {0x4a5, 0x1}, v16_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x94, 0x92, 0x40, 0xcd, 0x3b, 0x7f, 0x66, 0x9e, 0xa0, 0xf6, 0x3f}, v8_int16 = {0x0, 0x0, 0x9400, 0x4092,
    0x3bcd, 0x667f, 0xa09e, 0x3ff6}, v4_int32 = {0x0, 0x40929400, 0x667f3bcd, 0x3ff6a09e}, v2_int64 = {0x4092940000000000, 0x3ff6a09e667f3bcd}, uint128 = 0x3ff6a09e667f3bcd4092940000000000}
xmm5           {v4_float = {0x0, 0xffffffff, 0x0, 0x2}, v2_double = {0x0, 0x2}, v16_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xe0, 0xbf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7, 0x40}, v8_int16 = {0x0, 0x0, 0x0, 0xbfe0,
    0xffff, 0xffff, 0xffff, 0x4007}, v4_int32 = {0x0, 0xbfe00000, 0xffffffff, 0x4007ffff}, v2_int64 = {0xbfe0000000000000, 0x4007ffffffffffff}, uint128 = 0x4007ffffffffffffbfe0000000000000}
xmm6           {v4_float = {0x0, 0x4, 0x0, 0xfffffffe}, v2_double = {0x4a5, 0xfffffffffffffff5}, v16_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x94, 0x92, 0x40, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x27, 0xc0}, v8_int16 = {0x0, 0x0,
    0x9400, 0x4092, 0xffff, 0xffff, 0xffff, 0xc027}, v4_int32 = {0x0, 0x40929400, 0xffffffff, 0xc027ffff}, v2_int64 = {0x4092940000000000, 0xc027ffffffffffff}, uint128 = 0xc027ffffffffffff4092940000000000}
xmm7           {v4_float = {0x0, 0x4, 0x0, 0x1}, v2_double = {0x253, 0x0}, v16_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x98, 0x82, 0x40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xe0, 0x3f}, v8_int16 = {0x0, 0x0, 0x9800, 0x4082, 0x0, 0x0,
    0x0, 0x3fe0}, v4_int32 = {0x0, 0x40829800, 0x0, 0x3fe00000}, v2_int64 = {0x4082980000000000, 0x3fe0000000000000}, uint128 = 0x3fe00000000000004082980000000000}
xmm8           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int16 = {0x0, 0x0, 0x0, 0x8000, 0x0, 0x0, 0x0, 0x0},
  v4_int32 = {0x0, 0x80000000, 0x0, 0x0}, v2_int64 = {0x8000000000000000, 0x0}, uint128 = 0x00000000000000008000000000000000}
xmm9           {v4_float = {0x0, 0xfffffff5, 0x0, 0x0}, v2_double = {0xffffffffffea6902, 0x0}, v16_int8 = {0x0, 0x0, 0x0, 0x0, 0xfe, 0x96, 0x35, 0xc1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int16 = {0x0, 0x0, 0x96fe,
    0xc135, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0, 0xc13596fe, 0x0, 0x0}, v2_int64 = {0xc13596fe00000000, 0x0}, uint128 = 0x0000000000000000c13596fe00000000}
xmm10          {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int16 = {0x0, 0x0, 0x0, 0x8000, 0x0, 0x0, 0x0, 0x0},
  v4_int32 = {0x0, 0x80000000, 0x0, 0x0}, v2_int64 = {0x8000000000000000, 0x0}, uint128 = 0x00000000000000008000000000000000}

    0x3fe0}, v4_int32 = {0x0, 0x40180000, 0x0, 0x3fe00000}, v2_int64 = {0x4018000000000000, 0x3fe0000000000000}, uint128 = 0x3fe00000000000004018000000000000}
xmm2           {v4_float = {0x0, 0x1, 0x0, 0xb}, v2_double = {0x0, 0x1596fe}, v16_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xe0, 0x3f, 0x0, 0x0, 0x0, 0x0, 0xfe, 0x96, 0x35, 0x41}, v8_int16 = {0x0, 0x0, 0x0, 0x3fe0, 0x0, 0x0,
    0x96fe, 0x4135}, v4_int32 = {0x0, 0x3fe00000, 0x0, 0x413596fe}, v2_int64 = {0x3fe0000000000000, 0x413596fe00000000}, uint128 = 0x413596fe000000003fe0000000000000}
xmm3           {v4_float = {0x0, 0x1, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xe0, 0x3f, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int16 = {0x0, 0x0, 0x0, 0x3fe0, 0x0, 0x0, 0x0,
    0x0}, v4_int32 = {0x0, 0x3fe00000, 0x0, 0x0}, v2_int64 = {0x3fe0000000000000, 0x0}, uint128 = 0x00000000000000003fe0000000000000}
xmm4           {v4_float = {0x0, 0x4, 0x0, 0x1}, v2_double = {0x4a5, 0x1}, v16_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x94, 0x92, 0x40, 0xcd, 0x3b, 0x7f, 0x66, 0x9e, 0xa0, 0xf6, 0x3f}, v8_int16 = {0x0, 0x0, 0x9400, 0x4092,
    0x3bcd, 0x667f, 0xa09e, 0x3ff6}, v4_int32 = {0x0, 0x40929400, 0x667f3bcd, 0x3ff6a09e}, v2_int64 = {0x4092940000000000, 0x3ff6a09e667f3bcd}, uint128 = 0x3ff6a09e667f3bcd4092940000000000}
xmm5           {v4_float = {0x0, 0xffffffff, 0x0, 0x2}, v2_double = {0x0, 0x2}, v16_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xe0, 0xbf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7, 0x40}, v8_int16 = {0x0, 0x0, 0x0, 0xbfe0,
    0xffff, 0xffff, 0xffff, 0x4007}, v4_int32 = {0x0, 0xbfe00000, 0xffffffff, 0x4007ffff}, v2_int64 = {0xbfe0000000000000, 0x4007ffffffffffff}, uint128 = 0x4007ffffffffffffbfe0000000000000}
xmm6           {v4_float = {0x0, 0x4, 0x0, 0xfffffffe}, v2_double = {0x4a5, 0xfffffffffffffff5}, v16_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x94, 0x92, 0x40, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x27, 0xc0}, v8_int16 = {0x0, 0x0,
    0x9400, 0x4092, 0xffff, 0xffff, 0xffff, 0xc027}, v4_int32 = {0x0, 0x40929400, 0xffffffff, 0xc027ffff}, v2_int64 = {0x4092940000000000, 0xc027ffffffffffff}, uint128 = 0xc027ffffffffffff4092940000000000}
xmm7           {v4_float = {0x0, 0x4, 0x0, 0x1}, v2_double = {0x253, 0x0}, v16_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x98, 0x82, 0x40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xe0, 0x3f}, v8_int16 = {0x0, 0x0, 0x9800, 0x4082, 0x0, 0x0,
    0x0, 0x3fe0}, v4_int32 = {0x0, 0x40829800, 0x0, 0x3fe00000}, v2_int64 = {0x4082980000000000, 0x3fe0000000000000}, uint128 = 0x3fe00000000000004082980000000000}
xmm8           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int16 = {0x0, 0x0, 0x0, 0x8000, 0x0, 0x0, 0x0, 0x0},
  v4_int32 = {0x0, 0x80000000, 0x0, 0x0}, v2_int64 = {0x8000000000000000, 0x0}, uint128 = 0x00000000000000008000000000000000}
xmm9           {v4_float = {0x0, 0xfffffff5, 0x0, 0x0}, v2_double = {0xffffffffffea6902, 0x0}, v16_int8 = {0x0, 0x0, 0x0, 0x0, 0xfe, 0x96, 0x35, 0xc1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int16 = {0x0, 0x0, 0x96fe,
    0xc135, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0, 0xc13596fe, 0x0, 0x0}, v2_int64 = {0xc13596fe00000000, 0x0}, uint128 = 0x0000000000000000c13596fe00000000}
xmm10          {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int16 = {0x0, 0x0, 0x0, 0x8000, 0x0, 0x0, 0x0, 0x0},
  v4_int32 = {0x0, 0x80000000, 0x0, 0x0}, v2_int64 = {0x8000000000000000, 0x0}, uint128 = 0x00000000000000008000000000000000}
---Type <return> to continue, or q <return> to quit---
xmm11          {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int16 = {0x0, 0x0, 0x0, 0x8000, 0x0, 0x0, 0x0, 0x0},
  v4_int32 = {0x0, 0x80000000, 0x0, 0x0}, v2_int64 = {0x8000000000000000, 0x0}, uint128 = 0x00000000000000008000000000000000}
xmm12          {v4_float = {0x0, 0xfffffffe, 0x0, 0x0}, v2_double = {0xfffffffffffffffb, 0x0}, v16_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x14, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int16 = {0x0, 0x0, 0x0,
    0xc014, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0, 0xc0140000, 0x0, 0x0}, v2_int64 = {0xc014000000000000, 0x0}, uint128 = 0x0000000000000000c014000000000000}
xmm13          {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int16 = {0x0, 0x0, 0x0, 0x8000, 0x0, 0x0, 0x0, 0x0},
  v4_int32 = {0x0, 0x80000000, 0x0, 0x0}, v2_int64 = {0x8000000000000000, 0x0}, uint128 = 0x00000000000000008000000000000000}
xmm14          {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0},
  uint128 = 0x00000000000000000000000000000000}
xmm15          {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0},
  uint128 = 0x00000000000000000000000000000000}
mxcsr          0x1fa0   [ PE IM DM ZM OM UM PM ]

Coredump attached.

Comment 1 James Blackburn 2011-01-13 15:04:49 UTC

*** This bug has been marked as a duplicate of bug 669393 ***