Description of problem: rsyslog sometimes crashes with coredump, rsyslogd dead but pid file exists


gdb:

Version-Release number of selected component (if applicable): rsyslog-5.8.10-6.el6


warning: Unable to find dynamic linker breakpoint function.
GDB will be unable to debug shared library initializers
and track explicitly loaded dynamic code.
Core was generated by `/sbin/rsyslogd -i /var/run/syslogd.pid -c 5'.
Program terminated with signal 11, Segmentation fault.
#0  qDeqLinkedList (pThis=0x7f10dc263330, ppUsr=0x7f10d4fdbce8) at queue.c:566
566                *ppUsr = pEntry->pUsr;
(gdb) bt
#0  qDeqLinkedList (pThis=0x7f10dc263330, ppUsr=0x7f10d4fdbce8) at queue.c:566
#1  0x00007f10daec1105 in qqueueDeq (pThis=0x7f10dc263330, pWti=<value optimized out>) at queue.c:956
#2  DequeueConsumableElements (pThis=0x7f10dc263330, pWti=<value optimized out>) at queue.c:1461
#3  DequeueConsumable (pThis=0x7f10dc263330, pWti=<value optimized out>) at queue.c:1505
#4  0x0000000000000001 in ?? ()
#5  0x00007f10dc2635a8 in ?? ()
#6  0x00007f10d4fdbce0 in ?? ()
#7  0x0000001000dd0480 in ?? ()
#8  0x00007f10dc263440 in ?? ()
#9  0x00000000000003c0 in ?? ()
#10 0x00007f10dc263d98 in ?? ()
#11 0x00007f10dc2635a0 in ?? ()
#12 0x00007f10dc263d70 in ?? ()
#13 0x00000010dc264300 in ?? ()
#14 0x00007f10d4fdbcd8 in ?? ()
#15 0x00007f10dc263350 in ?? ()
#16 0x0000000052239cfd in ?? ()
#17 0x00007f10d0034560 in ?? ()
#18 0x0000000052239cfd in ?? ()
#19 0x00007f10d0009ac0 in ?? ()
#20 0x0000000000000000 in ?? ()





rsyslog.conf:

# rsyslog v5 configuration file

# For more information see /usr/share/doc/rsyslog-*/rsyslog_conf.html
# If you experience problems, see http://www.rsyslog.com/doc/troubleshoot.html

#### MODULES ####

$ModLoad imuxsock # provides support for local system logging (e.g. via logger command)
$ModLoad imklog   # provides kernel logging support (previously done by rklogd)
#$ModLoad immark  # provides --MARK-- message capability

# Provides UDP syslog reception
#$ModLoad imudp
#$UDPServerRun 514

# Provides TCP syslog reception
#$ModLoad imtcp
#$InputTCPServerRun 514


#### GLOBAL DIRECTIVES ####

# Use default timestamp format
$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat

$SystemLogRateLimitInterval 0

# File syncing capability is disabled by default. This feature is usually not required,
# not useful and an extreme performance hit
#$ActionFileEnableSync on

# Include all config files in /etc/rsyslog.d/
$IncludeConfig /etc/rsyslog.d/*.conf


#### RULES ####

# Log all kernel messages to the console.
# Logging much else clutters up the screen.
#kern.*                                                 /dev/console

# Log anything (except mail) of level info or higher.
# Don't log private authentication messages!
*.info;mail.none;authpriv.none;cron.none;auth.!info;user.none;local0.none;local1.none;local2.none;local3.none;local4.none;local5.none;local6.none;local7.none     /var/log/messages

# The authpriv file has restricted access.
authpriv.*;auth.info                                    /var/log/secure

# Log all the mail messages in one place.
mail.*                                                  -/var/log/maillog

# Log cron stuff
cron.*                                                  /var/log/cron

# Save boot messages also to boot.log
local7.*                                                /var/log/boot.log


### start forwarding rule ###
$WorkDirectory /var/lib/rsyslog # where to place spool files
$ActionQueueFileName bb.aa.cc.cz # unique name prefix for spool files
$ActionQueueMaxDiskSpace 128m # 1gb space limit (use as much as possible)
$ActionQueueSaveOnShutdown on # save messages to disk on shutdown
$ActionQueueType LinkedList   # run asynchronously
$ActionResumeRetryCount -1    # infinite retries if host is down
$ActionQueueTimeoutEnqueue 1
*.* @@bb.aa.cc.cz:2526
### end forwarding rule ###

### start forwarding rule ###
$WorkDirectory /var/lib/rsyslog # where to place spool files
$ActionQueueFileName xx.aa.cc.cz # unique name prefix for spool files
$ActionQueueMaxDiskSpace 128m # 1gb space limit (use as much as possible)
$ActionQueueSaveOnShutdown on # save messages to disk on shutdown
$ActionQueueType LinkedList   # run asynchronously
$ActionResumeRetryCount -1    # infinite retries if host is down
$ActionQueueTimeoutEnqueue 1
*.* @@xx.aa.cc.cz:2526
### end forwarding rule ###