Description of problem: *__GI___assert_fail (assertion=0x2b505abfac44 "false", file=0x2b505ac0a8d8 "qpid/sys/DispatchHandle.cpp", line=344, the variable "state" has unexpected value "qpid::sys::DispatchHandle::INACTIVE" Stack traces: Thread 11 (process 11228): #0 0x0000003994cd1f58 in epoll_wait () from /lib64/libc.so.6 #1 0x00002b505ab9bb36 in qpid::sys::Poller::wait (this=0x1c40ffe0, timeout=<value optimized out>) at qpid/sys/epoll/EpollPoller.cpp:304 #2 0x00002b505abdf688 in qpid::sys::Dispatcher::run (this=0x7fff5048c1d0) at qpid/sys/Dispatcher.cpp:38 #3 0x00002b505a6e7fe6 in qpid::broker::Broker::run (this=<value optimized out>) at qpid/broker/Broker.cpp:297 #4 0x000000000040901b in QpiddDaemon::child (this=0x7fff5048c720) at posix/QpiddBroker.cpp:128 #5 0x00002b505a718f7e in qpid::broker::Daemon::fork (this=0x7fff5048c720) at qpid/broker/Daemon.cpp:85 #6 0x0000000000406963 in QpiddBroker::execute (this=<value optimized out>, options=<value optimized out>) at posix/QpiddBroker.cpp:158 #7 0x0000000000405468 in main (argc=14, argv=0x7fff5048cd18) at qpidd.cpp:77 Current language: auto; currently c Thread 10 (process 11229): #0 0x000000399580a687 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002b505a78a3c6 in qpid::broker::Timer::run (this=0x1c419a50) at ./qpid/sys/posix/Condition.h:69 #2 0x00002b505ab9386a in qpid::sys::(anonymous namespace)::runRunnable () at /usr/include/boost/any.hpp:46 #3 0x00000039958062f7 in start_thread (arg=<value optimized out>) at pthread_create.c:296 #4 0x0000003994cd1b6d in clone () from /lib64/libc.so.6 Thread 9 (process 11230): #0 0x000000399580a687 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002b505a78a3c6 in qpid::broker::Timer::run (this=0x1c419b08) at ./qpid/sys/posix/Condition.h:69 #2 0x00002b505ab9386a in qpid::sys::(anonymous namespace)::runRunnable () at /usr/include/boost/any.hpp:46 #3 0x00000039958062f7 in start_thread (arg=<value optimized out>) at pthread_create.c:296 #4 0x0000003994cd1b6d in clone () from /lib64/libc.so.6 Thread 8 (process 11233): #0 0x0000003994cd1f58 in epoll_wait () from /lib64/libc.so.6 #1 0x00002b505ab9bb36 in qpid::sys::Poller::wait (this=0x1c40ffe0, timeout=<value optimized out>) at qpid/sys/epoll/EpollPoller.cpp:304 #2 0x00002b505abdf688 in qpid::sys::Dispatcher::run (this=0x7fff5048c1d0) at qpid/sys/Dispatcher.cpp:38 #3 0x00002b505ab9386a in qpid::sys::(anonymous namespace)::runRunnable () at /usr/include/boost/any.hpp:46 #4 0x00000039958062f7 in start_thread (arg=<value optimized out>) at pthread_create.c:296 #5 0x0000003994cd1b6d in clone () from /lib64/libc.so.6 Thread 7 (process 11234): #0 0x0000003994cd1f58 in epoll_wait () from /lib64/libc.so.6 #1 0x00002b505ab9bb36 in qpid::sys::Poller::wait (this=0x1c40ffe0, timeout=<value optimized out>) at qpid/sys/epoll/EpollPoller.cpp:304 #2 0x00002b505abdf688 in qpid::sys::Dispatcher::run (this=0x7fff5048c1d0) at qpid/sys/Dispatcher.cpp:38 #3 0x00002b505ab9386a in qpid::sys::(anonymous namespace)::runRunnable () at /usr/include/boost/any.hpp:46 #4 0x00000039958062f7 in start_thread (arg=<value optimized out>) at pthread_create.c:296 #5 0x0000003994cd1b6d in clone () from /lib64/libc.so.6 Thread 6 (process 11235): #0 0x0000003994cd1f58 in epoll_wait () from /lib64/libc.so.6 #1 0x00002b505ab9bb36 in qpid::sys::Poller::wait (this=0x1c40ffe0, timeout=<value optimized out>) at qpid/sys/epoll/EpollPoller.cpp:304 #2 0x00002b505abdf688 in qpid::sys::Dispatcher::run (this=0x7fff5048c1d0) at qpid/sys/Dispatcher.cpp:38 #3 0x00002b505ab9386a in qpid::sys::(anonymous namespace)::runRunnable () at /usr/include/boost/any.hpp:46 #4 0x00000039958062f7 in start_thread (arg=<value optimized out>) at pthread_create.c:296 #5 0x0000003994cd1b6d in clone () from /lib64/libc.so.6 Thread 5 (process 11236): #0 0x0000003994cd1f58 in epoll_wait () from /lib64/libc.so.6 #1 0x00002b505ab9bb36 in qpid::sys::Poller::wait (this=0x1c40ffe0, timeout=<value optimized out>) at qpid/sys/epoll/EpollPoller.cpp:304 #2 0x00002b505abdf688 in qpid::sys::Dispatcher::run (this=0x7fff5048c1d0) at qpid/sys/Dispatcher.cpp:38 #3 0x00002b505ab9386a in qpid::sys::(anonymous namespace)::runRunnable () at /usr/include/boost/any.hpp:46 #4 0x00000039958062f7 in start_thread (arg=<value optimized out>) at pthread_create.c:296 #5 0x0000003994cd1b6d in clone () from /lib64/libc.so.6 Thread 4 (process 11237): #0 *__GI___libc_free (mem=0x2aaac1b7e940) at malloc.c:3503 #1 0x00002b505abb8a26 in ~AMQContentBody (this=0x2aaac1b6ff70) at /usr/lib/gcc/x86_64-redhat-linux/4.1.2/../../../../include/c++/4.1.2/bits/basic_string.h:233 #2 0x00002b505a71dc26 in ~BodyHolder (this=0x2aaac1b6ff60) at ./qpid/framing/Blob.h:186 #3 0x00002b505abb99d1 in qpid::framing::AMQFrame::decode (this=0x2aaaac042070, buffer=@0x45056ba0) at ./qpid/RefCounted.h:42 #4 0x00002b505abd0068 in qpid::framing::FrameDecoder::decode (this=0x2aaaac042070, buffer=@0x45055e87) at qpid/framing/FrameDecoder.cpp:43 #5 0x00002b505aee634b in qpid::cluster::Connection::deliverBuffer (this=0x2aaaac041f50, buf=@0x45056ba0) at qpid/cluster/Connection.cpp:204 #6 0x00002b505aed14e2 in qpid::cluster::Cluster::deliveredEvent (this=0x1c41ab20, e=@0x221392a0) at qpid/cluster/Cluster.cpp:237 #7 0x00002b505aecd280 in qpid::cluster::Cluster::delivered (this=0x1c41ab20, events=@0x1c41b140) at /usr/include/boost/bind/mem_fn_template.hpp:149 #8 0x00002b505aed74ca in boost::function1<void, std::deque<qpid::cluster::Event, std::allocator<qpid::cluster::Event> >&, std::allocator<void> >::operator() ( this=0x0, a0=@0x45055e87) at /usr/include/boost/function/function_template.hpp:576 #9 0x00002b505aedbe5f in qpid::sys::PollableQueue<qpid::cluster::Event>::dispatch (this=0x1c41afc0, h=@0x1c41b058) at ./qpid/sys/PollableQueue.h:127 #10 0x00002b505abe148f in boost::function1<void, qpid::sys::DispatchHandle&, std::allocator<boost::function_base> >::operator() (this=0x0, a0=@0x45055e87) at /usr/include/boost/function/function_template.hpp:576 #11 0x00002b505abe088f in qpid::sys::DispatchHandle::processEvent (this=0x1c41b058, type=<value optimized out>) at qpid/sys/DispatchHandle.cpp:353 #12 0x00002b505abdf6ae in qpid::sys::Dispatcher::run (this=0x7fff5048c1d0) at qpid/sys/Poller.h:105 #13 0x00002b505ab9386a in qpid::sys::(anonymous namespace)::runRunnable () at /usr/include/boost/any.hpp:46 #14 0x00000039958062f7 in start_thread (arg=<value optimized out>) at pthread_create.c:296 #15 0x0000003994cd1b6d in clone () from /lib64/libc.so.6 Thread 3 (process 11239): #0 0x0000003994cd1f58 in epoll_wait () from /lib64/libc.so.6 #1 0x00002b505ab9bb36 in qpid::sys::Poller::wait (this=0x1c40ffe0, timeout=<value optimized out>) at qpid/sys/epoll/EpollPoller.cpp:304 #2 0x00002b505abdf688 in qpid::sys::Dispatcher::run (this=0x7fff5048c1d0) at qpid/sys/Dispatcher.cpp:38 #3 0x00002b505ab9386a in qpid::sys::(anonymous namespace)::runRunnable () at /usr/include/boost/any.hpp:46 #4 0x00000039958062f7 in start_thread (arg=<value optimized out>) at pthread_create.c:296 #5 0x0000003994cd1b6d in clone () from /lib64/libc.so.6 Thread 2 (process 11240): #0 0x0000003994cd1f58 in epoll_wait () from /lib64/libc.so.6 #1 0x00002b505ab9bb36 in qpid::sys::Poller::wait (this=0x1c40ffe0, timeout=<value optimized out>) at qpid/sys/epoll/EpollPoller.cpp:304 #2 0x00002b505abdf688 in qpid::sys::Dispatcher::run (this=0x7fff5048c1d0) at qpid/sys/Dispatcher.cpp:38 #3 0x00002b505ab9386a in qpid::sys::(anonymous namespace)::runRunnable () at /usr/include/boost/any.hpp:46 #4 0x00000039958062f7 in start_thread (arg=<value optimized out>) at pthread_create.c:296 #5 0x0000003994cd1b6d in clone () from /lib64/libc.so.6 Thread 1 (process 11238): #0 0x0000003994c30155 in *__GI_raise (sig=<value optimized out>) at ../nptl/sysdeps/unix/sysv/linux/raise.c:64 #1 0x0000003994c31bf0 in *__GI_abort () at abort.c:88 #2 0x0000003994c295d6 in *__GI___assert_fail (assertion=0x2b505abfac44 "false", file=0x2b505ac0a8d8 "qpid/sys/DispatchHandle.cpp", line=344, function=0x2b505ac0a980 "virtual void qpid::sys::DispatchHandle::processEvent(qpid::sys::Poller::EventType)") at assert.c:78 #3 0x00002b505abe062c in qpid::sys::DispatchHandle::processEvent (this=0x2aaaac0011c8, type=qpid::sys::Poller::READABLE) at qpid/sys/DispatchHandle.cpp:344 #4 0x00002b505abdf6ae in qpid::sys::Dispatcher::run (this=0x7fff5048c1d0) at qpid/sys/Poller.h:105 #5 0x00002b505ab9386a in qpid::sys::(anonymous namespace)::runRunnable () at /usr/include/boost/any.hpp:46 #6 0x00000039958062f7 in start_thread (arg=<value optimized out>) at pthread_create.c:296 #7 0x0000003994cd1b6d in clone () from /lib64/libc.so.6
Produced by running a 4 node cluster on mrg7-10 and then on mrg10: while ./tsxtest --messages 30000 --rate 10000 ; do true ; done
Cluster brokers were started with: QPIDD="$BUILD/qpidd --cluster-name=aconway \ --load-module=$BUILD/.libs/cluster.so --no-module-dir \ --no-data-dir --auth=no --mgmt-enable=no \ --log-to-stderr=yes --log-to-file=$LOG \ --log-enable=info+ --log-enable=debug+:cluster \ --tcp-nodelay --cluster-read-max=1 \
*** This bug has been marked as a duplicate of bug 467822 ***