Attachment 910005 Details for Bug 1109488 – all-glance-thread.txt

all-glance-thread.txt

glance-hang.txt (text/plain), 70.78 KB, created by Attila Fazekas on 2014-06-18 13:12:55 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Attila Fazekas

Created: 2014-06-18 13:12:55 UTC

Size: 70.78 KB

patch

obsolete

>[root@afazekas-epel-qpid-1 ~]# ps aux |grep glance
>root     11470  0.0  0.0 112640   972 pts/10   S+   08:59   0:00 grep --color=auto glance
>stack    14361  0.0  0.4 178744 36156 pts/8    S+   Jun17   0:01 /usr/bin/python /usr/bin/glance-registry --config-file=/etc/glance/glance-registry.conf
>stack    14396  0.0  0.5 271664 40412 pts/8    S+   Jun17   0:00 /usr/bin/python /usr/bin/glance-registry --config-file=/etc/glance/glance-registry.conf
>stack    14401  0.0  0.5 192720 45868 pts/9    S+   Jun17   0:01 /usr/bin/python /usr/bin/glance-api --config-file=/etc/glance/glance-api.conf
>stack    14411  0.0  0.5 195404 45704 pts/9    S+   Jun17   0:00 /usr/bin/python /usr/bin/glance-api --config-file=/etc/glance/glance-api.conf
>[root@afazekas-epel-qpid-1 ~]# ps aux |grep [g]lance | awk '{print $2}' | while read PID; do echo PID:$PID; gdb </dev/null -p $PID -silent --nx --batch -ex "set pagination off" -ex "info threads"         -ex "thread apply all py-bt full"; done
>PID:14361
>[Thread debugging using libthread_db enabled]
>Using host libthread_db library "/lib64/libthread_db.so.1".
>0x00007f6e42140c02 in wait () from /lib64/libpthread.so.0
>  Id   Target Id         Frame 
>* 1    Thread 0x7f6e42923740 (LWP 14361) "glance-registry" 0x00007f6e42140c02 in wait () from /lib64/libpthread.so.0
>
>Thread 1 (Thread 0x7f6e42923740 (LWP 14361)):
>#1 <built-in function wait>
>#3 Frame 0x2432f00, for file /opt/stack/new/glance/glance/common/wsgi.py, line 258, in wait_on_children (self=<Server(sock=<GreenSocket(shutdown=<instancemethod at remote 0x23893c0>, fileno=<instancemethod at remote 0x172e780>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x172efa0>, setsockopt=<instancemethod at remote 0x2389280>, bind=<instancemethod at remote 0x14293c0>, fd=<_socketobject at remote 0x15949f0>, getsockname=<instancemethod at remote 0x1429550>, _timeout=None, close=<instancemethod at remote 0x1429320>, listen=<instancemethod at remote 0x205c690>) at remote 0x19db9d0>, application=<AuthProtocol(admin_token=None, LOG=<Logger(name='keystoneclient.middleware.auth_token', parent=<RootLogger(name='root', parent=None, handlers=[<ColorHandler(stream=<file at remote 0x7f6e428fe1e0>, level=0, lock=<_RLock(_Verbose__verbose=False, _RLock__owner=None, _RLock__block=<Semaphore(counter=1, _waiters=set([])) at remote 0x174d150>, _RLock__count=0) at remote 0x1746e10>, _name=None, filters=[], fo...(truncated)
>    pid, status = os.wait()
>#7 Frame 0x2432d20, for file /opt/stack/new/glance/glance/common/wsgi.py, line 285, in wait (self=<Server(sock=<GreenSocket(shutdown=<instancemethod at remote 0x23893c0>, fileno=<instancemethod at remote 0x172e780>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x172efa0>, setsockopt=<instancemethod at remote 0x2389280>, bind=<instancemethod at remote 0x14293c0>, fd=<_socketobject at remote 0x15949f0>, getsockname=<instancemethod at remote 0x1429550>, _timeout=None, close=<instancemethod at remote 0x1429320>, listen=<instancemethod at remote 0x205c690>) at remote 0x19db9d0>, application=<AuthProtocol(admin_token=None, LOG=<Logger(name='keystoneclient.middleware.auth_token', parent=<RootLogger(name='root', parent=None, handlers=[<ColorHandler(stream=<file at remote 0x7f6e428fe1e0>, level=0, lock=<_RLock(_Verbose__verbose=False, _RLock__owner=None, _RLock__block=<Semaphore(counter=1, _waiters=set([])) at remote 0x174d150>, _RLock__count=0) at remote 0x1746e10>, _name=None, filters=[], formatter=<Con...(truncated)
>    self.wait_on_children()
>#11 Frame 0x11bb280, for file /opt/stack/new/glance/glance/cmd/registry.py, line 52, in main (server=<Server(sock=<GreenSocket(shutdown=<instancemethod at remote 0x23893c0>, fileno=<instancemethod at remote 0x172e780>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x172efa0>, setsockopt=<instancemethod at remote 0x2389280>, bind=<instancemethod at remote 0x14293c0>, fd=<_socketobject at remote 0x15949f0>, getsockname=<instancemethod at remote 0x1429550>, _timeout=None, close=<instancemethod at remote 0x1429320>, listen=<instancemethod at remote 0x205c690>) at remote 0x19db9d0>, application=<AuthProtocol(admin_token=None, LOG=<Logger(name='keystoneclient.middleware.auth_token', parent=<RootLogger(name='root', parent=None, handlers=[<ColorHandler(stream=<file at remote 0x7f6e428fe1e0>, level=0, lock=<_RLock(_Verbose__verbose=False, _RLock__owner=None, _RLock__block=<Semaphore(counter=1, _waiters=set([])) at remote 0x174d150>, _RLock__count=0) at remote 0x1746e10>, _name=None, filters=[], formatter=<C...(truncated)
>    server.wait()
>#14 Frame 0xc1f650, for file /usr/bin/glance-registry, line 10, in <module> ()
>    sys.exit(main())
>PID:14396
>[Thread debugging using libthread_db enabled]
>Using host libthread_db library "/lib64/libthread_db.so.1".
>0x00007f6e41753e0d in poll () from /lib64/libc.so.6
>  Id   Target Id         Frame 
>* 1    Thread 0x7f6e42923740 (LWP 14396) "glance-registry" 0x00007f6e41753e0d in poll () from /lib64/libc.so.6
>
>Thread 1 (Thread 0x7f6e42923740 (LWP 14396)):
>#2 <built-in method poll of select.poll object at remote 0x2391530>
>#4 Frame 0x2435ca0, for file /usr/lib/python2.7/site-packages/eventlet/hubs/poll.py, line 73, in do_poll (self=<Hub(next_timers=[], clock=<built-in function time>, debug_exceptions=True, debug_blocking_resolution=1, modify=<built-in method modify of select.poll object at remote 0x2391530>, listeners={'read': {3: <FdListener(fileno=3, cb=<built-in method switch of greenlet.greenlet object at remote 0x7f6e4040e230>, evtype='read') at remote 0x23ab710>}, 'write': {}}, debug_blocking=False, running=True, timers_canceled=0, greenlet=<greenlet.greenlet at remote 0x11594b0>, stopping=False, timers=[], poll=<select.poll at remote 0x2391530>, secondaries={'read': {}, 'write': {}}, lclass=<type at remote 0x243c960>) at remote 0x23ab7d0>, seconds=<float at remote 0x128ba90>)
>    return self.poll.poll(int(seconds * 1000.0))
>#7 Frame 0x2435a70, for file /usr/lib/python2.7/site-packages/eventlet/hubs/poll.py, line 84, in wait (self=<Hub(next_timers=[], clock=<built-in function time>, debug_exceptions=True, debug_blocking_resolution=1, modify=<built-in method modify of select.poll object at remote 0x2391530>, listeners={'read': {3: <FdListener(fileno=3, cb=<built-in method switch of greenlet.greenlet object at remote 0x7f6e4040e230>, evtype='read') at remote 0x23ab710>}, 'write': {}}, debug_blocking=False, running=True, timers_canceled=0, greenlet=<greenlet.greenlet at remote 0x11594b0>, stopping=False, timers=[], poll=<select.poll at remote 0x2391530>, secondaries={'read': {}, 'write': {}}, lclass=<type at remote 0x243c960>) at remote 0x23ab7d0>, seconds=<float at remote 0x128ba90>, readers={...}, writers={...})
>    presult = self.do_poll(seconds)
>#11 Frame 0x24350e0, for file /usr/lib/python2.7/site-packages/eventlet/hubs/hub.py, line 236, in run (self=<Hub(next_timers=[], clock=<built-in function time>, debug_exceptions=True, debug_blocking_resolution=1, modify=<built-in method modify of select.poll object at remote 0x2391530>, listeners={'read': {3: <FdListener(fileno=3, cb=<built-in method switch of greenlet.greenlet object at remote 0x7f6e4040e230>, evtype='read') at remote 0x23ab710>}, 'write': {}}, debug_blocking=False, running=True, timers_canceled=0, greenlet=<greenlet.greenlet at remote 0x11594b0>, stopping=False, timers=[], poll=<select.poll at remote 0x2391530>, secondaries={'read': {}, 'write': {}}, lclass=<type at remote 0x243c960>) at remote 0x23ab7d0>, a=(), kw={}, wakeup_when=None, sleep_time=<float at remote 0x128ba90>)
>    self.wait(sleep_time)
>#20 <built-in method switch of greenlet.greenlet object at remote 0x11594b0>
>#22 Frame 0x2434b70, for file /usr/lib/python2.7/site-packages/eventlet/hubs/hub.py, line 187, in switch (self=<Hub(next_timers=[], clock=<built-in function time>, debug_exceptions=True, debug_blocking_resolution=1, modify=<built-in method modify of select.poll object at remote 0x2391530>, listeners={'read': {3: <FdListener(fileno=3, cb=<built-in method switch of greenlet.greenlet object at remote 0x7f6e4040e230>, evtype='read') at remote 0x23ab710>}, 'write': {}}, debug_blocking=False, running=True, timers_canceled=0, greenlet=<greenlet.greenlet at remote 0x11594b0>, stopping=False, timers=[], poll=<select.poll at remote 0x2391530>, secondaries={'read': {}, 'write': {}}, lclass=<type at remote 0x243c960>) at remote 0x23ab7d0>, cur=<greenlet.greenlet at remote 0x7f6e4040e230>, switch_out=None)
>    return self.greenlet.switch()
>#25 Frame 0x243a290, for file /usr/lib/python2.7/site-packages/eventlet/hubs/__init__.py, line 155, in trampoline (fd=<_socketobject at remote 0x15949f0>, read=True, write=None, timeout=None, timeout_exc=<timeout at remote 0x2392a78>, t=None, hub=<Hub(next_timers=[], clock=<built-in function time>, debug_exceptions=True, debug_blocking_resolution=1, modify=<built-in method modify of select.poll object at remote 0x2391530>, listeners={'read': {3: <FdListener(fileno=3, cb=<built-in method switch of greenlet.greenlet object at remote 0x7f6e4040e230>, evtype='read') at remote 0x23ab710>}, 'write': {}}, debug_blocking=False, running=True, timers_canceled=0, greenlet=<greenlet.greenlet at remote 0x11594b0>, stopping=False, timers=[], poll=<select.poll at remote 0x2391530>, secondaries={'read': {}, 'write': {}}, lclass=<type at remote 0x243c960>) at remote 0x23ab7d0>, current=<greenlet.greenlet at remote 0x7f6e4040e230>, fileno=3, listener=<...>)
>    return hub.switch()
>#29 Frame 0x2439910, for file /usr/lib/python2.7/site-packages/eventlet/greenio.py, line 183, in accept (self=<GreenSocket(shutdown=<instancemethod at remote 0x23893c0>, fileno=<instancemethod at remote 0x172e780>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x172efa0>, setsockopt=<instancemethod at remote 0x2389280>, bind=<instancemethod at remote 0x14293c0>, fd=<_socketobject at remote 0x15949f0>, getsockname=<instancemethod at remote 0x1429550>, _timeout=None, close=<instancemethod at remote 0x1429320>, listen=<instancemethod at remote 0x205c690>) at remote 0x19db9d0>, fd=<...>, res=None)
>    timeout_exc=socket.timeout("timed out"))
>#32 Frame 0x2438b70, for file /usr/lib/python2.7/site-packages/eventlet/wsgi.py, line 693, in server (sock=<GreenSocket(shutdown=<instancemethod at remote 0x23893c0>, fileno=<instancemethod at remote 0x172e780>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x172efa0>, setsockopt=<instancemethod at remote 0x2389280>, bind=<instancemethod at remote 0x14293c0>, fd=<_socketobject at remote 0x15949f0>, getsockname=<instancemethod at remote 0x1429550>, _timeout=None, close=<instancemethod at remote 0x1429320>, listen=<instancemethod at remote 0x205c690>) at remote 0x19db9d0>, site=<AuthProtocol(admin_token=u'MIIVvAYJKoZIhvcNAQcCoIIVrTCCFakCAQExCTAHBgUrDgMCGjCCFBIGCSqGSIb3DQEHAaCCFAMEghP-eyJhY2Nlc3MiOiB7InRva2VuIjogeyJpc3N1ZWRfYXQiOiAiMjAxNC0wNi0xN1QxMzo0ODowNS4zMDk4NTgiLCAiZXhwaXJlcyI6ICIyMDE0LTA2LTE3VDE0OjQ4OjA1WiIsICJpZCI6ICJwbGFjZWhvbGRlciIsICJ0ZW5hbnQiOiB7ImRlc2NyaXB0aW9uIjogbnVsbCwgImVuYWJsZWQiOiB0cnVlLCAiaWQiOiAiMjc4M2MyN2Q5YzM2NDMxY2E1NTFkMWI4Y2ZjNGM4YjAiLCAibmFtZSI6ICJzZXJ2aWNlIn19LCAic2VydmljZU...(truncated)
>    client_socket = sock.accept()
>#36 Frame 0x2433e50, for file /opt/stack/new/glance/glance/common/wsgi.py, line 328, in run_server (self=<Server(sock=<GreenSocket(shutdown=<instancemethod at remote 0x23893c0>, fileno=<instancemethod at remote 0x172e780>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x172efa0>, setsockopt=<instancemethod at remote 0x2389280>, bind=<instancemethod at remote 0x14293c0>, fd=<_socketobject at remote 0x15949f0>, getsockname=<instancemethod at remote 0x1429550>, _timeout=None, close=<instancemethod at remote 0x1429320>, listen=<instancemethod at remote 0x205c690>) at remote 0x19db9d0>, application=<AuthProtocol(admin_token=u'MIIVvAYJKoZIhvcNAQcCoIIVrTCCFakCAQExCTAHBgUrDgMCGjCCFBIGCSqGSIb3DQEHAaCCFAMEghP-eyJhY2Nlc3MiOiB7InRva2VuIjogeyJpc3N1ZWRfYXQiOiAiMjAxNC0wNi0xN1QxMzo0ODowNS4zMDk4NTgiLCAiZXhwaXJlcyI6ICIyMDE0LTA2LTE3VDE0OjQ4OjA1WiIsICJpZCI6ICJwbGFjZWhvbGRlciIsICJ0ZW5hbnQiOiB7ImRlc2NyaXB0aW9uIjogbnVsbCwgImVuYWJsZWQiOiB0cnVlLCAiaWQiOiAiMjc4M2MyN2Q5YzM2NDMxY2E1NTFkMWI4Y2ZjNGM4YjAiLCAibmFtZSI6ICJzZXJ2aWNl...(truncated)
>    debug=False)
>#40 Frame 0x2432b50, for file /opt/stack/new/glance/glance/common/wsgi.py, line 300, in run_child (self=<Server(sock=<GreenSocket(shutdown=<instancemethod at remote 0x23893c0>, fileno=<instancemethod at remote 0x172e780>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x172efa0>, setsockopt=<instancemethod at remote 0x2389280>, bind=<instancemethod at remote 0x14293c0>, fd=<_socketobject at remote 0x15949f0>, getsockname=<instancemethod at remote 0x1429550>, _timeout=None, close=<instancemethod at remote 0x1429320>, listen=<instancemethod at remote 0x205c690>) at remote 0x19db9d0>, application=<AuthProtocol(admin_token=u'MIIVvAYJKoZIhvcNAQcCoIIVrTCCFakCAQExCTAHBgUrDgMCGjCCFBIGCSqGSIb3DQEHAaCCFAMEghP-eyJhY2Nlc3MiOiB7InRva2VuIjogeyJpc3N1ZWRfYXQiOiAiMjAxNC0wNi0xN1QxMzo0ODowNS4zMDk4NTgiLCAiZXhwaXJlcyI6ICIyMDE0LTA2LTE3VDE0OjQ4OjA1WiIsICJpZCI6ICJwbGFjZWhvbGRlciIsICJ0ZW5hbnQiOiB7ImRlc2NyaXB0aW9uIjogbnVsbCwgImVuYWJsZWQiOiB0cnVlLCAiaWQiOiAiMjc4M2MyN2Q5YzM2NDMxY2E1NTFkMWI4Y2ZjNGM4YjAiLCAibmFtZSI6ICJzZXJ2aWNlI...(truncated)
>    self.run_server()
>#44 Frame 0x1b74b10, for file /opt/stack/new/glance/glance/common/wsgi.py, line 250, in start (self=<Server(sock=<GreenSocket(shutdown=<instancemethod at remote 0x23893c0>, fileno=<instancemethod at remote 0x172e780>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x172efa0>, setsockopt=<instancemethod at remote 0x2389280>, bind=<instancemethod at remote 0x14293c0>, fd=<_socketobject at remote 0x15949f0>, getsockname=<instancemethod at remote 0x1429550>, _timeout=None, close=<instancemethod at remote 0x1429320>, listen=<instancemethod at remote 0x205c690>) at remote 0x19db9d0>, application=<AuthProtocol(admin_token=u'MIIVvAYJKoZIhvcNAQcCoIIVrTCCFakCAQExCTAHBgUrDgMCGjCCFBIGCSqGSIb3DQEHAaCCFAMEghP-eyJhY2Nlc3MiOiB7InRva2VuIjogeyJpc3N1ZWRfYXQiOiAiMjAxNC0wNi0xN1QxMzo0ODowNS4zMDk4NTgiLCAiZXhwaXJlcyI6ICIyMDE0LTA2LTE3VDE0OjQ4OjA1WiIsICJpZCI6ICJwbGFjZWhvbGRlciIsICJ0ZW5hbnQiOiB7ImRlc2NyaXB0aW9uIjogbnVsbCwgImVuYWJsZWQiOiB0cnVlLCAiaWQiOiAiMjc4M2MyN2Q5YzM2NDMxY2E1NTFkMWI4Y2ZjNGM4YjAiLCAibmFtZSI6ICJzZXJ2aWNlIn19L...(truncated)
>    self.run_child()
>#48 Frame 0x11bb280, for file /opt/stack/new/glance/glance/cmd/registry.py, line 51, in main (server=<Server(sock=<GreenSocket(shutdown=<instancemethod at remote 0x23893c0>, fileno=<instancemethod at remote 0x172e780>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x172efa0>, setsockopt=<instancemethod at remote 0x2389280>, bind=<instancemethod at remote 0x14293c0>, fd=<_socketobject at remote 0x15949f0>, getsockname=<instancemethod at remote 0x1429550>, _timeout=None, close=<instancemethod at remote 0x1429320>, listen=<instancemethod at remote 0x205c690>) at remote 0x19db9d0>, application=<AuthProtocol(admin_token=u'MIIVvAYJKoZIhvcNAQcCoIIVrTCCFakCAQExCTAHBgUrDgMCGjCCFBIGCSqGSIb3DQEHAaCCFAMEghP-eyJhY2Nlc3MiOiB7InRva2VuIjogeyJpc3N1ZWRfYXQiOiAiMjAxNC0wNi0xN1QxMzo0ODowNS4zMDk4NTgiLCAiZXhwaXJlcyI6ICIyMDE0LTA2LTE3VDE0OjQ4OjA1WiIsICJpZCI6ICJwbGFjZWhvbGRlciIsICJ0ZW5hbnQiOiB7ImRlc2NyaXB0aW9uIjogbnVsbCwgImVuYWJsZWQiOiB0cnVlLCAiaWQiOiAiMjc4M2MyN2Q5YzM2NDMxY2E1NTFkMWI4Y2ZjNGM4YjAiLCAibmFtZSI6ICJzZXJ2aWNlIn19...(truncated)
>    default_port=9191)
>#51 Frame 0xc1f650, for file /usr/bin/glance-registry, line 10, in <module> ()
>    sys.exit(main())
>PID:14401
>[Thread debugging using libthread_db enabled]
>Using host libthread_db library "/lib64/libthread_db.so.1".
>0x00007f00c666cc02 in wait () from /lib64/libpthread.so.0
>  Id   Target Id         Frame 
>* 1    Thread 0x7f00c6e4f740 (LWP 14401) "glance-api" 0x00007f00c666cc02 in wait () from /lib64/libpthread.so.0
>
>Thread 1 (Thread 0x7f00c6e4f740 (LWP 14401)):
>#1 <built-in function wait>
>#3 Frame 0x38eaa80, for file /opt/stack/new/glance/glance/common/wsgi.py, line 258, in wait_on_children (self=<Server(sock=<GreenSocket(shutdown=<instancemethod at remote 0x42a4640>, fileno=<instancemethod at remote 0x2f7fbe0>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x2f7fe60>, setsockopt=<instancemethod at remote 0x347de60>, bind=<instancemethod at remote 0x2f7fc30>, fd=<_socketobject at remote 0x3845ec0>, getsockname=<instancemethod at remote 0x2f7fe10>, _timeout=None, close=<instancemethod at remote 0x2ee6a50>, listen=<instancemethod at remote 0x3545050>) at remote 0x3936f90>, application=<VersionNegotiationFilter(application=<AuthProtocol(admin_token=None, LOG=<Logger(name='keystoneclient.middleware.auth_token', parent=<RootLogger(name='root', parent=None, handlers=[<ColorHandler(stream=<file at remote 0x7f00c6e2a1e0>, level=0, lock=<_RLock(_Verbose__verbose=False, _RLock__owner=None, _RLock__block=<Semaphore(counter=1, _waiters=set([])) at remote 0x33d7950>, _RLock__count=0) at remote ...(truncated)
>    pid, status = os.wait()
>#7 Frame 0x38ea8a0, for file /opt/stack/new/glance/glance/common/wsgi.py, line 285, in wait (self=<Server(sock=<GreenSocket(shutdown=<instancemethod at remote 0x42a4640>, fileno=<instancemethod at remote 0x2f7fbe0>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x2f7fe60>, setsockopt=<instancemethod at remote 0x347de60>, bind=<instancemethod at remote 0x2f7fc30>, fd=<_socketobject at remote 0x3845ec0>, getsockname=<instancemethod at remote 0x2f7fe10>, _timeout=None, close=<instancemethod at remote 0x2ee6a50>, listen=<instancemethod at remote 0x3545050>) at remote 0x3936f90>, application=<VersionNegotiationFilter(application=<AuthProtocol(admin_token=None, LOG=<Logger(name='keystoneclient.middleware.auth_token', parent=<RootLogger(name='root', parent=None, handlers=[<ColorHandler(stream=<file at remote 0x7f00c6e2a1e0>, level=0, lock=<_RLock(_Verbose__verbose=False, _RLock__owner=None, _RLock__block=<Semaphore(counter=1, _waiters=set([])) at remote 0x33d7950>, _RLock__count=0) at remote 0x33d7910>, ...(truncated)
>    self.wait_on_children()
>#11 Frame 0x3414f10, for file /opt/stack/new/glance/glance/cmd/api.py, line 64, in main (server=<Server(sock=<GreenSocket(shutdown=<instancemethod at remote 0x42a4640>, fileno=<instancemethod at remote 0x2f7fbe0>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x2f7fe60>, setsockopt=<instancemethod at remote 0x347de60>, bind=<instancemethod at remote 0x2f7fc30>, fd=<_socketobject at remote 0x3845ec0>, getsockname=<instancemethod at remote 0x2f7fe10>, _timeout=None, close=<instancemethod at remote 0x2ee6a50>, listen=<instancemethod at remote 0x3545050>) at remote 0x3936f90>, application=<VersionNegotiationFilter(application=<AuthProtocol(admin_token=None, LOG=<Logger(name='keystoneclient.middleware.auth_token', parent=<RootLogger(name='root', parent=None, handlers=[<ColorHandler(stream=<file at remote 0x7f00c6e2a1e0>, level=0, lock=<_RLock(_Verbose__verbose=False, _RLock__owner=None, _RLock__block=<Semaphore(counter=1, _waiters=set([])) at remote 0x33d7950>, _RLock__count=0) at remote 0x33d7910>, _na...(truncated)
>    server.wait()
>#14 Frame 0x2513650, for file /usr/bin/glance-api, line 10, in <module> ()
>    sys.exit(main())
>PID:14411
>[Thread debugging using libthread_db enabled]
>Using host libthread_db library "/lib64/libthread_db.so.1".
>0x00007f00c5c7fdf0 in __poll_nocancel () from /lib64/libc.so.6
>  Id   Target Id         Frame 
>* 1    Thread 0x7f00c6e4f740 (LWP 14411) "glance-api" 0x00007f00c5c7fdf0 in __poll_nocancel () from /lib64/libc.so.6
>
>Thread 1 (Thread 0x7f00c6e4f740 (LWP 14411)):
>#2 <built-in method poll of select.poll object at remote 0x43aa238>
>#4 Frame 0x4931570, for file /usr/lib/python2.7/site-packages/qpid/compat.py, line 80, in select (rlist=set([<...>]), wlist=set([]), xlist=(), timeout=None, fd_count=1, rset=set([<...>]), wset=set([]), xset=set([]), poller=<select.poll at remote 0x43aa238>, rwset=set([]), ro=<...>, fd_map={6: <...>}, o=<...>)
>    active = poller.poll(timeout)
>#8 Frame 0x4931340, for file /usr/lib/python2.7/site-packages/qpid/selector.py, line 124, in run (self=<Selector(thread=<Thread(_Thread__ident=73962192, _Thread__block=<_Condition(_Verbose__verbose=False, _Condition__lock=<Semaphore(counter=1, _waiters=set([])) at remote 0x464ef90>, acquire=<instancemethod at remote 0x44de3c0>, _Condition__waiters=[], release=<instancemethod at remote 0x43641e0>) at remote 0x464efd0>, _Thread__name='Thread-1', _Thread__daemonic=True, _Thread__started=<_Event(_Verbose__verbose=False, _Event__flag=True, _Event__cond=<_Condition(_Verbose__verbose=False, _Condition__lock=<Semaphore(counter=1, _waiters=set([])) at remote 0x464eed0>, acquire=<instancemethod at remote 0x44de230>, _Condition__waiters=[], release=<instancemethod at remote 0x46ab410>) at remote 0x464ef10>) at remote 0x464ee90>, _Thread__stderr=<file at remote 0x7f00c6e2a1e0>, _Thread__target=<instancemethod at remote 0x4692f50>, _Thread__kwargs={}, _Verbose__verbose=False, _Thread__args=(), _Thread__stopped=False, _Thread...(truncated)
>    rd, wr, ex = select(self.reading, self.writing, (), timeout)
>#13 Frame 0x4931170, for file /usr/lib64/python2.7/threading.py, line 764, in run (self=<Thread(_Thread__ident=73962192, _Thread__block=<_Condition(_Verbose__verbose=False, _Condition__lock=<Semaphore(counter=1, _waiters=set([])) at remote 0x464ef90>, acquire=<instancemethod at remote 0x44de3c0>, _Condition__waiters=[], release=<instancemethod at remote 0x43641e0>) at remote 0x464efd0>, _Thread__name='Thread-1', _Thread__daemonic=True, _Thread__started=<_Event(_Verbose__verbose=False, _Event__flag=True, _Event__cond=<_Condition(_Verbose__verbose=False, _Condition__lock=<Semaphore(counter=1, _waiters=set([])) at remote 0x464eed0>, acquire=<instancemethod at remote 0x44de230>, _Condition__waiters=[], release=<instancemethod at remote 0x46ab410>) at remote 0x464ef10>) at remote 0x464ee90>, _Thread__stderr=<file at remote 0x7f00c6e2a1e0>, _Thread__target=<instancemethod at remote 0x4692f50>, _Thread__kwargs={}, _Verbose__verbose=False, _Thread__args=(), _Thread__stopped=False, _Thread__initialized=True) at remote 0x4...(truncated)
>    self.__target(*self.__args, **self.__kwargs)
>#16 Frame 0x4930c20, for file /usr/lib64/python2.7/threading.py, line 811, in __bootstrap_inner (self=<Thread(_Thread__ident=73962192, _Thread__block=<_Condition(_Verbose__verbose=False, _Condition__lock=<Semaphore(counter=1, _waiters=set([])) at remote 0x464ef90>, acquire=<instancemethod at remote 0x44de3c0>, _Condition__waiters=[], release=<instancemethod at remote 0x43641e0>) at remote 0x464efd0>, _Thread__name='Thread-1', _Thread__daemonic=True, _Thread__started=<_Event(_Verbose__verbose=False, _Event__flag=True, _Event__cond=<_Condition(_Verbose__verbose=False, _Condition__lock=<Semaphore(counter=1, _waiters=set([])) at remote 0x464eed0>, acquire=<instancemethod at remote 0x44de230>, _Condition__waiters=[], release=<instancemethod at remote 0x46ab410>) at remote 0x464ef10>) at remote 0x464ee90>, _Thread__stderr=<file at remote 0x7f00c6e2a1e0>, _Thread__target=<instancemethod at remote 0x4692f50>, _Thread__kwargs={}, _Verbose__verbose=False, _Thread__args=(), _Thread__stopped=False, _Thread__initialized=True)...(truncated)
>    self.run()
>#19 Frame 0x4930a50, for file /usr/lib64/python2.7/threading.py, line 784, in __bootstrap (self=<Thread(_Thread__ident=73962192, _Thread__block=<_Condition(_Verbose__verbose=False, _Condition__lock=<Semaphore(counter=1, _waiters=set([])) at remote 0x464ef90>, acquire=<instancemethod at remote 0x44de3c0>, _Condition__waiters=[], release=<instancemethod at remote 0x43641e0>) at remote 0x464efd0>, _Thread__name='Thread-1', _Thread__daemonic=True, _Thread__started=<_Event(_Verbose__verbose=False, _Event__flag=True, _Event__cond=<_Condition(_Verbose__verbose=False, _Condition__lock=<Semaphore(counter=1, _waiters=set([])) at remote 0x464eed0>, acquire=<instancemethod at remote 0x44de230>, _Condition__waiters=[], release=<instancemethod at remote 0x46ab410>) at remote 0x464ef10>) at remote 0x464ee90>, _Thread__stderr=<file at remote 0x7f00c6e2a1e0>, _Thread__target=<instancemethod at remote 0x4692f50>, _Thread__kwargs={}, _Verbose__verbose=False, _Thread__args=(), _Thread__stopped=False, _Thread__initialized=True) at re...(truncated)
>    self.__bootstrap_inner()
>#24 Frame 0x4940a00, for file /usr/lib/python2.7/site-packages/eventlet/green/thread.py, line 27, in __thread_body (func=<instancemethod at remote 0x4692730>, args=(), kwargs={})
>    func(*args, **kwargs)
>#33 Frame 0x46c9f70, for file /usr/lib/python2.7/site-packages/eventlet/hubs/timer.py, line 56, in __call__ (self=<Timer(seconds=0, tpl=(<built-in method switch of greenlet.greenlet object at remote 0x46892d0>, (<instancemethod at remote 0x4692730>, (), {}), {}), called=True) at remote 0x4654050>, args=(...), cb=<built-in method switch of greenlet.greenlet object at remote 0x46892d0>, kw={...})
>    cb(*args, **kw)
>#43 Frame 0x46c95c0, for file /usr/lib/python2.7/site-packages/eventlet/hubs/hub.py, line 346, in fire_timers (self=<Hub(next_timers=[(<float at remote 0x47375a0>, <Timer(seconds=0, tpl=(<instancemethod at remote 0x46925f0>, (), {}), called=False) at remote 0x46540d0>)], clock=<built-in function time>, debug_exceptions=True, debug_blocking_resolution=1, modify=<built-in method modify of select.poll object at remote 0x43a9fa8>, listeners={'read': {4: <FdListener(fileno=4, cb=<built-in method switch of greenlet.greenlet object at remote 0x7f00c493a730>, evtype='read') at remote 0x44d8b10>}, 'write': {}}, debug_blocking=False, running=True, timers_canceled=2, greenlet=<greenlet.greenlet at remote 0x33c8910>, stopping=False, timers=[(<float at remote 0x4737630>, <Timer(seconds=<float at remote 0x4737678>, called=True) at remote 0x46a9210>), (<float at remote 0x4737690>, <Timer(seconds=<float at remote 0x4737660>, called=True) at remote 0x46a9190>)], poll=<select.poll at remote 0x43a9fa8>, secondaries={'read': {}, 'wr...(truncated)
>    timer()
>#46 Frame 0x46c93c0, for file /usr/lib/python2.7/site-packages/eventlet/hubs/hub.py, line 226, in run (self=<Hub(next_timers=[(<float at remote 0x47375a0>, <Timer(seconds=0, tpl=(<instancemethod at remote 0x46925f0>, (), {}), called=False) at remote 0x46540d0>)], clock=<built-in function time>, debug_exceptions=True, debug_blocking_resolution=1, modify=<built-in method modify of select.poll object at remote 0x43a9fa8>, listeners={'read': {4: <FdListener(fileno=4, cb=<built-in method switch of greenlet.greenlet object at remote 0x7f00c493a730>, evtype='read') at remote 0x44d8b10>}, 'write': {}}, debug_blocking=False, running=True, timers_canceled=2, greenlet=<greenlet.greenlet at remote 0x33c8910>, stopping=False, timers=[(<float at remote 0x4737630>, <Timer(seconds=<float at remote 0x4737678>, called=True) at remote 0x46a9210>), (<float at remote 0x4737690>, <Timer(seconds=<float at remote 0x4737660>, called=True) at remote 0x46a9190>)], poll=<select.poll at remote 0x43a9fa8>, secondaries={'read': {}, 'write': {}...(truncated)
>    self.fire_timers(self.clock())
>#55 <built-in method switch of greenlet.greenlet object at remote 0x33c8910>
>#57 Frame 0x46deb20, for file /usr/lib/python2.7/site-packages/eventlet/hubs/hub.py, line 187, in switch (self=<Hub(next_timers=[(<float at remote 0x47375a0>, <Timer(seconds=0, tpl=(<instancemethod at remote 0x46925f0>, (), {}), called=False) at remote 0x46540d0>)], clock=<built-in function time>, debug_exceptions=True, debug_blocking_resolution=1, modify=<built-in method modify of select.poll object at remote 0x43a9fa8>, listeners={'read': {4: <FdListener(fileno=4, cb=<built-in method switch of greenlet.greenlet object at remote 0x7f00c493a730>, evtype='read') at remote 0x44d8b10>}, 'write': {}}, debug_blocking=False, running=True, timers_canceled=2, greenlet=<greenlet.greenlet at remote 0x33c8910>, stopping=False, timers=[(<float at remote 0x4737630>, <Timer(seconds=<float at remote 0x4737678>, called=True) at remote 0x46a9210>), (<float at remote 0x4737690>, <Timer(seconds=<float at remote 0x4737660>, called=True) at remote 0x46a9190>)], poll=<select.poll at remote 0x43a9fa8>, secondaries={'read': {}, 'write':...(truncated)
>    return self.greenlet.switch()
>#60 Frame 0x46dd150, for file /usr/lib/python2.7/site-packages/eventlet/hubs/__init__.py, line 155, in trampoline (fd=<_socketobject at remote 0x3845ec0>, read=True, write=None, timeout=None, timeout_exc=<timeout at remote 0x38a8530>, t=None, hub=<Hub(next_timers=[(<float at remote 0x47375a0>, <Timer(seconds=0, tpl=(<instancemethod at remote 0x46925f0>, (), {}), called=False) at remote 0x46540d0>)], clock=<built-in function time>, debug_exceptions=True, debug_blocking_resolution=1, modify=<built-in method modify of select.poll object at remote 0x43a9fa8>, listeners={'read': {4: <FdListener(fileno=4, cb=<built-in method switch of greenlet.greenlet object at remote 0x7f00c493a730>, evtype='read') at remote 0x44d8b10>}, 'write': {}}, debug_blocking=False, running=True, timers_canceled=2, greenlet=<greenlet.greenlet at remote 0x33c8910>, stopping=False, timers=[(<float at remote 0x4737630>, <Timer(seconds=<float at remote 0x4737678>, called=True) at remote 0x46a9210>), (<float at remote 0x4737690>, <Timer(seconds=<fl...(truncated)
>    return hub.switch()
>#64 Frame 0x46c7d30, for file /usr/lib/python2.7/site-packages/eventlet/greenio.py, line 183, in accept (self=<GreenSocket(shutdown=<instancemethod at remote 0x42a4640>, fileno=<instancemethod at remote 0x2f7fbe0>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x2f7fe60>, setsockopt=<instancemethod at remote 0x347de60>, bind=<instancemethod at remote 0x2f7fc30>, fd=<_socketobject at remote 0x3845ec0>, getsockname=<instancemethod at remote 0x2f7fe10>, _timeout=None, close=<instancemethod at remote 0x2ee6a50>, listen=<instancemethod at remote 0x3545050>) at remote 0x3936f90>, fd=<...>, res=None)
>    timeout_exc=socket.timeout("timed out"))
>#67 Frame 0x46c6dc0, for file /usr/lib/python2.7/site-packages/eventlet/wsgi.py, line 693, in server (sock=<GreenSocket(shutdown=<instancemethod at remote 0x42a4640>, fileno=<instancemethod at remote 0x2f7fbe0>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x2f7fe60>, setsockopt=<instancemethod at remote 0x347de60>, bind=<instancemethod at remote 0x2f7fc30>, fd=<_socketobject at remote 0x3845ec0>, getsockname=<instancemethod at remote 0x2f7fe10>, _timeout=None, close=<instancemethod at remote 0x2ee6a50>, listen=<instancemethod at remote 0x3545050>) at remote 0x3936f90>, site=<VersionNegotiationFilter(application=<AuthProtocol(admin_token=u'MIIVvAYJKoZIhvcNAQcCoIIVrTCCFakCAQExCTAHBgUrDgMCGjCCFBIGCSqGSIb3DQEHAaCCFAMEghP-eyJhY2Nlc3MiOiB7InRva2VuIjogeyJpc3N1ZWRfYXQiOiAiMjAxNC0wNi0xN1QxMzo0ODowNC44ODM1OTkiLCAiZXhwaXJlcyI6ICIyMDE0LTA2LTE3VDE0OjQ4OjA0WiIsICJpZCI6ICJwbGFjZWhvbGRlciIsICJ0ZW5hbnQiOiB7ImRlc2NyaXB0aW9uIjogbnVsbCwgImVuYWJsZWQiOiB0cnVlLCAiaWQiOiAiMjc4M2MyN2Q5YzM2NDMxY2E1NTFkMWI4Y2ZjNGM4YjAiLCAi...(truncated)
>    client_socket = sock.accept()
>#71 Frame 0x46c6810, for file /opt/stack/new/glance/glance/common/wsgi.py, line 328, in run_server (self=<Server(sock=<GreenSocket(shutdown=<instancemethod at remote 0x42a4640>, fileno=<instancemethod at remote 0x2f7fbe0>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x2f7fe60>, setsockopt=<instancemethod at remote 0x347de60>, bind=<instancemethod at remote 0x2f7fc30>, fd=<_socketobject at remote 0x3845ec0>, getsockname=<instancemethod at remote 0x2f7fe10>, _timeout=None, close=<instancemethod at remote 0x2ee6a50>, listen=<instancemethod at remote 0x3545050>) at remote 0x3936f90>, application=<VersionNegotiationFilter(application=<AuthProtocol(admin_token=u'MIIVvAYJKoZIhvcNAQcCoIIVrTCCFakCAQExCTAHBgUrDgMCGjCCFBIGCSqGSIb3DQEHAaCCFAMEghP-eyJhY2Nlc3MiOiB7InRva2VuIjogeyJpc3N1ZWRfYXQiOiAiMjAxNC0wNi0xN1QxMzo0ODowNC44ODM1OTkiLCAiZXhwaXJlcyI6ICIyMDE0LTA2LTE3VDE0OjQ4OjA0WiIsICJpZCI6ICJwbGFjZWhvbGRlciIsICJ0ZW5hbnQiOiB7ImRlc2NyaXB0aW9uIjogbnVsbCwgImVuYWJsZWQiOiB0cnVlLCAiaWQiOiAiMjc4M2MyN2Q5YzM2NDMxY2E1NTFkMW...(truncated)
>    debug=False)
>#75 Frame 0x342ac20, for file /opt/stack/new/glance/glance/common/wsgi.py, line 300, in run_child (self=<Server(sock=<GreenSocket(shutdown=<instancemethod at remote 0x42a4640>, fileno=<instancemethod at remote 0x2f7fbe0>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x2f7fe60>, setsockopt=<instancemethod at remote 0x347de60>, bind=<instancemethod at remote 0x2f7fc30>, fd=<_socketobject at remote 0x3845ec0>, getsockname=<instancemethod at remote 0x2f7fe10>, _timeout=None, close=<instancemethod at remote 0x2ee6a50>, listen=<instancemethod at remote 0x3545050>) at remote 0x3936f90>, application=<VersionNegotiationFilter(application=<AuthProtocol(admin_token=u'MIIVvAYJKoZIhvcNAQcCoIIVrTCCFakCAQExCTAHBgUrDgMCGjCCFBIGCSqGSIb3DQEHAaCCFAMEghP-eyJhY2Nlc3MiOiB7InRva2VuIjogeyJpc3N1ZWRfYXQiOiAiMjAxNC0wNi0xN1QxMzo0ODowNC44ODM1OTkiLCAiZXhwaXJlcyI6ICIyMDE0LTA2LTE3VDE0OjQ4OjA0WiIsICJpZCI6ICJwbGFjZWhvbGRlciIsICJ0ZW5hbnQiOiB7ImRlc2NyaXB0aW9uIjogbnVsbCwgImVuYWJsZWQiOiB0cnVlLCAiaWQiOiAiMjc4M2MyN2Q5YzM2NDMxY2E1NTFkMWI...(truncated)
>    self.run_server()
>#79 Frame 0x462e960, for file /opt/stack/new/glance/glance/common/wsgi.py, line 250, in start (self=<Server(sock=<GreenSocket(shutdown=<instancemethod at remote 0x42a4640>, fileno=<instancemethod at remote 0x2f7fbe0>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x2f7fe60>, setsockopt=<instancemethod at remote 0x347de60>, bind=<instancemethod at remote 0x2f7fc30>, fd=<_socketobject at remote 0x3845ec0>, getsockname=<instancemethod at remote 0x2f7fe10>, _timeout=None, close=<instancemethod at remote 0x2ee6a50>, listen=<instancemethod at remote 0x3545050>) at remote 0x3936f90>, application=<VersionNegotiationFilter(application=<AuthProtocol(admin_token=u'MIIVvAYJKoZIhvcNAQcCoIIVrTCCFakCAQExCTAHBgUrDgMCGjCCFBIGCSqGSIb3DQEHAaCCFAMEghP-eyJhY2Nlc3MiOiB7InRva2VuIjogeyJpc3N1ZWRfYXQiOiAiMjAxNC0wNi0xN1QxMzo0ODowNC44ODM1OTkiLCAiZXhwaXJlcyI6ICIyMDE0LTA2LTE3VDE0OjQ4OjA0WiIsICJpZCI6ICJwbGFjZWhvbGRlciIsICJ0ZW5hbnQiOiB7ImRlc2NyaXB0aW9uIjogbnVsbCwgImVuYWJsZWQiOiB0cnVlLCAiaWQiOiAiMjc4M2MyN2Q5YzM2NDMxY2E1NTFkMWI4Y2Z...(truncated)
>    self.run_child()
>#83 Frame 0x3414f10, for file /opt/stack/new/glance/glance/cmd/api.py, line 63, in main (server=<Server(sock=<GreenSocket(shutdown=<instancemethod at remote 0x42a4640>, fileno=<instancemethod at remote 0x2f7fbe0>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x2f7fe60>, setsockopt=<instancemethod at remote 0x347de60>, bind=<instancemethod at remote 0x2f7fc30>, fd=<_socketobject at remote 0x3845ec0>, getsockname=<instancemethod at remote 0x2f7fe10>, _timeout=None, close=<instancemethod at remote 0x2ee6a50>, listen=<instancemethod at remote 0x3545050>) at remote 0x3936f90>, application=<VersionNegotiationFilter(application=<AuthProtocol(admin_token=u'MIIVvAYJKoZIhvcNAQcCoIIVrTCCFakCAQExCTAHBgUrDgMCGjCCFBIGCSqGSIb3DQEHAaCCFAMEghP-eyJhY2Nlc3MiOiB7InRva2VuIjogeyJpc3N1ZWRfYXQiOiAiMjAxNC0wNi0xN1QxMzo0ODowNC44ODM1OTkiLCAiZXhwaXJlcyI6ICIyMDE0LTA2LTE3VDE0OjQ4OjA0WiIsICJpZCI6ICJwbGFjZWhvbGRlciIsICJ0ZW5hbnQiOiB7ImRlc2NyaXB0aW9uIjogbnVsbCwgImVuYWJsZWQiOiB0cnVlLCAiaWQiOiAiMjc4M2MyN2Q5YzM2NDMxY2E1NTFkMWI4Y2ZjNGM...(truncated)
>    server.start(config.load_paste_app('glance-api'), default_port=9292)
>#86 Frame 0x2513650, for file /usr/bin/glance-api, line 10, in <module> ()
>    sys.exit(main())
>[root@afazekas-epel-qpid-1 ~]# ps aux |grep glance
>root     11537  0.0  0.0 112640   976 pts/10   S+   09:00   0:00 grep --color=auto glance
>stack    14361  0.0  0.4 178744 36156 pts/8    S+   Jun17   0:01 /usr/bin/python /usr/bin/glance-registry --config-file=/etc/glance/glance-registry.conf
>stack    14396  0.0  0.5 271664 40412 pts/8    S+   Jun17   0:00 /usr/bin/python /usr/bin/glance-registry --config-file=/etc/glance/glance-registry.conf
>stack    14401  0.0  0.5 192720 45868 pts/9    S+   Jun17   0:01 /usr/bin/python /usr/bin/glance-api --config-file=/etc/glance/glance-api.conf
>stack    14411  0.0  0.5 195404 45704 pts/9    S+   Jun17   0:00 /usr/bin/python /usr/bin/glance-api --config-file=/etc/glance/glance-api.conf
>[root@afazekas-epel-qpid-1 ~]# ps aux |grep glance
>root     11470  0.0  0.0 112640   972 pts/10   S+   08:59   0:00 grep --color=auto glance
>stack    14361  0.0  0.4 178744 36156 pts/8    S+   Jun17   0:01 /usr/bin/python /usr/bin/glance-registry --config-file=/etc/glance/glance-registry.conf
>stack    14396  0.0  0.5 271664 40412 pts/8    S+   Jun17   0:00 /usr/bin/python /usr/bin/glance-registry --config-file=/etc/glance/glance-registry.conf
>stack    14401  0.0  0.5 192720 45868 pts/9    S+   Jun17   0:01 /usr/bin/python /usr/bin/glance-api --config-file=/etc/glance/glance-api.conf
>stack    14411  0.0  0.5 195404 45704 pts/9    S+   Jun17   0:00 /usr/bin/python /usr/bin/glance-api --config-file=/etc/glance/glance-api.conf
>[root@afazekas-epel-qpid-1 ~]# ps aux |grep [g]lance | awk '{print $2}' | while read PID; do echo PID:$PID; gdb </dev/null -p $PID -silent --nx --batch -ex "set pagination off" -ex "info threads"         -ex "thread apply all py-bt full"; done
>PID:14361
>[Thread debugging using libthread_db enabled]
>Using host libthread_db library "/lib64/libthread_db.so.1".
>0x00007f6e42140c02 in wait () from /lib64/libpthread.so.0
>  Id   Target Id         Frame 
>* 1    Thread 0x7f6e42923740 (LWP 14361) "glance-registry" 0x00007f6e42140c02 in wait () from /lib64/libpthread.so.0
>
>Thread 1 (Thread 0x7f6e42923740 (LWP 14361)):
>#1 <built-in function wait>
>#3 Frame 0x2432f00, for file /opt/stack/new/glance/glance/common/wsgi.py, line 258, in wait_on_children (self=<Server(sock=<GreenSocket(shutdown=<instancemethod at remote 0x23893c0>, fileno=<instancemethod at remote 0x172e780>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x172efa0>, setsockopt=<instancemethod at remote 0x2389280>, bind=<instancemethod at remote 0x14293c0>, fd=<_socketobject at remote 0x15949f0>, getsockname=<instancemethod at remote 0x1429550>, _timeout=None, close=<instancemethod at remote 0x1429320>, listen=<instancemethod at remote 0x205c690>) at remote 0x19db9d0>, application=<AuthProtocol(admin_token=None, LOG=<Logger(name='keystoneclient.middleware.auth_token', parent=<RootLogger(name='root', parent=None, handlers=[<ColorHandler(stream=<file at remote 0x7f6e428fe1e0>, level=0, lock=<_RLock(_Verbose__verbose=False, _RLock__owner=None, _RLock__block=<Semaphore(counter=1, _waiters=set([])) at remote 0x174d150>, _RLock__count=0) at remote 0x1746e10>, _name=None, filters=[], fo...(truncated)
>    pid, status = os.wait()
>#7 Frame 0x2432d20, for file /opt/stack/new/glance/glance/common/wsgi.py, line 285, in wait (self=<Server(sock=<GreenSocket(shutdown=<instancemethod at remote 0x23893c0>, fileno=<instancemethod at remote 0x172e780>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x172efa0>, setsockopt=<instancemethod at remote 0x2389280>, bind=<instancemethod at remote 0x14293c0>, fd=<_socketobject at remote 0x15949f0>, getsockname=<instancemethod at remote 0x1429550>, _timeout=None, close=<instancemethod at remote 0x1429320>, listen=<instancemethod at remote 0x205c690>) at remote 0x19db9d0>, application=<AuthProtocol(admin_token=None, LOG=<Logger(name='keystoneclient.middleware.auth_token', parent=<RootLogger(name='root', parent=None, handlers=[<ColorHandler(stream=<file at remote 0x7f6e428fe1e0>, level=0, lock=<_RLock(_Verbose__verbose=False, _RLock__owner=None, _RLock__block=<Semaphore(counter=1, _waiters=set([])) at remote 0x174d150>, _RLock__count=0) at remote 0x1746e10>, _name=None, filters=[], formatter=<Con...(truncated)
>    self.wait_on_children()
>#11 Frame 0x11bb280, for file /opt/stack/new/glance/glance/cmd/registry.py, line 52, in main (server=<Server(sock=<GreenSocket(shutdown=<instancemethod at remote 0x23893c0>, fileno=<instancemethod at remote 0x172e780>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x172efa0>, setsockopt=<instancemethod at remote 0x2389280>, bind=<instancemethod at remote 0x14293c0>, fd=<_socketobject at remote 0x15949f0>, getsockname=<instancemethod at remote 0x1429550>, _timeout=None, close=<instancemethod at remote 0x1429320>, listen=<instancemethod at remote 0x205c690>) at remote 0x19db9d0>, application=<AuthProtocol(admin_token=None, LOG=<Logger(name='keystoneclient.middleware.auth_token', parent=<RootLogger(name='root', parent=None, handlers=[<ColorHandler(stream=<file at remote 0x7f6e428fe1e0>, level=0, lock=<_RLock(_Verbose__verbose=False, _RLock__owner=None, _RLock__block=<Semaphore(counter=1, _waiters=set([])) at remote 0x174d150>, _RLock__count=0) at remote 0x1746e10>, _name=None, filters=[], formatter=<C...(truncated)
>    server.wait()
>#14 Frame 0xc1f650, for file /usr/bin/glance-registry, line 10, in <module> ()
>    sys.exit(main())
>PID:14396
>[Thread debugging using libthread_db enabled]
>Using host libthread_db library "/lib64/libthread_db.so.1".
>0x00007f6e41753e0d in poll () from /lib64/libc.so.6
>  Id   Target Id         Frame 
>* 1    Thread 0x7f6e42923740 (LWP 14396) "glance-registry" 0x00007f6e41753e0d in poll () from /lib64/libc.so.6
>
>Thread 1 (Thread 0x7f6e42923740 (LWP 14396)):
>#2 <built-in method poll of select.poll object at remote 0x2391530>
>#4 Frame 0x2435ca0, for file /usr/lib/python2.7/site-packages/eventlet/hubs/poll.py, line 73, in do_poll (self=<Hub(next_timers=[], clock=<built-in function time>, debug_exceptions=True, debug_blocking_resolution=1, modify=<built-in method modify of select.poll object at remote 0x2391530>, listeners={'read': {3: <FdListener(fileno=3, cb=<built-in method switch of greenlet.greenlet object at remote 0x7f6e4040e230>, evtype='read') at remote 0x23ab710>}, 'write': {}}, debug_blocking=False, running=True, timers_canceled=0, greenlet=<greenlet.greenlet at remote 0x11594b0>, stopping=False, timers=[], poll=<select.poll at remote 0x2391530>, secondaries={'read': {}, 'write': {}}, lclass=<type at remote 0x243c960>) at remote 0x23ab7d0>, seconds=<float at remote 0x128ba90>)
>    return self.poll.poll(int(seconds * 1000.0))
>#7 Frame 0x2435a70, for file /usr/lib/python2.7/site-packages/eventlet/hubs/poll.py, line 84, in wait (self=<Hub(next_timers=[], clock=<built-in function time>, debug_exceptions=True, debug_blocking_resolution=1, modify=<built-in method modify of select.poll object at remote 0x2391530>, listeners={'read': {3: <FdListener(fileno=3, cb=<built-in method switch of greenlet.greenlet object at remote 0x7f6e4040e230>, evtype='read') at remote 0x23ab710>}, 'write': {}}, debug_blocking=False, running=True, timers_canceled=0, greenlet=<greenlet.greenlet at remote 0x11594b0>, stopping=False, timers=[], poll=<select.poll at remote 0x2391530>, secondaries={'read': {}, 'write': {}}, lclass=<type at remote 0x243c960>) at remote 0x23ab7d0>, seconds=<float at remote 0x128ba90>, readers={...}, writers={...})
>    presult = self.do_poll(seconds)
>#11 Frame 0x24350e0, for file /usr/lib/python2.7/site-packages/eventlet/hubs/hub.py, line 236, in run (self=<Hub(next_timers=[], clock=<built-in function time>, debug_exceptions=True, debug_blocking_resolution=1, modify=<built-in method modify of select.poll object at remote 0x2391530>, listeners={'read': {3: <FdListener(fileno=3, cb=<built-in method switch of greenlet.greenlet object at remote 0x7f6e4040e230>, evtype='read') at remote 0x23ab710>}, 'write': {}}, debug_blocking=False, running=True, timers_canceled=0, greenlet=<greenlet.greenlet at remote 0x11594b0>, stopping=False, timers=[], poll=<select.poll at remote 0x2391530>, secondaries={'read': {}, 'write': {}}, lclass=<type at remote 0x243c960>) at remote 0x23ab7d0>, a=(), kw={}, wakeup_when=None, sleep_time=<float at remote 0x128ba90>)
>    self.wait(sleep_time)
>#20 <built-in method switch of greenlet.greenlet object at remote 0x11594b0>
>#22 Frame 0x2434b70, for file /usr/lib/python2.7/site-packages/eventlet/hubs/hub.py, line 187, in switch (self=<Hub(next_timers=[], clock=<built-in function time>, debug_exceptions=True, debug_blocking_resolution=1, modify=<built-in method modify of select.poll object at remote 0x2391530>, listeners={'read': {3: <FdListener(fileno=3, cb=<built-in method switch of greenlet.greenlet object at remote 0x7f6e4040e230>, evtype='read') at remote 0x23ab710>}, 'write': {}}, debug_blocking=False, running=True, timers_canceled=0, greenlet=<greenlet.greenlet at remote 0x11594b0>, stopping=False, timers=[], poll=<select.poll at remote 0x2391530>, secondaries={'read': {}, 'write': {}}, lclass=<type at remote 0x243c960>) at remote 0x23ab7d0>, cur=<greenlet.greenlet at remote 0x7f6e4040e230>, switch_out=None)
>    return self.greenlet.switch()
>#25 Frame 0x243a290, for file /usr/lib/python2.7/site-packages/eventlet/hubs/__init__.py, line 155, in trampoline (fd=<_socketobject at remote 0x15949f0>, read=True, write=None, timeout=None, timeout_exc=<timeout at remote 0x2392a78>, t=None, hub=<Hub(next_timers=[], clock=<built-in function time>, debug_exceptions=True, debug_blocking_resolution=1, modify=<built-in method modify of select.poll object at remote 0x2391530>, listeners={'read': {3: <FdListener(fileno=3, cb=<built-in method switch of greenlet.greenlet object at remote 0x7f6e4040e230>, evtype='read') at remote 0x23ab710>}, 'write': {}}, debug_blocking=False, running=True, timers_canceled=0, greenlet=<greenlet.greenlet at remote 0x11594b0>, stopping=False, timers=[], poll=<select.poll at remote 0x2391530>, secondaries={'read': {}, 'write': {}}, lclass=<type at remote 0x243c960>) at remote 0x23ab7d0>, current=<greenlet.greenlet at remote 0x7f6e4040e230>, fileno=3, listener=<...>)
>    return hub.switch()
>#29 Frame 0x2439910, for file /usr/lib/python2.7/site-packages/eventlet/greenio.py, line 183, in accept (self=<GreenSocket(shutdown=<instancemethod at remote 0x23893c0>, fileno=<instancemethod at remote 0x172e780>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x172efa0>, setsockopt=<instancemethod at remote 0x2389280>, bind=<instancemethod at remote 0x14293c0>, fd=<_socketobject at remote 0x15949f0>, getsockname=<instancemethod at remote 0x1429550>, _timeout=None, close=<instancemethod at remote 0x1429320>, listen=<instancemethod at remote 0x205c690>) at remote 0x19db9d0>, fd=<...>, res=None)
>    timeout_exc=socket.timeout("timed out"))
>#32 Frame 0x2438b70, for file /usr/lib/python2.7/site-packages/eventlet/wsgi.py, line 693, in server (sock=<GreenSocket(shutdown=<instancemethod at remote 0x23893c0>, fileno=<instancemethod at remote 0x172e780>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x172efa0>, setsockopt=<instancemethod at remote 0x2389280>, bind=<instancemethod at remote 0x14293c0>, fd=<_socketobject at remote 0x15949f0>, getsockname=<instancemethod at remote 0x1429550>, _timeout=None, close=<instancemethod at remote 0x1429320>, listen=<instancemethod at remote 0x205c690>) at remote 0x19db9d0>, site=<AuthProtocol(admin_token=u'MIIVvAYJKoZIhvcNAQcCoIIVrTCCFakCAQExCTAHBgUrDgMCGjCCFBIGCSqGSIb3DQEHAaCCFAMEghP-eyJhY2Nlc3MiOiB7InRva2VuIjogeyJpc3N1ZWRfYXQiOiAiMjAxNC0wNi0xN1QxMzo0ODowNS4zMDk4NTgiLCAiZXhwaXJlcyI6ICIyMDE0LTA2LTE3VDE0OjQ4OjA1WiIsICJpZCI6ICJwbGFjZWhvbGRlciIsICJ0ZW5hbnQiOiB7ImRlc2NyaXB0aW9uIjogbnVsbCwgImVuYWJsZWQiOiB0cnVlLCAiaWQiOiAiMjc4M2MyN2Q5YzM2NDMxY2E1NTFkMWI4Y2ZjNGM4YjAiLCAibmFtZSI6ICJzZXJ2aWNlIn19LCAic2VydmljZU...(truncated)
>    client_socket = sock.accept()
>#36 Frame 0x2433e50, for file /opt/stack/new/glance/glance/common/wsgi.py, line 328, in run_server (self=<Server(sock=<GreenSocket(shutdown=<instancemethod at remote 0x23893c0>, fileno=<instancemethod at remote 0x172e780>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x172efa0>, setsockopt=<instancemethod at remote 0x2389280>, bind=<instancemethod at remote 0x14293c0>, fd=<_socketobject at remote 0x15949f0>, getsockname=<instancemethod at remote 0x1429550>, _timeout=None, close=<instancemethod at remote 0x1429320>, listen=<instancemethod at remote 0x205c690>) at remote 0x19db9d0>, application=<AuthProtocol(admin_token=u'MIIVvAYJKoZIhvcNAQcCoIIVrTCCFakCAQExCTAHBgUrDgMCGjCCFBIGCSqGSIb3DQEHAaCCFAMEghP-eyJhY2Nlc3MiOiB7InRva2VuIjogeyJpc3N1ZWRfYXQiOiAiMjAxNC0wNi0xN1QxMzo0ODowNS4zMDk4NTgiLCAiZXhwaXJlcyI6ICIyMDE0LTA2LTE3VDE0OjQ4OjA1WiIsICJpZCI6ICJwbGFjZWhvbGRlciIsICJ0ZW5hbnQiOiB7ImRlc2NyaXB0aW9uIjogbnVsbCwgImVuYWJsZWQiOiB0cnVlLCAiaWQiOiAiMjc4M2MyN2Q5YzM2NDMxY2E1NTFkMWI4Y2ZjNGM4YjAiLCAibmFtZSI6ICJzZXJ2aWNl...(truncated)
>    debug=False)
>#40 Frame 0x2432b50, for file /opt/stack/new/glance/glance/common/wsgi.py, line 300, in run_child (self=<Server(sock=<GreenSocket(shutdown=<instancemethod at remote 0x23893c0>, fileno=<instancemethod at remote 0x172e780>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x172efa0>, setsockopt=<instancemethod at remote 0x2389280>, bind=<instancemethod at remote 0x14293c0>, fd=<_socketobject at remote 0x15949f0>, getsockname=<instancemethod at remote 0x1429550>, _timeout=None, close=<instancemethod at remote 0x1429320>, listen=<instancemethod at remote 0x205c690>) at remote 0x19db9d0>, application=<AuthProtocol(admin_token=u'MIIVvAYJKoZIhvcNAQcCoIIVrTCCFakCAQExCTAHBgUrDgMCGjCCFBIGCSqGSIb3DQEHAaCCFAMEghP-eyJhY2Nlc3MiOiB7InRva2VuIjogeyJpc3N1ZWRfYXQiOiAiMjAxNC0wNi0xN1QxMzo0ODowNS4zMDk4NTgiLCAiZXhwaXJlcyI6ICIyMDE0LTA2LTE3VDE0OjQ4OjA1WiIsICJpZCI6ICJwbGFjZWhvbGRlciIsICJ0ZW5hbnQiOiB7ImRlc2NyaXB0aW9uIjogbnVsbCwgImVuYWJsZWQiOiB0cnVlLCAiaWQiOiAiMjc4M2MyN2Q5YzM2NDMxY2E1NTFkMWI4Y2ZjNGM4YjAiLCAibmFtZSI6ICJzZXJ2aWNlI...(truncated)
>    self.run_server()
>#44 Frame 0x1b74b10, for file /opt/stack/new/glance/glance/common/wsgi.py, line 250, in start (self=<Server(sock=<GreenSocket(shutdown=<instancemethod at remote 0x23893c0>, fileno=<instancemethod at remote 0x172e780>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x172efa0>, setsockopt=<instancemethod at remote 0x2389280>, bind=<instancemethod at remote 0x14293c0>, fd=<_socketobject at remote 0x15949f0>, getsockname=<instancemethod at remote 0x1429550>, _timeout=None, close=<instancemethod at remote 0x1429320>, listen=<instancemethod at remote 0x205c690>) at remote 0x19db9d0>, application=<AuthProtocol(admin_token=u'MIIVvAYJKoZIhvcNAQcCoIIVrTCCFakCAQExCTAHBgUrDgMCGjCCFBIGCSqGSIb3DQEHAaCCFAMEghP-eyJhY2Nlc3MiOiB7InRva2VuIjogeyJpc3N1ZWRfYXQiOiAiMjAxNC0wNi0xN1QxMzo0ODowNS4zMDk4NTgiLCAiZXhwaXJlcyI6ICIyMDE0LTA2LTE3VDE0OjQ4OjA1WiIsICJpZCI6ICJwbGFjZWhvbGRlciIsICJ0ZW5hbnQiOiB7ImRlc2NyaXB0aW9uIjogbnVsbCwgImVuYWJsZWQiOiB0cnVlLCAiaWQiOiAiMjc4M2MyN2Q5YzM2NDMxY2E1NTFkMWI4Y2ZjNGM4YjAiLCAibmFtZSI6ICJzZXJ2aWNlIn19L...(truncated)
>    self.run_child()
>#48 Frame 0x11bb280, for file /opt/stack/new/glance/glance/cmd/registry.py, line 51, in main (server=<Server(sock=<GreenSocket(shutdown=<instancemethod at remote 0x23893c0>, fileno=<instancemethod at remote 0x172e780>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x172efa0>, setsockopt=<instancemethod at remote 0x2389280>, bind=<instancemethod at remote 0x14293c0>, fd=<_socketobject at remote 0x15949f0>, getsockname=<instancemethod at remote 0x1429550>, _timeout=None, close=<instancemethod at remote 0x1429320>, listen=<instancemethod at remote 0x205c690>) at remote 0x19db9d0>, application=<AuthProtocol(admin_token=u'MIIVvAYJKoZIhvcNAQcCoIIVrTCCFakCAQExCTAHBgUrDgMCGjCCFBIGCSqGSIb3DQEHAaCCFAMEghP-eyJhY2Nlc3MiOiB7InRva2VuIjogeyJpc3N1ZWRfYXQiOiAiMjAxNC0wNi0xN1QxMzo0ODowNS4zMDk4NTgiLCAiZXhwaXJlcyI6ICIyMDE0LTA2LTE3VDE0OjQ4OjA1WiIsICJpZCI6ICJwbGFjZWhvbGRlciIsICJ0ZW5hbnQiOiB7ImRlc2NyaXB0aW9uIjogbnVsbCwgImVuYWJsZWQiOiB0cnVlLCAiaWQiOiAiMjc4M2MyN2Q5YzM2NDMxY2E1NTFkMWI4Y2ZjNGM4YjAiLCAibmFtZSI6ICJzZXJ2aWNlIn19...(truncated)
>    default_port=9191)
>#51 Frame 0xc1f650, for file /usr/bin/glance-registry, line 10, in <module> ()
>    sys.exit(main())
>PID:14401
>[Thread debugging using libthread_db enabled]
>Using host libthread_db library "/lib64/libthread_db.so.1".
>0x00007f00c666cc02 in wait () from /lib64/libpthread.so.0
>  Id   Target Id         Frame 
>* 1    Thread 0x7f00c6e4f740 (LWP 14401) "glance-api" 0x00007f00c666cc02 in wait () from /lib64/libpthread.so.0
>
>Thread 1 (Thread 0x7f00c6e4f740 (LWP 14401)):
>#1 <built-in function wait>
>#3 Frame 0x38eaa80, for file /opt/stack/new/glance/glance/common/wsgi.py, line 258, in wait_on_children (self=<Server(sock=<GreenSocket(shutdown=<instancemethod at remote 0x42a4640>, fileno=<instancemethod at remote 0x2f7fbe0>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x2f7fe60>, setsockopt=<instancemethod at remote 0x347de60>, bind=<instancemethod at remote 0x2f7fc30>, fd=<_socketobject at remote 0x3845ec0>, getsockname=<instancemethod at remote 0x2f7fe10>, _timeout=None, close=<instancemethod at remote 0x2ee6a50>, listen=<instancemethod at remote 0x3545050>) at remote 0x3936f90>, application=<VersionNegotiationFilter(application=<AuthProtocol(admin_token=None, LOG=<Logger(name='keystoneclient.middleware.auth_token', parent=<RootLogger(name='root', parent=None, handlers=[<ColorHandler(stream=<file at remote 0x7f00c6e2a1e0>, level=0, lock=<_RLock(_Verbose__verbose=False, _RLock__owner=None, _RLock__block=<Semaphore(counter=1, _waiters=set([])) at remote 0x33d7950>, _RLock__count=0) at remote ...(truncated)
>    pid, status = os.wait()
>#7 Frame 0x38ea8a0, for file /opt/stack/new/glance/glance/common/wsgi.py, line 285, in wait (self=<Server(sock=<GreenSocket(shutdown=<instancemethod at remote 0x42a4640>, fileno=<instancemethod at remote 0x2f7fbe0>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x2f7fe60>, setsockopt=<instancemethod at remote 0x347de60>, bind=<instancemethod at remote 0x2f7fc30>, fd=<_socketobject at remote 0x3845ec0>, getsockname=<instancemethod at remote 0x2f7fe10>, _timeout=None, close=<instancemethod at remote 0x2ee6a50>, listen=<instancemethod at remote 0x3545050>) at remote 0x3936f90>, application=<VersionNegotiationFilter(application=<AuthProtocol(admin_token=None, LOG=<Logger(name='keystoneclient.middleware.auth_token', parent=<RootLogger(name='root', parent=None, handlers=[<ColorHandler(stream=<file at remote 0x7f00c6e2a1e0>, level=0, lock=<_RLock(_Verbose__verbose=False, _RLock__owner=None, _RLock__block=<Semaphore(counter=1, _waiters=set([])) at remote 0x33d7950>, _RLock__count=0) at remote 0x33d7910>, ...(truncated)
>    self.wait_on_children()
>#11 Frame 0x3414f10, for file /opt/stack/new/glance/glance/cmd/api.py, line 64, in main (server=<Server(sock=<GreenSocket(shutdown=<instancemethod at remote 0x42a4640>, fileno=<instancemethod at remote 0x2f7fbe0>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x2f7fe60>, setsockopt=<instancemethod at remote 0x347de60>, bind=<instancemethod at remote 0x2f7fc30>, fd=<_socketobject at remote 0x3845ec0>, getsockname=<instancemethod at remote 0x2f7fe10>, _timeout=None, close=<instancemethod at remote 0x2ee6a50>, listen=<instancemethod at remote 0x3545050>) at remote 0x3936f90>, application=<VersionNegotiationFilter(application=<AuthProtocol(admin_token=None, LOG=<Logger(name='keystoneclient.middleware.auth_token', parent=<RootLogger(name='root', parent=None, handlers=[<ColorHandler(stream=<file at remote 0x7f00c6e2a1e0>, level=0, lock=<_RLock(_Verbose__verbose=False, _RLock__owner=None, _RLock__block=<Semaphore(counter=1, _waiters=set([])) at remote 0x33d7950>, _RLock__count=0) at remote 0x33d7910>, _na...(truncated)
>    server.wait()
>#14 Frame 0x2513650, for file /usr/bin/glance-api, line 10, in <module> ()
>    sys.exit(main())
>PID:14411
>[Thread debugging using libthread_db enabled]
>Using host libthread_db library "/lib64/libthread_db.so.1".
>0x00007f00c5c7fdf0 in __poll_nocancel () from /lib64/libc.so.6
>  Id   Target Id         Frame 
>* 1    Thread 0x7f00c6e4f740 (LWP 14411) "glance-api" 0x00007f00c5c7fdf0 in __poll_nocancel () from /lib64/libc.so.6
>
>Thread 1 (Thread 0x7f00c6e4f740 (LWP 14411)):
>#2 <built-in method poll of select.poll object at remote 0x43aa238>
>#4 Frame 0x4931570, for file /usr/lib/python2.7/site-packages/qpid/compat.py, line 80, in select (rlist=set([<...>]), wlist=set([]), xlist=(), timeout=None, fd_count=1, rset=set([<...>]), wset=set([]), xset=set([]), poller=<select.poll at remote 0x43aa238>, rwset=set([]), ro=<...>, fd_map={6: <...>}, o=<...>)
>    active = poller.poll(timeout)
>#8 Frame 0x4931340, for file /usr/lib/python2.7/site-packages/qpid/selector.py, line 124, in run (self=<Selector(thread=<Thread(_Thread__ident=73962192, _Thread__block=<_Condition(_Verbose__verbose=False, _Condition__lock=<Semaphore(counter=1, _waiters=set([])) at remote 0x464ef90>, acquire=<instancemethod at remote 0x44de3c0>, _Condition__waiters=[], release=<instancemethod at remote 0x43641e0>) at remote 0x464efd0>, _Thread__name='Thread-1', _Thread__daemonic=True, _Thread__started=<_Event(_Verbose__verbose=False, _Event__flag=True, _Event__cond=<_Condition(_Verbose__verbose=False, _Condition__lock=<Semaphore(counter=1, _waiters=set([])) at remote 0x464eed0>, acquire=<instancemethod at remote 0x44de230>, _Condition__waiters=[], release=<instancemethod at remote 0x46ab410>) at remote 0x464ef10>) at remote 0x464ee90>, _Thread__stderr=<file at remote 0x7f00c6e2a1e0>, _Thread__target=<instancemethod at remote 0x4692f50>, _Thread__kwargs={}, _Verbose__verbose=False, _Thread__args=(), _Thread__stopped=False, _Thread...(truncated)
>    rd, wr, ex = select(self.reading, self.writing, (), timeout)
>#13 Frame 0x4931170, for file /usr/lib64/python2.7/threading.py, line 764, in run (self=<Thread(_Thread__ident=73962192, _Thread__block=<_Condition(_Verbose__verbose=False, _Condition__lock=<Semaphore(counter=1, _waiters=set([])) at remote 0x464ef90>, acquire=<instancemethod at remote 0x44de3c0>, _Condition__waiters=[], release=<instancemethod at remote 0x43641e0>) at remote 0x464efd0>, _Thread__name='Thread-1', _Thread__daemonic=True, _Thread__started=<_Event(_Verbose__verbose=False, _Event__flag=True, _Event__cond=<_Condition(_Verbose__verbose=False, _Condition__lock=<Semaphore(counter=1, _waiters=set([])) at remote 0x464eed0>, acquire=<instancemethod at remote 0x44de230>, _Condition__waiters=[], release=<instancemethod at remote 0x46ab410>) at remote 0x464ef10>) at remote 0x464ee90>, _Thread__stderr=<file at remote 0x7f00c6e2a1e0>, _Thread__target=<instancemethod at remote 0x4692f50>, _Thread__kwargs={}, _Verbose__verbose=False, _Thread__args=(), _Thread__stopped=False, _Thread__initialized=True) at remote 0x4...(truncated)
>    self.__target(*self.__args, **self.__kwargs)
>#16 Frame 0x4930c20, for file /usr/lib64/python2.7/threading.py, line 811, in __bootstrap_inner (self=<Thread(_Thread__ident=73962192, _Thread__block=<_Condition(_Verbose__verbose=False, _Condition__lock=<Semaphore(counter=1, _waiters=set([])) at remote 0x464ef90>, acquire=<instancemethod at remote 0x44de3c0>, _Condition__waiters=[], release=<instancemethod at remote 0x43641e0>) at remote 0x464efd0>, _Thread__name='Thread-1', _Thread__daemonic=True, _Thread__started=<_Event(_Verbose__verbose=False, _Event__flag=True, _Event__cond=<_Condition(_Verbose__verbose=False, _Condition__lock=<Semaphore(counter=1, _waiters=set([])) at remote 0x464eed0>, acquire=<instancemethod at remote 0x44de230>, _Condition__waiters=[], release=<instancemethod at remote 0x46ab410>) at remote 0x464ef10>) at remote 0x464ee90>, _Thread__stderr=<file at remote 0x7f00c6e2a1e0>, _Thread__target=<instancemethod at remote 0x4692f50>, _Thread__kwargs={}, _Verbose__verbose=False, _Thread__args=(), _Thread__stopped=False, _Thread__initialized=True)...(truncated)
>    self.run()
>#19 Frame 0x4930a50, for file /usr/lib64/python2.7/threading.py, line 784, in __bootstrap (self=<Thread(_Thread__ident=73962192, _Thread__block=<_Condition(_Verbose__verbose=False, _Condition__lock=<Semaphore(counter=1, _waiters=set([])) at remote 0x464ef90>, acquire=<instancemethod at remote 0x44de3c0>, _Condition__waiters=[], release=<instancemethod at remote 0x43641e0>) at remote 0x464efd0>, _Thread__name='Thread-1', _Thread__daemonic=True, _Thread__started=<_Event(_Verbose__verbose=False, _Event__flag=True, _Event__cond=<_Condition(_Verbose__verbose=False, _Condition__lock=<Semaphore(counter=1, _waiters=set([])) at remote 0x464eed0>, acquire=<instancemethod at remote 0x44de230>, _Condition__waiters=[], release=<instancemethod at remote 0x46ab410>) at remote 0x464ef10>) at remote 0x464ee90>, _Thread__stderr=<file at remote 0x7f00c6e2a1e0>, _Thread__target=<instancemethod at remote 0x4692f50>, _Thread__kwargs={}, _Verbose__verbose=False, _Thread__args=(), _Thread__stopped=False, _Thread__initialized=True) at re...(truncated)
>    self.__bootstrap_inner()
>#24 Frame 0x4940a00, for file /usr/lib/python2.7/site-packages/eventlet/green/thread.py, line 27, in __thread_body (func=<instancemethod at remote 0x4692730>, args=(), kwargs={})
>    func(*args, **kwargs)
>#33 Frame 0x46c9f70, for file /usr/lib/python2.7/site-packages/eventlet/hubs/timer.py, line 56, in __call__ (self=<Timer(seconds=0, tpl=(<built-in method switch of greenlet.greenlet object at remote 0x46892d0>, (<instancemethod at remote 0x4692730>, (), {}), {}), called=True) at remote 0x4654050>, args=(...), cb=<built-in method switch of greenlet.greenlet object at remote 0x46892d0>, kw={...})
>    cb(*args, **kw)
>#43 Frame 0x46c95c0, for file /usr/lib/python2.7/site-packages/eventlet/hubs/hub.py, line 346, in fire_timers (self=<Hub(next_timers=[(<float at remote 0x47375a0>, <Timer(seconds=0, tpl=(<instancemethod at remote 0x46925f0>, (), {}), called=False) at remote 0x46540d0>)], clock=<built-in function time>, debug_exceptions=True, debug_blocking_resolution=1, modify=<built-in method modify of select.poll object at remote 0x43a9fa8>, listeners={'read': {4: <FdListener(fileno=4, cb=<built-in method switch of greenlet.greenlet object at remote 0x7f00c493a730>, evtype='read') at remote 0x44d8b10>}, 'write': {}}, debug_blocking=False, running=True, timers_canceled=2, greenlet=<greenlet.greenlet at remote 0x33c8910>, stopping=False, timers=[(<float at remote 0x4737630>, <Timer(seconds=<float at remote 0x4737678>, called=True) at remote 0x46a9210>), (<float at remote 0x4737690>, <Timer(seconds=<float at remote 0x4737660>, called=True) at remote 0x46a9190>)], poll=<select.poll at remote 0x43a9fa8>, secondaries={'read': {}, 'wr...(truncated)
>    timer()
>#46 Frame 0x46c93c0, for file /usr/lib/python2.7/site-packages/eventlet/hubs/hub.py, line 226, in run (self=<Hub(next_timers=[(<float at remote 0x47375a0>, <Timer(seconds=0, tpl=(<instancemethod at remote 0x46925f0>, (), {}), called=False) at remote 0x46540d0>)], clock=<built-in function time>, debug_exceptions=True, debug_blocking_resolution=1, modify=<built-in method modify of select.poll object at remote 0x43a9fa8>, listeners={'read': {4: <FdListener(fileno=4, cb=<built-in method switch of greenlet.greenlet object at remote 0x7f00c493a730>, evtype='read') at remote 0x44d8b10>}, 'write': {}}, debug_blocking=False, running=True, timers_canceled=2, greenlet=<greenlet.greenlet at remote 0x33c8910>, stopping=False, timers=[(<float at remote 0x4737630>, <Timer(seconds=<float at remote 0x4737678>, called=True) at remote 0x46a9210>), (<float at remote 0x4737690>, <Timer(seconds=<float at remote 0x4737660>, called=True) at remote 0x46a9190>)], poll=<select.poll at remote 0x43a9fa8>, secondaries={'read': {}, 'write': {}...(truncated)
>    self.fire_timers(self.clock())
>#55 <built-in method switch of greenlet.greenlet object at remote 0x33c8910>
>#57 Frame 0x46deb20, for file /usr/lib/python2.7/site-packages/eventlet/hubs/hub.py, line 187, in switch (self=<Hub(next_timers=[(<float at remote 0x47375a0>, <Timer(seconds=0, tpl=(<instancemethod at remote 0x46925f0>, (), {}), called=False) at remote 0x46540d0>)], clock=<built-in function time>, debug_exceptions=True, debug_blocking_resolution=1, modify=<built-in method modify of select.poll object at remote 0x43a9fa8>, listeners={'read': {4: <FdListener(fileno=4, cb=<built-in method switch of greenlet.greenlet object at remote 0x7f00c493a730>, evtype='read') at remote 0x44d8b10>}, 'write': {}}, debug_blocking=False, running=True, timers_canceled=2, greenlet=<greenlet.greenlet at remote 0x33c8910>, stopping=False, timers=[(<float at remote 0x4737630>, <Timer(seconds=<float at remote 0x4737678>, called=True) at remote 0x46a9210>), (<float at remote 0x4737690>, <Timer(seconds=<float at remote 0x4737660>, called=True) at remote 0x46a9190>)], poll=<select.poll at remote 0x43a9fa8>, secondaries={'read': {}, 'write':...(truncated)
>    return self.greenlet.switch()
>#60 Frame 0x46dd150, for file /usr/lib/python2.7/site-packages/eventlet/hubs/__init__.py, line 155, in trampoline (fd=<_socketobject at remote 0x3845ec0>, read=True, write=None, timeout=None, timeout_exc=<timeout at remote 0x38a8530>, t=None, hub=<Hub(next_timers=[(<float at remote 0x47375a0>, <Timer(seconds=0, tpl=(<instancemethod at remote 0x46925f0>, (), {}), called=False) at remote 0x46540d0>)], clock=<built-in function time>, debug_exceptions=True, debug_blocking_resolution=1, modify=<built-in method modify of select.poll object at remote 0x43a9fa8>, listeners={'read': {4: <FdListener(fileno=4, cb=<built-in method switch of greenlet.greenlet object at remote 0x7f00c493a730>, evtype='read') at remote 0x44d8b10>}, 'write': {}}, debug_blocking=False, running=True, timers_canceled=2, greenlet=<greenlet.greenlet at remote 0x33c8910>, stopping=False, timers=[(<float at remote 0x4737630>, <Timer(seconds=<float at remote 0x4737678>, called=True) at remote 0x46a9210>), (<float at remote 0x4737690>, <Timer(seconds=<fl...(truncated)
>    return hub.switch()
>#64 Frame 0x46c7d30, for file /usr/lib/python2.7/site-packages/eventlet/greenio.py, line 183, in accept (self=<GreenSocket(shutdown=<instancemethod at remote 0x42a4640>, fileno=<instancemethod at remote 0x2f7fbe0>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x2f7fe60>, setsockopt=<instancemethod at remote 0x347de60>, bind=<instancemethod at remote 0x2f7fc30>, fd=<_socketobject at remote 0x3845ec0>, getsockname=<instancemethod at remote 0x2f7fe10>, _timeout=None, close=<instancemethod at remote 0x2ee6a50>, listen=<instancemethod at remote 0x3545050>) at remote 0x3936f90>, fd=<...>, res=None)
>    timeout_exc=socket.timeout("timed out"))
>#67 Frame 0x46c6dc0, for file /usr/lib/python2.7/site-packages/eventlet/wsgi.py, line 693, in server (sock=<GreenSocket(shutdown=<instancemethod at remote 0x42a4640>, fileno=<instancemethod at remote 0x2f7fbe0>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x2f7fe60>, setsockopt=<instancemethod at remote 0x347de60>, bind=<instancemethod at remote 0x2f7fc30>, fd=<_socketobject at remote 0x3845ec0>, getsockname=<instancemethod at remote 0x2f7fe10>, _timeout=None, close=<instancemethod at remote 0x2ee6a50>, listen=<instancemethod at remote 0x3545050>) at remote 0x3936f90>, site=<VersionNegotiationFilter(application=<AuthProtocol(admin_token=u'MIIVvAYJKoZIhvcNAQcCoIIVrTCCFakCAQExCTAHBgUrDgMCGjCCFBIGCSqGSIb3DQEHAaCCFAMEghP-eyJhY2Nlc3MiOiB7InRva2VuIjogeyJpc3N1ZWRfYXQiOiAiMjAxNC0wNi0xN1QxMzo0ODowNC44ODM1OTkiLCAiZXhwaXJlcyI6ICIyMDE0LTA2LTE3VDE0OjQ4OjA0WiIsICJpZCI6ICJwbGFjZWhvbGRlciIsICJ0ZW5hbnQiOiB7ImRlc2NyaXB0aW9uIjogbnVsbCwgImVuYWJsZWQiOiB0cnVlLCAiaWQiOiAiMjc4M2MyN2Q5YzM2NDMxY2E1NTFkMWI4Y2ZjNGM4YjAiLCAi...(truncated)
>    client_socket = sock.accept()
>#71 Frame 0x46c6810, for file /opt/stack/new/glance/glance/common/wsgi.py, line 328, in run_server (self=<Server(sock=<GreenSocket(shutdown=<instancemethod at remote 0x42a4640>, fileno=<instancemethod at remote 0x2f7fbe0>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x2f7fe60>, setsockopt=<instancemethod at remote 0x347de60>, bind=<instancemethod at remote 0x2f7fc30>, fd=<_socketobject at remote 0x3845ec0>, getsockname=<instancemethod at remote 0x2f7fe10>, _timeout=None, close=<instancemethod at remote 0x2ee6a50>, listen=<instancemethod at remote 0x3545050>) at remote 0x3936f90>, application=<VersionNegotiationFilter(application=<AuthProtocol(admin_token=u'MIIVvAYJKoZIhvcNAQcCoIIVrTCCFakCAQExCTAHBgUrDgMCGjCCFBIGCSqGSIb3DQEHAaCCFAMEghP-eyJhY2Nlc3MiOiB7InRva2VuIjogeyJpc3N1ZWRfYXQiOiAiMjAxNC0wNi0xN1QxMzo0ODowNC44ODM1OTkiLCAiZXhwaXJlcyI6ICIyMDE0LTA2LTE3VDE0OjQ4OjA0WiIsICJpZCI6ICJwbGFjZWhvbGRlciIsICJ0ZW5hbnQiOiB7ImRlc2NyaXB0aW9uIjogbnVsbCwgImVuYWJsZWQiOiB0cnVlLCAiaWQiOiAiMjc4M2MyN2Q5YzM2NDMxY2E1NTFkMW...(truncated)
>    debug=False)
>#75 Frame 0x342ac20, for file /opt/stack/new/glance/glance/common/wsgi.py, line 300, in run_child (self=<Server(sock=<GreenSocket(shutdown=<instancemethod at remote 0x42a4640>, fileno=<instancemethod at remote 0x2f7fbe0>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x2f7fe60>, setsockopt=<instancemethod at remote 0x347de60>, bind=<instancemethod at remote 0x2f7fc30>, fd=<_socketobject at remote 0x3845ec0>, getsockname=<instancemethod at remote 0x2f7fe10>, _timeout=None, close=<instancemethod at remote 0x2ee6a50>, listen=<instancemethod at remote 0x3545050>) at remote 0x3936f90>, application=<VersionNegotiationFilter(application=<AuthProtocol(admin_token=u'MIIVvAYJKoZIhvcNAQcCoIIVrTCCFakCAQExCTAHBgUrDgMCGjCCFBIGCSqGSIb3DQEHAaCCFAMEghP-eyJhY2Nlc3MiOiB7InRva2VuIjogeyJpc3N1ZWRfYXQiOiAiMjAxNC0wNi0xN1QxMzo0ODowNC44ODM1OTkiLCAiZXhwaXJlcyI6ICIyMDE0LTA2LTE3VDE0OjQ4OjA0WiIsICJpZCI6ICJwbGFjZWhvbGRlciIsICJ0ZW5hbnQiOiB7ImRlc2NyaXB0aW9uIjogbnVsbCwgImVuYWJsZWQiOiB0cnVlLCAiaWQiOiAiMjc4M2MyN2Q5YzM2NDMxY2E1NTFkMWI...(truncated)
>    self.run_server()
>#79 Frame 0x462e960, for file /opt/stack/new/glance/glance/common/wsgi.py, line 250, in start (self=<Server(sock=<GreenSocket(shutdown=<instancemethod at remote 0x42a4640>, fileno=<instancemethod at remote 0x2f7fbe0>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x2f7fe60>, setsockopt=<instancemethod at remote 0x347de60>, bind=<instancemethod at remote 0x2f7fc30>, fd=<_socketobject at remote 0x3845ec0>, getsockname=<instancemethod at remote 0x2f7fe10>, _timeout=None, close=<instancemethod at remote 0x2ee6a50>, listen=<instancemethod at remote 0x3545050>) at remote 0x3936f90>, application=<VersionNegotiationFilter(application=<AuthProtocol(admin_token=u'MIIVvAYJKoZIhvcNAQcCoIIVrTCCFakCAQExCTAHBgUrDgMCGjCCFBIGCSqGSIb3DQEHAaCCFAMEghP-eyJhY2Nlc3MiOiB7InRva2VuIjogeyJpc3N1ZWRfYXQiOiAiMjAxNC0wNi0xN1QxMzo0ODowNC44ODM1OTkiLCAiZXhwaXJlcyI6ICIyMDE0LTA2LTE3VDE0OjQ4OjA0WiIsICJpZCI6ICJwbGFjZWhvbGRlciIsICJ0ZW5hbnQiOiB7ImRlc2NyaXB0aW9uIjogbnVsbCwgImVuYWJsZWQiOiB0cnVlLCAiaWQiOiAiMjc4M2MyN2Q5YzM2NDMxY2E1NTFkMWI4Y2Z...(truncated)
>    self.run_child()
>#83 Frame 0x3414f10, for file /opt/stack/new/glance/glance/cmd/api.py, line 63, in main (server=<Server(sock=<GreenSocket(shutdown=<instancemethod at remote 0x42a4640>, fileno=<instancemethod at remote 0x2f7fbe0>, act_non_blocking=False, getsockopt=<instancemethod at remote 0x2f7fe60>, setsockopt=<instancemethod at remote 0x347de60>, bind=<instancemethod at remote 0x2f7fc30>, fd=<_socketobject at remote 0x3845ec0>, getsockname=<instancemethod at remote 0x2f7fe10>, _timeout=None, close=<instancemethod at remote 0x2ee6a50>, listen=<instancemethod at remote 0x3545050>) at remote 0x3936f90>, application=<VersionNegotiationFilter(application=<AuthProtocol(admin_token=u'MIIVvAYJKoZIhvcNAQcCoIIVrTCCFakCAQExCTAHBgUrDgMCGjCCFBIGCSqGSIb3DQEHAaCCFAMEghP-eyJhY2Nlc3MiOiB7InRva2VuIjogeyJpc3N1ZWRfYXQiOiAiMjAxNC0wNi0xN1QxMzo0ODowNC44ODM1OTkiLCAiZXhwaXJlcyI6ICIyMDE0LTA2LTE3VDE0OjQ4OjA0WiIsICJpZCI6ICJwbGFjZWhvbGRlciIsICJ0ZW5hbnQiOiB7ImRlc2NyaXB0aW9uIjogbnVsbCwgImVuYWJsZWQiOiB0cnVlLCAiaWQiOiAiMjc4M2MyN2Q5YzM2NDMxY2E1NTFkMWI4Y2ZjNGM...(truncated)
>    server.start(config.load_paste_app('glance-api'), default_port=9292)
>#86 Frame 0x2513650, for file /usr/bin/glance-api, line 10, in <module> ()
>    sys.exit(main())
>[root@afazekas-epel-qpid-1 ~]# ps aux |grep glance
>root     11537  0.0  0.0 112640   976 pts/10   S+   09:00   0:00 grep --color=auto glance
>stack    14361  0.0  0.4 178744 36156 pts/8    S+   Jun17   0:01 /usr/bin/python /usr/bin/glance-registry --config-file=/etc/glance/glance-registry.conf
>stack    14396  0.0  0.5 271664 40412 pts/8    S+   Jun17   0:00 /usr/bin/python /usr/bin/glance-registry --config-file=/etc/glance/glance-registry.conf
>stack    14401  0.0  0.5 192720 45868 pts/9    S+   Jun17   0:01 /usr/bin/python /usr/bin/glance-api --config-file=/etc/glance/glance-api.conf
>stack    14411  0.0  0.5 195404 45704 pts/9    S+   Jun17   0:00 /usr/bin/python /usr/bin/glance-api --config-file=/etc/glance/glance-api.conf
>

Actions: View

Attachments on bug 1109488: 908781 | 910005