The glusterfs client process crashed in 2.0.10rc2. error-gen was used with ENOMEM.
Script for the sanity of glusterfs was being run during the crash.

This is the backtrace of the core generated.


[?1034hGNU gdb 6.8
Copyright (C) 2008 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-slackware-linux"...

warning: Can't read pathname for load map: Input/output error.
Reading symbols from /opt/glusterfs/2.0.10rc2/lib/libglusterfs.so.0...done.
Loaded symbols for /opt/glusterfs/2.0.10rc2/lib/libglusterfs.so.0
Reading symbols from /lib64/libdl.so.2...done.
Loaded symbols for /lib64/libdl.so.2
Reading symbols from /lib64/libpthread.so.0...done.
Loaded symbols for /lib64/libpthread.so.0
Reading symbols from /lib64/libc.so.6...done.
Loaded symbols for /lib64/libc.so.6
Reading symbols from /lib64/ld-linux-x86-64.so.2...done.
Loaded symbols for /lib64/ld-linux-x86-64.so.2
Reading symbols from /opt/glusterfs/2.0.10rc2/lib/glusterfs/2.0.10rc2/xlator/protocol/client.so...done.
Loaded symbols for /opt/glusterfs/2.0.10rc2/lib/glusterfs/2.0.10rc2/xlator/protocol/client.so
Reading symbols from /opt/glusterfs/2.0.10rc2/lib/glusterfs/2.0.10rc2/xlator/cluster/replicate.so...done.
Loaded symbols for /opt/glusterfs/2.0.10rc2/lib/glusterfs/2.0.10rc2/xlator/cluster/replicate.so
Reading symbols from /opt/glusterfs/2.0.10rc2/lib/glusterfs/2.0.10rc2/xlator/cluster/distribute.so...done.
Loaded symbols for /opt/glusterfs/2.0.10rc2/lib/glusterfs/2.0.10rc2/xlator/cluster/distribute.so
Reading symbols from /opt/glusterfs/2.0.10rc2/lib/glusterfs/2.0.10rc2/xlator/performance/write-behind.so...done.
Loaded symbols for /opt/glusterfs/2.0.10rc2/lib/glusterfs/2.0.10rc2/xlator/performance/write-behind.so
Reading symbols from /opt/glusterfs/2.0.10rc2/lib/glusterfs/2.0.10rc2/xlator/performance/read-ahead.so...done.
Loaded symbols for /opt/glusterfs/2.0.10rc2/lib/glusterfs/2.0.10rc2/xlator/performance/read-ahead.so
Reading symbols from /opt/glusterfs/2.0.10rc2/lib/glusterfs/2.0.10rc2/xlator/performance/io-cache.so...done.
Loaded symbols for /opt/glusterfs/2.0.10rc2/lib/glusterfs/2.0.10rc2/xlator/performance/io-cache.so
Reading symbols from /opt/glusterfs/2.0.10rc2/lib/glusterfs/2.0.10rc2/xlator/performance/quick-read.so...done.
Loaded symbols for /opt/glusterfs/2.0.10rc2/lib/glusterfs/2.0.10rc2/xlator/performance/quick-read.so
Reading symbols from /opt/glusterfs/2.0.10rc2/lib/glusterfs/2.0.10rc2/xlator/performance/stat-prefetch.so...done.
Loaded symbols for /opt/glusterfs/2.0.10rc2/lib/glusterfs/2.0.10rc2/xlator/performance/stat-prefetch.so
Reading symbols from /opt/glusterfs/2.0.10rc2/lib/glusterfs/2.0.10rc2/xlator/mount/fuse.so...done.
Loaded symbols for /opt/glusterfs/2.0.10rc2/lib/glusterfs/2.0.10rc2/xlator/mount/fuse.so
Reading symbols from /lib64/libfuse.so.2...done.
Loaded symbols for /lib64/libfuse.so.2
Reading symbols from /lib64/librt.so.1...done.
Loaded symbols for /lib64/librt.so.1
Reading symbols from /opt/glusterfs/2.0.10rc2/lib/glusterfs/2.0.10rc2/transport/socket.so...done.
Loaded symbols for /opt/glusterfs/2.0.10rc2/lib/glusterfs/2.0.10rc2/transport/socket.so
Reading symbols from /lib64/libnss_files.so.2...done.
Loaded symbols for /lib64/libnss_files.so.2
Reading symbols from /usr/lib64/libgcc_s.so.1...done.
Loaded symbols for /usr/lib64/libgcc_s.so.1
Core was generated by `/opt/glusterfs/2.0.10rc2/sbin/glusterfs -f cl_dist_repl.vol /mnt/ipod/ -l /tmp/'.
Program terminated with signal 11, Segmentation fault.
[New process 7663]
[New process 7667]
[New process 7664]
#0  0x00007f3e9b0b865c in client_lk (frame=0x7f3e8c887b20, this=0x610cc0, fd=0x7f3e947b39d0, cmd=6, flock=0x0)
    at ../../../../../xlators/protocol/client/src/client-protocol.c:2873
2873		switch (flock->l_type) {
(gdb) bt
#0  0x00007f3e9b0b865c in client_lk (frame=0x7f3e8c887b20, this=0x610cc0, fd=0x7f3e947b39d0, cmd=6, flock=0x0)
    at ../../../../../xlators/protocol/client/src/client-protocol.c:2873
#1  0x00007f3e9ae71079 in afr_lk_cbk (frame=0x7f3e8c71c2f0, cookie=0x0, this=0x611a30, op_ret=-1, op_errno=77, lock=0x0)
    at ../../../../../xlators/cluster/afr/src/afr.c:2064
#2  0x00007f3e9b0b85ad in client_lk (frame=0x7f3e8c895320, this=0x610820, fd=0x7f3e947b39d0, cmd=6, flock=0x7f3e947ac6d0)
    at ../../../../../xlators/protocol/client/src/client-protocol.c:2857
#3  0x00007f3e9ae7169e in afr_lk (frame=0x7f3e8c71c2f0, this=0x611a30, fd=0x7f3e947b39d0, cmd=6, flock=0x7f3e947ac6d0)
    at ../../../../../xlators/cluster/afr/src/afr.c:2121
#4  0x00007f3e9ac539ba in dht_lk (frame=0x7f3e8c9acd80, this=0x611c80, fd=0x7f3e947b39d0, cmd=6, flock=0x7f3e947ac6d0)
    at ../../../../../xlators/cluster/dht/src/dht-common.c:1956
#5  0x00007f3e9c2755df in default_lk (frame=0x7f3e8c770970, this=0x612550, fd=0x7f3e947b39d0, cmd=6, lock=0x7f3e947ac6d0)
    at ../../../libglusterfs/src/defaults.c:1183
#6  0x00007f3e9c2755df in default_lk (frame=0x7f3e8cb186d0, this=0x612eb0, fd=0x7f3e947b39d0, cmd=6, lock=0x7f3e947ac6d0)
    at ../../../libglusterfs/src/defaults.c:1183
#7  0x00007f3e9a624bc3 in ioc_lk (frame=0x7f3e8c7155e0, this=0x6137f0, fd=0x7f3e947b39d0, cmd=6, lock=0x7f3e947ac6d0)
    at ../../../../../xlators/performance/io-cache/src/io-cache.c:1106
#8  0x00007f3e9a41cd11 in qr_lk_helper (frame=0x7f3e947abc70, this=0x614120, fd=0x7f3e947b39d0, cmd=6, lock=0x7f3e947ac6d0)
    at ../../../../../xlators/performance/quick-read/src/quick-read.c:2097
#9  0x00007f3e9c2854f4 in call_resume_wind (stub=0x7f3e947ac690) at ../../../libglusterfs/src/call-stub.c:2547
#10 0x00007f3e9c28995f in call_resume (stub=0x7f3e947ac690) at ../../../libglusterfs/src/call-stub.c:4169
#11 0x00007f3e9a41637a in qr_open_cbk (frame=0x7f3e947abc70, cookie=0x7f3e947abcc0, this=0x614120, op_ret=0, op_errno=117, 
    fd=0x7f3e947b39d0) at ../../../../../xlators/performance/quick-read/src/quick-read.c:402
#12 0x00007f3e9a623056 in ioc_open_cbk (frame=0x7f3e947abcc0, cookie=0x7f3e94ab3c20, this=0x6137f0, op_ret=0, op_errno=117, 
    fd=0x7f3e947b39d0) at ../../../../../xlators/performance/io-cache/src/io-cache.c:552
#13 0x00007f3e9a82cbed in ra_open_cbk (frame=0x7f3e94ab3c20, cookie=0x7f3e94ab3c70, this=0x612eb0, op_ret=0, op_errno=117, 
    fd=0x7f3e947b39d0) at ../../../../../xlators/performance/read-ahead/src/read-ahead.c:106
#14 0x00007f3e9aa37b62 in wb_open_cbk (frame=0x7f3e94ab3c70, cookie=0x7f3e9479b700, this=0x612550, op_ret=0, op_errno=117, 
    fd=0x7f3e947b39d0) at ../../../../../xlators/performance/write-behind/src/write-behind.c:982
#15 0x00007f3e9ac51f1c in dht_fd_cbk (frame=0x7f3e9479b700, cookie=0x7f3e946d5320, this=0x611c80, op_ret=0, op_errno=5, 
    fd=0x7f3e947b39d0) at ../../../../../xlators/cluster/dht/src/dht-common.c:1684
#16 0x00007f3e9ae6b637 in afr_open_cbk (frame=0x7f3e946d5320, cookie=0x0, this=0x611a30, op_ret=-1, op_errno=5, fd=0x7f3e947b39d0)
    at ../../../../../xlators/cluster/afr/src/afr.c:829
#17 0x00007f3e9b0bc7d0 in client_open_cbk (frame=0x7f3e948104c0, hdr=0x7f3e8c9a60d0, hdrlen=40, iobuf=0x0)
    at ../../../../../xlators/protocol/client/src/client-protocol.c:4098
#18 0x00007f3e9b0c16cd in protocol_client_interpret (this=0x610820, trans=0x619a40, hdr_p=0x7f3e8c9a60d0 "", hdrlen=40, iobuf=0x0)
    at ../../../../../xlators/protocol/client/src/client-protocol.c:6056
#19 0x00007f3e9b0c2345 in protocol_client_pollin (this=0x610820, trans=0x619a40)
    at ../../../../../xlators/protocol/client/src/client-protocol.c:6347
#20 0x00007f3e9b0c24d9 in notify (this=0x610820, event=2, data=0x619a40)
    at ../../../../../xlators/protocol/client/src/client-protocol.c:6390
#21 0x00007f3e991b8edd in socket_event_poll_in (this=0x619a40) at ../../../../transport/socket/src/socket.c:714
#22 0x00007f3e991b91e0 in socket_event_handler (fd=13, idx=5, data=0x619a40, poll_in=1, poll_out=0, poll_err=0)
    at ../../../../transport/socket/src/socket.c:814
#23 0x00007f3e9c28da6b in event_dispatch_epoll_handler (event_pool=0x60a310, events=0x61ce40, i=0)
    at ../../../libglusterfs/src/event.c:804
#24 0x00007f3e9c28dc3a in event_dispatch_epoll (event_pool=0x60a310) at ../../../libglusterfs/src/event.c:867
#25 0x00007f3e9c28df4b in event_dispatch (event_pool=0x60a310) at ../../../libglusterfs/src/event.c:975
#26 0x0000000000405d88 in main (argc=6, argv=0x7fffafd6ad38) at ../../../glusterfsd/src/glusterfsd.c:1328
(gdb) l
2868			gf_log (this->name, GF_LOG_DEBUG,
2869				"Unknown cmd (%d)!", gf_cmd);
2870			goto unwind;
2871		}
2872	
2873		switch (flock->l_type) {
2874		case F_RDLCK:
2875			gf_type = GF_LK_F_RDLCK;
2876			break;
2877		case F_WRLCK:
(gdb) p flock
$1 = (struct flock *) 0x0
(gdb) f 1
#1  0x00007f3e9ae71079 in afr_lk_cbk (frame=0x7f3e8c71c2f0, cookie=0x0, this=0x611a30, op_ret=-1, op_errno=77, lock=0x0)
    at ../../../../../xlators/cluster/afr/src/afr.c:2064
2064			STACK_WIND_COOKIE (frame, afr_lk_cbk, (void *) (long) child_index,
(gdb) p lock
$2 = (struct flock *) 0x0
(gdb) f 2
#2  0x00007f3e9b0b85ad in client_lk (frame=0x7f3e8c895320, this=0x610820, fd=0x7f3e947b39d0, cmd=6, flock=0x7f3e947ac6d0)
    at ../../../../../xlators/protocol/client/src/client-protocol.c:2857
2857			STACK_UNWIND(frame, -1, EBADFD, NULL);
(gdb) q