gnfs  crashed during fileop with 4x3 distributed-replicated setup with 3.1.0qa5.

Log file can be found /share/tickets/1315/1315.log
core file-
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
(gdb) bt full
#0  0x00002aaaab568343 in pthread_mutex_lock () from /lib64/libpthread.so.0
No symbol table info available.
#1  0x00002aaaaacee55d in inode_ref (inode=0x4514a70) at inode.c:478
    table = (inode_table_t *) 0x0
#2  0x00002aaaaace0463 in loc_copy (dst=0x4514a78, src=0x2aaaaac4b408) at
xlator.c:1023
    ret = -1
#3  0x00002aaaacc8c3ce in dht_lookup (frame=0x2aaaac3354a0, this=0x63d050,
loc=0x2aaaaac4b408, xattr_req=0x0) at dht-common.c:815
    subvol = (xlator_t *) 0x0
    hashed_subvol = (xlator_t *) 0x0
    cached_subvol = (xlator_t *) 0x0
    local = (dht_local_t *) 0x4514a70
    conf = (dht_conf_t *) 0x63f920
    ret = -1
    op_errno = -1
    layout = (dht_layout_t *) 0x0
    i = 0
    call_cnt = 0
    __FUNCTION__ = "dht_lookup"
#4  0x00002aaaacec3518 in nfs_fop_lookup (nfsx=0x63e080, xl=0x63d050,
nfu=0x40affd10, loc=0x2aaaaac4b408, cbk=0x2aaaacee3cc8
<nfs3_fh_resolve_entry_lookup_cbk>, 
    local=0x2aaaaac4b020) at nfs-fops.c:284
    _new = (call_frame_t *) 0x2aaaac3354a0
    old_THIS = (xlator_t *) 0x2aaaaaf22560
    tmp_cbk = (fop_lookup_cbk_t) 0x2aaaacec2e21 <nfs_fop_lookup_cbk>
    frame = (call_frame_t *) 0x2aaaac536e78
    ret = -14
    nfl = (struct nfs_fop_local *) 0x2aaaaab25280
    __FUNCTION__ = "nfs_fop_lookup"
#5  0x00002aaaacecb58a in nfs_lookup (nfsx=0x63e080, xl=0x63d050,
nfu=0x40affd10, pathloc=0x2aaaaac4b408, cbk=0x2aaaacee3cc8
<nfs3_fh_resolve_entry_lookup_cbk>, 
    local=0x2aaaaac4b020) at nfs-generics.c:86
    ret = -14
#6  0x00002aaaacee5097 in nfs3_fh_resolve_entry_hard (cs=0x2aaaaac4b020) at
nfs3-helpers.c:2949
    ret = -2
    nfu = {uid = 0, gids = {0 <repeats 17 times>}, ngrps = 1}
    __FUNCTION__ = "nfs3_fh_resolve_entry_hard"
#7  0x00002aaaacee5232 in nfs3_fh_resolve_entry (cs=0x2aaaaac4b020) at
nfs3-helpers.c:2995
    ret = -14
#8  0x00002aaaacee537b in nfs3_fh_resolve_and_resume (cs=0x2aaaaac4b020,
fh=0x40affed0, entry=0x40afff10 "fileop_dir_29_8_13", 
Missing separate debuginfos, use: debuginfo-install gcc.x86_64 glibc.x86_64
 resum_fn=0x2aaaaced5838 <nfs3_mkdir_resume>) at nfs3-helpers.c:3022
    ret = -14
#9  0x00002aaaaced5d68 in nfs3_mkdir (req=0x67f230, dirfh=0x40affed0,
name=0x40afff10 "fileop_dir_29_8_13", sattr=0x40affe88) at nfs3.c:2439
    vol = (xlator_t *) 0x63d050
    stat = NFS3ERR_SERVERFAULT
    ret = -14
    nfs3 = (struct nfs3_state *) 0x65a270
    cs = (nfs3_call_state_t *) 0x2aaaaac4b020
    __FUNCTION__ = "nfs3_mkdir"
#10 0x00002aaaaced5ece in nfs3svc_mkdir (req=0x67f230) at nfs3.c:2473
    name = "fileop_dir_29_8_13\000\000\000\000\000\000\000
�@\000\000\000\000\000\020\000\000\000\000\000\000,\017\177��*\000\000\000\020\000\000\000\000\000\000\200٬��*\000\000`\002�@",
'\0' <repeats 21 times>, "
�@\000\000\000\000\000\020\000\000\000\000\000\000,\017\177��*\000\000����",
'\0' <repeats 28 times>, "P�b", '\0' <repeats 13 times>,
"`\002�@\000\000\000\0001\017�*\000\000�Y���*\000\000\034U���*\000\000\000\000\000\000\000\000\000\000��P\004\000\000\000\000p\237P\004\000\000\000\000P\000�@\000\000\000\000P\000�@\000\000\000\000"...
    dirfh = {ident = ":O", hashcount = 2, xlatorid = 0, gen =
5505199980916578812, ino = 634077276, entryhash = {257, 62611, 0 <repeats 19
times>}}
    args = {where = {dir = {data = {data_len = 26, data_val = 0x40affed0
":O\002"}}, name = 0x40afff10 "fileop_dir_29_8_13"}, attributes = {mode =
{set_it = 1, 
      set_mode3_u = {mode = 493}}, uid = {set_it = 0, set_uid3_u = {uid = 0}},
gid = {set_it = 0, set_gid3_u = {gid = 0}}, size = {set_it = 0, set_size3_u = {
        size = 0}}, atime = {set_it = DONT_CHANGE, set_atime_u = {atime =
{seconds = 0, nseconds = 0}}}, mtime = {set_it = DONT_CHANGE, set_mtime_u =
{mtime = {
          seconds = 0, nseconds = 0}}}}}
    ret = -1
    __FUNCTION__ = "nfs3svc_mkdir"
#11 0x00002aaaaceebaec in nfs_rpcsvc_handle_rpc_call (conn=0x64b450) at
../../../../xlators/nfs/lib/src//rpcsvc.c:1901
    actor = (rpcsvc_actor_t *) 0x2aaaad104140
    req = (rpcsvc_request_t *) 0x67f230
    ret = -1
#12 0x00002aaaaceec7ee in nfs_rpcsvc_record_update_state (conn=0x64b450,
dataread=0) at ../../../../xlators/nfs/lib/src//rpcsvc.c:2383
    rs = (rpcsvc_record_state_t *) 0x64b468
    svc = (rpcsvc_t *) 0x0
    __FUNCTION__ = "nfs_rpcsvc_record_update_state"
#13 0x00002aaaaceec944 in nfs_rpcsvc_conn_data_poll_in (conn=0x64b450) at
../../../../xlators/nfs/lib/src//rpcsvc.c:2426
    dataread = 124
    readsize = 124
    readaddr = 0x2aaaac134000 "���g"
    ret = -1
    __FUNCTION__ = "nfs_rpcsvc_conn_data_poll_in"
#14 0x00002aaaaceecd68 in nfs_rpcsvc_conn_data_handler (fd=21, idx=3,
data=0x64b450, poll_in=1, poll_out=0, poll_err=0)
    at ../../../../xlators/nfs/lib/src//rpcsvc.c:2555
    conn = (rpcsvc_conn_t *) 0x64b450
    ret = 0
#15 0x00002aaaaad02165 in event_dispatch_epoll_handler (event_pool=0x63f2f0,
events=0x659360, i=0) at event.c:812
    event_data = (struct event_data *) 0x659364
    handler = (event_handler_t) 0x2aaaaceecce1 <nfs_rpcsvc_conn_data_handler>
    data = (void *) 0x64b450
    idx = 3
    ret = -1
    __FUNCTION__ = "event_dispatch_epoll_handler"
#16 0x00002aaaaad02338 in event_dispatch_epoll (event_pool=0x63f2f0) at
event.c:876
    events = (struct epoll_event *) 0x659360
    size = 1
    i = 0
    ret = 1
    __FUNCTION__ = "event_dispatch_epoll"
#17 0x00002aaaaad02608 in event_dispatch (event_pool=0x63f2f0) at event.c:984
    ret = -1
    __FUNCTION__ = "event_dispatch"
#18 0x00002aaaacee8236 in nfs_rpcsvc_stage_proc (arg=0x62b300) at
../../../../xlators/nfs/lib/src//rpcsvc.c:64
    stg = (rpcsvc_stage_t *) 0x62b300
#19 0x00002aaaab566407 in start_thread () from /lib64/libpthread.so.0
No symbol table info available.
#20 0x00002aaaab84fb0d in clone () from /lib64/libc.so.6
No symbol table info available.