nfs server crashed in afr_lookup_save_gfid since gfid was NULL.

This is the backtrace.


 bt
#0  0x00007fde992a6d05 in raise (sig=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:64
#1  0x00007fde992aaab6 in abort () at abort.c:92
#2  0x00007fde9929f7c5 in __assert_fail (assertion=0x7fde96b9626d "new && !uuid_is_null (new)", file=<value optimized out>, line=195, 
    function=<value optimized out>) at assert.c:81
#3  0x00007fde96b800ce in afr_lookup_save_gfid (dst=0x7fde8c162748 "", new=0x0, inode=0x7fde95934224)
    at ../../../../../xlators/cluster/afr/src/afr-common.c:195
#4  0x00007fde96b844ae in afr_lookup (frame=0x7fde9878141c, this=0xc2f970, loc=0x7fde8c089878, xattr_req=0x0)
    at ../../../../../xlators/cluster/afr/src/afr-common.c:1703
#5  0x00007fde9691ab75 in dht_rmdir_is_subvol_empty (frame=0x7fde987881a8, this=0xc31890, entries=0x7fff42c01770, src=0xc2f970)
    at ../../../../../xlators/cluster/dht/src/dht-common.c:4499
#6  0x00007fde9691ac99 in dht_rmdir_readdirp_cbk (frame=0x7fde987881a8, cookie=0x7fde9878d074, this=0xc31890, op_ret=3, op_errno=2, 
    entries=0x7fff42c01770) at ../../../../../xlators/cluster/dht/src/dht-common.c:4527
#7  0x00007fde96b39137 in afr_readdirp_cbk (frame=0x7fde9878d074, cookie=0x1, this=0xc2f970, op_ret=3, op_errno=2, entries=0x7fff42c01770)
    at ../../../../../xlators/cluster/afr/src/afr-dir-read.c:641
#8  0x00007fde97a540b8 in client3_1_readdirp_cbk (req=0x7fde92dfdb98, iov=0x7fde92dfdbd8, count=1, myframe=0x7fde98781bcc)
    at ../../../../../xlators/protocol/client/src/client3_1-fops.c:1939
#9  0x00007fde99c560be in rpc_clnt_handle_reply (clnt=0xc83cc0, pollin=0x7fde8c0dd120) at ../../../../rpc/rpc-lib/src/rpc-clnt.c:741
#10 0x00007fde99c563fa in rpc_clnt_notify (trans=0xc83e70, mydata=0xc83cf0, event=RPC_TRANSPORT_MSG_RECEIVED, data=0x7fde8c0dd120)
    at ../../../../rpc/rpc-lib/src/rpc-clnt.c:854
#11 0x00007fde99c527b7 in rpc_transport_notify (this=0xc83e70, event=RPC_TRANSPORT_MSG_RECEIVED, data=0x7fde8c0dd120)
    at ../../../../rpc/rpc-lib/src/rpc-transport.c:919
#12 0x00007fde93304ca0 in socket_event_poll_in (this=0xc83e70) at ../../../../../rpc/rpc-transport/socket/src/socket.c:1647
#13 0x00007fde93305214 in socket_event_handler (fd=11, idx=1, data=0xc83e70, poll_in=1, poll_out=0, poll_err=0)
    at ../../../../../rpc/rpc-transport/socket/src/socket.c:1762
#14 0x00007fde99eacc45 in event_dispatch_epoll_handler (event_pool=0xc1bbc0, events=0xc94310, i=0) at ../../../libglusterfs/src/event.c:794
#15 0x00007fde99eace5f in event_dispatch_epoll (event_pool=0xc1bbc0) at ../../../libglusterfs/src/event.c:856
#16 0x00007fde99ead1d1 in event_dispatch (event_pool=0xc1bbc0) at ../../../libglusterfs/src/event.c:956
#17 0x00000000004072c0 in main (argc=7, argv=0x7fff42c01ce8) at ../../../glusterfsd/src/glusterfsd.c:1509
(gdb) f 3
#3  0x00007fde96b800ce in afr_lookup_save_gfid (dst=0x7fde8c162748 "", new=0x0, inode=0x7fde95934224)
    at ../../../../../xlators/cluster/afr/src/afr-common.c:195
195	                GF_ASSERT (new && !uuid_is_null (new));
(gdb) p new
$6 = (void *) 0x0
(gdb) l
190	afr_lookup_save_gfid (uuid_t dst, void* new, inode_t *inode)
191	{
192	        if (inode && !uuid_is_null (inode->gfid)) {
193	                uuid_copy (dst, inode->gfid);
194	        } else {
195	                GF_ASSERT (new && !uuid_is_null (new));
196	                uuid_copy (dst, new);
197	        }
198	}
199	
(gdb) f 5
#5  0x00007fde9691ab75 in dht_rmdir_is_subvol_empty (frame=0x7fde987881a8, this=0xc31890, entries=0x7fff42c01770, src=0xc2f970)
    at ../../../../../xlators/cluster/dht/src/dht-common.c:4499
4499	                STACK_WIND (lookup_frame, dht_rmdir_lookup_cbk,
(gdb) l
4494	                {
4495	                        local->call_cnt++;
4496	                }
4497	                UNLOCK (&frame->lock);
4498	
4499	                STACK_WIND (lookup_frame, dht_rmdir_lookup_cbk,
4500	                            src, src->fops->lookup,
4501	                            &lookup_local->loc, NULL);
4502	                ret++;
4503	        }
(gdb) f 4
#4  0x00007fde96b844ae in afr_lookup (frame=0x7fde9878141c, this=0xc2f970, loc=0x7fde8c089878, xattr_req=0x0)
    at ../../../../../xlators/cluster/afr/src/afr-common.c:1703
1703	        afr_lookup_save_gfid (local->cont.lookup.gfid_req, gfid_req,
(gdb) p *loc
$7 = {path = 0x7fde8c125c90 "/clients/client0/~dmtmp/WORD/~WRL0004.TMP", name = 0x7fde8c125cad "~WRL0004.TMP", inode = 0x7fde95934224, 
  parent = 0x7fde9591e93c, gfid = '\000' <repeats 15 times>, pargfid = '\000' <repeats 15 times>, ino = 0}
(gdb) p *loc->inode
$8 = {table = 0xc53690, gfid = '\000' <repeats 15 times>, lock = 1, nlookup = 0, ref = 2, ino = 0, ia_type = IA_INVAL, fd_list = {
    next = 0x7fde95934264, prev = 0x7fde95934264}, dentry_list = {next = 0x7fde95934274, prev = 0x7fde95934274}, hash = {
    next = 0x7fde95934284, prev = 0x7fde95934284}, list = {next = 0x7fde9591dc44, prev = 0xc536f0}, _ctx = 0x7fde8c157ce0}
(gdb) 

In afr_lookup the xattr_req is NULL and both loc->gfid and loc->inode->gfid is NULL. Whenever a new call is originated from intermediate xlators (i.e. the call is not from the application), then loc->gfid should be filled in. In this case dht should have filled in loc->gfid before sending the lookup call and afr should have checked loc->gfid for filling in the gfid.