Bug 765371 (GLUSTER-3639)

After the tests of bug 765369, tried to remove everything from the mount point by doing rm -rf from both fuse and nfs clients simultaneously. The fuse client crashed with the following backtrace.

Core was generated by `/usr/local/sbin/glusterfs --volfile-id=mirror --volfile-server=10.1.11.73 /clie'.
Program terminated with signal 6, Aborted.
#0  0x00007f270a314a75 in raise () from /lib/libc.so.6
(gdb) bt
#0  0x00007f270a314a75 in raise () from /lib/libc.so.6
#1  0x00007f270a3185c0 in abort () from /lib/libc.so.6
#2  0x00007f270a30d941 in __assert_fail () from /lib/libc.so.6
#3  0x00007f2704bb93e0 in afr_inode_set_read_ctx (this=0x7f26edf52ea0, inode=0x7f269bd24d20, read_child=-1, fresh_children=0x12b8180)
    at ../../../../../xlators/cluster/afr/src/afr-common.c:417
#4  0x00007f2704ba87f1 in afr_sh_entry_fix (frame=0x7f270955a37c, this=0x7f26edf52ea0)
    at ../../../../../xlators/cluster/afr/src/afr-self-heal-entry.c:2259
#5  0x00007f2704ba88c1 in afr_sh_entry_lookup_cbk (frame=0x7f270955a37c, cookie=0x1, this=0x7f26edf52ea0, op_ret=-1, op_errno=2, 
    inode=0x7f269bd24d20, buf=0x7fffb9b6ca10, xattr=0x0, postparent=0x7fffb9b6c9a0)
    at ../../../../../xlators/cluster/afr/src/afr-self-heal-entry.c:2288
#6  0x00007f2704dfc01e in client3_1_lookup_cbk (req=0x7f269c2fc060, iov=0x7f269c2fc0a0, count=1, myframe=0x7f27097d8778)
    at ../../../../../xlators/protocol/client/src/client3_1-fops.c:2255
#7  0x00007f270acb130d in rpc_clnt_handle_reply (clnt=0x7f26edf5fc10, pollin=0x15c3da0) at ../../../../rpc/rpc-lib/src/rpc-clnt.c:789
#8  0x00007f270acb165a in rpc_clnt_notify (trans=0x7f26edf5ff40, mydata=0x7f26edf5fc40, event=RPC_TRANSPORT_MSG_RECEIVED, data=0x15c3da0)
    at ../../../../rpc/rpc-lib/src/rpc-clnt.c:902
#9  0x00007f270acad6b8 in rpc_transport_notify (this=0x7f26edf5ff40, event=RPC_TRANSPORT_MSG_RECEIVED, data=0x15c3da0)
    at ../../../../rpc/rpc-lib/src/rpc-transport.c:498
#10 0x00007f2707e327d0 in socket_event_poll_in (this=0x7f26edf5ff40) at ../../../../../rpc/rpc-transport/socket/src/socket.c:1675
#11 0x00007f2707e32d54 in socket_event_handler (fd=340, idx=221, data=0x7f26edf5ff40, poll_in=1, poll_out=0, poll_err=0)
    at ../../../../../rpc/rpc-transport/socket/src/socket.c:1790
#12 0x00007f270af072fa in event_dispatch_epoll_handler (event_pool=0xf1f7c0, events=0xf24d70, i=0) at ../../../libglusterfs/src/event.c:794
#13 0x00007f270af0751d in event_dispatch_epoll (event_pool=0xf1f7c0) at ../../../libglusterfs/src/event.c:856
#14 0x00007f270af078a8 in event_dispatch (event_pool=0xf1f7c0) at ../../../libglusterfs/src/event.c:956
#15 0x000000000040837c in main (argc=4, argv=0x7fffb9b6cfc8) at ../../../glusterfsd/src/glusterfsd.c:1592
(gdb) f 3
#3  0x00007f2704bb93e0 in afr_inode_set_read_ctx (this=0x7f26edf52ea0, inode=0x7f269bd24d20, read_child=-1, fresh_children=0x12b8180)
    at ../../../../../xlators/cluster/afr/src/afr-common.c:417
417             GF_ASSERT (read_child >= 0);
(gdb) p read_child
$1 = -1
(gdb) f 4
#4  0x00007f2704ba87f1 in afr_sh_entry_fix (frame=0x7f270955a37c, this=0x7f26edf52ea0)
    at ../../../../../xlators/cluster/afr/src/afr-self-heal-entry.c:2259
2259            afr_inode_set_read_ctx (this, sh->inode, sh->source,
(gdb) p sh->source
$2 = -1
(gdb) l
2254            sh->source = source;
2255
2256            afr_reset_children (sh->fresh_children, priv->child_count);
2257            afr_get_fresh_children (sh->success_children, sh->sources,
2258                                    sh->fresh_children, priv->child_count);
2259            afr_inode_set_read_ctx (this, sh->inode, sh->source,
2260                                    sh->fresh_children);
2261
2262
2263    heal:
(gdb) source
source command requires file name of file to source.
(gdb)  p source
$3 = -1
(gdb) l afr_sh_entry_fix
2217    }
2218
2219
2220    int
2221    afr_sh_entry_fix (call_frame_t *frame, xlator_t *this)
2222    {
2223            afr_local_t     *local = NULL;
2224            afr_self_heal_t *sh = NULL;
2225            afr_private_t   *priv = NULL;
2226            int              source = 0;
(gdb) 
2227
2228            int nsources = 0;
2229
2230            local = frame->local;
2231            sh = &local->self_heal;
2232            priv = this->private;
2233
2234            if (sh->forced_merge) {
2235                    sh->source = -1;
2236                    goto heal;
(gdb) 
2237            }
2238
2239            nsources = afr_build_sources (this, sh->xattr, sh->buf,
2240                                          sh->pending_matrix, sh->sources,
2241                                          sh->success_children,
2242                                          AFR_ENTRY_TRANSACTION);
2243            if (nsources == 0) {
2244                    gf_log (this->name, GF_LOG_TRACE,
2245                            "No self-heal needed for %s",
2246                            local->loc.path);
(gdb) 
2247
2248                    afr_sh_entry_finish (frame, this);
2249                    return 0;
2250            }
2251
2252            source = afr_sh_select_source (sh->sources, priv->child_count);
2253
2254            sh->source = source;


 l afr_build_sources
607
608     int
609     afr_build_sources (xlator_t *this, dict_t **xattr, struct iatt *bufs,
610                        int32_t **pending_matrix, int32_t *sources,
611                        int32_t *success_children, afr_transaction_type type)
612     {
613             afr_private_t           *priv = NULL;
614             afr_self_heal_type      sh_type    = AFR_SELF_HEAL_INVALID;
615             int                     nsources   = -1;
616
(gdb) 
617             priv = this->private;
618
619             if (afr_get_children_count (success_children, priv->child_count) == 0)
620                     goto out;
621
622             afr_build_pending_matrix (priv->pending_key, pending_matrix,
623                                       xattr, type, priv->child_count);
624
625             sh_type = afr_self_heal_type_for_transaction (type);
626             if (AFR_SELF_HEAL_INVALID == sh_type)
(gdb) 
627                     goto out;
628
629             afr_sh_print_pending_matrix (pending_matrix, this);
630
631             nsources = afr_mark_sources (sources, pending_matrix, bufs,
632                                          priv->child_count, sh_type,
633                                          success_children, this->name);
634     out:
635             return nsources;
636     }

Here nsources can be -1 if it cannot identify the sources properly. But we are checking nsources only for 0 after calling afr_build_sources and instead of returning we are continuing with read_child set to -1.

The crash is observed because the entry-self-heal goes ahead with the self-heal even when the lookups failed, this has already been fixed as part of 3557 patch.
Marking it resolved fixed

CHANGE: http://review.gluster.com/556 (Change-Id: Ifdf0db71594ce526ad85c21103726798d9aceef4) merged in master by Vijay Bellur (vijay)