Hide Forgot
nfs server crashed due to assert since the read_child was -1. Setup: 2 replica setup 1 fuse client executing sanity script 1 nfs client executing sanity script volume set operations in a loop brick kill, sleep and bring brick up find | xargs stat on fuse mount point. nfs server crashed with the following backtrace. Core was generated by `/usr/local/sbin/glusterfs -f /etc/glusterd/nfs/nfs-server.vol -p /etc/glusterd/'. Program terminated with signal 6, Aborted. #0 0x0000003eb5e30265 in raise () from /lib64/libc.so.6 (gdb) bt #0 0x0000003eb5e30265 in raise () from /lib64/libc.so.6 #1 0x0000003eb5e31d10 in abort () from /lib64/libc.so.6 #2 0x0000003eb5e296e6 in __assert_fail () from /lib64/libc.so.6 #3 0x00002aaaaad0f3f1 in afr_transaction_rm_stale_children (frame=0x2ad8a0c430d4, this=0x64b34c0, inode=0x2aaaabcdf2dc, type=AFR_DATA_TRANSACTION) at ../../../../../xlators/cluster/afr/src/afr-transaction.c:414 #4 0x00002aaaaad0f625 in afr_changelog_post_op (frame=0x2ad8a0c430d4, this=0x64b34c0) at ../../../../../xlators/cluster/afr/src/afr-transaction.c:492 #5 0x00002aaaaad135e8 in afr_transaction_resume (frame=0x2ad8a0c430d4, this=0x64b34c0) at ../../../../../xlators/cluster/afr/src/afr-transaction.c:1187 #6 0x00002aaaaad045ca in afr_writev_wind_cbk (frame=0x2ad8a0c430d4, cookie=0x1, this=0x64b34c0, op_ret=65536, op_errno=0, prebuf=0x7fff175596d0, postbuf=0x7fff17559660) at ../../../../../xlators/cluster/afr/src/afr-inode-write.c:124 #7 0x00002aaaaaac20d3 in client3_1_writev_cbk (req=0x2aaaad72a04c, iov=0x2aaaad72a08c, count=1, myframe=0x2ad8a09920f8) at ../../../../../xlators/protocol/client/src/client3_1-fops.c:692 #8 0x00002ad89fd0125e in rpc_clnt_handle_reply (clnt=0x64c6ec0, pollin=0x64d4bb0) at ../../../../rpc/rpc-lib/src/rpc-clnt.c:789 #9 0x00002ad89fd01586 in rpc_clnt_notify (trans=0x64c71f0, mydata=0x64c6ef0, event=RPC_TRANSPORT_MSG_RECEIVED, data=0x64d4bb0) at ../../../../rpc/rpc-lib/src/rpc-clnt.c:902 #10 0x00002ad89fcfd9f3 in rpc_transport_notify (this=0x64c71f0, event=RPC_TRANSPORT_MSG_RECEIVED, data=0x64d4bb0) at ../../../../rpc/rpc-lib/src/rpc-transport.c:498 #11 0x00002aaaad4daea7 in socket_event_poll_in (this=0x64c71f0) at ../../../../../rpc/rpc-transport/socket/src/socket.c:1675 #12 0x00002aaaad4db3e9 in socket_event_handler (fd=10, idx=4, data=0x64c71f0, poll_in=1, poll_out=0, poll_err=0) at ../../../../../rpc/rpc-transport/socket/src/socket.c:1790 #13 0x00002ad89faa984c in event_dispatch_epoll_handler (event_pool=0x64ab960, events=0x64c9e30, i=0) at ../../../libglusterfs/src/event.c:794 #14 0x00002ad89faa9a51 in event_dispatch_epoll (event_pool=0x64ab960) at ../../../libglusterfs/src/event.c:856 #15 0x00002ad89faa9dab in event_dispatch (event_pool=0x64ab960) at ../../../libglusterfs/src/event.c:956 #16 0x000000000040784d in main (argc=7, argv=0x7fff17559d68) at ../../../glusterfsd/src/glusterfsd.c:1592 (gdb) f 3 #3 0x00002aaaaad0f3f1 in afr_transaction_rm_stale_children (frame=0x2ad8a0c430d4, this=0x64b34c0, inode=0x2aaaabcdf2dc, type=AFR_DATA_TRANSACTION) at ../../../../../xlators/cluster/afr/src/afr-transaction.c:414 414 GF_ASSERT (read_child >= 0); (gdb) p read_child $1 = -1 (gdb) f 4 #4 0x00002aaaaad0f625 in afr_changelog_post_op (frame=0x2ad8a0c430d4, this=0x64b34c0) at ../../../../../xlators/cluster/afr/src/afr-transaction.c:492 492 afr_transaction_rm_stale_children (frame, this, (gdb) l 487 488 __mark_down_children (local->pending, priv->child_count, 489 local->child_up, local->transaction.type); 490 491 if (local->fd) 492 afr_transaction_rm_stale_children (frame, this, 493 local->fd->inode, 494 local->transaction.type); 495 496 xattr = alloca (priv->child_count * sizeof (*xattr)); (gdb) (gdb) 375 afr_local_t *local = NULL; 376 int **pending = NULL; 377 int idx = 0; 378 int32_t *stale_children = NULL; 379 int32_t *fresh_children = NULL; 380 gf_boolean_t rm_stale_children = _gf_false; 381 382 idx = afr_index_for_transaction_type (type); 383 384 priv = this->private; (gdb) 385 local = frame->local; 386 pending = local->pending; 387 388 stale_children = afr_children_create (priv->child_count); 389 if (!stale_children) 390 goto out; 391 392 fresh_children = local->fresh_children; 393 read_child = afr_inode_get_read_ctx (this, inode, fresh_children); 394 (gdb) 395 GF_ASSERT (read_child >= 0); 396 397 if (pending[read_child][idx] == 0) 398 read_child = -1; 399 400 for (i = 0; i < priv->child_count; i++) { 401 if (!afr_is_child_present (fresh_children, 402 priv->child_count, i)) 403 continue; 404 if (pending[i][idx] == 0) { (gdb) p fresh_children[0] $2 = -1 (gdb) p fresh_children[1] $3 = -1 (gdb)
CHANGE: http://review.gluster.com/740 (Change-Id: I00c714a89575023f6dbdd3430dcbf191e5d08019) merged in master by Vijay Bellur (vijay)