While running fio over nfs mount point with 2 replicates, when server1 was killed the nfs server crashed.

fio config file

[job1]
filename=/jbod/anush/client1/testfile
size=10G
rw=write
bs=64K

NFS log file:

[2010-03-16 23:02:48] T [nfs.c:239:nfs_user_create] nfs: gid: 0
[2010-03-16 23:02:48] T [nfs.c:239:nfs_user_create] nfs: gid: 1
[2010-03-16 23:02:48] T [nfs.c:239:nfs_user_create] nfs: gid: 2
[2010-03-16 23:02:48] T [nfs.c:239:nfs_user_create] nfs: gid: 3
[2010-03-16 23:02:48] T [nfs.c:239:nfs_user_create] nfs: gid: 4
[2010-03-16 23:02:48] T [nfs.c:239:nfs_user_create] nfs: gid: 6
[2010-03-16 23:02:48] T [nfs.c:239:nfs_user_create] nfs: gid: 10
[2010-03-16 23:02:48] T [nfs-fops.c:109:nfs_create_frame] nfs: uid: 0, gid 0, gids: 7
[2010-03-16 23:02:48] T [nfs-fops.c:111:nfs_create_frame] nfs: gid: 0
[2010-03-16 23:02:48] T [nfs-fops.c:111:nfs_create_frame] nfs: gid: 1
[2010-03-16 23:02:48] T [nfs-fops.c:111:nfs_create_frame] nfs: gid: 2
[2010-03-16 23:02:48] T [nfs-fops.c:111:nfs_create_frame] nfs: gid: 3
[2010-03-16 23:02:48] T [nfs-fops.c:111:nfs_create_frame] nfs: gid: 4
[2010-03-16 23:02:48] T [nfs-fops.c:111:nfs_create_frame] nfs: gid: 6
[2010-03-16 23:02:48] T [nfs-fops.c:111:nfs_create_frame] nfs: gid: 10
[2010-03-16 23:02:48] T [rpcsvc.c:2371:rpcsvc_conn_data_poll_in] rpc-service: conn: 0x1f937380, readsize: 4, dataread: 4
[2010-03-16 23:02:48] T [rpcsvc.c:1065:rpcsvc_record_read_complete_fraghdr] rpc-service: Received fragment size: 65680
[2010-03-16 23:02:48] T [rpcsvc.c:1068:rpcsvc_record_read_complete_fraghdr] rpc-service: Vectored RPC header, remaining: 32
[2010-03-16 23:02:48] T [rpcsvc.c:2371:rpcsvc_conn_data_poll_in] rpc-service: conn: 0x1f937380, readsize: 32, dataread: 32
[2010-03-16 23:02:48] T [rpcsvc.c:2305:rpcsvc_record_update_state] rpc-service: Vectored frag
[2010-03-16 23:02:48] T [rpcsvc.c:1177:rpcsvc_record_read_partial_frag] rpc-service: Fragment remaining: 0
[2010-03-16 23:02:48] T [rpcsvc.c:2268:rpcsvc_handle_vectored_frag] rpc-service: Vectored frag complete
[2010-03-16 23:02:48] T [rpcsvc.c:2052:rpcsvc_update_vectored_barerpc] rpc-service: Vectored RPC cred remaining: 56
[2010-03-16 23:02:48] T [rpcsvc.c:2371:rpcsvc_conn_data_poll_in] rpc-service: conn: 0x1f937380, readsize: 56, dataread: 56
[2010-03-16 23:02:48] T [rpcsvc.c:2305:rpcsvc_record_update_state] rpc-service: Vectored frag
[2010-03-16 23:02:48] T [rpcsvc.c:1177:rpcsvc_record_read_partial_frag] rpc-service: Fragment remaining: 0
[2010-03-16 23:02:48] T [rpcsvc.c:2268:rpcsvc_handle_vectored_frag] rpc-service: Vectored frag complete
[2010-03-16 23:02:48] T [rpcsvc.c:2029:rpcsvc_update_vectored_cred] rpc-service: Vectored RPC verfsz remaining: 8
[2010-03-16 23:02:48] T [rpcsvc.c:2371:rpcsvc_conn_data_poll_in] rpc-service: conn: 0x1f937380, readsize: 8, dataread: 8
[2010-03-16 23:02:48] T [rpcsvc.c:2305:rpcsvc_record_update_state] rpc-service: Vectored frag
[2010-03-16 23:02:48] T [rpcsvc.c:1177:rpcsvc_record_read_partial_frag] rpc-service: Fragment remaining: 0
[2010-03-16 23:02:48] T [rpcsvc.c:2268:rpcsvc_handle_vectored_frag] rpc-service: Vectored frag complete
[2010-03-16 23:02:48] T [rpcsvc.c:2005:rpcsvc_update_vectored_verfsz] rpc-service: Vectored RPC preparing call
[2010-03-16 23:02:48] T [rpcsvc-auth.c:248:rpcsvc_auth_request_init] rpc-service: Auth handler: AUTH_UNIX
[2010-03-16 23:02:48] D [rpcsvc.c:1792:rpcsvc_request_create] rpc-service: RPC XID: 52e2953, Ver: 2, Program: 100003, ProgVers: 3, Proc: 7
[2010-03-16 23:02:48] T [auth-unix.c:65:auth_unix_authenticate] rpc-service: Auth Info: machine name: brick6, uid: 0, gid: 0
[2010-03-16 23:02:48] D [rpcsvc.c:1267:rpcsvc_program_actor] rpc-service: Actor found: NFS3 - WRITE
[2010-03-16 23:02:48] T [rpcsvc.c:1964:rpcsvc_handle_vectored_prep_rpc_call] rpc-service: Vectored RPC proc header remaining: 4
[2010-03-16 23:02:48] T [rpcsvc.c:2371:rpcsvc_conn_data_poll_in] rpc-service: conn: 0x1f937380, readsize: 4, dataread: 4
[2010-03-16 23:02:48] T [rpcsvc.c:2305:rpcsvc_record_update_state] rpc-service: Vectored frag
[2010-03-16 23:02:48] T [rpcsvc.c:1177:rpcsvc_record_read_partial_frag] rpc-service: Fragment remaining: 0
[2010-03-16 23:02:48] T [rpcsvc.c:2268:rpcsvc_handle_vectored_frag] rpc-service: Vectored frag complete
[2010-03-16 23:02:48] D [rpcsvc.c:1267:rpcsvc_program_actor] rpc-service: Actor found: NFS3 - WRITE
[2010-03-16 23:02:48] T [rpcsvc.c:2112:rpcsvc_handle_vectored_rpc_call] rpc-service: Vectored RPC proc remaining: 24
[2010-03-16 23:02:48] T [rpcsvc.c:2371:rpcsvc_conn_data_poll_in] rpc-service: conn: 0x1f937380, readsize: 24, dataread: 24
[2010-03-16 23:02:48] T [rpcsvc.c:2305:rpcsvc_record_update_state] rpc-service: Vectored frag
[2010-03-16 23:02:48] T [rpcsvc.c:1177:rpcsvc_record_read_partial_frag] rpc-service: Fragment remaining: 0
[2010-03-16 23:02:48] T [rpcsvc.c:2268:rpcsvc_handle_vectored_frag] rpc-service: Vectored frag complete
[2010-03-16 23:02:48] D [rpcsvc.c:1267:rpcsvc_program_actor] rpc-service: Actor found: NFS3 - WRITE
[2010-03-16 23:02:48] T [rpcsvc.c:2112:rpcsvc_handle_vectored_rpc_call] rpc-service: Vectored RPC proc remaining: 20
[2010-03-16 23:02:48] T [rpcsvc.c:2371:rpcsvc_conn_data_poll_in] rpc-service: conn: 0x1f937380, readsize: 20, dataread: 20
/lib64/libc.so.6[0x333da301b0]
[2010-03-16 23:02:48] T [rpcsvc.c:2305:rpcsvc_record_update_state] rpc-service: Vectored frag
[2010-03-16 23:02:48] T [rpcsvc.c:1177:rpcsvc_record_read_partial_frag] rpc-service: Fragment remaining: 0
/opt/glusterfs/gnfs/lib/glusterfs/3.0.0git/xlator/nfs/server.so(nfs3_call_state_wipe+0x22)[0x2aaaab052432]
[2010-03-16 23:02:48] T [rpcsvc.c:2268:rpcsvc_handle_vectored_frag] rpc-service: Vectored frag complete
/opt/glusterfs/gnfs/lib/glusterfs/3.0.0git/xlator/nfs/server.so(nfs3svc_write_cbk+0xcd)[0x2aaaab054afd]
[2010-03-16 23:02:48] D [rpcsvc.c:1267:rpcsvc_program_actor] rpc-service: Actor found: NFS3 - WRITE
/opt/glusterfs/gnfs/lib/glusterfs/3.0.0git/xlator/nfs/server.so(nfs_fop_writev_cbk+0x9b)[0x2aaaab0455db]
[2010-03-16 23:02:48] T [rpcsvc.c:2108:rpcsvc_handle_vectored_rpc_call] rpc-service: Vectored RPC buf remaining: 65536
/opt/glusterfs/gnfs/lib/glusterfs/3.0.0git/xlator/cluster/replicate.so(afr_writev_unwind+0xd2)[0x2aaaaae11c22]
/opt/glusterfs/gnfs/lib/glusterfs/3.0.0git/xlator/cluster/replicate.so(afr_writev_wind_cbk+0xba)[0x2aaaaae1359a]
/opt/glusterfs/gnfs/lib/glusterfs/3.0.0git/xlator/protocol/client.so(client_write_cbk+0x289)[0x2aaaaabedb69]
/opt/glusterfs/gnfs/lib/glusterfs/3.0.0git/xlator/protocol/client.so(protocol_client_pollin+0xca)[0x2aaaaabda0ba]
/opt/glusterfs/gnfs/lib/glusterfs/3.0.0git/xlator/protocol/client.so(notify+0x212)[0x2aaaaabe14d2]
/opt/glusterfs/gnfs/lib/libglusterfs.so.0(xlator_notify+0x43)[0x2afcbc046473]
/opt/glusterfs/gnfs/lib/glusterfs/3.0.0git/transport/socket.so(socket_event_handler+0xd3)[0x2aaaab6e6073]
/opt/glusterfs/gnfs/lib/libglusterfs.so.0[0x2afcbc0600e5]
/opt/glusterfs/gnfs/sbin/glusterfs(main+0x9d8)[0x404158]
/lib64/libc.so.6(__libc_start_main+0xf4)[0x333da1d8b4]
/opt/glusterfs/gnfs/sbin/glusterfs[0x402709]


bt:

(gdb) bt
#0  nfs3_call_state_wipe (cs=0x2aaaab5af050) at nfs3.c:195
#1  0x00002aaaab054afd in nfs3svc_write_cbk (frame=<value optimized out>, cookie=<value optimized out>, this=<value optimized out>, 
    op_ret=<value optimized out>, op_errno=0, prebuf=0x1f93bd80, postbuf=0x1f93be10) at nfs3.c:1634
#2  0x00002aaaab0455db in nfs_fop_writev_cbk (frame=0x1f939348, cookie=0x1f885720, this=0x1f885720, op_ret=65536, op_errno=0, prebuf=0x1f93bd80, 
    postbuf=0x1f93be10) at nfs-fops.c:1105
#3  0x00002aaaaae11c22 in afr_writev_unwind (frame=<value optimized out>, this=<value optimized out>) at afr-inode-write.c:75
#4  0x00002aaaaae1359a in afr_writev_wind_cbk (frame=0x1f93b808, cookie=<value optimized out>, this=0x1f885720, op_ret=65536, op_errno=0, 
    prebuf=0x7fff5808f580, postbuf=0x7fff5808f4f0) at afr-inode-write.c:138
#5  0x00002aaaaabedb69 in client_write_cbk (frame=0x2aaab0016610, hdr=<value optimized out>, hdrlen=<value optimized out>, iobuf=<value optimized out>)
    at client-protocol.c:4455
#6  0x00002aaaaabda0ba in protocol_client_pollin (this=0x1f885350, trans=0x1f8a7750) at client-protocol.c:6827
#7  0x00002aaaaabe14d2 in notify (this=0x1f936df0, event=2, data=0x1f8a7750) at client-protocol.c:6946
#8  0x00002afcbc046473 in ?? () from /opt/glusterfs/gnfs/lib/libglusterfs.so.0
#9  0x000000001f8a7750 in ?? ()
#10 0x0000000000000000 in ?? ()

*** This bug has been marked as a duplicate of bug 757 ***