Mount crashing in an 'ASSERT' that checks the inode size in the function ec-inode-write.c Program terminated with signal 11, Segmentation fault. #0 0x00007f5502715dcb in ec_manager_truncate (fop=0x7f53ff654910, state=<optimized out>) at ec-inode-write.c:1475 1475 GF_ASSERT(ec_get_inode_size(fop, fop->locks[0].lock->loc.inode, This is the corresponding thread: Thread 1 (Thread 0x7f54f907a700 (LWP 31806)): #0 0x00007f5502715dcb in ec_manager_truncate (fop=0x7f53ff654910, state=<optimized out>) at ec-inode-write.c:1475 #1 0x00007f55026f399b in __ec_manager (fop=0x7f53ff654910, error=0) at ec-common.c:2698 #2 0x00007f55026f3b78 in ec_resume (fop=0x7f53ff654910, error=0) at ec-common.c:481 #3 0x00007f55026f3c9f in ec_complete (fop=0x7f53ff654910) at ec-common.c:554 #4 0x00007f5502711d0c in ec_inode_write_cbk (frame=<optimized out>, this=0x7f54fc186380, cookie=0x3, op_ret=0, op_errno=0, prestat=0x7f54f9079920, poststat=0x7f54f9079990, xdata=0x0) at ec-inode-write.c:156 #5 0x00007f550298224c in client3_3_ftruncate_cbk (req=<optimized out>, iov=<optimized out>, count=<optimized out>, myframe=0x7f5488ba7870) at client-rpc-fops.c:1415 #6 0x00007f5510476960 in rpc_clnt_handle_reply (clnt=clnt@entry=0x7f54fc4a1330, pollin=pollin@entry=0x7f549b65dc30) at rpc-clnt.c:778 #7 0x00007f5510476d03 in rpc_clnt_notify (trans=<optimized out>, mydata=0x7f54fc4a1360, event=<optimized out>, data=0x7f549b65dc30) at rpc-clnt.c:971 #8 0x00007f5510472a73 in rpc_transport_notify (this=this@entry=0x7f54fc4a1500, event=event@entry=RPC_TRANSPORT_MSG_RECEIVED, data=data@entry=0x7f549b65dc30) at rpc-transport.c:538 #9 0x00007f5505067566 in socket_event_poll_in (this=this@entry=0x7f54fc4a1500, notify_handled=<optimized out>) at socket.c:2315 #10 0x00007f5505069b0c in socket_event_handler (fd=90, idx=99, gen=472, data=0x7f54fc4a1500, poll_in=1, poll_out=0, poll_err=0) at socket.c:2467 #11 0x00007f551070c7e4 in event_dispatch_epoll_handler (event=0x7f54f9079e80, event_pool=0x5625cf18aa30) at event-epoll.c:583 #12 event_dispatch_epoll_worker (data=0x7f54fc296580) at event-epoll.c:659 #13 0x00007f550f50ddd5 in start_thread (arg=0x7f54f907a700) at pthread_create.c:307 #14 0x00007f550edd5ead in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 We're crashing in this part of the code, specifically: (gdb) l 1470 1471 /* This shouldn't fail because we have the inode locked. */ 1472 /* Inode size doesn't need to be updated under locks, because 1473 * conflicting operations won't be in-flight 1474 */ 1475 GF_ASSERT(ec_get_inode_size(fop, fop->locks[0].lock->loc.inode, 1476 &cbk->iatt[0].ia_size)); 1477 cbk->iatt[1].ia_size = fop->user_size; 1478 /* This shouldn't fail because we have the inode locked. */ 1479 GF_ASSERT(ec_set_inode_size(fop, fop->locks[0].lock->loc.inode, (gdb) p *cbk $7 = {list = {next = 0x7f53ff654950, prev = 0x7f53ff654950}, answer_list = {next = 0x7f53ff654960, prev = 0x7f53ff654960}, fop = 0x7f53ff654910, next = 0x0, idx = 3, op_ret = 0, op_errno = 0, count = 1, mask = 8, xdata = 0x0, dict = 0x0, int32 = 0, uintptr = {0, 0, 0}, size = 0, version = {0, 0}, inode = 0x0, fd = 0x0, statvfs = {f_bsize = 0, f_frsize = 0, f_blocks = 0, f_bfree = 0, f_bavail = 0, f_files = 0, f_ffree = 0, f_favail = 0, f_fsid = 0, f_flag = 0, f_namemax = 0, __f_spare = {0, 0, 0, 0, 0, 0}}, iatt = {{ia_ino = 12285952560967103824, ia_gfid = "\337\b\247-\b\344Fݪ\200x«\276\265P", ia_dev = 2224, ia_type = IA_IFREG, ia_prot = {suid = 0 '\000', sgid = 0 '\000', sticky = 0 '\000', owner = {read = 1 '\001', write = 1 '\001', exec = 1 '\001'}, group = {read = 1 '\001', write = 0 '\000', exec = 1 '\001'}, other = {read = 1 '\001', write = 0 '\000', exec = 1 '\001'}}, ia_nlink = 1, ia_uid = 0, ia_gid = 0, ia_rdev = 0, ia_size = 491520, ia_blksize = 4096, ia_blocks = 3840, ia_atime = 1557032019, ia_atime_nsec = 590833985, ia_mtime = 1557032498, ia_mtime_nsec = 824769499, ia_ctime = 1557032498, ia_ctime_nsec = 824769499}, {ia_ino = 12285952560967103824, ia_gfid = "\337\b\247-\b\344Fݪ\200x«\276\265P", ia_dev = 2224, ia_type = IA_IFREG, ia_prot = {suid = 0 '\000', sgid = 0 '\000', sticky = 0 '\000', owner = {read = 1 '\001', write = 1 '\001', exec = 1 '\001'}, group = {read = 1 '\001', write = 0 '\000', exec = 1 '\001'}, other = {read = 1 '\001', write = 0 '\000', exec = 1 '\001'}}, ia_nlink = 1, ia_uid = 0, ia_gid = 0, ia_rdev = 0, ia_size = 0, ia_blksize = 4096, ia_blocks = 0, ia_atime = 1557032019, ia_atime_nsec = 590833985, ia_mtime = 1557032498, ia_mtime_nsec = 824769499, ia_ctime = 1557032498, ia_ctime_nsec = 824769499}, {ia_ino = 0, ia_gfid = '\000' <repeats 15 times>, ia_dev = 0, ia_type = IA_INVAL, ia_prot = {suid = 0 '\000', sgid = 0 '\000', sticky = 0 '\000', owner = {read = 0 '\000', write = 0 '\000', exec = 0 '\000'}, group = {read = 0 '\000', write = 0 '\000', exec = 0 '\000'}, other = {read = 0 '\000', write = 0 '\000', exec = 0 '\000'}}, ia_nlink = 0, ia_uid = 0, ia_gid = 0, ia_rdev = 0, ia_size = 0, ia_blksize = 0, ia_blocks = 0, ia_atime = 0, ia_atime_nsec = 0, ia_mtime = 0, ia_mtime_nsec = 0, ia_ctime = 0, ia_ctime_nsec = 0}, {ia_ino = 0, ia_gfid = '\000' <repeats 15 times>, ia_dev = 0, ia_type = IA_INVAL, ia_prot = {suid = 0 '\000', sgid = 0 '\000', sticky = 0 '\000', owner = {read = 0 '\000', write = 0 '\000', exec = 0 '\000'}, group = { read = 0 '\000', write = 0 '\000', exec = 0 '\000'}, other = {read = 0 '\000', write = 0 '\000', exec = 0 '\000'}}, ia_nlink = 0, ia_uid = 0, ia_gid = 0, ia_rdev = 0, ia_size = 0, ia_blksize = 0, ia_blocks = 0, ia_atime = 0, ia_atime_nsec = 0, ia_mtime = 0, ia_mtime_nsec = 0, ia_ctime = 0, ia_ctime_nsec = 0}, {ia_ino = 0, ia_gfid = '\000' <repeats 15 times>, ia_dev = 0, ia_type = IA_INVAL, ia_prot = {suid = 0 '\000', sgid = 0 '\000', sticky = 0 '\000', owner = {read = 0 '\000', write = 0 '\000', exec = 0 '\000'}, group = {read = 0 '\000', write = 0 '\000', exec = 0 '\000'}, other = {read = 0 '\000', write = 0 '\000', exec = 0 '\000'}}, ia_nlink = 0, ia_uid = 0, ia_gid = 0, ia_rdev = 0, ia_size = 0, ia_blksize = 0, ia_blocks = 0, ia_atime = 0, ia_atime_nsec = 0, ia_mtime = 0, ia_mtime_nsec = 0, ia_ctime = 0, ia_ctime_nsec = 0}}, flock = {l_type = 0, l_whence = 0, l_start = 0, l_len = 0, l_pid = 0, l_owner = {len = 0, data = '\000' <repeats 1023 times>}}, vector = 0x0, buffers = 0x0, str = 0x0, entries = {{list = {next = 0x7f54429a3188, prev = 0x7f54429a3188}, {next = 0x7f54429a3188, prev = 0x7f54429a3188}}, d_ino = 0, d_off = 0, d_len = 0, d_type = 0, d_stat = {ia_ino = 0, ia_gfid = '\000' <repeats 15 times>, ia_dev = 0, ia_type = IA_INVAL, ia_prot = {suid = 0 '\000', sgid = 0 '\000', sticky = 0 '\000', owner = {read = 0 '\000', write = 0 '\000', exec = 0 '\000'}, group = {read = 0 '\000', write = 0 '\000', exec = 0 '\000'}, other = {read = 0 '\000', write = 0 '\000', exec = 0 '\000'}}, ia_nlink = 0, ia_uid = 0, ia_gid = 0, ia_rdev = 0, ia_size = 0, ia_blksize = 0, ia_blocks = 0, ia_atime = 0, ia_atime_nsec = 0, ia_mtime = 0, ia_mtime_nsec = 0, ia_ctime = 0, ia_ctime_nsec = 0}, dict = 0x0, inode = 0x0, d_name = 0x7f54429a3230 ""}, offset = 0, what = GF_SEEK_DATA} (gdb) p *cbk->fop $8 = {id = 24, refs = 3, state = 4, minimum = 1, expected = 1, winds = 0, jobs = 1, error = 0, parent = 0x7f532c197d80, xl = 0x7f54fc186380, req_frame = 0x7f532c048c60, frame = 0x7f54700662d0, cbk_list = { next = 0x7f54429a2a10, prev = 0x7f54429a2a10}, answer_list = {next = 0x7f54429a2a20, prev = 0x7f54429a2a20}, pending_list = {next = 0x7f533007acc0, prev = 0x7f5477976ac0}, answer = 0x7f54429a2a10, lock_count = 0, locked = 0, locks = {{lock = 0x0, fop = 0x0, owner_list = {next = 0x7f53ff6549a0, prev = 0x7f53ff6549a0}, wait_list = {next = 0x7f53ff6549b0, prev = 0x7f53ff6549b0}, update = {_gf_false, _gf_false}, dirty = {_gf_false, _gf_false}, optimistic_changelog = _gf_false, base = 0x0, size = 0, waiting_flags = 0, fl_start = 0, fl_end = 0}, {lock = 0x0, fop = 0x0, owner_list = {next = 0x7f53ff654a10, prev = 0x7f53ff654a10}, wait_list = { next = 0x7f53ff654a20, prev = 0x7f53ff654a20}, update = {_gf_false, _gf_false}, dirty = {_gf_false, _gf_false}, optimistic_changelog = _gf_false, base = 0x0, size = 0, waiting_flags = 0, fl_start = 0, fl_end = 0}}, first_lock = 0, lock = {spinlock = 0, mutex = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 0, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 39 times>, __align = 0}}, flags = 0, first = 0, mask = 8, healing = 0, remaining = 0, received = 8, good = 8, uid = 0, gid = 0, wind = 0x7f5502710ae0 <ec_wind_ftruncate>, handler = 0x7f5502715c50 <ec_manager_truncate>, resume = 0x0, cbks = {access = 0x7f550270af50 <ec_open_truncate_cbk>, create = 0x7f550270af50 <ec_open_truncate_cbk>, discard = 0x7f550270af50 <ec_open_truncate_cbk>, entrylk = 0x7f550270af50 <ec_open_truncate_cbk>, fentrylk = 0x7f550270af50 <ec_open_truncate_cbk>, fallocate = 0x7f550270af50 <ec_open_truncate_cbk>, flush = 0x7f550270af50 <ec_open_truncate_cbk>, fsync = 0x7f550270af50 <ec_open_truncate_cbk>, fsyncdir = 0x7f550270af50 <ec_open_truncate_cbk>, getxattr = 0x7f550270af50 <ec_open_truncate_cbk>, fgetxattr = 0x7f550270af50 <ec_open_truncate_cbk>, heal = 0x7f550270af50 <ec_open_truncate_cbk>, fheal = 0x7f550270af50 <ec_open_truncate_cbk>, inodelk = 0x7f550270af50 <ec_open_truncate_cbk>, finodelk = 0x7f550270af50 <ec_open_truncate_cbk>, link = 0x7f550270af50 <ec_open_truncate_cbk>, lk = 0x7f550270af50 <ec_open_truncate_cbk>, lookup = 0x7f550270af50 <ec_open_truncate_cbk>, mkdir = 0x7f550270af50 <ec_open_truncate_cbk>, mknod = 0x7f550270af50 <ec_open_truncate_cbk>, open = 0x7f550270af50 <ec_open_truncate_cbk>, opendir = 0x7f550270af50 <ec_open_truncate_cbk>, readdir = 0x7f550270af50 <ec_open_truncate_cbk>, readdirp = 0x7f550270af50 <ec_open_truncate_cbk>, readlink = 0x7f550270af50 <ec_open_truncate_cbk>, readv = 0x7f550270af50 <ec_open_truncate_cbk>, removexattr = 0x7f550270af50 <ec_open_truncate_cbk>, fremovexattr = 0x7f550270af50 <ec_open_truncate_cbk>, rename = 0x7f550270af50 <ec_open_truncate_cbk>, rmdir = 0x7f550270af50 <ec_open_truncate_cbk>, setattr = 0x7f550270af50 <ec_open_truncate_cbk>, fsetattr = 0x7f550270af50 <ec_open_truncate_cbk>, setxattr = 0x7f550270af50 <ec_open_truncate_cbk>, fsetxattr = 0x7f550270af50 <ec_open_truncate_cbk>, stat = 0x7f550270af50 <ec_open_truncate_cbk>, fstat = 0x7f550270af50 <ec_open_truncate_cbk>, statfs = 0x7f550270af50 <ec_open_truncate_cbk>, symlink = 0x7f550270af50 <ec_open_truncate_cbk>, truncate = 0x7f550270af50 <ec_open_truncate_cbk>, ftruncate = 0x7f550270af50 <ec_open_truncate_cbk>, unlink = 0x7f550270af50 <ec_open_truncate_cbk>, writev = 0x7f550270af50 <ec_open_truncate_cbk>, xattrop = 0x7f550270af50 <ec_open_truncate_cbk>, fxattrop = 0x7f550270af50 <ec_open_truncate_cbk>, zerofill = 0x7f550270af50 <ec_open_truncate_cbk>, seek = 0x7f550270af50 <ec_open_truncate_cbk>, ipc = 0x7f550270af50 <ec_open_truncate_cbk>}, data = 0x7f5477976a60, heal = 0x0, healer = {next = 0x7f53ff654b08, prev = 0x7f53ff654b08}, user_size = 0, head = 0, use_fd = 1, xdata = 0x0, dict = 0x0, int32 = 0, uint32 = 0, size = 0, offset = 0, mode = {0, 0}, entrylk_cmd = ENTRYLK_LOCK, entrylk_type = ENTRYLK_RDLCK, xattrop_flags = GF_XATTROP_ADD_ARRAY, dev = 0, inode = 0x0, fd = 0x7f54dfb900a0, iatt = {ia_ino = 0, ia_gfid = '\000' <repeats 15 times>, ia_dev = 0, ia_type = IA_INVAL, ia_prot = { suid = 0 '\000', sgid = 0 '\000', sticky = 0 '\000', owner = {read = 0 '\000', write = 0 '\000', exec = 0 '\000'}, group = {read = 0 '\000', write = 0 '\000', exec = 0 '\000'}, other = {read = 0 '\000', write = 0 '\000', exec = 0 '\000'}}, ia_nlink = 0, ia_uid = 0, ia_gid = 0, ia_rdev = 0, ia_size = 0, ia_blksize = 0, ia_blocks = 0, ia_atime = 0, ia_atime_nsec = 0, ia_mtime = 0, ia_mtime_nsec = 0, ia_ctime = 0, ia_ctime_nsec = 0}, str = {0x0, 0x0}, loc = {{path = 0x0, name = 0x0, inode = 0x0, parent = 0x0, gfid = '\000' <repeats 15 times>, pargfid = '\000' <repeats 15 times>}, {path = 0x0, name = 0x0, inode = 0x0, parent = 0x0, gfid = '\000' <repeats 15 times>, pargfid = '\000' <repeats 15 times>}}, flock = {l_type = 0, l_whence = 0, l_start = 0, l_len = 0, l_pid = 0, l_owner = {len = 0, data = '\000' <repeats 1023 times>}}, vector = 0x0, buffers = 0x0, seek = GF_SEEK_DATA, errstr = 0x0} Checking further the lock: (gdb) p fop->locks[0] $5 = {lock = 0x0, fop = 0x0, owner_list = {next = 0x7f53ff6549a0, prev = 0x7f53ff6549a0}, wait_list = {next = 0x7f53ff6549b0, prev = 0x7f53ff6549b0}, update = {_gf_false, _gf_false}, dirty = {_gf_false, _gf_false}, optimistic_changelog = _gf_false, base = 0x0, size = 0, waiting_flags = 0, fl_start = 0, fl_end = 0} (gdb) p fop->locks[0].lock $6 = (ec_lock_t *) 0x0 (gdb) p fop->locks[0].lock->loc.inode Cannot access memory at address 0x90
REVIEW: https://review.gluster.org/22660 (cluster/ec: Reopen shouldn't happen with O_TRUNC) merged (#1) on master by Pranith Kumar Karampuri
REVIEW: https://review.gluster.org/22674 (tests: Test openfd heal doesn't truncate files) posted (#1) for review on master by Pranith Kumar Karampuri
REVIEW: https://review.gluster.org/22674 (tests: Test openfd heal doesn't truncate files) merged (#8) on master by Pranith Kumar Karampuri