Hide Forgot
Description of problem: DHT : rename of file fails with error 'mv: cannot move `<source>' to a subdirectory of itself, `<destination>' - for source file hashed sub-vol is down and cached sub-vol is up ; for destination file hashed and cached sub-vol is same and up Version-Release number of selected component (if applicable): 3.4.0.30rhs-2.el6_4.x86_64 How reproducible: not always Steps to Reproduce: 1) had a Dist volume mounted as FUSE , created few files; perform rename operation. [root@DHT1 bricks]# gluster v info test1 Volume Name: test1 Type: Distribute Volume ID: 20aa042f-302a-4f25-9382-79eaff30d0a5 Status: Started Number of Bricks: 3 Transport-type: tcp Bricks: Brick1: 10.70.37.195:/rhs/brick1/t1 Brick2: 10.70.37.195:/rhs/brick1/t2 Brick3: 10.70.37.66:/rhs/brick1/t1 [root@rhs-client22 test1]# mount | grep test1 10.70.37.66:/test1 on /mnt/test1 type fuse.glusterfs (rw,default_permissions,allow_other,max_read=131072) [root@rhs-client22 ~]# cd /mnt/test1; touch f{1..10} [root@rhs-client22 test1]# for i in {1..20}; do mv f$i fnew$i; done 2) bring one brick down by killing process There are no active volume tasks [root@DHT1 bricks]# kill -9 16136 [root@DHT1 bricks]# gluster v status test1 Status of volume: test1 Gluster process Port Online Pid ------------------------------------------------------------------------------ Brick 10.70.37.195:/rhs/brick1/t1 49159 Y 16081 Brick 10.70.37.195:/rhs/brick1/t2 N/A N 16136 Brick 10.70.37.66:/rhs/brick1/t1 49156 Y 20337 NFS Server on localhost 2049 Y 16148 NFS Server on 10.70.37.66 2049 Y 20491 There are no active volume tasks 3) try to delete file - cached sub-vol is up and hashed is down. It gives error. [root@rhs-client22 test1]# ls fnew1 fnew10 fnew2 fnew3 fnew5 fnew6 fnew7 fnew9 [root@rhs-client22 test1]# rm -f fnew10 rm: cannot remove `fnew10': Invalid argument down brick:- [root@DHT1 bricks]# ls -l /rhs/brick1/t2 total 0 ---------T 2 root root 0 Sep 4 09:59 fnew10 -rw-r--r-- 2 root root 0 Sep 4 2013 fnew4 ---------T 2 root root 0 Sep 4 09:59 fnew6 -rw-r--r-- 2 root root 0 Sep 4 2013 fnew8 ---------T 2 root root 0 Sep 4 09:59 fnew9 up brick:- [root@DHT1 bricks]# ls -l /rhs/brick1/t1 total 0 -rw-r--r-- 2 root root 0 Sep 4 2013 fnew1 -rw-r--r-- 2 root root 0 Sep 4 2013 fnew10 -rw-r--r-- 2 root root 0 Sep 4 2013 fnew2 ---------T 2 root root 0 Sep 4 09:59 fnew3 -rw-r--r-- 2 root root 0 Sep 4 2013 fnew5 ---------T 2 root root 0 Sep 4 09:59 fnew7 4) created one more file - abc [root@rhs-client22 test1]# ls fnew1 fnew10 fnew2 fnew3 fnew5 fnew6 fnew7 fnew9 [root@rhs-client22 test1]# touch abc 5) rename file(cached sub-vol is up and hashed is down) to this new file [root@rhs-client22 test1]# mv fnew10 abc mv: overwrite `abc'? y mv: cannot move `fnew10' to a subdirectory of itself, `abc' Actual results: rename of file fails with error 'mv: cannot move `<source>' to a subdirectory of itself, `<destination>' - for source file hashed sub-vol is down and cached sub-vol is up ; for destination file hashed and cached sub-vol is same and up Expected results: for source file if cached sub-vol is up and for destination file if hashed and cached ; both sub-vol is up then rename should not fail Additional info: log snippet tail -f /var/log/glusterfs/mnt-test1.log [2013-09-04 08:02:31.628736] I [rpc-clnt.c:1687:rpc_clnt_reconfig] 0-test1-client-1: changing port to 49160 (from 0) [2013-09-04 08:02:31.632486] E [socket.c:2158:socket_connect_finish] 0-test1-client-1: connection to 10.70.37.195:49160 failed (Connection refused) [2013-09-04 08:02:35.632651] I [rpc-clnt.c:1687:rpc_clnt_reconfig] 0-test1-client-1: changing port to 49160 (from 0) [2013-09-04 08:02:35.636348] E [socket.c:2158:socket_connect_finish] 0-test1-client-1: connection to 10.70.37.195:49160 failed (Connection refused) [2013-09-04 08:02:39.532686] W [client-rpc-fops.c:2604:client3_3_lookup_cbk] 0-test1-client-1: remote operation failed: Transport endpoint is not connected. Path: / (00000000-0000-0000-0000-000000000001) [2013-09-04 08:02:39.534607] W [client-rpc-fops.c:2604:client3_3_lookup_cbk] 0-test1-client-1: remote operation failed: Transport endpoint is not connected. Path: /fnew10 (94b9ac49-b31f-4d47-b51c-2fc721d8c15d) [2013-09-04 08:02:39.638544] I [rpc-clnt.c:1687:rpc_clnt_reconfig] 0-test1-client-1: changing port to 49160 (from 0) [2013-09-04 08:02:39.642238] E [socket.c:2158:socket_connect_finish] 0-test1-client-1: connection to 10.70.37.195:49160 failed (Connection refused) [2013-09-04 08:02:42.365734] W [client-rpc-fops.c:2604:client3_3_lookup_cbk] 0-test1-client-1: remote operation failed: Transport endpoint is not connected. Path: / (00000000-0000-0000-0000-000000000001) [2013-09-04 08:02:42.367338] W [dht-layout.c:179:dht_layout_search] 0-test1-dht: no subvolume for hash (value) = 1022996023 [2013-09-04 08:02:42.367390] I [dht-rename.c:874:dht_rename] 0-test1-dht: no subvolume in layout for path=/fnew10 [2013-09-04 08:02:42.367433] W [fuse-bridge.c:2170:fuse_rename_cbk] 0-glusterfs-fuse: 356: /fnew10 -> /abc => -1 (Invalid argument) [2013-09-04 08:02:43.644527] I [rpc-clnt.c:1687:rpc_clnt_reconfig] 0-test1-client-1: changing port to 49160 (from 0) [2013-09-04 08:02:43.650580] E [socket.c:2158:socket_connect_finish] 0-test1-client-1: connection to 10.70.37.195:49160 failed (Connection refused) [2013-09-04 08:02:47.649503] I [rpc-clnt.c:1687:rpc_clnt_reconfig] 0-test1-client-1: changing port to 49160 (from 0) [2013-09-04 08:02:47.653216] E [socket.c:2158:socket_connect_finish] 0-test1-client-1: connection to 10.70.37.195:49160 failed (Connection refused)