Does this happen only with quota on?

Saurabh, Do you still have the setup? because i could not reproduce it with a simple distribute setup, so I was thinking if it is something specific to the setup.

Krishna,
I am having the machines but the same volume is not available.

Machines are 10.1.11.130/131 with 3.2.5qa3 installed

Volume Name: dist1
Type: Distribute
Status: Started
Number of Bricks: 2
Transport-type: tcp
Bricks:
Brick1: 10.1.11.130:/export/dist1
Brick2: 10.1.11.131:/export/dist2


root     10960     1  0 05:43 ?        00:00:00 /root/325/inst/sbin/glusterfsd --xlator-option dist1-server.listen-port=24031 -s localhost --volfile-id dist1.10.1.11.130.export-dist1 -p /etc/glusterd/vols/dist1/run/10.1.11.130-export-dist1.pid -S /tmp/81d548359c2ce537dab2321dde71a18b.socket --brick-name /export/dist1 --brick-port 24031 -l /root/325/inst/var/log/glusterfs/bricks/export-dist1.log



[root@RHEL6 inst]# ls -l /proc/10960/fd
total 0
lrwx------. 1 root root 64 Oct 31 05:52 0 -> /dev/null
lrwx------. 1 root root 64 Oct 31 05:52 1 -> /dev/null
lrwx------. 1 root root 64 Oct 31 05:52 10 -> /tmp/tmpfiIYpni (deleted)
lrwx------. 1 root root 64 Oct 31 05:52 11 -> socket:[4083842]
lrwx------. 1 root root 64 Oct 31 05:52 12 -> socket:[4083942]
lrwx------. 1 root root 64 Oct 31 05:52 13 -> /export/dist1/iozone.tmp (deleted)
lrwx------. 1 root root 64 Oct 31 05:52 14 -> /export/dist1/iozone.tmp (deleted)
lrwx------. 1 root root 64 Oct 31 05:52 15 -> /export/dist1/iozone.tmp (deleted)
lrwx------. 1 root root 64 Oct 31 05:52 16 -> /export/dist1/iozone.tmp (deleted)
lrwx------. 1 root root 64 Oct 31 05:52 17 -> /export/dist1/iozone.tmp (deleted)
lrwx------. 1 root root 64 Oct 31 05:52 2 -> /dev/null
lrwx------. 1 root root 64 Oct 31 05:52 3 -> anon_inode:[eventpoll]
l-wx------. 1 root root 64 Oct 31 05:52 4 -> /root/325/inst/var/log/glusterfs/bricks/export-dist1.log
lrwx------. 1 root root 64 Oct 31 05:52 5 -> /etc/glusterd/vols/dist1/run/10.1.11.130-export-dist1.pid
lrwx------. 1 root root 64 Oct 31 05:52 6 -> socket:[4083807]
lrwx------. 1 root root 64 Oct 31 05:52 7 -> socket:[4083823]
lrwx------. 1 root root 64 Oct 31 05:52 8 -> socket:[4083824]
lrwx------. 1 root root 64 Oct 31 05:52 9 -> socket:[4083937]



root     11865     1  0 05:43 ?        00:00:00 /root/325/inst//sbin/glusterfsd --xlator-option dist1-server.listen-port=24027 -s localhost --volfile-id dist1.10.1.11.131.export-dist2 -p /etc/glusterd/vols/dist1/run/10.1.11.131-export-dist2.pid -S /tmp/34467ab1ef1b456a6d926b9f1a411002.socket --brick-name /export/dist2 --brick-port 24027 -l /root/325/inst//var/log/glusterfs/bricks/export-dist2.log



[root@RHEL6 glusterfs-3.2.5qa4]# ls -l /proc/11865/fd
total 0
lrwx------. 1 root root 64 Oct 31 05:51 0 -> /dev/null
lrwx------. 1 root root 64 Oct 31 05:51 1 -> /dev/null
lrwx------. 1 root root 64 Oct 31 05:51 10 -> /tmp/tmpfZembV4 (deleted)
lrwx------. 1 root root 64 Oct 31 05:51 11 -> socket:[2749616]
lrwx------. 1 root root 64 Oct 31 05:51 12 -> socket:[2749720]
lrwx------. 1 root root 64 Oct 31 05:51 2 -> /dev/null
lrwx------. 1 root root 64 Oct 31 05:51 3 -> anon_inode:[eventpoll]
l-wx------. 1 root root 64 Oct 31 05:51 4 -> /root/325/inst/var/log/glusterfs/bricks/export-dist2.log
lrwx------. 1 root root 64 Oct 31 05:51 5 -> /etc/glusterd/vols/dist1/run/10.1.11.131-export-dist2.pid
lrwx------. 1 root root 64 Oct 31 05:51 6 -> socket:[2749551]
lrwx------. 1 root root 64 Oct 31 05:51 7 -> socket:[2749567]
lrwx------. 1 root root 64 Oct 31 05:51 8 -> socket:[2749568]
lrwx------. 1 root root 64 Oct 31 05:51 9 -> socket:[2749709]



[root@RHEL6 nfs-test]# mount

10.1.11.130:/dist1 on /mnt/nfs-test type nfs (rw,vers=3,nolock,addr=10.1.11.130)



[root@RHEL6 nfs-test]# pwd
/mnt/nfs-test
[root@RHEL6 nfs-test]# ls
[root@RHEL6 nfs-test]# 


[iobuf.global.iobuf_pool.arena.1.active_iobuf.1]
iobuf.global.iobuf_pool.arena.1.active_iobuf.1.ref=1
iobuf.global.iobuf_pool.arena.1.active_iobuf.1.ptr=0x7f07fd072000

[iobuf.global.iobuf_pool.arena.1.active_iobuf.2]
iobuf.global.iobuf_pool.arena.1.active_iobuf.2.ref=1
iobuf.global.iobuf_pool.arena.1.active_iobuf.2.ptr=0x7f07fd0b2000

Saurabh,
What I just observed is, the deleted iozone.tmp files that you see here:

[root@RHEL6 ~]# ls -l /proc/20891/fd
total 0
lrwx------. 1 root root 64 Oct 21 06:50 0 -> /dev/null
lrwx------. 1 root root 64 Oct 21 06:50 1 -> /dev/null
lrwx------. 1 root root 64 Oct 21 06:50 10 -> /tmp/tmpfPFHaWx (deleted)
lrwx------. 1 root root 64 Oct 21 06:50 11 -> socket:[2548258]
lrwx------. 1 root root 64 Oct 21 06:50 12 -> socket:[2548325]
lrwx------. 1 root root 64 Oct 21 06:50 13 -> /export/dr-1/iozone.tmp (deleted)
lrwx------. 1 root root 64 Oct 21 06:50 14 -> /export/dr-1/iozone.tmp (deleted)
lrwx------. 1 root root 64 Oct 21 06:50 15 -> /export/dr-1/iozone.tmp (deleted)
lrwx------. 1 root root 64 Oct 21 06:50 16 -> /export/dr-1/iozone.tmp (deleted)


gets removed after a while. There is a delay as this deletion happens in the background and done by a janitor thread on the server.

Can you confirm?

Krishna, even I have observed the same behavior.

In that case I will close the bug :-)