Bug 765533 (GLUSTER-3801)

Summary: Self-heal of files/dirs with absence of gfid's when a look up operation made on the files/dirs sets the xattrs of the source files/dirs to "-1"
Product: [Community] GlusterFS Reporter: Shwetha H Panduranga <shwetha>
Component: replicateAssignee: Pranith Kumar K <pkarampu>
Status: CLOSED WORKSFORME QA Contact:
Severity: medium Docs Contact:
Priority: medium    
Version: 3.2.4CC: gluster-bugs, vijay
Target Milestone: ---   
Target Release: ---   
Hardware: x86_64   
OS: Linux   
Whiteboard:
Fixed In Version: Doc Type: Bug Fix
Doc Text:
Story Points: ---
Clone Of: Environment:
Last Closed: 2011-12-16 09:27:06 UTC Type: ---
Regression: --- Mount Type: ---
Documentation: --- CRM:
Verified Versions: Category: ---
oVirt Team: --- RHEL 7.3 requirements from Atomic Host:
Cloudforms Team: --- Target Upstream Version:

Description Shwetha H Panduranga 2011-11-10 03:44:04 UTC
Test Case:-
----------
[Set-up]
1. Create a brick1 with directories/files in it
2. Create a brick2 with no data
3. Create a replicate volume with replcia 2 , brick1 and brick2
4. start the volume
5. Bring down brick1
6. mount to volume created from client

[Execution]
1. execute "find . |xargs stat" 
2. Bring brick1 up
3. ls -la <directory> { name of directory which we added to brick1 before volume was created }

[Expected Output]
1. self-heal of the directory successful
2. md5sum of the brick1 and brick2 matches.
3. xattrs of files/directories matched. 

The xattrs of the files/directories after completion of self-heal are different.

Brick1:-
-------
[root@SERVER1 ~]# getfattr  -R -d -m . -ehex /opt/export/
getfattr: Removing leading '/' from absolute path names
# file: opt/export/
trusted.gfid=0x00000000000000000000000000000001
trusted.glusterfs.test=0x776f726b696e6700

# file: opt/export//gluster
security.selinux=0x756e636f6e66696e65645f753a6f626a6563745f723a66696c655f743a733000
trusted.afr.replicate-client-0=0x00000000ffffffff00000000
trusted.afr.replicate-client-1=0x000000000000000000000001
trusted.gfid=0x53843e12ca7641698ca9ce18e0cf38a4

# file: opt/export//gluster/glusterfs-core-3.2.4-1.el6.x86_64.rpm
security.selinux=0x756e636f6e66696e65645f753a6f626a6563745f723a66696c655f743a733000
trusted.afr.replicate-client-0=0x00000000ffffffff00000000
trusted.afr.replicate-client-1=0x000000000000000000000000
trusted.gfid=0x001e0a04a8f74973890dc1b53ddc2127

# file: opt/export//gluster/glusterfs-core-3.2.5qa6-1.x86_64.rpm
security.selinux=0x756e636f6e66696e65645f753a6f626a6563745f723a66696c655f743a733000
trusted.afr.replicate-client-0=0x00000000ffffffff00000000
trusted.afr.replicate-client-1=0x000000000000000000000000
trusted.gfid=0x9898b5941f7d40549f13110b7211b855

# file: opt/export//gluster/glusterfs-core-3.2.5qa2-1.x86_64.rpm
security.selinux=0x756e636f6e66696e65645f753a6f626a6563745f723a66696c655f743a733000
trusted.afr.replicate-client-0=0x00000000ffffffff00000000
trusted.afr.replicate-client-1=0x000000000000000000000000
trusted.gfid=0x254b2d5fedce41469b0ccffa4a5ef7de

# file: opt/export//gluster/glusterfs-debuginfo-3.2.5qa4-1.x86_64.rpm
security.selinux=0x756e636f6e66696e65645f753a6f626a6563745f723a66696c655f743a733000
trusted.afr.replicate-client-0=0x00000000ffffffff00000000
trusted.afr.replicate-client-1=0x000000000000000000000000
trusted.gfid=0x727c9b51353847daad59dda9d96f9a39

# file: opt/export//gluster/glusterfs-fuse-3.2.5qa2-1.x86_64.rpm
security.selinux=0x756e636f6e66696e65645f753a6f626a6563745f723a66696c655f743a733000
trusted.afr.replicate-client-0=0x00000000ffffffff00000000
trusted.afr.replicate-client-1=0x000000000000000000000000
trusted.gfid=0x6e732e137cc04cb180fe514f6061f7a8

# file: opt/export//gluster/glusterfs-core-3.2.5qa4-1.x86_64.rpm
security.selinux=0x756e636f6e66696e65645f753a6f626a6563745f723a66696c655f743a733000
trusted.afr.replicate-client-0=0x00000000ffffffff00000000
trusted.afr.replicate-client-1=0x000000000000000000000000
trusted.gfid=0x4bec161ea78242b2836ad2909fc06ec7

# file: opt/export//gluster/glusterfs-debuginfo-3.2.5qa6-1.x86_64.rpm
security.selinux=0x756e636f6e66696e65645f753a6f626a6563745f723a66696c655f743a733000
trusted.afr.replicate-client-0=0x00000000ffffffff00000000
trusted.afr.replicate-client-1=0x000000000000000000000000
trusted.gfid=0xb91b527ba63640c999b978f0b8421a89

Brick2:-
-------
[root@SERVER2 ~]# getfattr  -R -d -m . -ehex /opt/export/
getfattr: Removing leading '/' from absolute path names
# file: opt/export/
trusted.gfid=0x00000000000000000000000000000001
trusted.glusterfs.test=0x776f726b696e6700

# file: opt/export//gluster
security.selinux=0x756e636f6e66696e65645f753a6f626a6563745f723a66696c655f743a733000
trusted.afr.replicate-client-0=0x000000000000000000000000
trusted.afr.replicate-client-1=0x000000000000000000000000
trusted.gfid=0x53843e12ca7641698ca9ce18e0cf38a4

# file: opt/export//gluster/glusterfs-core-3.2.4-1.el6.x86_64.rpm
security.selinux=0x756e636f6e66696e65645f753a6f626a6563745f723a66696c655f743a733000
trusted.afr.replicate-client-0=0x000000000000000000000000
trusted.afr.replicate-client-1=0x000000000000000000000000
trusted.gfid=0x001e0a04a8f74973890dc1b53ddc2127

# file: opt/export//gluster/glusterfs-core-3.2.5qa6-1.x86_64.rpm
security.selinux=0x756e636f6e66696e65645f753a6f626a6563745f723a66696c655f743a733000
trusted.afr.replicate-client-0=0x000000000000000000000000
trusted.afr.replicate-client-1=0x000000000000000000000000
trusted.gfid=0x9898b5941f7d40549f13110b7211b855

# file: opt/export//gluster/glusterfs-core-3.2.5qa2-1.x86_64.rpm
security.selinux=0x756e636f6e66696e65645f753a6f626a6563745f723a66696c655f743a733000
trusted.afr.replicate-client-0=0x000000000000000000000000
trusted.afr.replicate-client-1=0x000000000000000000000000
trusted.gfid=0x254b2d5fedce41469b0ccffa4a5ef7de

# file: opt/export//gluster/glusterfs-debuginfo-3.2.5qa4-1.x86_64.rpm
security.selinux=0x756e636f6e66696e65645f753a6f626a6563745f723a66696c655f743a733000
trusted.afr.replicate-client-0=0x000000000000000000000000
trusted.afr.replicate-client-1=0x000000000000000000000000
trusted.gfid=0x727c9b51353847daad59dda9d96f9a39

# file: opt/export//gluster/glusterfs-fuse-3.2.5qa2-1.x86_64.rpm
security.selinux=0x756e636f6e66696e65645f753a6f626a6563745f723a66696c655f743a733000
trusted.afr.replicate-client-0=0x000000000000000000000000
trusted.afr.replicate-client-1=0x000000000000000000000000
trusted.gfid=0x6e732e137cc04cb180fe514f6061f7a8

# file: opt/export//gluster/glusterfs-core-3.2.5qa4-1.x86_64.rpm
security.selinux=0x756e636f6e66696e65645f753a6f626a6563745f723a66696c655f743a733000
trusted.afr.replicate-client-0=0x000000000000000000000000
trusted.afr.replicate-client-1=0x000000000000000000000000
trusted.gfid=0x4bec161ea78242b2836ad2909fc06ec7

# file: opt/export//gluster/glusterfs-debuginfo-3.2.5qa6-1.x86_64.rpm
security.selinux=0x756e636f6e66696e65645f753a6f626a6563745f723a66696c655f743a733000
trusted.afr.replicate-client-0=0x000000000000000000000000
trusted.afr.replicate-client-1=0x000000000000000000000000
trusted.gfid=0xb91b527ba63640c999b978f0b8421a89

Comment 1 Pranith Kumar K 2011-11-14 07:26:19 UTC
Shwetha,
    I scripted the steps, I dont see the xattr problem. Are you sure none of the steps are missed?.
Here is the script:
#!/bin/bash -x
rm -rf /home/repl1 /home/repl2
mkdir -p /home/repl1/a
touch /home/repl1/a/{1..10}
mkdir -p /home/repl2
gluster volume create vol replica 2 `hostname`:/home/repl1 `hostname`:/home/repl2
gluster volume start vol
sleep 3
kill -9 `cat /etc/glusterd/vols/vol/run/$HOSTNAME-home-repl1.pid`
ps aux | grep gluster
mount -t glusterfs `hostname`:/vol /mnt/client
cd /mnt/client
find . | xargs stat
gluster volume start vol force
sleep 20
ls -la a
getfattr -d -m . -e hex /home/repl1/a/*
getfattr -d -m . -e hex /home/repl2/a/*

Here is the output of a sample file:
# file: home/repl1/a/4
security.selinux=0x756e636f6e66696e65645f753a6f626a6563745f723a686f6d655f726f6f745f743a733000
trusted.afr.vol-client-0=0x000000000000000000000000
trusted.afr.vol-client-1=0x000000000000000000000000
trusted.gfid=0x85bf556a03f8411fbede20c5891872bb

# file: home/repl2/a/4
security.selinux=0x756e636f6e66696e65645f753a6f626a6563745f723a686f6d655f726f6f745f743a733000
trusted.afr.vol-client-0=0x000000000000000000000000
trusted.afr.vol-client-1=0x000000000000000000000000
trusted.gfid=0x85bf556a03f8411fbede20c5891872bb