posted by Liam Slusser <lslusser>, on gluster-users.

I believe i posted this here before but never got any replies.  I'm in
the middle of upgrading to Gluster 2.0.9 and would like to move away
from having to use fuse to serve up files out of apache so im working
again on getting boosting working correctly.

Everything appears to load and work fine but i always get permission
denied, 403, in my apache logs.  Works fine under fuse.  I'm running
Apache under the user nobody which does have read access to all the
files.

I'm not sure where to start debugging this?  Ideas?

thanks,
liam

I can run things from the command line...

[root@server]# LD_PRELOAD="/home/gluster/apps/glusterfs-2.0.9/lib/glusterfs/glusterfs-booster.so"
file /pub/data/tnsc/test/test.mp3
/pub/data/tnsc/test/test.mp3: MPEG ADTS, layer III, v1, 128 kBits,
44.1 kHz, JntStereo

[root@server]# LD_PRELOAD="/home/gluster/apps/glusterfs-2.0.9/lib/glusterfs/glusterfs-booster.so"
ls -agl /pub/data/tnsc/test/test.mp3
ls: /pub/data/tnsc/test/test.mp3: Invalid argument
-rw-r--r-- 1 user 3692251 Aug 27  2007 /pub/data/tnsc/test/test.mp3

[root@server]# wget -S http://x.x.x.x/tnsc/test/test.mp3
--2010-01-08 13:34:55--  http://x.x.x.x/tnsc/test/test.mp3
Connecting to x.x.x.x:80... connected.
HTTP request sent, awaiting response...
 HTTP/1.1 403 Forbidden
 Date: Fri, 08 Jan 2010 21:34:55 GMT
 Server: Apache/2.2.14 (Unix)
 Content-Length: 228
 Keep-Alive: timeout=5, max=100
 Connection: Keep-Alive
 Content-Type: text/html; charset=iso-8859-1
2010-01-08 13:34:55 ERROR 403: Forbidden.

Apache access log:

x.x.x.x - - [08/Jan/2010:12:50:46 -0800] "GET /tnsc/test/test.mp3
HTTP/1.0" 403 228 "-" "Wget/1.11.4 Red Hat modified"

Apache error log:

[Fri Jan 08 12:50:46 2010] [error] [client x..x.x.x] (22)Invalid
argument: file permissions deny server access:
/pub/data/tnsc/test/test.mp3

Gluster error log:

[2010-01-08 12:50:46] D [libglusterfsclient.c:4866:glusterfs_stat]
libglusterfsclient: path /pub/data/tnsc/test/test.mp3
[2010-01-08 12:50:46] D
[libglusterfsclient.c:1541:_libgf_vmp_search_entry]
libglusterfsclient: VMP Search: path /pub/data/tnsc/test/test.mp3,
type: LongestPrefix
[2010-01-08 12:50:46] D
[libglusterfsclient.c:1628:libgf_vmp_search_entry] libglusterfsclient:
VMP Entry found: path :/pub/data/tnsc/test/test.mp3 vmp: /pub/
[2010-01-08 12:50:46] D [libglusterfsclient.c:4788:__glusterfs_stat]
libglusterfsclient: path /data/tnsc/test/test.mp3, op: 2
[2010-01-08 12:50:46] D
[libglusterfsclient.c:869:libgf_resolve_path_light]
libglusterfsclient: Path: /data/tnsc/test/test.mp3, Resolved Path:
/data/tnsc/test/test.mp3
[2010-01-08 12:50:46] D
[libglusterfsclient-dentry.c:389:libgf_client_path_lookup]
libglusterfsclient: resolved path(/data/tnsc/test/test.mp3) to
1118653312/1118655564
[2010-01-08 12:50:46] D [libglusterfsclient.c:4710:libgf_client_stat]
libglusterfsclient: Cache will be used
[2010-01-08 12:50:46] D [libglusterfsclient.c:3001:glusterfs_open]
libglusterfsclient: path /pub/data/tnsc/test/test.mp3
[2010-01-08 12:50:46] D
[libglusterfsclient.c:1541:_libgf_vmp_search_entry]
libglusterfsclient: VMP Search: path /pub/data/tnsc/test/test.mp3,
type: LongestPrefix
[2010-01-08 12:50:46] D
[libglusterfsclient.c:1628:libgf_vmp_search_entry] libglusterfsclient:
VMP Entry found: path :/pub/data/tnsc/test/test.mp3 vmp: /pub/
[2010-01-08 12:50:46] D
[libglusterfsclient.c:869:libgf_resolve_path_light]
libglusterfsclient: Path: /data/tnsc/test/test.mp3, Resolved Path:
/data/tnsc/test/test.mp3
[2010-01-08 12:50:46] D
[libglusterfsclient-dentry.c:389:libgf_client_path_lookup]
libglusterfsclient: resolved path(/data/tnsc/test/test.mp3) to
1118653312/1118655564
[2010-01-08 12:50:46] D [libglusterfsclient.c:2752:libgf_client_open]
libglusterfsclient: open: path /data/tnsc/test/test.mp3, status: 0,
errno 117

Booster.fstab:

/home/gluster/apps/glusterfs-2.0.9/etc/glusterfs/glusterfs.vol-pub.booster
/pub/ glusterfs
subvolume=cache,logfile=/home/gluster/apps/glusterfs-2.0.9/var/log/glusterfs/booster-pub.log,loglevel=DEBUG,attr_timeout=0

glusterfs.vol-pub.booster:

volume brick1a
 type protocol/client
 option transport-type tcp
 option remote-host x.x.x.30
 option remote-subvolume brick1a
end-volume

volume brick2a
 type protocol/client
 option transport-type tcp
 option remote-host x.x.x.35
 option remote-subvolume brick2a
end-volume

volume replicate
 type cluster/replicate
 subvolumes brick1a brick2a
end-volume

volume iothreads
 type performance/io-threads
 option thread-count 32
 subvolumes replicate
end-volume

volume readahead
 type performance/read-ahead
 option page-count 16       # cache per file  = (page-count x page-size)
 option force-atime-update off
 subvolumes iothreads
end-volume

volume cache
 type performance/io-cache
 option cache-size 512MB
 subvolumes readahead
end-volume

httpd.conf:

Alias /tnsc/ "/pub/data/tnsc/"
<Directory "/pub/data/tnsc/">
   Options FollowSymLinks
   AllowOverride None
   Order allow,deny
   Allow from all
</Directory>

I also noticed this while trying to debug this
issue...Notice the md5sum do not match up below?

On the fuse mounted system:

[root@server test]# ls -al test.mp3
-rw-r--r-- 1 user group 3692251 Aug 27  2007 test.mp3

[root@server test]# md5sum test.mp3
d480d794882c814ae1a2426b79cf8b3e  test.mp3

Using glusterfs-boost.so:

[root@server tmp]#
LD_PRELOAD="/home/gluster/apps/glusterfs-2.0.9/lib/glusterfs/glusterfs-booster.so"
ls -al /pub/data/tnsc/test/test.mp3
ls: /pub/data/tnsc/test/test.mp3: Invalid argument
-rw-r--r-- 1 tcode tcode 3692251 Aug 27  2007 /pub/data/tnsc/test/test.mp3

[root@server tmp]#
LD_PRELOAD="/home/gluster/apps/glusterfs-2.0.9/lib/glusterfs/glusterfs-booster.so"
cp /pub/data/tnsc/test/test.mp3 /tmp/test.mp3

[root@server tmp]# md5sum /tmp/test.mp3
9bff3bb90b6897fc19b6b4658b83f3f8  /tmp/test.mp3

[root@server tmp]# ls -al /tmp/test.mp3
-rw-r--r-- 1 root root 3690496 Jan 11 14:10 /tmp/test.mp3

logs from server:

server1:

[2010-01-11 22:09:39] N [server-protocol.c:7056:mop_setvolume] server:
accepted client from 192.168.12.72:1014
[2010-01-11 22:09:39] N [server-protocol.c:7056:mop_setvolume] server:
accepted client from 192.168.12.72:1015
[2010-01-11 22:09:39] N [server-protocol.c:7056:mop_setvolume] server:
accepted client from 192.168.12.72:1010
[2010-01-11 22:09:39] N [server-protocol.c:7056:mop_setvolume] server:
accepted client from 192.168.12.72:1011
[2010-01-11 22:09:39] N [server-protocol.c:7056:mop_setvolume] server:
accepted client from 192.168.12.72:1006
[2010-01-11 22:09:39] N [server-protocol.c:7056:mop_setvolume] server:
accepted client from 192.168.12.72:1007
[2010-01-11 22:09:39] N [server-protocol.c:7056:mop_setvolume] server:
accepted client from 192.168.12.72:1002
[2010-01-11 22:09:39] N [server-protocol.c:7056:mop_setvolume] server:
accepted client from 192.168.12.72:1005
[2010-01-11 22:09:39] N [server-protocol.c:7056:mop_setvolume] server:
accepted client from 192.168.12.72:1001
[2010-01-11 22:09:39] N [server-protocol.c:7056:mop_setvolume] server:
accepted client from 192.168.12.72:1004
[2010-01-11 22:09:49] E [posix.c:270:posix_lookup] server1: lstat on
/data/tnsc/test/test.mp3/.htaccess failed: Not a directory

server2:

[2010-01-11 22:14:12] N [server-protocol.c:7056:mop_setvolume] server:
accepted client from 192.168.12.72:1017
[2010-01-11 22:14:12] N [server-protocol.c:7056:mop_setvolume] server:
accepted client from 192.168.12.72:1016
[2010-01-11 22:14:12] N [server-protocol.c:7056:mop_setvolume] server:
accepted client from 192.168.12.72:993
[2010-01-11 22:14:12] N [server-protocol.c:7056:mop_setvolume] server:
accepted client from 192.168.12.72:992
[2010-01-11 22:14:12] N [server-protocol.c:7056:mop_setvolume] server:
accepted client from 192.168.12.72:987
[2010-01-11 22:14:12] N [server-protocol.c:7056:mop_setvolume] server:
accepted client from 192.168.12.72:984
[2010-01-11 22:14:12] N [server-protocol.c:7056:mop_setvolume] server:
accepted client from 192.168.12.72:983
[2010-01-11 22:14:12] N [server-protocol.c:7056:mop_setvolume] server:
accepted client from 192.168.12.72:981
[2010-01-11 22:14:12] N [server-protocol.c:7056:mop_setvolume] server:
accepted client from 192.168.12.72:982
[2010-01-11 22:14:12] N [server-protocol.c:7056:mop_setvolume] server:
accepted client from 192.168.12.72:980
[2010-01-11 22:14:14] E [posix.c:270:posix_lookup] server2: lstat on
/data/tnsc/test/test.mp3/.htaccess failed: Not a directory