Bug 1264334

Strace info:

[root@dht-rhs-24 blk_char_dev]# strace mknod MYFIFO p
execve("/usr/bin/mknod", ["mknod", "MYFIFO", "p"], [/* 30 vars */]) = 0
brk(0)                                  = 0x13e5000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fdb9b1bb000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=35047, ...}) = 0
mmap(NULL, 35047, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fdb9b1b2000
close(3)                                = 0
open("/lib64/libselinux.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240d\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=147096, ...}) = 0
mmap(NULL, 2246808, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fdb9ad76000
mprotect(0x7fdb9ad97000, 2097152, PROT_NONE) = 0
mmap(0x7fdb9af97000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x21000) = 0x7fdb9af97000
mmap(0x7fdb9af99000, 6296, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fdb9af99000
close(3)                                = 0
open("/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\34\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=2107760, ...}) = 0
mmap(NULL, 3932736, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fdb9a9b5000
mprotect(0x7fdb9ab6b000, 2097152, PROT_NONE) = 0
mmap(0x7fdb9ad6b000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1b6000) = 0x7fdb9ad6b000
mmap(0x7fdb9ad71000, 16960, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fdb9ad71000
close(3)                                = 0
open("/lib64/libpcre.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360\25\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=398272, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fdb9b1b1000
mmap(NULL, 2490888, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fdb9a754000
mprotect(0x7fdb9a7b3000, 2097152, PROT_NONE) = 0
mmap(0x7fdb9a9b3000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x5f000) = 0x7fdb9a9b3000
close(3)                                = 0
open("/lib64/liblzma.so.5", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0000/\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=153184, ...}) = 0
mmap(NULL, 2245240, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fdb9a52f000
mprotect(0x7fdb9a553000, 2093056, PROT_NONE) = 0
mmap(0x7fdb9a752000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x23000) = 0x7fdb9a752000
close(3)                                = 0
open("/lib64/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320\16\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=19512, ...}) = 0
mmap(NULL, 2109744, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fdb9a32b000
mprotect(0x7fdb9a32e000, 2093056, PROT_NONE) = 0
mmap(0x7fdb9a52d000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7fdb9a52d000
close(3)                                = 0
open("/lib64/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240l\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=141616, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fdb9b1b0000
mmap(NULL, 2208864, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fdb9a10f000
mprotect(0x7fdb9a125000, 2097152, PROT_NONE) = 0
mmap(0x7fdb9a325000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x16000) = 0x7fdb9a325000
mmap(0x7fdb9a327000, 13408, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fdb9a327000
close(3)                                = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fdb9b1af000
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fdb9b1ad000
arch_prctl(ARCH_SET_FS, 0x7fdb9b1ad800) = 0
mprotect(0x7fdb9ad6b000, 16384, PROT_READ) = 0
mprotect(0x7fdb9a325000, 4096, PROT_READ) = 0
mprotect(0x7fdb9a52d000, 4096, PROT_READ) = 0
mprotect(0x7fdb9a752000, 4096, PROT_READ) = 0
mprotect(0x7fdb9a9b3000, 4096, PROT_READ) = 0
mprotect(0x7fdb9af97000, 4096, PROT_READ) = 0
mprotect(0x60e000, 4096, PROT_READ)     = 0
mprotect(0x7fdb9b1bc000, 4096, PROT_READ) = 0
munmap(0x7fdb9b1b2000, 35047)           = 0
set_tid_address(0x7fdb9b1adad0)         = 29730
set_robust_list(0x7fdb9b1adae0, 24)     = 0
rt_sigaction(SIGRTMIN, {0x7fdb9a115780, [], SA_RESTORER|SA_SIGINFO, 0x7fdb9a11e130}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {0x7fdb9a115810, [], SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x7fdb9a11e130}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
statfs("/sys/fs/selinux", {f_type=0xf97cff8c, f_bsize=4096, f_blocks=0, f_bfree=0, f_bavail=0, f_files=0, f_ffree=0, f_fsid={0, 0}, f_namelen=255, f_frsize=4096}) = 0
statfs("/sys/fs/selinux", {f_type=0xf97cff8c, f_bsize=4096, f_blocks=0, f_bfree=0, f_bavail=0, f_files=0, f_ffree=0, f_fsid={0, 0}, f_namelen=255, f_frsize=4096}) = 0
stat("/sys/fs/selinux", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
brk(0)                                  = 0x13e5000
brk(0x1406000)                          = 0x1406000
open("/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=106065056, ...}) = 0
mmap(NULL, 106065056, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fdb93be8000
close(3)                                = 0
mknod("MYFIFO", S_IFIFO|0666)           = -1 EACCES (Permission denied)
open("/usr/share/locale/locale.alias", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=2502, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fdb9b1ba000
read(3, "# Locale name alias data base.\n#"..., 4096) = 2502
read(3, "", 4096)                       = 0
close(3)                                = 0
munmap(0x7fdb9b1ba000, 4096)            = 0
open("/usr/share/locale/en_US.UTF-8/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en_US.utf8/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en_US/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en.UTF-8/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en.utf8/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/lib64/charset.alias", O_RDONLY|O_NOFOLLOW) = -1 ENOENT (No such file or directory)
write(2, "mknod: ", 7mknod: )                  = 7
write(2, "\342\200\230MYFIFO\342\200\231", 12‘MYFIFO’) = 12
open("/usr/share/locale/en_US.UTF-8/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en_US.utf8/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en_US/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en.UTF-8/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en.utf8/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
write(2, ": Permission denied", 19: Permission denied)     = 19
write(2, "\n", 1
)                       = 1
close(1)                                = 0
close(2)                                = 0
exit_group(1)                           = ?
+++ exited with 1 +++
[root@dht-rhs-24 blk_char_dev]# 

Saving logs and sosreports @http://rhsqe-repo.lab.eng.blr.redhat.com/sosreports/1264334

I tried this on another cluster and there it worked fine.

Looks like this is some SELinux permission issue as I see avc failures in the log files while running this test on beaker.

Here is a snippet of one error message:

Info: Searching AVC errors produced since 1442399189.96 (Wed Sep 16 06:26:29 2015)
Searching logs...
Running '/usr/bin/env LC_ALL=en_US.UTF-8 /sbin/ausearch -m AVC -m USER_AVC -m SELINUX_ERR -ts 09/16/2015 06:26:29 < /dev/null >/mnt/testarea/tmp.rhts-db-submit-result.wVDnmS 2>&1'
----
time->Wed Sep 16 06:26:55 2015
type=SYSCALL msg=audit(1442399215.522:14331): arch=c000003e syscall=133 success=no exit=-13 a0=7f3214edb770 a1=61a4 a2=d2a a3=7f3229092ac0 items=0 ppid=1 pid=17394 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm="glusterfsd" exe="/usr/sbin/glusterfsd" subj=system_u:system_r:glusterd_t:s0 key=(null)
type=AVC msg=audit(1442399215.522:14331): avc:  denied  { create } for  pid=17394 comm="glusterfsd" name="block_200674_nfs" scontext=system_u:system_r:glusterd_t:s0 tcontext=system_u:object_r:unlabeled_t:s0 tclass=blk_file
Fail: AVC messages found.
Checking for errors...
Using stronger AVC checks.
	Define empty RHTS_OPTION_STRONGER_AVC parameter if this causes any problems.
Running 'cat /mnt/testarea/tmp.rhts-db-submit-result.wVDnmS | /sbin/ausearch -m AVC -m SELINUX_ERR'
Fail: AVC messages found.
Running 'cat %s | /sbin/ausearch -m USER_AVC >/mnt/testarea/tmp.rhts-db-submit-result.c3G2a3 2>&1'
Info: No AVC messages found.
/bin/grep 'avc: ' /mnt/testarea/dmesg.log | /bin/grep --invert-match TESTOUT.log
No AVC messages found in dmesg
Running '/usr/sbin/sestatus'
SELinux status:                 enabled
SELinuxfs mount:                /sys/fs/selinux
SELinux root directory:         /etc/selinux
Loaded policy name:             targeted
Current mode:                   enforcing
Mode from config file:          enforcing
Policy MLS status:              enabled
Policy deny_unknown status:     allowed
Max kernel policy version:      28
Running 'rpm -q selinux-policy || true'
selinux-policy-3.13.1-23.el7_1.18.noarch


The job link of the test on beaker:

https://beaker.engineering.redhat.com/jobs/1083777

This is essentially a SELinux policy issue.
I changed the context and it worked. Closing the bug. The test scripts need to be modified to setup the right SE Linux context.