Using strace & ps I've gathered further diagnostic info on this process's behaviour:

[root@core-01 ~]# strace -p 13931

Process 13931 attached - interrupt to quit
restart_syscall(<... resuming interrupted call ...>) = 1
accept(13, {sa_family=AF_FILE, NULL}, [2]) = 16
setsockopt(16, SOL_TCP, TCP_NODELAY, [1], 4) = -1 EOPNOTSUPP (Operation not supported)
fcntl(16, F_GETFD)                      = 0
fcntl(16, F_SETFD, FD_CLOEXEC)          = 0
fcntl(16, F_GETFL)                      = 0x2 (flags O_RDWR)
fcntl(16, F_SETFL, O_RDWR|O_NONBLOCK)   = 0
brk(0x2735000)                          = 0x2735000
getsockopt(16, SOL_SOCKET, SO_PEERCRED, "\362@\0\0\0\0\0\0\0\0\0\0", [12]) = 0
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0\34", 4)                = 4
read(16, " \0\200\206\0\0\0\1\0\0\0B\0\0\0\0\0\0\0\0\0\0\0\0", 24) = 24
brk(0x2775000)                          = 0x2775000
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=5, revents=POLLIN}])
read(5, "\0", 1)                        = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 1 ([{fd=16, revents=POLLOUT}])
write(16, "\0\0\0@ \0\200\206\0\0\0\1\0\0\0B\0\0\0\1\0\0\0\0\0\0\0\0\0\0\0\1"..., 64) = 64
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0(", 4)                  = 4
read(16, " \0\200\206\0\0\0\1\0\0\0\1\0\0\0\0\0\0\0\1\0\0\0\0\0\0\0\1\0\0\0\0"..., 36) = 36
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=5, revents=POLLIN}])
read(5, "\0", 1)                        = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 1 ([{fd=16, revents=POLLOUT}])
write(16, "\0\0\0008 \0\200\206\0\0\0\1\0\0\0\1\0\0\0\1\0\0\0\1\0\0\0\0\0\0\0\1"..., 56) = 56
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0\34", 4)                = 4
read(16, " \0\200\206\0\0\0\1\0\0\0n\0\0\0\0\0\0\0\2\0\0\0\0", 24) = 24
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=5, revents=POLLIN}])
read(5, "\0", 1)                        = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 1 ([{fd=16, revents=POLLOUT}])
write(16, "\0\0\0L \0\200\206\0\0\0\1\0\0\0n\0\0\0\1\0\0\0\2\0\0\0\0\0\0\0\16"..., 76) = 76
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0\34", 4)                = 4
read(16, " \0\200\206\0\0\0\1\0\0\0\3\0\0\0\0\0\0\0\3\0\0\0\0", 24) = 24
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x26d4520, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x26d4520, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
futex(0x26d4520, FUTEX_WAKE_PRIVATE, 1) = 0
futex(0x25ee440, FUTEX_WAKE_PRIVATE, 1) = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 1 ([{fd=16, revents=POLLOUT}])
write(16, "\0\0\0@ \0\200\206\0\0\0\1\0\0\0\3\0\0\0\1\0\0\0\3\0\0\0\0\0\0\0\4"..., 64) = 64
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0\34", 4)                = 4
read(16, " \0\200\206\0\0\0\1\0\0\0\4\0\0\0\0\0\0\0\4\0\0\0\0", 24) = 24
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=5, revents=POLLIN}])
read(5, "\0", 1)                        = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 1 ([{fd=16, revents=POLLOUT}])
write(16, "\0\0\0@ \0\200\206\0\0\0\1\0\0\0\4\0\0\0\1\0\0\0\4\0\0\0\0\0\0\0\0"..., 64) = 64
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0\34", 4)                = 4
read(16, " \0\200\206\0\0\0\1\0\0\0\2\0\0\0\0\0\0\0\5\0\0\0\0", 24) = 24
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=5, revents=POLLIN}])
read(5, "\0", 1)                        = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 1 ([{fd=16, revents=POLLOUT}])
write(16, "\0\0\0008 \0\200\206\0\0\0\1\0\0\0\2\0\0\0\1\0\0\0\5\0\0\0\0\0\0\0\0"..., 56) = 56
close(16)                               = 0
brk(0x26f5000)                          = 0x26f5000
close(18)                               = 0
close(17)                               = 0
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}], 10, -1) = 1 ([{fd=13, revents=POLLIN}])
accept(13, {sa_family=AF_FILE, NULL}, [2]) = 16
setsockopt(16, SOL_TCP, TCP_NODELAY, [1], 4) = -1 EOPNOTSUPP (Operation not supported)
fcntl(16, F_GETFD)                      = 0
fcntl(16, F_SETFD, FD_CLOEXEC)          = 0
fcntl(16, F_GETFL)                      = 0x2 (flags O_RDWR)
fcntl(16, F_SETFL, O_RDWR|O_NONBLOCK)   = 0
brk(0x2735000)                          = 0x2735000
getsockopt(16, SOL_SOCKET, SO_PEERCRED, "\3A\0\0\0\0\0\0\0\0\0\0", [12]) = 0
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0\34", 4)                = 4
read(16, " \0\200\206\0\0\0\1\0\0\0B\0\0\0\0\0\0\0\0\0\0\0\0", 24) = 24
brk(0x2775000)                          = 0x2775000
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=5, revents=POLLIN}])
read(5, "\0", 1)                        = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 1 ([{fd=16, revents=POLLOUT}])
write(16, "\0\0\0@ \0\200\206\0\0\0\1\0\0\0B\0\0\0\1\0\0\0\0\0\0\0\0\0\0\0\1"..., 64) = 64
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0(", 4)                  = 4
read(16, " \0\200\206\0\0\0\1\0\0\0\1\0\0\0\0\0\0\0\1\0\0\0\0\0\0\0\1\0\0\0\0"..., 36) = 36
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=5, revents=POLLIN}])
read(5, "\0", 1)                        = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 1 ([{fd=16, revents=POLLOUT}])
write(16, "\0\0\0008 \0\200\206\0\0\0\1\0\0\0\1\0\0\0\1\0\0\0\1\0\0\0\0\0\0\0\1"..., 56) = 56
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0\34", 4)                = 4
read(16, " \0\200\206\0\0\0\1\0\0\0n\0\0\0\0\0\0\0\2\0\0\0\0", 24) = 24
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x26d4520, FUTEX_WAKE_PRIVATE, 1) = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=5, revents=POLLIN}])
read(5, "\0", 1)                        = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 1 ([{fd=16, revents=POLLOUT}])
write(16, "\0\0\0L \0\200\206\0\0\0\1\0\0\0n\0\0\0\1\0\0\0\2\0\0\0\0\0\0\0\16"..., 76) = 76
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0\34", 4)                = 4
read(16, " \0\200\206\0\0\0\1\0\0\0\3\0\0\0\0\0\0\0\3\0\0\0\0", 24) = 24
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x26d4520, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x681d40, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
futex(0x681d40, FUTEX_WAKE_PRIVATE, 1)  = 0
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 2 ([{fd=5, revents=POLLIN}, {fd=16, revents=POLLOUT}])
read(5, "\0", 1)                        = 1
write(16, "\0\0\0@ \0\200\206\0\0\0\1\0\0\0\3\0\0\0\1\0\0\0\3\0\0\0\0\0\0\0\4"..., 64) = 64
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0\34", 4)                = 4
read(16, " \0\200\206\0\0\0\1\0\0\0\4\0\0\0\0\0\0\0\4\0\0\0\0", 24) = 24
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=5, revents=POLLIN}])
read(5, "\0", 1)                        = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 1 ([{fd=16, revents=POLLOUT}])
write(16, "\0\0\0@ \0\200\206\0\0\0\1\0\0\0\4\0\0\0\1\0\0\0\4\0\0\0\0\0\0\0\0"..., 64) = 64
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0\34", 4)                = 4
read(16, " \0\200\206\0\0\0\1\0\0\0\2\0\0\0\0\0\0\0\5\0\0\0\0", 24) = 24
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=5, revents=POLLIN}])
read(5, "\0", 1)                        = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 1 ([{fd=16, revents=POLLOUT}])
write(16, "\0\0\0008 \0\200\206\0\0\0\1\0\0\0\2\0\0\0\1\0\0\0\5\0\0\0\0\0\0\0\0"..., 56) = 56
close(16)                               = 0
brk(0x26f5000)                          = 0x26f5000
close(18)                               = 0
close(17)                               = 0
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}], 10, -1) = 1 ([{fd=13, revents=POLLIN}])
accept(13, {sa_family=AF_FILE, NULL}, [2]) = 16
setsockopt(16, SOL_TCP, TCP_NODELAY, [1], 4) = -1 EOPNOTSUPP (Operation not supported)
fcntl(16, F_GETFD)                      = 0
fcntl(16, F_SETFD, FD_CLOEXEC)          = 0
fcntl(16, F_GETFL)                      = 0x2 (flags O_RDWR)
fcntl(16, F_SETFL, O_RDWR|O_NONBLOCK)   = 0
brk(0x2735000)                          = 0x2735000
getsockopt(16, SOL_SOCKET, SO_PEERCRED, "\7A\0\0\0\0\0\0\0\0\0\0", [12]) = 0
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0\34", 4)                = 4
read(16, " \0\200\206\0\0\0\1\0\0\0B\0\0\0\0\0\0\0\0\0\0\0\0", 24) = 24
brk(0x2775000)                          = 0x2775000
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=5, revents=POLLIN}])
read(5, "\0", 1)                        = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 1 ([{fd=16, revents=POLLOUT}])
write(16, "\0\0\0@ \0\200\206\0\0\0\1\0\0\0B\0\0\0\1\0\0\0\0\0\0\0\0\0\0\0\1"..., 64) = 64
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0(", 4)                  = 4
read(16, " \0\200\206\0\0\0\1\0\0\0\1\0\0\0\0\0\0\0\1\0\0\0\0\0\0\0\1\0\0\0\0"..., 36) = 36
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x26d4520, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x26d4520, FUTEX_WAIT_PRIVATE, 2, NULL) = 0
futex(0x26d4520, FUTEX_WAKE_PRIVATE, 1) = 0
futex(0x25ee440, FUTEX_WAKE_PRIVATE, 1) = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=5, revents=POLLIN}])
read(5, "\0", 1)                        = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 1 ([{fd=16, revents=POLLOUT}])
write(16, "\0\0\0008 \0\200\206\0\0\0\1\0\0\0\1\0\0\0\1\0\0\0\1\0\0\0\0\0\0\0\1"..., 56) = 56
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0\34", 4)                = 4
read(16, " \0\200\206\0\0\0\1\0\0\0n\0\0\0\0\0\0\0\2\0\0\0\0", 24) = 24
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x26d4520, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x681d40, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
futex(0x681d40, FUTEX_WAKE_PRIVATE, 1)  = 0
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 2 ([{fd=5, revents=POLLIN}, {fd=16, revents=POLLOUT}])
read(5, "\0", 1)                        = 1
write(16, "\0\0\0L \0\200\206\0\0\0\1\0\0\0n\0\0\0\1\0\0\0\2\0\0\0\0\0\0\0\16"..., 76) = 76
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0,", 4)                  = 4
read(16, " \0\200\206\0\0\0\1\0\0\0\27\0\0\0\0\0\0\0\3\0\0\0\0\0\0\0\vblue"..., 40) = 40
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x25ee440, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
futex(0x25ee440, FUTEX_WAKE_PRIVATE, 1) = 0
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 1 ([{fd=16, revents=POLLOUT}])
write(16, "\0\0\0\\ \0\200\206\0\0\0\1\0\0\0\27\0\0\0\1\0\0\0\3\0\0\0\0\0\0\0\v"..., 92) = 92
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0@", 4)                  = 4
read(16, " \0\200\206\0\0\0\1\0\0\0\20\0\0\0\0\0\0\0\4\0\0\0\0\0\0\0\vblue"..., 60) = 60
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x26d4520, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x7f868c0dc600, FUTEX_WAKE_PRIVATE, 1) = 0
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=5, revents=POLLIN}])
read(5, "\0", 1)                        = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 1 ([{fd=16, revents=POLLOUT}])
write(16, "\0\0\0X \0\200\206\0\0\0\1\0\0\0\20\0\0\0\1\0\0\0\4\0\0\0\0\0\0\0\1"..., 88) = 88
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0\34", 4)                = 4
read(16, " \0\200\206\0\0\0\1\0\0\0\2\0\0\0\0\0\0\0\5\0\0\0\0", 24) = 24
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x26d4520, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x26d4520, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
futex(0x26d4520, FUTEX_WAKE_PRIVATE, 1) = 0
futex(0x25ee440, FUTEX_WAKE_PRIVATE, 1) = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 1 ([{fd=16, revents=POLLOUT}])
write(16, "\0\0\0008 \0\200\206\0\0\0\1\0\0\0\2\0\0\0\1\0\0\0\5\0\0\0\0\0\0\0\0"..., 56) = 56
close(16)                               = 0
brk(0x26f5000)                          = 0x26f5000
close(18)                               = 0
close(17)                               = 0
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}], 10, -1) = 1 ([{fd=13, revents=POLLIN}])
accept(13, {sa_family=AF_FILE, NULL}, [2]) = 16
setsockopt(16, SOL_TCP, TCP_NODELAY, [1], 4) = -1 EOPNOTSUPP (Operation not supported)
fcntl(16, F_GETFD)                      = 0
fcntl(16, F_SETFD, FD_CLOEXEC)          = 0
fcntl(16, F_GETFL)                      = 0x2 (flags O_RDWR)
fcntl(16, F_SETFL, O_RDWR|O_NONBLOCK)   = 0
brk(0x2735000)                          = 0x2735000
getsockopt(16, SOL_SOCKET, SO_PEERCRED, "\nA\0\0\0\0\0\0\0\0\0\0", [12]) = 0
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0\34", 4)                = 4
read(16, " \0\200\206\0\0\0\1\0\0\0B\0\0\0\0\0\0\0\0\0\0\0\0", 24) = 24
brk(0x2775000)                          = 0x2775000
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=5, revents=POLLIN}])
read(5, "\0", 1)                        = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 1 ([{fd=16, revents=POLLOUT}])
write(16, "\0\0\0@ \0\200\206\0\0\0\1\0\0\0B\0\0\0\1\0\0\0\0\0\0\0\0\0\0\0\1"..., 64) = 64
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0(", 4)                  = 4
read(16, " \0\200\206\0\0\0\1\0\0\0\1\0\0\0\0\0\0\0\1\0\0\0\0\0\0\0\1\0\0\0\0"..., 36) = 36
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x26d4520, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x26d4520, FUTEX_WAIT_PRIVATE, 2, NULL) = 0
futex(0x26d4520, FUTEX_WAKE_PRIVATE, 1) = 0
futex(0x25ee440, FUTEX_WAKE_PRIVATE, 1) = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=5, revents=POLLIN}])
read(5, "\0", 1)                        = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 1 ([{fd=16, revents=POLLOUT}])
write(16, "\0\0\0008 \0\200\206\0\0\0\1\0\0\0\1\0\0\0\1\0\0\0\1\0\0\0\0\0\0\0\1"..., 56) = 56
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0\34", 4)                = 4
read(16, " \0\200\206\0\0\0\1\0\0\0n\0\0\0\0\0\0\0\2\0\0\0\0", 24) = 24
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x26d4520, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x26d4520, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
futex(0x26d4520, FUTEX_WAKE_PRIVATE, 1) = 0
futex(0x25ee440, FUTEX_WAKE_PRIVATE, 1) = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 1 ([{fd=16, revents=POLLOUT}])
write(16, "\0\0\0L \0\200\206\0\0\0\1\0\0\0n\0\0\0\1\0\0\0\2\0\0\0\0\0\0\0\16"..., 76) = 76
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0(", 4)                  = 4
read(16, " \0\200\206\0\0\0\1\0\0\0\27\0\0\0\0\0\0\0\3\0\0\0\0\0\0\0\10rhel"..., 36) = 36
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x26d4520, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x7f868c0dc600, FUTEX_WAKE_PRIVATE, 1) = 0
futex(0x681d40, FUTEX_WAKE_PRIVATE, 1)  = 1
futex(0x26d4520, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
futex(0x26d4520, FUTEX_WAKE_PRIVATE, 1) = 0
futex(0x25ee440, FUTEX_WAKE_PRIVATE, 1) = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 1 ([{fd=16, revents=POLLOUT}])
write(16, "\0\0\0X \0\200\206\0\0\0\1\0\0\0\27\0\0\0\1\0\0\0\3\0\0\0\0\0\0\0\10"..., 88) = 88
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0<", 4)                  = 4
read(16, " \0\200\206\0\0\0\1\0\0\0\20\0\0\0\0\0\0\0\4\0\0\0\0\0\0\0\10rhel"..., 56) = 56
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x26d4520, FUTEX_WAKE_PRIVATE, 1) = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=5, revents=POLLIN}])
read(5, "\0", 1)                        = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 1 ([{fd=16, revents=POLLOUT}])
write(16, "\0\0\0X \0\200\206\0\0\0\1\0\0\0\20\0\0\0\1\0\0\0\4\0\0\0\0\0\0\0\1"..., 88) = 88
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0\34", 4)                = 4
read(16, " \0\200\206\0\0\0\1\0\0\0\2\0\0\0\0\0\0\0\5\0\0\0\0", 24) = 24
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x26d4520, FUTEX_WAKE_PRIVATE, 1) = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 2 ([{fd=5, revents=POLLIN}, {fd=16, revents=POLLOUT}])
read(5, "\0", 1)                        = 1
write(16, "\0\0\0008 \0\200\206\0\0\0\1\0\0\0\2\0\0\0\1\0\0\0\5\0\0\0\0\0\0\0\0"..., 56) = 56
close(16)                               = 0
brk(0x26f5000)                          = 0x26f5000
close(18)                               = 0
close(17)                               = 0
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}], 10, -1) = 1 ([{fd=13, revents=POLLIN}])
accept(13, {sa_family=AF_FILE, NULL}, [2]) = 16
setsockopt(16, SOL_TCP, TCP_NODELAY, [1], 4) = -1 EOPNOTSUPP (Operation not supported)
fcntl(16, F_GETFD)                      = 0
fcntl(16, F_SETFD, FD_CLOEXEC)          = 0
fcntl(16, F_GETFL)                      = 0x2 (flags O_RDWR)
fcntl(16, F_SETFL, O_RDWR|O_NONBLOCK)   = 0
brk(0x2735000)                          = 0x2735000
getsockopt(16, SOL_SOCKET, SO_PEERCRED, "\26A\0\0\0\0\0\0\0\0\0\0", [12]) = 0
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0\34", 4)                = 4
read(16, " \0\200\206\0\0\0\1\0\0\0B\0\0\0\0\0\0\0\0\0\0\0\0", 24) = 24
brk(0x2775000)                          = 0x2775000
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x26d4520, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x681d40, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
futex(0x681d40, FUTEX_WAKE_PRIVATE, 1)  = 0
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 2 ([{fd=5, revents=POLLIN}, {fd=16, revents=POLLOUT}])
read(5, "\0", 1)                        = 1
write(16, "\0\0\0@ \0\200\206\0\0\0\1\0\0\0B\0\0\0\1\0\0\0\0\0\0\0\0\0\0\0\1"..., 64) = 64
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0(", 4)                  = 4
read(16, " \0\200\206\0\0\0\1\0\0\0\1\0\0\0\0\0\0\0\1\0\0\0\0\0\0\0\1\0\0\0\0"..., 36) = 36
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x26d4520, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x26d4520, FUTEX_WAIT_PRIVATE, 2, NULL) = 0
futex(0x26d4520, FUTEX_WAKE_PRIVATE, 1) = 0
futex(0x25ee440, FUTEX_WAKE_PRIVATE, 1) = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 1 ([{fd=16, revents=POLLOUT}])
write(16, "\0\0\0008 \0\200\206\0\0\0\1\0\0\0\1\0\0\0\1\0\0\0\1\0\0\0\0\0\0\0\1"..., 56) = 56
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0\34", 4)                = 4
read(16, " \0\200\206\0\0\0\1\0\0\0n\0\0\0\0\0\0\0\2\0\0\0\0", 24) = 24
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x26d4520, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x681d40, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
futex(0x681d40, FUTEX_WAKE_PRIVATE, 1)  = 0
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 2 ([{fd=5, revents=POLLIN}, {fd=16, revents=POLLOUT}])
read(5, "\0", 1)                        = 1
write(16, "\0\0\0L \0\200\206\0\0\0\1\0\0\0n\0\0\0\1\0\0\0\2\0\0\0\0\0\0\0\16"..., 76) = 76
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0\34", 4)                = 4
read(16, " \0\200\206\0\0\0\1\0\0\0\3\0\0\0\0\0\0\0\3\0\0\0\0", 24) = 24
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x26d4520, FUTEX_WAKE_PRIVATE, 1) = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 2 ([{fd=5, revents=POLLIN}, {fd=16, revents=POLLOUT}])
read(5, "\0", 1)                        = 1
write(16, "\0\0\0@ \0\200\206\0\0\0\1\0\0\0\3\0\0\0\1\0\0\0\3\0\0\0\0\0\0\0\4"..., 64) = 64
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0\34", 4)                = 4
read(16, " \0\200\206\0\0\0\1\0\0\0\4\0\0\0\0\0\0\0\4\0\0\0\0", 24) = 24
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x26d4520, FUTEX_WAKE_PRIVATE, 1) = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 2 ([{fd=5, revents=POLLIN}, {fd=16, revents=POLLOUT}])
read(5, "\0", 1)                        = 1
write(16, "\0\0\0@ \0\200\206\0\0\0\1\0\0\0\4\0\0\0\1\0\0\0\4\0\0\0\0\0\0\0\0"..., 64) = 64
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0\34", 4)                = 4
read(16, " \0\200\206\0\0\0\1\0\0\0\2\0\0\0\0\0\0\0\5\0\0\0\0", 24) = 24
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x26d4520, FUTEX_WAKE_PRIVATE, 1) = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 2 ([{fd=5, revents=POLLIN}, {fd=16, revents=POLLOUT}])
read(5, "\0", 1)                        = 1
write(16, "\0\0\0008 \0\200\206\0\0\0\1\0\0\0\2\0\0\0\1\0\0\0\5\0\0\0\0\0\0\0\0"..., 56) = 56
close(16)                               = 0
brk(0x26f5000)                          = 0x26f5000
close(18)                               = 0
close(17)                               = 0
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}], 10, -1) = 1 ([{fd=13, revents=POLLIN}])
accept(13, {sa_family=AF_FILE, NULL}, [2]) = 16
setsockopt(16, SOL_TCP, TCP_NODELAY, [1], 4) = -1 EOPNOTSUPP (Operation not supported)
fcntl(16, F_GETFD)                      = 0
fcntl(16, F_SETFD, FD_CLOEXEC)          = 0
fcntl(16, F_GETFL)                      = 0x2 (flags O_RDWR)
fcntl(16, F_SETFL, O_RDWR|O_NONBLOCK)   = 0
brk(0x2735000)                          = 0x2735000
getsockopt(16, SOL_SOCKET, SO_PEERCRED, "\34A\0\0\0\0\0\0\0\0\0\0", [12]) = 0
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0\34", 4)                = 4
read(16, " \0\200\206\0\0\0\1\0\0\0B\0\0\0\0\0\0\0\0\0\0\0\0", 24) = 24
brk(0x2775000)                          = 0x2775000
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x26d4520, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x681d40, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
futex(0x681d40, FUTEX_WAKE_PRIVATE, 1)  = 0
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 2 ([{fd=5, revents=POLLIN}, {fd=16, revents=POLLOUT}])
read(5, "\0", 1)                        = 1
write(16, "\0\0\0@ \0\200\206\0\0\0\1\0\0\0B\0\0\0\1\0\0\0\0\0\0\0\0\0\0\0\1"..., 64) = 64
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0(", 4)                  = 4
read(16, " \0\200\206\0\0\0\1\0\0\0\1\0\0\0\0\0\0\0\1\0\0\0\0\0\0\0\1\0\0\0\0"..., 36) = 36
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x26d4520, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x26d4520, FUTEX_WAIT_PRIVATE, 2, NULL) = 0
futex(0x26d4520, FUTEX_WAKE_PRIVATE, 1) = 0
futex(0x25ee440, FUTEX_WAKE_PRIVATE, 1) = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=5, revents=POLLIN}])
read(5, "\0", 1)                        = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 1 ([{fd=16, revents=POLLOUT}])
write(16, "\0\0\0008 \0\200\206\0\0\0\1\0\0\0\1\0\0\0\1\0\0\0\1\0\0\0\0\0\0\0\1"..., 56) = 56
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0\34", 4)                = 4
read(16, " \0\200\206\0\0\0\1\0\0\0n\0\0\0\0\0\0\0\2\0\0\0\0", 24) = 24
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=5, revents=POLLIN}])
read(5, "\0", 1)                        = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 1 ([{fd=16, revents=POLLOUT}])
write(16, "\0\0\0L \0\200\206\0\0\0\1\0\0\0n\0\0\0\1\0\0\0\2\0\0\0\0\0\0\0\16"..., 76) = 76
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0(", 4)                  = 4
read(16, " \0\200\206\0\0\0\1\0\0\0\27\0\0\0\0\0\0\0\3\0\0\0\0\0\0\0\10winx"..., 36) = 36
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x26d4520, FUTEX_WAKE_PRIVATE, 1) = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=5, revents=POLLIN}])
read(5, "\0", 1)                        = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 1 ([{fd=16, revents=POLLOUT}])
write(16, "\0\0\0X \0\200\206\0\0\0\1\0\0\0\27\0\0\0\1\0\0\0\3\0\0\0\0\0\0\0\10"..., 88) = 88
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0<", 4)                  = 4
read(16, " \0\200\206\0\0\0\1\0\0\0\20\0\0\0\0\0\0\0\4\0\0\0\0\0\0\0\10winx"..., 56) = 56
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x26d4520, FUTEX_WAKE_PRIVATE, 1) = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=5, revents=POLLIN}])
read(5, "\0", 1)                        = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 1 ([{fd=16, revents=POLLOUT}])
write(16, "\0\0\0X \0\200\206\0\0\0\1\0\0\0\20\0\0\0\1\0\0\0\4\0\0\0\0\0\0\0\1"..., 88) = 88
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=16, revents=POLLIN}])
read(16, "\0\0\0\34", 4)                = 4
read(16, " \0\200\206\0\0\0\1\0\0\0\2\0\0\0\0\0\0\0\5\0\0\0\0", 24) = 24
futex(0x25ee46c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x25ee468, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN}], 11, -1) = 1 ([{fd=5, revents=POLLIN}])
read(5, "\0", 1)                        = 1
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}, {fd=16, events=POLLIN|POLLOUT}], 11, -1) = 1 ([{fd=16, revents=POLLOUT}])
write(16, "\0\0\0008 \0\200\206\0\0\0\1\0\0\0\2\0\0\0\1\0\0\0\5\0\0\0\0\0\0\0\0"..., 56) = 56
close(16)                               = 0
brk(0x26f5000)                          = 0x26f5000
close(18)                               = 0
close(17)                               = 0
poll([{fd=5, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLERR|POLLHUP}, {fd=10, events=POLLERR|POLLHUP}, {fd=11, events=POLLERR|POLLHUP}, {fd=12, events=POLLIN}, {fd=3, events=POLLIN}, {fd=13, events=POLLIN|POLLERR|POLLHUP}, {fd=14, events=POLLIN|POLLERR|POLLHUP}, {fd=15, events=POLLIN}], 10, -1^C <unfinished ...>

At the end of every one of this poll interval, the rss of this process is consistently growing:

[root@core-01 ~]# while true; do ps -eo rss,pid,cmd|egrep libvirtd|egrep -v egrep; sleep 30; done
11236 13931 libvirtd --daemon
11240 13931 libvirtd --daemon
11240 13931 libvirtd --daemon
11244 13931 libvirtd --daemon
11276 13931 libvirtd --daemon
11276 13931 libvirtd --daemon
11276 13931 libvirtd --daemon
11276 13931 libvirtd --daemon
11280 13931 libvirtd --daemon
11280 13931 libvirtd --daemon
11280 13931 libvirtd --daemon
11280 13931 libvirtd --daemon
11284 13931 libvirtd --daemon

I have also tested this on my laptop; and noticed everytime I open and close virtmanager rss memory doubles

 7048  9758 libvirtd --daemon  -started libvirt
 24648  9758 libvirtd --daemon -opened virt-manager; closed virt-manager
 41920  9758 libvirtd --daemon -opened virt-manager; closed virt-manager
 44272  9758 libvirtd --daemon -opened virt-manager; closed virt-manager
 64916  9758 libvirtd --daemon -opened virt-manager

Colleague tested on Fedora 11 - same problem exists- but on a lower scale rather than 20M per iteration on F12, 1M on F11; how to reproduce... open and close libvirt 100 times and your machine will crash!

What version of libvirt are you running ? i.e. output of 
rpm -q libvirt

Daniel

libvirt-0.7.1-15.fc12.x86_64 shall I try rawhide, as I said this appears on Fedora 11 as well but not as severe (or in test case less virtual machine) - you should be able to reproduce quite easily.

I was able to reproduce the problem from comment #3 on an F12 machine.
Then I updated to a local build of libvirt-0.7.4 , restarted libvirtd
service and ran the same experiment, and the leak seems gone.

http://libvirt.org/git/?p=libvirt.git&a=search&h=e7c78b0a94a2c67ff7947a205100ed544bd8fab3&st=commit&s=leak

there have been a number of memory leak fixes in the last few weeks,
this probably explains that, so yes I would suggest to update to a rebuild
of 0.7.4, and tell us if you still see the issue,

  thanks,

Daniel

I can still reproduce the problem on latest GIT trees, so I don't think 0.7.4  fixes it (at least not entirely - there's always the possibility of multiple bugs )

I've narrowed this down to being a problem relating to the interface drivers, since you turn virRegisterInterfaceDriver() into a no-op all the leaks go away.

I have found & fixed bug in netcf, but it turns out this only impacts libvirtd when run as non-root.

https://bugzilla.redhat.com/show_bug.cgi?id=540472

So my comment #8 is probably incorrect wrt your scenario of running libvirtd as root.

Can you provide the $HOME/.virt-manager/virt-manager.log file, and also the /var/log/messages

There are two independent bugs here one with open and closing virt-manager; the other for polling the virtual machines. (refer to comment 1 and comment 3).

After running libvirt for ~7 hours you can see the effects from comment 1 taking place.

I dont see anything in virt-manager.log or messages to be useful, this is happening on two independent machines; as well as laptop. Eventually OOM is called and start killing off random processes.

  PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND            
13931 root      20   0 4749m 4.3g 3756 S  0.0 55.5   0:54.81 libvirtd           
 3710 qemu      20   0  721m 472m 1284 S  7.1  5.9 121:48.89 qemu-kvm           
 3462 qemu      20   0 2412m 435m 1272 S  5.3  5.4 157:13.18 qemu-kvm           
19904 qemu      20   0  795m 391m 1280 S  4.6  4.9  91:48.67 qemu-kvm

One additional observation from the original description; can you check rss from libvirtd and compare it to pmap total.

Please provide the virt-manager.log file, because I want to see how virt-manager is connecting.

Created attachment 373258 [details]
virt-manager.log

I can confirm seeing this same problem as per comment#1. Four VM's running, three idle, one active (upgrading F11 -> F12). Left overnight (about 8 hours), libvirt was taking up 40% RAM (of 8GB) and the system was all the way out in swap.
Recoverable, when turning off all the VM's and restarting libvirtd service, but means I can't work with VM's like I could on F11.

$ rpm -q libvirt
libvirt-0.7.1-15.fc12.x86_64

Attaching the virt-manager logs as well. (This system was updated to F12, from F11, on the 29th, early afternoon. You can see the gap in the virt-manager logs between 14:09 and 18:13.)

I can do some testing, so if there is a updates-testing or koji build that can be installed on F12, I'm game to run it.

/Anders

Created attachment 374703 [details]
virt-manager logs archive

Made an archive as the problem spans the two logs.

> Recoverable, when turning off all the VM's and restarting libvirtd service, but
> means I can't work with VM's like I could on F11.

FYI, you can safely restart  libvirtd while VMs are running - i'll leave them running untouched and redetect them when it starts again.

Hi Daniel,

I can restart libvirtd and the vm's wont be effected, however when other services depend on libvirtd ie - rgmanager its a different story and causes all sorts of problems.

I'm still trying to figure out what's different about your environment, since I don't see any obvious leaks on my F12 system, apart from the netcf things I mentioned which can't account for many GB of leaking. 

Can you show me how your virt-manager is configured, by attaching output of:

  # gconftool-2 -R /apps/virt-manager


And, then get a trace of what libivrt APIs it is running over the space of about 20 seconds by running


  # LIBVIRT_LOG_FILTERS="1:libvirt" LIBVIRT_LOG_OUTPUTS="1:stderr" virt-manager --no-fork

I, too, am seeing the leaks on Fedora 12. Tried libvirt 0.7.1. and 0.7.4 with the same result. In under 8 hours, libvirt will consume near 5GB of RAM. Only happens while there are guests running on the system (I've got 8) and happens faster if I leave virt-manager running.

Am using rgmanager to start/stop/migrate my guests so I suspect it's rather frequent checks of the guest state is aggravating the issue further.

Just for "fun" I ran the above mentioned virt-manager --no-fork and dumped the contents along with ps auxx output into the attached log file. This system is presently using libvirt 0.7.1.

Created attachment 374740 [details]
Output from virt-manager --no-fork run

Well that virt-manager log identified one difference with my setup - my virt-manager was not querying disk or network device statistics, but even enabling that doesn't trigger any memory leak.

I'm attaching a short python program - I'd appreciate info on whether this python program causes libvirtd to leak - make sure no other apps like virt-manager / rgmanager  are connected to libvirt while running this - i want this demo program being the only app running with libvirt. Make sure at least a couple of guests are running too.

Created attachment 374752 [details]
Python program to try & trigger a leak

Simple run as root

 # python leak.py

Had 3 guests running and ran the leak.py script. Output being attached.

Looped some ps output at the same time in a different terminal and did not see a leak.

Created attachment 374766 [details]
Output from leak.py

# gconftool-2 -R /apps/virt-manager
 system-tray = false
 /apps/virt-manager/urls:
  kickstart = []
  url-list-length = 10
  media = [http://astromirror.uchicago.edu/fedora/linux/releases/9/Fedora/x86_64/os/,http://download.fedora.redhat.com/pub/fedora/linux/releases/8/Fedora/x86_64/os/,http://download.fedora.redhat.com/pub/fedora/linux/releases/8/Fedora/i386/os/,http://mirrors.kernel.org/fedora/releases/9/Fedora/x86_64/,http://centos.omnispring.com/5.1/os/x86_64/,http://centos.omnispring.com/5.1/os/x86_64/isolinux/,http://centos.omnispring.com/5.1/os/x86_64,http://centos.omnispring.com/,http://centos.take2hosting.com/5.1/os/x86_64/images,http://centos.take2hosting.com/5.1/os/x86_64/]
 /apps/virt-manager/vmlist-fields:
  cpu_usage = false
  domain_id = true
  memory_usage = false
  virtual_cpus = true
  stats-type = 0
 /apps/virt-manager/paths:
  default-screenshot-path = 
  default-save-path = 
  default-image-path = 
  default-restore-path = 
  default-media-path = 
 /apps/virt-manager/stats:
  history-length = 120
  enable-net-poll = true
  update-interval = 1
  enable-disk-poll = true
  enable-mem-poll = true
  enable-cpu-poll = true
 /apps/virt-manager/console:
  scaling = 1
  keygrab = 2
  grab-notify = false
  popup = 1
 /apps/virt-manager/new-vm:
  local-sound = true
  remote-sound = false
 /apps/virt-manager/connections:
  uris = [qemu+tls://virt0-backup/system,qemu:///system]
  autoconnect = []
 /apps/virt-manager/details:
  show-toolbar = false

Opps, I didn't actually need the output from leak.py, I just wanted to know if libvirtd leaked while running it :-)

Anyway, since it didn't leak, at least that bit matches behaviour I see. So it does not appear to be related to the domain XML, Info, block stats, or interface stats functions. That doesn't really leave me any other ideas for what's causing the leak though.

I think we need to try and get a valgrind trace of the libvirtd daemon. So if someone here is able to help....

Start a few VMs when libvirtd is running normally, then without stopping the VMs, as root, run:

 # debuginfo-install libvirt
 # service libvirtd stop
 # valgrind --leak-check=full /usr/sbin/libvirtd


and then bring up virt-manager and connect it to libvirtd, and let it run for a few minutes. Then Ctrl-C the valgrind process and attach the logs it prints out to this BZ.

Just put # virsh domstate <domainname> into a perpetual while loop for 6 domains:

# while :; do virsh domstate database ; virsh domstate imap ; virsh domstate imap1-cluster ; virsh domstate imap2-cluster ; virsh domstate mail ; virsh domstate www; done

Not that it matters, but the domains were all stopped.

Ran a separate ps auxx loop to for libvirtd's memory consumption:

# while : ; do ps auxww | grep libvirtd | grep -v grep ; sleep 2 ; done

And the output from the libvirtd while loop:

root      1158  0.0  0.1 540408 10400 ?        Sl   10:10   0:00 libvirtd --daemon
root      1158  0.0  0.1 540408 10400 ?        Sl   10:10   0:00 libvirtd --daemon
root      1158  0.0  0.1 540408 10400 ?        Sl   10:10   0:00 libvirtd --daemon
root      1158  0.0  0.1 540408 10400 ?        Sl   10:10   0:00 libvirtd --daemon
root      1158  0.0  0.1 540408 10400 ?        Sl   10:10   0:00 libvirtd --daemon
root      1158  0.0  0.1 540408 10400 ?        Sl   10:10   0:00 libvirtd --daemon
root      1158  0.0  0.1 540408 10404 ?        Sl   10:10   0:00 libvirtd --daemon
-- started # virsh domstate loop ------
root      1158  0.0  0.1 540920 10584 ?        Rl   10:10   0:01 libvirtd --daemon
root      1158  0.0  0.1 540408 10580 ?        Sl   10:10   0:02 libvirtd --daemon
root      1158  0.0  0.1 540920 10876 ?        Sl   10:10   0:03 libvirtd --daemon
root      1158  0.0  0.1 540920 10812 ?        Rl   10:10   0:03 libvirtd --daemon
root      1158  0.0  0.1 540408 10700 ?        Sl   10:10   0:04 libvirtd --daemon
root      1158  0.0  0.1 540408 10776 ?        Sl   10:10   0:05 libvirtd --daemon
root      1158  0.0  1.6 540920 133436 ?       Rl   10:10   0:06 libvirtd --daemon
root      1158  0.0  3.1 540920 259244 ?       Sl   10:10   0:07 libvirtd --daemon
root      1158  0.0  4.6 868604 384304 ?       Sl   10:10   0:08 libvirtd --daemon
root      1158  0.0  6.2 868088 509248 ?       Sl   10:10   0:09 libvirtd --daemon
root      1158  0.0  7.7 868088 634848 ?       Sl   10:10   0:10 libvirtd --daemon
root      1158  0.0  8.8 1195768 724556 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 828704 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 828704 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 828704 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 828704 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 829472 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 829472 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 829472 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 829472 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 829472 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 829472 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 829472 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 829472 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 829472 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 829472 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 829472 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 829472 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 829472 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 829472 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 829472 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 830236 ?      Sl   10:10   0:11 libvirtd --daemon

  --- Terminated # virsh domstate loop ----

root      1158  0.0 10.1 1195768 830236 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 830236 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 830236 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 830236 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 830236 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 830236 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 830236 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 830236 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 830236 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 830236 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 830236 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 830236 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 830236 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 830236 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 831004 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 831004 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 831004 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 831004 ?      Sl   10:10   0:11 libvirtd --daemon
root      1158  0.0 10.1 1195768 831004 ?      Sl   10:10   0:11 libvirtd --daemon

Created attachment 374865 [details]
valgrind --leak-check=full /usr/sbin/libvirtd output

# valgrind --leak-check=full /usr/sbin/libvirtd > ./valgrind-out.txt 2>&1

Then looped virsh domstate <dom1> ; virsh domstate <dom2> ; etc for a short while (20 seconds or so I'd guesstimate).

I'm experimenting the same after updating system from F11 x86_64+rawvirt repo
to F12.
Could it be anything related to users previously running rawvirt on f11, or
does it happen also to fresh F12 environments?
My system is an HP ProLiant BL480c G1 with 12Gb of ram.
I have 4 guests, all x86_64: 
2 with CentOS 5.3 and 1Gb of ram each
2 with CentOS 5.4 and 3,5Gb of ram each.

boot of host on 26/11 about 19:20 with the first two guests configured to
auto-start
On 27/11 at 12:03 I start the other two guests in centos 5.4
The day after at 06:55

Nov 28 03:27:06 virtfed logrotate: ALERT exited abnormally with [1]
Nov 28 06:55:01 virtfed kernel: qemu-kvm invoked oom-killer:
gfp_mask=0x201da, order=0, oomkilladj=0
Nov 28 06:55:01 virtfed kernel: qemu-kvm cpuset=/ mems_allowed=0
Nov 28 06:55:01 virtfed kernel: Pid: 405, comm: qemu-kvm Tainted: G
    W  2.6.31.5-127.fc12.x86_64 #1
Nov 28 06:55:01 virtfed kernel: Call Trace:
Nov 28 06:55:01 virtfed kernel: [<ffffffff8108a7d7>] ?
cpuset_print_task_mems_allowed+0x91/0x9d
Nov 28 06:55:01 virtfed kernel: [<ffffffff810c38e9>]
oom_kill_process+0x98/0x256
Nov 28 06:55:01 virtfed kernel: [<ffffffff810c3d6a>] ?
select_bad_process+0xa3/0x102
Nov 28 06:55:01 virtfed kernel: [<ffffffff810c3e53>] __out_of_memory+0x8a/0x99
Nov 28 06:55:01 virtfed kernel: [<ffffffff810c3fc5>] out_of_memory+0x163/0x195
Nov 28 06:55:01 virtfed kernel: [<ffffffff810c75c1>]
__alloc_pages_nodemask+0x491/0x584
Nov 28 06:55:01 virtfed kernel: [<ffffffff810eb1c4>]
alloc_pages_current+0x95/0x9e
Nov 28 06:55:01 virtfed kernel: [<ffffffff810c15c1>]
__page_cache_alloc+0x5f/0x61
Nov 28 06:55:01 virtfed kernel: [<ffffffff810c9235>]
__do_page_cache_readahead+0x98/0x176
Nov 28 06:55:01 virtfed kernel: [<ffffffff810c9334>] ra_submit+0x21/0x25
Nov 28 06:55:01 virtfed kernel: [<ffffffff810c1abd>] filemap_fault+0x193/0x317
Nov 28 06:55:01 virtfed kernel: [<ffffffff810d6641>] __do_fault+0x54/0x3c4
Nov 28 06:55:01 virtfed kernel: [<ffffffff81017bc1>] ? read_tsc+0x9/0x1b
Nov 28 06:55:01 virtfed kernel: [<ffffffff810d881a>]
handle_mm_fault+0x2f6/0x705
Nov 28 06:55:01 virtfed kernel: [<ffffffff8106a9ff>] ?
__hrtimer_start_range_ns+0x2bb/0x2cd
Nov 28 06:55:01 virtfed kernel: [<ffffffff8141cfe1>] do_page_fault+0x281/0x299
Nov 28 06:55:01 virtfed kernel: [<ffffffff8141af75>] page_fault+0x25/0x30
Nov 28 06:55:01 virtfed kernel: Mem-Info:
Nov 28 06:55:01 virtfed kernel: Node 0 DMA per-cpu:
Nov 28 06:55:01 virtfed kernel: CPU    0: hi:    0, btch:   1 usd:   0
Nov 28 06:55:01 virtfed kernel: CPU    1: hi:    0, btch:   1 usd:   0
Nov 28 06:55:01 virtfed kernel: CPU    2: hi:    0, btch:   1 usd:   0
Nov 28 06:55:01 virtfed kernel: CPU    3: hi:    0, btch:   1 usd:   0
Nov 28 06:55:01 virtfed kernel: CPU    4: hi:    0, btch:   1 usd:   0
Nov 28 06:55:01 virtfed kernel: CPU    5: hi:    0, btch:   1 usd:   0
Nov 28 06:55:01 virtfed kernel: CPU    6: hi:    0, btch:   1 usd:   0
Nov 28 06:55:01 virtfed kernel: CPU    7: hi:    0, btch:   1 usd:   0
Nov 28 06:55:01 virtfed kernel: Node 0 DMA32 per-cpu:
Nov 28 06:55:01 virtfed kernel: CPU    0: hi:  186, btch:  31 usd:  86
Nov 28 06:55:01 virtfed kernel: CPU    1: hi:  186, btch:  31 usd: 136
Nov 28 06:55:01 virtfed kernel: CPU    2: hi:  186, btch:  31 usd:  51
Nov 28 06:55:01 virtfed kernel: CPU    3: hi:  186, btch:  31 usd: 126
Nov 28 06:55:01 virtfed kernel: CPU    4: hi:  186, btch:  31 usd:  86
Nov 28 06:55:01 virtfed kernel: CPU    5: hi:  186, btch:  31 usd:  34
Nov 28 06:55:01 virtfed kernel: CPU    6: hi:  186, btch:  31 usd: 171
Nov 28 06:55:01 virtfed kernel: CPU    7: hi:  186, btch:  31 usd: 169
Nov 28 06:55:01 virtfed kernel: Node 0 Normal per-cpu:
Nov 28 06:55:01 virtfed kernel: CPU    0: hi:  186, btch:  31 usd: 173
Nov 28 06:55:01 virtfed kernel: CPU    1: hi:  186, btch:  31 usd:  42
Nov 28 06:55:01 virtfed kernel: CPU    2: hi:  186, btch:  31 usd: 160
Nov 28 06:55:01 virtfed kernel: CPU    3: hi:  186, btch:  31 usd: 109
Nov 28 06:55:01 virtfed kernel: CPU    4: hi:  186, btch:  31 usd: 112
Nov 28 06:55:01 virtfed kernel: CPU    5: hi:  186, btch:  31 usd: 112
Nov 28 06:55:01 virtfed kernel: CPU    6: hi:  186, btch:  31 usd: 174
Nov 28 06:55:01 virtfed kernel: CPU    7: hi:  186, btch:  31 usd: 118
Nov 28 06:55:01 virtfed kernel: Active_anon:2537459 active_file:3
inactive_anon:364350
Nov 28 06:55:01 virtfed kernel: inactive_file:160 unevictable:12562
dirty:0 writeback:0 unstable:0
Nov 28 06:55:01 virtfed kernel: free:16256 slab:29833 mapped:7872
pagetables:10697 bounce:0
Nov 28 06:55:01 virtfed kernel: Node 0 DMA free:15824kB min:16kB
low:20kB high:24kB active_anon:0kB inactive_anon:0kB active_file:0kB
inactive_file:0kB unevictable:0kB present:15320kB pages_scanned:0
all_unreclaimable? yes
Nov 28 06:55:01 virtfed kernel: lowmem_reserve[]: 0 3254 12092 12092
Nov 28 06:55:01 virtfed kernel: Node 0 DMA32 free:39060kB min:3784kB
low:4728kB high:5676kB active_anon:2510452kB inactive_anon:502304kB
active_file:12kB inactive_file:244kB unevictable:520kB
present:3332660kB pages_scanned:120 all_unreclaimable? no
Nov 28 06:55:01 virtfed kernel: lowmem_reserve[]: 0 0 8837 8837
Nov 28 06:55:01 virtfed kernel: Node 0 Normal free:10140kB min:10280kB
low:12848kB high:15420kB active_anon:7639384kB inactive_anon:955096kB
active_file:0kB inactive_file:396kB unevictable:49728kB
present:9049596kB pages_scanned:0 all_unreclaimable? no
Nov 28 06:55:01 virtfed kernel: lowmem_reserve[]: 0 0 0 0
Nov 28 06:55:01 virtfed kernel: Node 0 DMA: 2*4kB 1*8kB 2*16kB 1*32kB
2*64kB 0*128kB 1*256kB 0*512kB 1*1024kB 1*2048kB 3*4096kB = 15824kB
Nov 28 06:55:01 virtfed kernel: Node 0 DMA32: 3793*4kB 2168*8kB
68*16kB 11*32kB 4*64kB 6*128kB 3*256kB 2*512kB 2*1024kB 0*2048kB
0*4096kB = 38820kB
Nov 28 06:55:01 virtfed kernel: Node 0 Normal: 1948*4kB 5*8kB 0*16kB
0*32kB 2*64kB 0*128kB 1*256kB 0*512kB 0*1024kB 1*2048kB 0*4096kB =
10264kB
Nov 28 06:55:01 virtfed kernel: 26703 total pagecache pages
Nov 28 06:55:01 virtfed kernel: 18747 pages in swap cache
Nov 28 06:55:01 virtfed kernel: Swap cache stats: add 1236041, delete
1217294, find 52896/69574
Nov 28 06:55:01 virtfed kernel: Free swap  = 0kB
Nov 28 06:55:01 virtfed kernel: Total swap = 4194296kB
Nov 28 06:55:01 virtfed kernel: 3145727 pages RAM
Nov 28 06:55:01 virtfed kernel: 63020 pages reserved
Nov 28 06:55:01 virtfed kernel: 76044 pages shared
Nov 28 06:55:01 virtfed kernel: 3053641 pages non-shared
Nov 28 06:55:01 virtfed kernel: virbr0: port 3(vnet4) entering disabled state
Nov 28 06:55:01 virtfed kernel: device vnet4 left promiscuous mode
Nov 28 06:55:01 virtfed kernel: virbr0: port 3(vnet4) entering disabled state

About 1 hour and a half after, at 08:14 I get the same for the other 5.4 vm.
The same happens for other processes and the first 2 guests, taht are
configured to auto-start and they do so...

yesterday I restart the other 2 guests and this morning I have:
 PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND
 2904 root      19  -1 10.6g 9.8g 2712 S  0.0 83.6  16:12.42 libvirtd

  [root virtfed ~]# ps -eo rss,pid,cmd|egrep libvirtd|egrep -v egrep
10309532 2904 libvirtd --daemon --listen

I have libvirtd as a service inside rhcs, so that every 30 seconds I have
rgmanager doing a check for its status:
Dec  1 10:56:39 virtfed bash[19916]: Executing /etc/init.d/libvirtd status

Don't know if this can influence the memory leak we are experiencing with
libvirtd....

Created attachment 375011 [details]
valgrind log from overnight run with libvirt.

I ran overnight
# valgrind --log-file=libvirtd.log --leak-check=full -v --time-stamp=yes /usr/sbin/libvirtd
and the logfile is attached. I'd had three VMs running and I had virt-manager started but done nothing with it. By about 10pm, everything was still okay, and valgrind/libvirtd took about 340MB. By 5am this morning, OOM killer started up and killed off various things (my Satellite VM having 3GB RAM allocated was one of the things gone).

If /var/log/messages from last night and onwards would help, I can provide that too. By the time I got to my system at 9am this morning, it took 30 minutes to get in to it so that I could finally kill valgrind/libvirtd, at which point its virt size was 8718MB and RES was 6.8GB according to top. (On a 8GB RAM, 2GB Swap system...)

Is there any other information, like /proc/.../maps that would help?

/Anders

Re comment#31, I ran a plain F11 system, then preupgraded to F12, so I've not gone via the "rawvirt on F11" step.

After discussion on #virt IRC, it appears that the virt-manager monitoring may be a red-herring. There is a cluster-suite resource 'vm.sh' that is issuing 'virsh domstate $GUEST'  on a very frequent basis for all guests on a host. Since each invocation of virsh opens a libvirt connection, this will trigger the netcf memory leak bugs. With virsh being run so frequently, this will easily account for the huge memory usage of libvirtd please.

Please apply the following update ASAP, and restart the libvirtd daemon after installing it.

https://admin.fedoraproject.org/updates/netcf-0.1.5-1.fc12

I don't think this is the (only at least) solution.
In fact on my system I have no netcf at all.
[root@virtfed ~]# rpm -q netcf
package netcf is not installed

My guests' logic is outside rhcs checks. Part of my cluster.conf is this:

                <service domain="DRBDNODE1" autostart="1" name="DRBDNODE1">
                <script ref="CLVMD">
                        <script ref="LIBVIRTD"/>
                </script>
                </service>

So in my case rgmanager is only doing "service libvirtd status" every 30 seconds.
I noticed this:
1) service libvirtd restart
2) Just after this I run this loop
[root@virtfed ~]# while true
> do
> ps -eo rss,pid,cmd|egrep libvirtd|egrep -v egrep
> sleep 30
> done

At the beginning I get
92948 14049 libvirtd --daemon --listen
97332 14049 libvirtd --daemon --listen
100408 14049 libvirtd --daemon --listen
97596 14049 libvirtd --daemon --listen
100148 14049 libvirtd --daemon --listen
100424 14049 libvirtd --daemon --listen

3) ssh my host with -X switch and run virt-manager and open 2 graphical consoles  
suddenly I get libvirtd memory grow about 3,5MB every 30 seconds
Now, after 2 hours and a half I have
1061572 14049 libvirtd --daemon --listen

4) I close virt-manager, but memory consumed by libvirtd still grows and grows every 30 seconds....

1077148 14049 libvirtd --daemon --listen
1079716 14049 libvirtd --daemon --listen
1082800 14049 libvirtd --daemon --listen
1085988 14049 libvirtd --daemon --listen
1089212 14049 libvirtd --daemon --listen

You must have  netcf-libs  installed at the very least, since libvirtd links against it.

I've grabbed and installed the koji build of netcf-libs and I'm running with that at the moment. After an hour or so, libvirtd is still only taking up 20-30MB RES. Will push it a bit further and will start up virt-manager too, but I should know in about 5-6 hours if the main issue is gone.

Ok, yes; I have
netcf-libs-0.1.4-1.fc12.x86_64
Updated to netcf-libs-0.1.5-1.fc12.x86_64.rpm
After restarting libvirtd and opening virt-manager and two consoles I get this
at the moment:
 7472 13736 libvirtd --daemon --listen
29652 13736 libvirtd --daemon --listen
43048 13736 libvirtd --daemon --listen
35856 13736 libvirtd --daemon --listen
24712 13736 libvirtd --daemon --listen
27468 13736 libvirtd --daemon --listen
29316 13736 libvirtd --daemon --listen
38876 13736 libvirtd --daemon --listen
40788 13736 libvirtd --daemon --listen
46424 13736 libvirtd --daemon --listen
46424 13736 libvirtd --daemon --listen
43564 13736 libvirtd --daemon --listen
43912 13736 libvirtd --daemon --listen

Let's see how and if it grows indefinitely...

FYI, if anyone still sees a major, unbounded, memory leak after applying the netcf-libs-0.1.5  update, please collect some libvirtd logging

  service libvirtd stop
  edi /etc/libvirt/libvirtd.conf to set

    log_filters="1:libvirt"
    log_outputs="1:file:/var/log/libvirtd.log"

  service libvirtd start
 

then, do whatever you need todo to make the leak occur & let it run for a while, then attach the resulting /var/log/libvirtd.log to this bug


NB, it is normal to see libvirtd's memory usage flucuate +/-  20 MB  during use over a few minutes - the key is whether it grows without bound over a longer time

Have run overnight in the same circumstances that yesterday morning saw my system pretty much useless but this morning, libvirtd is sitting pretty at ~30MB RES memory still.

I will conclude that the problem I was seeing is alleviated with the netcf-libs-0.1.5 package.

Many thanks for the quick fix Daniel.

/Anders

I'm making this bug depend on the 2 netcf fixes, and once those bugs close, I will close this one too.

I confirm it is ok also in my environment with netcf-libs-0.1.5-1.fc12.x86_64.
Host has 12Gb of ram.
I've 5 guests running with mem configured respectively for 1024, 1024, 4096, 3512, 3512 Mb each.
virt-manager running and several vnc connections
rhcs running, so "service libvirtd status" running every 30 seconds
Now libvirtd up since more than 3 days and at about 30Mb too
ps -eo rss,pid,cmd|egrep libvirtd|egrep -v egrep
gives
30484 13736 libvirtd --daemon --listen

Also, ksm/ksmtuned are doing a great job, with now about 100k pages shared!

Thanks,
Gianluca

Those netcf bugs are closed, so closing this as CURRENT_RELEASE.