Hide Forgot
Encountered a segfault in pmie reported by ABRT on November 11th, 2016. Date: Mon, 21 Nov 2016 16:01:38 -0500 From: ABRT Daemon <DoNotReply> To: root@localhost Subject: [perf-dept-admins] [abrt] a crash has been detected again abrt_version: 2.0.8 cmdline: /usr/bin/pmie -c /etc/pcp/config.perf-dept.pmie -h perf34 -f -l /var/log/pcp/pmmgr/perf34/pmie.log event_log: executable: /usr/bin/pmie hostname: perf44 kernel: 2.6.32-642.4.2.el6.x86_64 last_occurrence: 1479762057 machineid: sosreport_uploader-dmidecode=b81f6ea55b6d3e92d16d3d59006a4e94a51580377ab7f7ca4e81817d45d91f16 pid: 24616 pkg_arch: x86_64 pkg_epoch: 0 pkg_fingerprint: 199E 2F91 FD43 1D51 pkg_name: pcp pkg_release: 6.el6 pkg_vendor: Red Hat, Inc. pkg_version: 3.10.9 pwd: /srv/tmp/pcp/log/pmmgr time: Fri 11 Nov 2016 04:38:49 PM EST uid: 494 username: pcp sosreport.tar.xz: Binary file, 2421272 bytes cgroup: :8:blkio:/ :7:net_cls:/ :6:freezer:/ :5:devices:/ :4:memory:/ :3:cpuacct:/ :2:cpu:/ :1:cpuset:/ core_backtrace: :{ "signal": 11 :, "executable": "/usr/bin/pmie" :, "stacktrace": : [ { "crash_thread": true : , "frames": : [ { "address": 140374761737468 : , "build_id": "24d3ab3db0f38c7515feadf82191651da4117a18" : , "build_id_offset": 1264892 : , "function_name": "__strncpy_ssse3" : , "file_name": "/lib64/libc.so.6" : } : , { "address": 140374766948518 : , "build_id": "e91b72fca363fb828f9817a26df21513e264eff4" : , "build_id_offset": 86182 : , "function_name": "pmGetContextHostName_r" : , "file_name": "/usr/lib64/libpcp.so.3" : } : , { "address": 140374771781783 : , "build_id": "9d1459f7f8dda86c0031a4dac4901e9e5b38aab3" : , "build_id_offset": 69783 : , "file_name": "/usr/bin/pmie" : } : , { "address": 140374771784147 : , "build_id": "9d1459f7f8dda86c0031a4dac4901e9e5b38aab3" : , "build_id_offset": 72147 : , "file_name": "/usr/bin/pmie" : } : , { "address": 140374771791037 : , "build_id": "9d1459f7f8dda86c0031a4dac4901e9e5b38aab3" : , "build_id_offset": 79037 : , "file_name": "/usr/bin/pmie" : } : , { "address": 140374771752112 : , "build_id": "9d1459f7f8dda86c0031a4dac4901e9e5b38aab3" : , "build_id_offset": 40112 : , "function_name": "main" : , "file_name": "/usr/bin/pmie" : } ] : } ] :} dso_list: :/usr/lib64/libnssutil3.so nss-util-3.21.0-2.el6.x86_64 (Red Hat, Inc.) 1465610096 :/lib64/libfreebl3.so nss-softokn-freebl-3.14.3-23.3.el6_8.x86_64 (Red Hat, Inc.) 1472050433 :/usr/lib64/libsasl2.so.2.0.23 cyrus-sasl-lib-2.1.23-15.el6_6.2.x86_64 (Red Hat, Inc.) 1465610096 :/lib64/libresolv-2.12.so glibc-2.12-1.192.el6.x86_64 (Red Hat, Inc.) 1465610088 :/lib64/libnss_files-2.12.so glibc-2.12-1.192.el6.x86_64 (Red Hat, Inc.) 1465610088 :/usr/bin/pmie pcp-3.10.9-6.el6.x86_64 (Red Hat, Inc.) 1465610174 :/lib64/libcrypt-2.12.so glibc-2.12-1.192.el6.x86_64 (Red Hat, Inc.) 1465610088 :/usr/lib64/libssl3.so nss-3.21.0-8.el6.x86_64 (Red Hat, Inc.) 1465610149 :/lib64/libnspr4.so nspr-4.11.0-1.el6.x86_64 (Red Hat, Inc.) 1465610095 :/usr/lib64/libnss3.so nss-3.21.0-8.el6.x86_64 (Red Hat, Inc.) 1465610149 :/lib64/libm-2.12.so glibc-2.12-1.192.el6.x86_64 (Red Hat, Inc.) 1465610088 :/usr/lib64/libpcp.so.3 pcp-libs-3.10.9-6.el6.x86_64 (Red Hat, Inc.) 1465610149 :/lib64/libdl-2.12.so glibc-2.12-1.192.el6.x86_64 (Red Hat, Inc.) 1465610088 :/lib64/libplds4.so nspr-4.11.0-1.el6.x86_64 (Red Hat, Inc.) 1465610095 :/usr/lib64/libavahi-common.so.3.5.1 avahi-libs-0.6.25-15.el6_8.1.x86_64 (Red Hat, Inc.) 1472050444 :/usr/lib64/libavahi-client.so.3.2.5 avahi-libs-0.6.25-15.el6_8.1.x86_64 (Red Hat, Inc.) 1472050444 :/lib64/libz.so.1.2.3 zlib-1.2.3-29.el6.x86_64 (Red Hat, Inc.) 1375371213 :/lib64/libplc4.so nspr-4.11.0-1.el6.x86_64 (Red Hat, Inc.) 1465610095 :/lib64/libdbus-1.so.3.4.0 dbus-libs-1:1.2.24-8.el6_6.x86_64 (Red Hat, Inc.) 1465610095 :/lib64/ld-2.12.so glibc-2.12-1.192.el6.x86_64 (Red Hat, Inc.) 1465610088 :/lib64/librt-2.12.so glibc-2.12-1.192.el6.x86_64 (Red Hat, Inc.) 1465610088 :/lib64/libc-2.12.so glibc-2.12-1.192.el6.x86_64 (Red Hat, Inc.) 1465610088 :/lib64/libnss_dns-2.12.so glibc-2.12-1.192.el6.x86_64 (Red Hat, Inc.) 1465610088 :/lib64/libpthread-2.12.so glibc-2.12-1.192.el6.x86_64 (Red Hat, Inc.) 1465610088 environ: :PCP_WHICH_PROG=/usr/bin/which :PCP_XCONFIRM_PROG=/usr/bin/pmconfirm :PCP_PLATFORM_PATHS=/usr/bin/X11:/usr/local/bin :PCP_SHARE_DIR=/usr/share/pcp :PCP_MAKE_PROG=/usr/bin/gmake :TERM=linux :'PCP_AWK_PROG=/usr/bin/gawk --posix' :PCP_MAN_DIR=/usr/share/man :PCP_TMP_DIR=/var/lib/pcp/tmp :PCP_PMCDRCLOCAL_PATH=/etc/pcp/pmcd/rc.local :PCP_PMMGROPTIONS_PATH=/etc/pcp/pmmgr/pmmgr.options :PCP_RUN_DIR=/var/run/pcp :PCP_INC_DIR=/usr/include/pcp :PCP_ECHO_C= :PCP_PLATFORM=linux :PCP_LIB32_DIR=/usr/lib :PCP_PMSNAPCONTROL_PATH=/etc/pcp/pmsnap/control :PCP_LOG_DIR=/var/log/pcp :PCP_BIN_DIR=/usr/bin :PCP_PMDAS_DIR=/var/lib/pcp/pmdas :PCP_ECHO_N=-n :PATH=/usr/libexec/pcp/bin:/sbin:/usr/sbin:/bin:/usr/bin :PCP_SYSCONF_DIR=/etc/pcp :PCP_PYTHON_PROG=python :_=/usr/libexec/pcp/bin/pmmgr :runlevel=3 :RUNLEVEL=3 :PCP_MPI_DIRS= :PCP_VERSION=3.10.9 :LANGSH_SOURCED=1 :PWD=/var/log/pcp/pmmgr :PCP_LIB_DIR=/usr/lib64 :PCP_DOC_DIR=/usr/share/doc/pcp-3.10.9 :LANG=en_US.UTF-8 :PCP_DEMOS_DIR=/usr/share/pcp/demos :PCP_GROUP=pcp :previous=N :PREVLEVEL=N :PCP_PMWEBDOPTIONS_PATH=/etc/pcp/pmwebd/pmwebd.options :PCP_PMCDOPTIONS_PATH=/etc/pcp/pmcd/pmcd.options :PCP_PERL_PROG=perl :CONSOLETYPE=vt :SHLVL=2 :PCP_BINADM_DIR=/usr/libexec/pcp/bin :HOME=/var/lib/pcp :PCP_PMLOGGERCONTROL_PATH=/etc/pcp/pmlogger/control :PCP_SYSLOG_PROG=logger :PCP_VAR_DIR=/var/lib/pcp :PCP_PMCDCONF_PATH=/etc/pcp/pmcd/pmcd.conf :PCP_PMIECONTROL_PATH=/etc/pcp/pmie/control :UPSTART_INSTANCE= :PCP_RC_DIR=/etc/rc.d/init.d :PCP_TMPFILE_DIR=/var/tmp :'PCP_CC_PROG=gcc ' :PCP_PS_PROG=ps :UPSTART_EVENTS=runlevel :PCP_HTML_DIR=/usr/share/doc/pcp-doc/html :PCP_SYSTEMDUNIT_DIR= :PCP_ENV_DONE=y :PCP_USER=pcp :PCP_PMPROXYOPTIONS_PATH=/etc/pcp/pmproxy/pmproxy.options :PCP_SASLCONF_DIR=/etc/sasl2 :UPSTART_JOB=rc :PCP_ETC_DIR=/etc :PCP_ECHO_PROG=/bin/echo :PCP_PS_ALL_FLAGS=-efw :PCP_SYSCONFIG_DIR=/etc/sysconfig :PCP_SORT_PROG=/bin/sort limits: :Limit Soft Limit Hard Limit Units :Max cpu time unlimited unlimited seconds :Max file size unlimited unlimited bytes :Max data size unlimited unlimited bytes :Max stack size 10485760 unlimited bytes :Max core file size 0 unlimited bytes :Max resident set unlimited unlimited bytes :Max processes 257538 257538 processes :Max open files 1024 4096 files :Max locked memory 65536 65536 bytes :Max address space unlimited unlimited bytes :Max file locks unlimited unlimited locks :Max pending signals 257538 257538 signals :Max msgqueue size 819200 819200 bytes :Max nice priority 0 0 :Max realtime priority 0 0 :Max realtime timeout unlimited unlimited us maps: :7fab892a8000-7fab892ad000 r-xp 00000000 fd:00 6422553 /lib64/libnss_dns-2.12.so :7fab892ad000-7fab894ac000 ---p 00005000 fd:00 6422553 /lib64/libnss_dns-2.12.so :7fab894ac000-7fab894ad000 r--p 00004000 fd:00 6422553 /lib64/libnss_dns-2.12.so :7fab894ad000-7fab894ae000 rw-p 00005000 fd:00 6422553 /lib64/libnss_dns-2.12.so :7fab894ae000-7fab894bb000 r-xp 00000000 fd:00 6434502 /lib64/libnss_files-2.12.so :7fab894bb000-7fab896ba000 ---p 0000d000 fd:00 6434502 /lib64/libnss_files-2.12.so :7fab896ba000-7fab896bb000 r--p 0000c000 fd:00 6434502 /lib64/libnss_files-2.12.so :7fab896bb000-7fab896bc000 rw-p 0000d000 fd:00 6434502 /lib64/libnss_files-2.12.so :7fab896bc000-7fab896be000 r-xp 00000000 fd:00 6422635 /lib64/libfreebl3.so :7fab896be000-7fab898bd000 ---p 00002000 fd:00 6422635 /lib64/libfreebl3.so :7fab898bd000-7fab898be000 r--p 00001000 fd:00 6422635 /lib64/libfreebl3.so :7fab898be000-7fab898bf000 rw-p 00002000 fd:00 6422635 /lib64/libfreebl3.so :7fab898bf000-7fab898ff000 r-xp 00000000 fd:00 6422688 /lib64/libdbus-1.so.3.4.0 :7fab898ff000-7fab89afe000 ---p 00040000 fd:00 6422688 /lib64/libdbus-1.so.3.4.0 :7fab89afe000-7fab89aff000 r--p 0003f000 fd:00 6422688 /lib64/libdbus-1.so.3.4.0 :7fab89aff000-7fab89b00000 rw-p 00040000 fd:00 6422688 /lib64/libdbus-1.so.3.4.0 :7fab89b00000-7fab89b07000 r-xp 00000000 fd:00 6422687 /lib64/libcrypt-2.12.so :7fab89b07000-7fab89d07000 ---p 00007000 fd:00 6422687 /lib64/libcrypt-2.12.so :7fab89d07000-7fab89d08000 r--p 00007000 fd:00 6422687 /lib64/libcrypt-2.12.so :7fab89d08000-7fab89d09000 rw-p 00008000 fd:00 6422687 /lib64/libcrypt-2.12.so :7fab89d09000-7fab89d37000 rw-p 00000000 00:00 0 :7fab89d37000-7fab89d4d000 r-xp 00000000 fd:00 6422619 /lib64/libresolv-2.12.so :7fab89d4d000-7fab89f4d000 ---p 00016000 fd:00 6422619 /lib64/libresolv-2.12.so :7fab89f4d000-7fab89f4e000 r--p 00016000 fd:00 6422619 /lib64/libresolv-2.12.so :7fab89f4e000-7fab89f4f000 rw-p 00017000 fd:00 6422619 /lib64/libresolv-2.12.so :7fab89f4f000-7fab89f51000 rw-p 00000000 00:00 0 :7fab89f51000-7fab89f66000 r-xp 00000000 fd:00 6422631 /lib64/libz.so.1.2.3 :7fab89f66000-7fab8a165000 ---p 00015000 fd:00 6422631 /lib64/libz.so.1.2.3 :7fab8a165000-7fab8a166000 r--p 00014000 fd:00 6422631 /lib64/libz.so.1.2.3 :7fab8a166000-7fab8a167000 rw-p 00015000 fd:00 6422631 /lib64/libz.so.1.2.3 :7fab8a167000-7fab8a16a000 r-xp 00000000 fd:00 6422644 /lib64/libplds4.so :7fab8a16a000-7fab8a369000 ---p 00003000 fd:00 6422644 /lib64/libplds4.so :7fab8a369000-7fab8a36a000 r--p 00002000 fd:00 6422644 /lib64/libplds4.so :7fab8a36a000-7fab8a36b000 rw-p 00003000 fd:00 6422644 /lib64/libplds4.so :7fab8a36b000-7fab8a36f000 r-xp 00000000 fd:00 6422643 /lib64/libplc4.so :7fab8a36f000-7fab8a56e000 ---p 00004000 fd:00 6422643 /lib64/libplc4.so :7fab8a56e000-7fab8a56f000 r--p 00003000 fd:00 6422643 /lib64/libplc4.so :7fab8a56f000-7fab8a570000 rw-p 00004000 fd:00 6422643 /lib64/libplc4.so :7fab8a570000-7fab8a596000 r-xp 00000000 fd:00 5113875 /usr/lib64/libnssutil3.so :7fab8a596000-7fab8a795000 ---p 00026000 fd:00 5113875 /usr/lib64/libnssutil3.so :7fab8a795000-7fab8a79b000 r--p 00025000 fd:00 5113875 /usr/lib64/libnssutil3.so :7fab8a79b000-7fab8a79c000 rw-p 0002b000 fd:00 5113875 /usr/lib64/libnssutil3.so :7fab8a79c000-7fab8a7a3000 r-xp 00000000 fd:00 6422639 /lib64/librt-2.12.so :7fab8a7a3000-7fab8a9a2000 ---p 00007000 fd:00 6422639 /lib64/librt-2.12.so :7fab8a9a2000-7fab8a9a3000 r--p 00006000 fd:00 6422639 /lib64/librt-2.12.so :7fab8a9a3000-7fab8a9a4000 rw-p 00007000 fd:00 6422639 /lib64/librt-2.12.so :7fab8a9a4000-7fab8a9a6000 r-xp 00000000 fd:00 6422605 /lib64/libdl-2.12.so :7fab8a9a6000-7fab8aba6000 ---p 00002000 fd:00 6422605 /lib64/libdl-2.12.so :7fab8aba6000-7fab8aba7000 r--p 00002000 fd:00 6422605 /lib64/libdl-2.12.so :7fab8aba7000-7fab8aba8000 rw-p 00003000 fd:00 6422605 /lib64/libdl-2.12.so :7fab8aba8000-7fab8abbf000 r-xp 00000000 fd:00 6422595 /lib64/libpthread-2.12.so :7fab8abbf000-7fab8adbf000 ---p 00017000 fd:00 6422595 /lib64/libpthread-2.12.so :7fab8adbf000-7fab8adc0000 r--p 00017000 fd:00 6422595 /lib64/libpthread-2.12.so :7fab8adc0000-7fab8adc1000 rw-p 00018000 fd:00 6422595 /lib64/libpthread-2.12.so :7fab8adc1000-7fab8adc5000 rw-p 00000000 00:00 0 :7fab8adc5000-7fab8add5000 r-xp 00000000 fd:00 5136798 /usr/lib64/libavahi-client.so.3.2.5 :7fab8add5000-7fab8afd4000 ---p 00010000 fd:00 5136798 /usr/lib64/libavahi-client.so.3.2.5 :7fab8afd4000-7fab8afd5000 rw-p 0000f000 fd:00 5136798 /usr/lib64/libavahi-client.so.3.2.5 :7fab8afd5000-7fab8afe0000 r-xp 00000000 fd:00 5118986 /usr/lib64/libavahi-common.so.3.5.1 :7fab8afe0000-7fab8b1e0000 ---p 0000b000 fd:00 5118986 /usr/lib64/libavahi-common.so.3.5.1 :7fab8b1e0000-7fab8b1e1000 rw-p 0000b000 fd:00 5118986 /usr/lib64/libavahi-common.so.3.5.1 :7fab8b1e1000-7fab8b1fa000 r-xp 00000000 fd:00 5111975 /usr/lib64/libsasl2.so.2.0.23 :7fab8b1fa000-7fab8b3f9000 ---p 00019000 fd:00 5111975 /usr/lib64/libsasl2.so.2.0.23 :7fab8b3f9000-7fab8b3fa000 r--p 00018000 fd:00 5111975 /usr/lib64/libsasl2.so.2.0.23 :7fab8b3fa000-7fab8b3fb000 rw-p 00019000 fd:00 5111975 /usr/lib64/libsasl2.so.2.0.23 :7fab8b3fb000-7fab8b434000 r-xp 00000000 fd:00 6422641 /lib64/libnspr4.so :7fab8b434000-7fab8b634000 ---p 00039000 fd:00 6422641 /lib64/libnspr4.so :7fab8b634000-7fab8b635000 r--p 00039000 fd:00 6422641 /lib64/libnspr4.so :7fab8b635000-7fab8b637000 rw-p 0003a000 fd:00 6422641 /lib64/libnspr4.so :7fab8b637000-7fab8b639000 rw-p 00000000 00:00 0 :7fab8b639000-7fab8b770000 r-xp 00000000 fd:00 5115185 /usr/lib64/libnss3.so :7fab8b770000-7fab8b970000 ---p 00137000 fd:00 5115185 /usr/lib64/libnss3.so :7fab8b970000-7fab8b975000 r--p 00137000 fd:00 5115185 /usr/lib64/libnss3.so :7fab8b975000-7fab8b977000 rw-p 0013c000 fd:00 5115185 /usr/lib64/libnss3.so :7fab8b977000-7fab8b979000 rw-p 00000000 00:00 0 :7fab8b979000-7fab8b9b7000 r-xp 00000000 fd:00 5115187 /usr/lib64/libssl3.so :7fab8b9b7000-7fab8bbb7000 ---p 0003e000 fd:00 5115187 /usr/lib64/libssl3.so :7fab8bbb7000-7fab8bbba000 r--p 0003e000 fd:00 5115187 /usr/lib64/libssl3.so :7fab8bbba000-7fab8bbbb000 rw-p 00041000 fd:00 5115187 /usr/lib64/libssl3.so :7fab8bbbb000-7fab8bbbc000 rw-p 00000000 00:00 0 :7fab8bbbc000-7fab8bd46000 r-xp 00000000 fd:00 6422592 /lib64/libc-2.12.so :7fab8bd46000-7fab8bf46000 ---p 0018a000 fd:00 6422592 /lib64/libc-2.12.so :7fab8bf46000-7fab8bf4a000 r--p 0018a000 fd:00 6422592 /lib64/libc-2.12.so :7fab8bf4a000-7fab8bf4c000 rw-p 0018e000 fd:00 6422592 /lib64/libc-2.12.so :7fab8bf4c000-7fab8bf50000 rw-p 00000000 00:00 0 :7fab8bf50000-7fab8bfd3000 r-xp 00000000 fd:00 6422652 /lib64/libm-2.12.so :7fab8bfd3000-7fab8c1d2000 ---p 00083000 fd:00 6422652 /lib64/libm-2.12.so :7fab8c1d2000-7fab8c1d3000 r--p 00082000 fd:00 6422652 /lib64/libm-2.12.so :7fab8c1d3000-7fab8c1d4000 rw-p 00083000 fd:00 6422652 /lib64/libm-2.12.so :7fab8c1d4000-7fab8c248000 r-xp 00000000 fd:00 5136799 /usr/lib64/libpcp.so.3 :7fab8c248000-7fab8c448000 ---p 00074000 fd:00 5136799 /usr/lib64/libpcp.so.3 :7fab8c448000-7fab8c44c000 rw-p 00074000 fd:00 5136799 /usr/lib64/libpcp.so.3 :7fab8c44c000-7fab8c451000 rw-p 00000000 00:00 0 :7fab8c451000-7fab8c471000 r-xp 00000000 fd:00 6422554 /lib64/ld-2.12.so :7fab8c64d000-7fab8c658000 rw-p 00000000 00:00 0 :7fab8c66b000-7fab8c66e000 rw-s 00000000 fc:00 4563406637 /srv/tmp/pcp/lib/pmie/24616 :7fab8c66e000-7fab8c670000 rw-p 00000000 00:00 0 :7fab8c670000-7fab8c672000 r--p 0001f000 fd:00 6422554 /lib64/ld-2.12.so :7fab8c672000-7fab8c673000 rw-p 00021000 fd:00 6422554 /lib64/ld-2.12.so :7fab8c673000-7fab8c674000 rw-p 00000000 00:00 0 :7fab8c674000-7fab8c6a1000 r-xp 00000000 fd:00 5147932 /usr/bin/pmie :7fab8c8a1000-7fab8c8a3000 r--p 0002d000 fd:00 5147932 /usr/bin/pmie :7fab8c8a3000-7fab8c8a4000 rw-p 0002f000 fd:00 5147932 /usr/bin/pmie :7fab8c8a4000-7fab8c8ab000 rw-p 00000000 00:00 0 :7fab8dda9000-7fab8ddda000 rw-p 00000000 00:00 0 [heap] :7ffff7313000-7ffff7328000 rw-p 00000000 00:00 0 [stack] :7ffff7398000-7ffff7399000 r-xp 00000000 00:00 0 [vdso] :ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall] open_fds: :0:/dev/null :pos: 0 :flags: 0100000 :1:/srv/tmp/pcp/log/pmmgr/pmmgr.log :pos: 1038784545 :flags: 0102001 :2:/srv/tmp/pcp/log/pmmgr/perf34/pmie.log :pos: 10720 :flags: 0100001 var_log_messages: :2016-11-11T03:58:13.103070-05:00 perf44 pcp-pmie[16957]: Some CPU busy executing in system mode: perf48: cpu8 - 84% sys :2016-11-11T05:16:13.100753-05:00 perf44 pcp-pmie[16957]: Some CPU busy executing in system mode: perf48: cpu4 - 75% sys :2016-11-11T05:36:13.101015-05:00 perf44 pcp-pmie[16957]: Some CPU busy executing in system mode: perf48: cpu4 - 81% sys :2016-11-11T05:46:13.102788-05:00 perf44 pcp-pmie[16957]: Some CPU busy executing in system mode: perf48: cpu8 - 69% sys :2016-11-11T06:12:13.101035-05:00 perf44 pcp-pmie[16957]: Some CPU busy executing in system mode: perf48: cpu2 - 68% sys :2016-11-11T07:18:13.102736-05:00 perf44 pcp-pmie[16957]: Some CPU busy executing in system mode: perf48: cpu4 - 67% sys :2016-11-11T07:36:13.101076-05:00 perf44 pcp-pmie[16957]: Some CPU busy executing in system mode: perf48: cpu6 - 78% sys :2016-11-11T07:48:13.100756-05:00 perf44 pcp-pmie[16957]: Some CPU busy executing in system mode: perf48: cpu4 - 81% sys :2016-11-11T08:22:13.102798-05:00 perf44 pcp-pmie[16957]: Some CPU busy executing in system mode: perf48: cpu4 - 62% sys :2016-11-11T09:40:13.100973-05:00 perf44 pcp-pmie[16957]: Some CPU busy executing in system mode: perf48: cpu6 - 69% sys :2016-11-11T16:07:32.179703-05:00 perf44 pcp-pmie[13013]: High aggregate context switch rate: perf34: 532870 ctxsw/s :2016-11-11T16:17:31.724358-05:00 perf44 pcp-pmie[13013]: High aggregate context switch rate: perf34: 623936 ctxsw/s :2016-11-11T16:29:31.310735-05:00 perf44 pcp-pmie[13013]: High aggregate context switch rate: perf34: 578571 ctxsw/s :2016-11-11T16:38:49.753511-05:00 perf44 kernel: pmie[24616]: segfault at 7fab00000020 ip 00007fab8bcf0cfc sp 00007ffff7326068 error 4 in libc-2.12.so[7fab8bbbc000+18a000] :2016-11-11T16:38:49.919271-05:00 perf44 abrt[30954]: Saved core dump of pid 24616 (/usr/bin/pmie) to /var/spool/abrt/ccpp-2016-11-11-16:38:49-24616 (827392 bytes)
Analysis of the coredump suggests memory corruption within libpcp. strncpy at line 180 of pcp-3.10.9's src/libpcp/src/context.c is being called with a bad value from the pmResult. pmie[24616]: segfault at 7fab00000020 ip 00007fab8bcf0cfc sp 00007ffff7326068 error 4 in libc-2.12.so[7fab8bbbc000+18a000] (gdb) bt #0 0x00007fab8bcf0cfc in __strncpy_ssse3 () at ../sysdeps/x86_64/multiarch/strcpy.S:235 #1 0x00007fab8c1e90a6 in pmGetContextHostName_r (ctxid=<value optimized out>, buf=0x7fab8c44b7e0 "", buflen=64) at /usr/include/bits/string3.h:121 #2 0x00007fab8c685097 in newContext (host=0x7fab8ddb82e8, hconn=0x7fab8ddab580 "perf34.perf.lab.eng.bos.redhat.com") at pragmatics.c:104 #3 0x00007fab8c6859d3 in reinitMetric (m=0x7fab8ddb82b0) at pragmatics.c:827 #4 0x00007fab8c6874bd in enable () at eval.c:164 #5 run () at eval.c:774 #6 0x00007fab8c67dcb0 in main (argc=8, argv=<value optimized out>) at pmie.c:939 (gdb) p *resp $1 = {timestamp = {tv_sec = 1478900329, tv_usec = 744935}, numpmid = 1, vset = {0x7fab8ddab3c0}} (gdb) p * resp->vset[0] $9 = {pmid = 251658272, numval = 1, valfmt = 0, vlist = {{inst = -1, value = {pval = 0x7fab00000028, lval = 40}}}} (gdb) p & resp->vset[0]->vlist[0]->value.pval->vbuf $7 = (char (*)[1]) 0x7fab0000002c (gdb) p resp->vset[0]->vlist[0]->value.pval->vbuf Cannot access memory at address 0x7fab0000002c 170 name = "pmcd.hostname"; 171 sts = pmLookupName(1, &name, &pmid); 172 if (sts >= 0) 173 sts = pmFetch(1, &pmid, &resp); 174 if (pmDebug & DBG_TRACE_CONTEXT) 175 fprintf(stderr, "pmGetContextHostName_r reset(%d) -> 0\n", original 175 ); 176 177 PM_TPD(curcontext) = original; 178 if (sts >= 0) { 179 if (resp->vset[0]->numval > 0) { /* pmcd.hostname present */ 180 strncpy(buf, resp->vset[0]->vlist[0].value.pval->vbuf, buflen); 181 pmFreeResult(resp); This code looks OK. I don't have a theory as to why the pval pointer could be wrong.
Moving out to 6.10
There's not enough information here to make forward progress & we've not had any other reports of this issue unfortunately. Root cause of the corruption could have been any of many things, including some outside of PCP. We'll keep an eye out for it happening again though.