Attachment 649269 Details for Bug 877553 – Some further analysis of crash

Some further analysis of crash

job-err.txt (text/plain), 35.79 KB, created by Ken Giusti on 2012-11-21 14:40:27 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Ken Giusti

Created: 2012-11-21 14:40:27 UTC

Size: 35.79 KB

patch

obsolete

>
>Stack:
>
>(gdb) bt
>#0  0x00000000004162c7 in getThreadStats (this=<value optimized out>, job=0x660a3320)
>    at /usr/src/debug/condor-7.8.6/src/condor_contrib/mgmt/qmf/daemons/Submission.h:79
>#1  dec_Removed (this=<value optimized out>, job=0x660a3320)
>    at /usr/src/debug/condor-7.8.6/src/condor_contrib/mgmt/qmf/daemons/Submission.h:192
>#2  SubmissionObject::Decrement (this=<value optimized out>, job=0x660a3320)
>    at /usr/src/debug/condor-7.8.6/src/condor_contrib/mgmt/qmf/daemons/SubmissionObject.cpp:141
>#3  0x0000000000410af6 in Job::~Job (this=0x660a3320, __in_chrg=<value optimized out>)
>    at /usr/src/debug/condor-7.8.6/src/condor_contrib/mgmt/qmf/daemons/Job.cpp:538
>#4  0x000000000041b8ff in global_reset ()
>    at /usr/src/debug/condor-7.8.6/src/condor_contrib/mgmt/qmf/daemons/Globals.cpp:32
>#5  0x000000000040fef3 in process_history_files ()
>    at /usr/src/debug/condor-7.8.6/src/condor_contrib/mgmt/qmf/daemons/HistoryProcessingUtils.cpp:65
>#6  0x0000003df1f7f78a in ClassAdLogReader::BulkLoad (this=0x8f35c8)
>    at /usr/src/debug/condor-7.8.6/src/condor_utils/ClassAdLogReader.cpp:109
>#7  0x0000003df1f7f8a8 in ClassAdLogReader::Poll (this=0x8f35c8)
>    at /usr/src/debug/condor-7.8.6/src/condor_utils/ClassAdLogReader.cpp:83
>#8  0x0000003df1fa25a0 in JobLogMirror::TimerHandler_JobLogPolling (this=0x8f35c0)
>    at /usr/src/debug/condor-7.8.6/src/condor_utils/JobLogMirror.cpp:74
>#9  0x0000003df20478c1 in TimerManager::Timeout (this=0x3df234dda0, 
>    pNumFired=0x7fffffffdc98, pruntime=0x7fffffffdc90)
>    at /usr/src/debug/condor-7.8.6/src/condor_daemon_core.V6/timer_manager.cpp:428
>#10 0x0000003df2055083 in DaemonCore::Driver (this=0x85d640)
>    at /usr/src/debug/condor-7.8.6/src/condor_daemon_core.V6/daemon_core.cpp:3152
>#11 0x0000003df20445d0 in dc_main (argc=1, argv=0x7fffffffe3b0)
>    at /usr/src/debug/condor-7.8.6/src/condor_daemon_core.V6/daemon_core_main.cpp:2410
>#12 0x000000356421ecdd in __libc_start_main (main=0x40dc30 <main(int, char**)>, argc=2, 
>    ubp_av=0x7fffffffe3a8, init=<value optimized out>, fini=<value optimized out>, 
>    rtld_fini=<value optimized out>, stack_end=0x7fffffffe398) at libc-start.c:226
>#13 0x000000000040d789 in _start ()
>
>
>
>
>
>
>(gdb) p *job
>$32 = {
>  m_submission = 0x35b0b0a0,   <<<< REMEMBER THIS!
>  m_live_job = 0x0, 
>  m_history_job = 0x660a3260, 
>  m_key = 0x4b358d40 "597910.0", 
>  m_status = 3
>}
>
>
>(gdb) p *job->m_submission
>$33 = {
>  <qpid::management::Manageable> = {
>    _vptr.Manageable = 0x428c50, 
>    static STATUS_OK = 0, 
>    static STATUS_UNKNOWN_OBJECT = 1, 
>    static STATUS_UNKNOWN_METHOD = 2, 
>    static STATUS_NOT_IMPLEMENTED = 3, 
>    static STATUS_PARAMETER_INVALID = 4, 
>    static STATUS_FEATURE_NOT_IMPLEMENTED = 5, 
>    static STATUS_FORBIDDEN = 6, 
>    static STATUS_EXCEPTION = 7, 
>    static STATUS_USER = 65536
>  }, 
>  members of SubmissionObject: 
>  m_idle = std::set with 0 elements, 
>  m_running = std::set with 0 elements, 
>  m_removed = std::set with 0 elements, 
>  m_completed = std::set with 0 elements, 
>  m_held = std::set with 0 elements, 
>  m_transferring_output = std::set with 0 elements, 
>  m_suspended = std::set with 0 elements, 
>  ownerSet = true, 
>  mgmtObject = 0x35b0b210
>}
>
>
>(gdb) p *job->m_submission->mgmtObject
>$34 = {
>  <qpid::management::ManagementObject> = {
>    <qpid::management::ManagementItem> = {
>      _vptr.ManagementItem = 0xbe02cae0, 
>      static TYPE_U8 = 1 '\001', 
>      static TYPE_U16 = 2 '\002', 
>      static TYPE_U32 = 3 '\003', 
>      static TYPE_U64 = 4 '\004', 
>      static TYPE_SSTR = 6 '\006', 
>      static TYPE_LSTR = 7 '\a', 
>      static TYPE_ABSTIME = 8 '\b', 
>      static TYPE_DELTATIME = 9 '\t', 
>      static TYPE_REF = 10 '\n', 
>      static TYPE_BOOL = 11 '\v', 
>      static TYPE_FLOAT = 12 '\f', 
>      static TYPE_DOUBLE = 13 '\r', 
>      static TYPE_UUID = 14 '\016', 
>      static TYPE_FTABLE = 15 '\017', 
>      static TYPE_S8 = 16 '\020', 
>      static TYPE_S16 = 17 '\021', 
>      static TYPE_S32 = 18 '\022', 
>      static TYPE_S64 = 19 '\023', 
>      static TYPE_LIST = 21 '\025', 
>      static ACCESS_RC = 1 '\001', 
>      static ACCESS_RW = 2 '\002', 
>      static ACCESS_RO = 3 '\003', 
>      static DIR_I = 1 '\001', 
>      static DIR_O = 2 '\002', 
>      static DIR_IO = 3 '\003', 
>      static FLAG_CONFIG = 1 '\001', 
>      static FLAG_INDEX = 2 '\002', 
>      static FLAG_END = 128 '\200', 
>      static CLASS_KIND_TABLE = 1 '\001', 
>      static CLASS_KIND_EVENT = 2 '\002'
>    }, 
>    members of qpid::management::ManagementObject: 
>    createTime = 2022864968, 
>    destroyTime = 1707925464, 
>    updateTime = 1707928737, 
>    objectId = {
>      agent = 0x70e05c18, 
>      first = 2728499586406975, 
>      second = 17179869195, 
>      agentEpoch = 1352538714, 
>      v2Key = "grid0.lab.bos.redhat.com#635278",   <<< HOPEFULLY NOT GARBAGE!
>      agentName = "mfarrell"
>    }, 
>    configChanged = 24,    <<<< garbage?
>    instChanged = 159,     <<<< garbage?
>    deleted = 189,     <<<< garbage?
>    coreObject = 0x583d9418,     <<<< garbage? (should be SubmissionObject addr)
>    accessLock = {
>      impl = 0x62fd18
>    }, 
>    flags = 6487320,
>    static nextThreadIndex = 0, 
>    forcePublish = false, 
>    static MD5_LEN = 16 '\020', 
>    static maxThreads = 1
>  }, 
>  members of qmf::com::redhat::grid::Submission: 
>  static packageName = "com.redhat.grid", 
>  static className = "submission", 
>  static md5Sum = "\275\355\"#\241\ac\217\220\356]\367\345'i\207", 
>  jobserverRef = {
>    agent = 0x62fd18, 
>    first = 65, 
>    second = 1, 
>    agentEpoch = 3313015600, 
>    v2Key = 
>    "\000\000\000\000\065\000\000\000\220\262\260\065", '\000' <repeats 20 times>, "p\211<}\000\000\000\000 \204\343N\000\000\000\000p\316\"V\000\000\000\000\260\204\353\265\000\000\000\000P\000\000\000\000\000\000\000\061\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\002", '\000' <repeats 15 times>, "16\000\000\000\000\000\000\200}\206\215\000\000\000\000Q\000\000\000\000\000\000\000,\000\000\000\000\000\000\000,", '\000' <repeats 15 times>, "grid0.lab.bos.redhat.com#615797.9#1352472779\000\000\000\000A\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\002", '\000' <repeats 11 times>, ".com48\000\065\067\071\067.10#1352472779\000\000\000@\000\000\000\000\000\000\000\061\000\000\000\000\000\000\000\b\000\000\000\000\000\000\000\b", '\000' <repeats 11 times>, ".commfarrell\000\000\000\000\000\000\000\000\061\000\000\000\000\000\000\000grid0.lab.bos.redhat.com#648253\000È\320\064\000\000\000\000A\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\320Ö\b", '\000' <repeats 12 times>, "\020\006&H\000\000\000\000\320Ù»\215\000\000\000\000\260?\266h\000\000\000\000@\000\000\000\000\000\000\000A\000\000\000\000\000\000\000\n\000\000\000\000\000\000\000\n", '\000' <repeats 15 times>..., 
>    agentName = 
>    "\000\000\000\000\000\000\000\000\220\262\260\065", '\000' <repeats 20 times>, " 8\370{\000\000\000\000\340\022z2\000\000\000\000\000\334B\323\v\337\304\022\201\000\000\000\000\000\000\000\260\066\305?\000\000\000\000Hx\222x\000\000\000\000\062\006\315e\000\000\000\000\372\022\315e\000\000\000\000\030\277\236r\000\000\000\000?\032\236P\216\261\t\000\016\000\000\000\004\000\000\000Z\032\236P.com\270\\\340p\000\000\000\000X]\340p\000\000\000\000\370\\\340p\000\000\000\000è¾r\000\000\000\000\030\375b\000\000\000\000\000\030\375b\000\000\000\000\000\030\375b\000\000\000\000\000\061\000\000\000\000\000\000\000\001\000\000\---Type <return> to continue, or q <return> to quit---
>000\000\000\000\000\240}P|", '\000' <repeats 20 times>"\320, |P|\000\000\000\000\061\000\000\000\000\000\000\000grid0.lab.bos.redhat.com#609359\000\377\377\377\377\000\000\000\000A", '\000' <repeats 15 times>"\260, \310\356\233\000\000\000\000Ð½yt\000\000\000\000 E\b\215\000\000\000\000\020\304\"W\000\000\000\000PQ#\260\000\000\000\000@\000\000\000\000\000\000\000A", '\000' <repeats 11 times>, "5\000\000\000@+\a\254", '\000' <repeats 20 times>, "P\371\351\204\000\000\000\000\000}P|\000\000\000\000\260\236\271=\000\000\000\000\061", '\000' <repeats 11 times>, "5\000\000\000\200\220\306\035", '\000' <repeats 20 times>, "p~P|\000\000\000\000\201\000\000\000\000\000\000\000\360\372\351\204\000\000\000\000Hx\222x\000\000\000\000\307\034\240'\000\000\000\000\212)\240'\000\000\000\000\350p\251\254\000\000\000\000\367ÃP]L\t\000\000\000\000\000\004\000\000\000"...
>  }, 
>, 
>  QDate = 1353449963000000000, 
>  perThreadStatsArray = 0x81,        <<<< garbage!!!
>  static METHOD_ECHO = 1, 
>  static METHOD_GETJOBSUMMARIES = 2
>}
>
>
>Analysis:
>
>Using       v2Key = "grid0.lab.bos.redhat.com#635278", 
>
>Hoping it is not garbage - look it up in g_submissions:
>
>
>  [0xb1b32f30 "grid0.lab.bos.redhat.com#635276"] = 0x669cc450,
>  [0x76b928b0 "grid0.lab.bos.redhat.com#635277"] = 0xaadfd9d0,
>  [0x1723a750 "grid0.lab.bos.redhat.com#635278"] = 0x449e4cb0,  <<< NOT 0x35b0b0a0 !
>  [0xa2ddbcb0 "grid0.lab.bos.redhat.com#635279"] = 0x6257f760,
>  [0x9c00d930 "grid0.lab.bos.redhat.com#635280"] = 0xbcd758c0,
>
>Suspect this means it (the SubmissionObject) has been freed - assuming g_submissions
>"owns" the pointer to the SubmissionObject, since the job never delete[s] it.
>
>
>
>Other issue?  Run condor_job_server under valgrind:
>
>
>==21852== Invalid read of size 1
>==21852==    at 0x4A077D2: strlen (mc_replace_strmem.c:282)
>==21852==    by 0x37FCC9CEFF: std::basic_string<char, std::char_traits<char>, std::allocator<char> >::basic_string(char const*, std::allocator<char> const&) (char_traits.h:263)
>==21852==    by 0x410798: Attribute::Attribute(Attribute::AttributeType, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x410E32: LiveJobImpl::Get(char const*, Attribute const*&) const (in /usr/sbin/condor_job_server)
>==21852==    by 0x4114F9: LiveJobImpl::GetStatus() const (in /usr/sbin/condor_job_server)
>==21852==    by 0x4122DC: LiveJobImpl::Set(char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x412C91: Job::Set(char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x41B2C7: JobServerJobLogConsumer::SetAttribute(char const*, char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x3DF1F7F725: ClassAdLogReader::IncrementalLoad() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1F7F8A7: ClassAdLogReader::Poll() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1FA259F: JobLogMirror::TimerHandler_JobLogPolling() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF20478C0: TimerManager::Timeout(int*, double*) (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==  Address 0x18b9973d8 is 24 bytes inside a block of size 26 free'd
>==21852==    at 0x4A0545F: operator delete(void*) (vg_replace_malloc.c:387)
>==21852==    by 0x37FCC9D4A8: std::basic_string<char, std::char_traits<char>, std::allocator<char> >::~basic_string() (basic_string.h:236)
>==21852==    by 0x410E15: LiveJobImpl::Get(char const*, Attribute const*&) const (in /usr/sbin/condor_job_server)
>==21852==    by 0x4114F9: LiveJobImpl::GetStatus() const (in /usr/sbin/condor_job_server)
>==21852==    by 0x4122DC: LiveJobImpl::Set(char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x412C91: Job::Set(char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x41B2C7: JobServerJobLogConsumer::SetAttribute(char const*, char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x3DF1F7F725: ClassAdLogReader::IncrementalLoad() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1F7F8A7: ClassAdLogReader::Poll() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1FA259F: JobLogMirror::TimerHandler_JobLogPolling() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF20478C0: TimerManager::Timeout(int*, double*) (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF2055082: DaemonCore::Driver() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852== 
>==21852== Invalid read of size 1
>==21852==    at 0x4A077E4: strlen (mc_replace_strmem.c:282)
>==21852==    by 0x37FCC9CEFF: std::basic_string<char, std::char_traits<char>, std::allocator<char> >::basic_string(char const*, std::allocator<char> const&) (char_traits.h:263)
>==21852==    by 0x410798: Attribute::Attribute(Attribute::AttributeType, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x410E32: LiveJobImpl::Get(char const*, Attribute const*&) const (in /usr/sbin/condor_job_server)
>==21852==    by 0x4114F9: LiveJobImpl::GetStatus() const (in /usr/sbin/condor_job_server)
>==21852==    by 0x4122DC: LiveJobImpl::Set(char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x412C91: Job::Set(char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x41B2C7: JobServerJobLogConsumer::SetAttribute(char const*, char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x3DF1F7F725: ClassAdLogReader::IncrementalLoad() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1F7F8A7: ClassAdLogReader::Poll() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1FA259F: JobLogMirror::TimerHandler_JobLogPolling() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF20478C0: TimerManager::Timeout(int*, double*) (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==  Address 0x18b9973d9 is 25 bytes inside a block of size 26 free'd
>==21852==    at 0x4A0545F: operator delete(void*) (vg_replace_malloc.c:387)
>==21852==    by 0x37FCC9D4A8: std::basic_string<char, std::char_traits<char>, std::allocator<char> >::~basic_string() (basic_string.h:236)
>==21852==    by 0x410E15: LiveJobImpl::Get(char const*, Attribute const*&) const (in /usr/sbin/condor_job_server)
>==21852==    by 0x4114F9: LiveJobImpl::GetStatus() const (in /usr/sbin/condor_job_server)
>==21852==    by 0x4122DC: LiveJobImpl::Set(char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x412C91: Job::Set(char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x41B2C7: JobServerJobLogConsumer::SetAttribute(char const*, char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x3DF1F7F725: ClassAdLogReader::IncrementalLoad() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1F7F8A7: ClassAdLogReader::Poll() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1FA259F: JobLogMirror::TimerHandler_JobLogPolling() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF20478C0: TimerManager::Timeout(int*, double*) (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF2055082: DaemonCore::Driver() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852== 
>==21852== Invalid read of size 1
>==21852==    at 0x37FCC9CE08: char* std::string::_S_construct<char const*>(char const*, char const*, std::allocator<char> const&, std::forward_iterator_tag) (char_traits.h:247)
>==21852==    by 0x37FCC9CF12: std::basic_string<char, std::char_traits<char>, std::allocator<char> >::basic_string(char const*, std::allocator<char> const&) (basic_string.h:1543)
>==21852==    by 0x410798: Attribute::Attribute(Attribute::AttributeType, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x410E32: LiveJobImpl::Get(char const*, Attribute const*&) const (in /usr/sbin/condor_job_server)
>==21852==    by 0x4114F9: LiveJobImpl::GetStatus() const (in /usr/sbin/condor_job_server)
>==21852==    by 0x4122DC: LiveJobImpl::Set(char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x412C91: Job::Set(char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x41B2C7: JobServerJobLogConsumer::SetAttribute(char const*, char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x3DF1F7F725: ClassAdLogReader::IncrementalLoad() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1F7F8A7: ClassAdLogReader::Poll() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1FA259F: JobLogMirror::TimerHandler_JobLogPolling() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF20478C0: TimerManager::Timeout(int*, double*) (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==  Address 0x18b9973d8 is 24 bytes inside a block of size 26 free'd
>==21852==    at 0x4A0545F: operator delete(void*) (vg_replace_malloc.c:387)
>==21852==    by 0x37FCC9D4A8: std::basic_string<char, std::char_traits<char>, std::allocator<char> >::~basic_string() (basic_string.h:236)
>==21852==    by 0x410E15: LiveJobImpl::Get(char const*, Attribute const*&) const (in /usr/sbin/condor_job_server)
>==21852==    by 0x4114F9: LiveJobImpl::GetStatus() const (in /usr/sbin/condor_job_server)
>==21852==    by 0x4122DC: LiveJobImpl::Set(char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x412C91: Job::Set(char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x41B2C7: JobServerJobLogConsumer::SetAttribute(char const*, char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x3DF1F7F725: ClassAdLogReader::IncrementalLoad() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1F7F8A7: ClassAdLogReader::Poll() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1FA259F: JobLogMirror::TimerHandler_JobLogPolling() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF20478C0: TimerManager::Timeout(int*, double*) (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF2055082: DaemonCore::Driver() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852== 
>==21852== Invalid read of size 1
>==21852==    at 0x4A08081: memcpy (mc_replace_strmem.c:628)
>==21852==    by 0x37FCC9CDDF: char* std::string::_S_construct<char const*>(char const*, char const*, std::allocator<char> const&, std::forward_iterator_tag) (char_traits.h:275)
>==21852==    by 0x37FCC9CF12: std::basic_string<char, std::char_traits<char>, std::allocator<char> >::basic_string(char const*, std::allocator<char> const&) (basic_string.h:1543)
>==21852==    by 0x410798: Attribute::Attribute(Attribute::AttributeType, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x410E32: LiveJobImpl::Get(char const*, Attribute const*&) const (in /usr/sbin/condor_job_server)
>==21852==    by 0x410FAD: LiveJobImpl::GetQDate() const (in /usr/sbin/condor_job_server)
>==21852==    by 0x412C9F: Job::Set(char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x41B2C7: JobServerJobLogConsumer::SetAttribute(char const*, char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x3DF1F7F725: ClassAdLogReader::IncrementalLoad() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1F7F8A7: ClassAdLogReader::Poll() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1FA259F: JobLogMirror::TimerHandler_JobLogPolling() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF20478C0: TimerManager::Timeout(int*, double*) (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==  Address 0x172c65b01 is 33 bytes inside a block of size 35 free'd
>==21852==    at 0x4A0545F: operator delete(void*) (vg_replace_malloc.c:387)
>==21852==    by 0x37FCC9D4A8: std::basic_string<char, std::char_traits<char>, std::allocator<char> >::~basic_string() (basic_string.h:236)
>==21852==    by 0x410E15: LiveJobImpl::Get(char const*, Attribute const*&) const (in /usr/sbin/condor_job_server)
>==21852==    by 0x410FAD: LiveJobImpl::GetQDate() const (in /usr/sbin/condor_job_server)
>==21852==    by 0x412C9F: Job::Set(char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x41B2C7: JobServerJobLogConsumer::SetAttribute(char const*, char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x3DF1F7F725: ClassAdLogReader::IncrementalLoad() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1F7F8A7: ClassAdLogReader::Poll() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1FA259F: JobLogMirror::TimerHandler_JobLogPolling() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF20478C0: TimerManager::Timeout(int*, double*) (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF2055082: DaemonCore::Driver() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF20445CF: dc_main(int, char**) (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852== 
>==21852== Invalid read of size 8
>==21852==    at 0x4A080AC: memcpy (mc_replace_strmem.c:628)
>==21852==    by 0x37FCC9CDDF: char* std::string::_S_construct<char const*>(char const*, char const*, std::allocator<char> const&, std::forward_iterator_tag) (char_traits.h:275)
>==21852==    by 0x37FCC9CF12: std::basic_string<char, std::char_traits<char>, std::allocator<char> >::basic_string(char const*, std::allocator<char> const&) (basic_string.h:1543)
>==21852==    by 0x410798: Attribute::Attribute(Attribute::AttributeType, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x410E32: LiveJobImpl::Get(char const*, Attribute const*&) const (in /usr/sbin/condor_job_server)
>==21852==    by 0x410FAD: LiveJobImpl::GetQDate() const (in /usr/sbin/condor_job_server)
>==21852==    by 0x412C9F: Job::Set(char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x41B2C7: JobServerJobLogConsumer::SetAttribute(char const*, char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x3DF1F7F725: ClassAdLogReader::IncrementalLoad() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1F7F8A7: ClassAdLogReader::Poll() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1FA259F: JobLogMirror::TimerHandler_JobLogPolling() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF20478C0: TimerManager::Timeout(int*, double*) (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==  Address 0x172c65af8 is 24 bytes inside a block of size 35 free'd
>==21852==    at 0x4A0545F: operator delete(void*) (vg_replace_malloc.c:387)
>==21852==    by 0x37FCC9D4A8: std::basic_string<char, std::char_traits<char>, std::allocator<char> >::~basic_string() (basic_string.h:236)
>==21852==    by 0x410E15: LiveJobImpl::Get(char const*, Attribute const*&) const (in /usr/sbin/condor_job_server)
>==21852==    by 0x410FAD: LiveJobImpl::GetQDate() const (in /usr/sbin/condor_job_server)
>==21852==    by 0x412C9F: Job::Set(char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x41B2C7: JobServerJobLogConsumer::SetAttribute(char const*, char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x3DF1F7F725: ClassAdLogReader::IncrementalLoad() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1F7F8A7: ClassAdLogReader::Poll() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1FA259F: JobLogMirror::TimerHandler_JobLogPolling() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF20478C0: TimerManager::Timeout(int*, double*) (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF2055082: DaemonCore::Driver() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF20445CF: dc_main(int, char**) (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852== 
>==21852== Invalid read of size 8
>==21852==    at 0x4A07F90: memcpy (mc_replace_strmem.c:628)
>==21852==    by 0x37FCC9CDDF: char* std::string::_S_construct<char const*>(char const*, char const*, std::allocator<char> const&, std::forward_iterator_tag) (char_traits.h:275)
>==21852==    by 0x37FCC9CF12: std::basic_string<char, std::char_traits<char>, std::allocator<char> >::basic_string(char const*, std::allocator<char> const&) (basic_string.h:1543)
>==21852==    by 0x410798: Attribute::Attribute(Attribute::AttributeType, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x410E32: LiveJobImpl::Get(char const*, Attribute const*&) const (in /usr/sbin/condor_job_server)
>==21852==    by 0x410FAD: LiveJobImpl::GetQDate() const (in /usr/sbin/condor_job_server)
>==21852==    by 0x412C9F: Job::Set(char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x41B2C7: JobServerJobLogConsumer::SetAttribute(char const*, char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x3DF1F7F725: ClassAdLogReader::IncrementalLoad() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1F7F8A7: ClassAdLogReader::Poll() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1FA259F: JobLogMirror::TimerHandler_JobLogPolling() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF20478C0: TimerManager::Timeout(int*, double*) (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==  Address 0x17030f428 is 24 bytes inside a block of size 35 free'd
>==21852==    at 0x4A0545F: operator delete(void*) (vg_replace_malloc.c:387)
>==21852==    by 0x37FCC9D4A8: std::basic_string<char, std::char_traits<char>, std::allocator<char> >::~basic_string() (basic_string.h:236)
>==21852==    by 0x410E15: LiveJobImpl::Get(char const*, Attribute const*&) const (in /usr/sbin/condor_job_server)
>==21852==    by 0x410FAD: LiveJobImpl::GetQDate() const (in /usr/sbin/condor_job_server)
>==21852==    by 0x412C9F: Job::Set(char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x41B2C7: JobServerJobLogConsumer::SetAttribute(char const*, char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x3DF1F7F725: ClassAdLogReader::IncrementalLoad() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1F7F8A7: ClassAdLogReader::Poll() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1FA259F: JobLogMirror::TimerHandler_JobLogPolling() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF20478C0: TimerManager::Timeout(int*, double*) (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF2055082: DaemonCore::Driver() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF20445CF: dc_main(int, char**) (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852== 
>==21852== Invalid read of size 2
>==21852==    at 0x4A07FE0: memcpy (mc_replace_strmem.c:628)
>==21852==    by 0x37FCC9CDDF: char* std::string::_S_construct<char const*>(char const*, char const*, std::allocator<char> const&, std::forward_iterator_tag) (char_traits.h:275)
>==21852==    by 0x37FCC9CF12: std::basic_string<char, std::char_traits<char>, std::allocator<char> >::basic_string(char const*, std::allocator<char> const&) (basic_string.h:1543)
>==21852==    by 0x410798: Attribute::Attribute(Attribute::AttributeType, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x410E32: LiveJobImpl::Get(char const*, Attribute const*&) const (in /usr/sbin/condor_job_server)
>==21852==    by 0x410FAD: LiveJobImpl::GetQDate() const (in /usr/sbin/condor_job_server)
>==21852==    by 0x412C9F: Job::Set(char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x41B2C7: JobServerJobLogConsumer::SetAttribute(char const*, char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x3DF1F7F725: ClassAdLogReader::IncrementalLoad() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1F7F8A7: ClassAdLogReader::Poll() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1FA259F: JobLogMirror::TimerHandler_JobLogPolling() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF20478C0: TimerManager::Timeout(int*, double*) (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==  Address 0x17030f430 is 32 bytes inside a block of size 35 free'd
>==21852==    at 0x4A0545F: operator delete(void*) (vg_replace_malloc.c:387)
>==21852==    by 0x37FCC9D4A8: std::basic_string<char, std::char_traits<char>, std::allocator<char> >::~basic_string() (basic_string.h:236)
>==21852==    by 0x410E15: LiveJobImpl::Get(char const*, Attribute const*&) const (in /usr/sbin/condor_job_server)
>==21852==    by 0x410FAD: LiveJobImpl::GetQDate() const (in /usr/sbin/condor_job_server)
>==21852==    by 0x412C9F: Job::Set(char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x41B2C7: JobServerJobLogConsumer::SetAttribute(char const*, char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x3DF1F7F725: ClassAdLogReader::IncrementalLoad() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1F7F8A7: ClassAdLogReader::Poll() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1FA259F: JobLogMirror::TimerHandler_JobLogPolling() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF20478C0: TimerManager::Timeout(int*, double*) (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF2055082: DaemonCore::Driver() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF20445CF: dc_main(int, char**) (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852== 
>==21852== Mismatched free() / delete / delete []
>==21852==    at 0x4A04E5D: operator delete[](void*) (vg_replace_malloc.c:409)
>==21852==    by 0x41B0A3: JobServerJobLogConsumer::DestroyClassAd(char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x3DF1F7F725: ClassAdLogReader::IncrementalLoad() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1F7F8A7: ClassAdLogReader::Poll() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1FA259F: JobLogMirror::TimerHandler_JobLogPolling() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF20478C0: TimerManager::Timeout(int*, double*) (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF2055082: DaemonCore::Driver() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF20445CF: dc_main(int, char**) (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x356421ECDC: (below main) (libc-start.c:226)
>==21852==  Address 0x1685243d0 is 0 bytes inside a block of size 11 alloc'd
>==21852==    at 0x4A05FDE: malloc (vg_replace_malloc.c:236)
>==21852==    by 0x35642802C1: strdup (strdup.c:43)
>==21852==    by 0x41B34F: JobServerJobLogConsumer::NewClassAd(char const*, char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x3DF1F7F725: ClassAdLogReader::IncrementalLoad() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1F7F8A7: ClassAdLogReader::Poll() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1FA259F: JobLogMirror::TimerHandler_JobLogPolling() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF20478C0: TimerManager::Timeout(int*, double*) (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF2055082: DaemonCore::Driver() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF20445CF: dc_main(int, char**) (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x356421ECDC: (below main) (libc-start.c:226)
>==21852== 
>==21852== Mismatched free() / delete / delete []
>==21852==    at 0x4A04E5D: operator delete[](void*) (vg_replace_malloc.c:409)
>==21852==    by 0x41B8FE: global_reset() (in /usr/sbin/condor_job_server)
>==21852==    by 0x40FEF2: process_history_files() (in /usr/sbin/condor_job_server)
>==21852==    by 0x3DF1F7F789: ClassAdLogReader::BulkLoad() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1F7F8A7: ClassAdLogReader::Poll() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1FA259F: JobLogMirror::TimerHandler_JobLogPolling() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF20478C0: TimerManager::Timeout(int*, double*) (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF2055082: DaemonCore::Driver() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF20445CF: dc_main(int, char**) (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x356421ECDC: (below main) (libc-start.c:226)
>==21852==  Address 0x173a7ef60 is 0 bytes inside a block of size 5 alloc'd
>==21852==    at 0x4A05FDE: malloc (vg_replace_malloc.c:236)
>==21852==    by 0x35642802C1: strdup (strdup.c:43)
>==21852==    by 0x41B34F: JobServerJobLogConsumer::NewClassAd(char const*, char const*, char const*) (in /usr/sbin/condor_job_server)
>==21852==    by 0x3DF1F7F725: ClassAdLogReader::IncrementalLoad() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1F7F8A7: ClassAdLogReader::Poll() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1FA259F: JobLogMirror::TimerHandler_JobLogPolling() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF20478C0: TimerManager::Timeout(int*, double*) (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF2055082: DaemonCore::Driver() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF20445CF: dc_main(int, char**) (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x356421ECDC: (below main) (libc-start.c:226)
>==21852== 
>==21852== Invalid read of size 8
>==21852==    at 0x37FCC69E60: std::_Rb_tree_increment(std::_Rb_tree_node_base*) (tree.cc:60)
>==21852==    by 0x41B92B: global_reset() (in /usr/sbin/condor_job_server)
>==21852==    by 0x40FEF2: process_history_files() (in /usr/sbin/condor_job_server)
>==21852==    by 0x3DF1F7F789: ClassAdLogReader::BulkLoad() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1F7F8A7: ClassAdLogReader::Poll() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1FA259F: JobLogMirror::TimerHandler_JobLogPolling() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF20478C0: TimerManager::Timeout(int*, double*) (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF2055082: DaemonCore::Driver() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF20445CF: dc_main(int, char**) (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x356421ECDC: (below main) (libc-start.c:226)
>==21852==  Address 0x19bbaa918 is 24 bytes inside a block of size 48 free'd
>==21852==    at 0x4A0545F: operator delete(void*) (vg_replace_malloc.c:387)
>==21852==    by 0x41B91B: global_reset() (in /usr/sbin/condor_job_server)
>==21852==    by 0x40FEF2: process_history_files() (in /usr/sbin/condor_job_server)
>==21852==    by 0x3DF1F7F789: ClassAdLogReader::BulkLoad() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1F7F8A7: ClassAdLogReader::Poll() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1FA259F: JobLogMirror::TimerHandler_JobLogPolling() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF20478C0: TimerManager::Timeout(int*, double*) (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF2055082: DaemonCore::Driver() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF20445CF: dc_main(int, char**) (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x356421ECDC: (below main) (libc-start.c:226)
>==21852== 
>==21852== Invalid read of size 8
>==21852==    at 0x37FCC69E80: std::_Rb_tree_increment(std::_Rb_tree_node_base*) (tree.cc:68)
>==21852==    by 0x41B92B: global_reset() (in /usr/sbin/condor_job_server)
>==21852==    by 0x40FEF2: process_history_files() (in /usr/sbin/condor_job_server)
>==21852==    by 0x3DF1F7F789: ClassAdLogReader::BulkLoad() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1F7F8A7: ClassAdLogReader::Poll() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1FA259F: JobLogMirror::TimerHandler_JobLogPolling() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF20478C0: TimerManager::Timeout(int*, double*) (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF2055082: DaemonCore::Driver() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF20445CF: dc_main(int, char**) (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x356421ECDC: (below main) (libc-start.c:226)
>==21852==  Address 0x19d23c128 is 8 bytes inside a block of size 48 free'd
>==21852==    at 0x4A0545F: operator delete(void*) (vg_replace_malloc.c:387)
>==21852==    by 0x41B91B: global_reset() (in /usr/sbin/condor_job_server)
>==21852==    by 0x40FEF2: process_history_files() (in /usr/sbin/condor_job_server)
>==21852==    by 0x3DF1F7F789: ClassAdLogReader::BulkLoad() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1F7F8A7: ClassAdLogReader::Poll() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF1FA259F: JobLogMirror::TimerHandler_JobLogPolling() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF20478C0: TimerManager::Timeout(int*, double*) (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF2055082: DaemonCore::Driver() (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x3DF20445CF: dc_main(int, char**) (in /usr/lib64/libcondor_utils_7_8_7.so)
>==21852==    by 0x356421ECDC: (below main) (libc-start.c:226)
>==21852== 
>==21852== 
>==21852== More than 10000000 total errors detected.  I'm not reporting any more.
>==21852== Final error counts will be inaccurate.  Go fix your program!
>==21852== Rerun with --error-limit=no to disable this cutoff.  Note
>==21852== that errors may occur in your program without prior warning from
>==21852== Valgrind, because errors are no longer being displayed.
>==21852== 
>
>

Actions: View

Attachments on bug 877553: 649269