Description of problem:

Customer is having very often crashes in ns-slapd.

While investigating the core file, I can see that there's a faling assert. But I cannot manage to understand which is the missing lib.

I am opening this bug only to find help to clarify the root cause if this core dump.

The relevant thread is:

#0  0x00007f327c3865d7 in __GI_raise (sig=25420) at ../nptl/sysdeps/unix/sysv/linux/raise.c:39
        resultvar = <optimized out>
        pid = -1
        selftid = 25440
#1  0x00007f327c387cc8 in __GI_abort () at abort.c:88
        save_stage = 2
        act = {__sigaction_handler = {sa_handler = 0x7ffe13bbddee, sa_sigaction = 0x7ffe13bbddee}, sa_mask = {__val = {139854810497648, 139854845906728, 384, 412, 139854809140899, 4, 139854232124208, 9797968672, 0, 0, 0, 0, 0, 21474836480, 139854853918720, 139854810509608}}, sa_flags = 2120814405, sa_restorer = 0x7f327e690fc0 <__PRETTY_FUNCTION__.4029>}
        sigs = {__val = {32, 0 <repeats 15 times>}}
#2  0x00007f327c37f546 in __assert_fail_base (fmt=0x7f327c4cf128 ".so.1 must be installed for pthread_cancel to work\n", assertion=0x7f327e690f45 "r == 0", file=<optimized out>, line=<optimized out>, function=<optimized out>) at assert.c:77
        result = <optimized out>
        old = <optimized out>
        buf = 0x7f327ee35000
        str = 0x7f3248004a60 ""
        total = 4096
#3  0x00007f327c37f5f2 in __GI___assert_fail (assertion=0x634c <Address 0x634c out of bounds>, file=0x7f327e690f28 "../../../include/k5-thread.h", line=384, function=0xffffffffffffffff <Address 0xffffffffffffffff out of bounds>) at assert.c:101
No locals.
#4  0x00007f324000f1c8 in ?? ()
No symbol table info available.
#5  0x00007f327e628783 in k5_mutex_lock (m=0x7f327e690f45) at ../../../include/k5-thread.h:384
No locals.
#6  0x00007f327e632931 in k5_mutex_lock (m=0x7f327e690f45) at ../../../include/k5-thread.h:384
        r = <optimized out>
#7  k5_cc_mutex_lock (context=0x7f324000f1c8, context@entry=0x7f3248012320, m=0x7f327e690f45, m@entry=0x7f3248012320) at ccbase.c:455
No locals.
#8  0x00007f327e639e0f in krb5_mcc_store (ctx=0x7f3248012320, id=<optimized out>, creds=<optimized out>) at cc_memory.c:623
        err = <optimized out>
        new_node = 0x7f32480049e0
        mptr = 0x7f324000f1c0
#9  0x00007f327e63de55 in krb5_cc_store_cred (context=context@entry=0x7f3248012320, cache=0x7f3248028fa0, creds=0x7f3248012b00) at ccfns.c:91
        ret = <optimized out>
        tkt = 0x7f3259d37ed0
        s1 = <optimized out>
        s2 = <optimized out>
#10 0x00007f327e64e9b3 in complete (context=context@entry=0x7f3248012320, ctx=ctx@entry=0x7f3248028cd0) at get_creds.c:443
No locals.
#11 0x00007f327e64f971 in step_non_referral (ctx=<optimized out>, context=<optimized out>) at get_creds.c:466
No locals.
#12 krb5_tkt_creds_step (context=context@entry=0x7f3248012320, ctx=ctx@entry=0x7f3248028cd0, in=in@entry=0x7f3259d37ee0, out=out@entry=0x7f3259d37ed0, realm=realm@entry=0x7f3259d37ef0, flags=flags@entry=0x7f3259d37ec8) at get_creds.c:1254
        no_input = <optimized out>
#13 0x00007f327e650407 in krb5_tkt_creds_get (context=context@entry=0x7f3248012320, ctx=0x7f3248028cd0) at get_creds.c:1190
        code = <optimized out>
        request = {magic = -1760647422, length = 0, data = 0x0}
        reply = {magic = -1760647422, length = 1050, data = 0x7f32480045b0 "m\202\004\026\060\202\004\022\240\003\002\001\005\241\003\002\001\r\242\202\001\002\060\201\377\060\201\374\241\004\002\002"}
        realm = {magic = -1760647422, length = 0, data = 0x0}
        flags = 0
        tcp_only = 0
        use_master = 1
#14 0x00007f327e65052c in krb5_get_credentials (context=context@entry=0x7f3248012320, options=options@entry=0, ccache=<optimized out>, in_creds=in_creds@entry=0x7f3259d38050, out_creds=out_creds@entry=0x7f3259d38048) at get_creds.c:1279
        code = 0
        ncreds = <optimized out>
        ctx = 0x7f3248028cd0
#15 0x00007f327607f451 in get_credentials (server=<optimized out>, out_creds=<synthetic pointer>, endtime=<optimized out>, now=1453817449, cred=0x7f3248029df0, context=<optimized out>) at init_sec_context.c:195
        flags = 0
        code = <optimized out>
        in_creds = {magic = 0, client = 0x7f3248009ef0, server = 0x7f3248029ce0, keyblock = {magic = 0, enctype = 0, length = 0, contents = 0x0}, times = {authtime = 0, starttime = 0, endtime = 0, renew_till = 0}, is_skey = 0, ticket_flags = 0, addresses = 0x0, ticket = {magic = 0, length = 0, data = 0x0}, second_ticket = {magic = 0, length = 0, data = 0x0}, authdata = 0x0}
        evidence_creds = {magic = 0, client = 0x0, server = 0x0, keyblock = {magic = 0, enctype = 0, length = 0, contents = 0x0}, times = {authtime = 0, starttime = 0, endtime = 0, renew_till = 0}, is_skey = 0, ticket_flags = 0, addresses = 0x0, ticket = {magic = 0, length = 0, data = 0x0}, second_ticket = {magic = 0, length = 0, data = 0x0}, authdata = 0x0}
        result_creds = 0x0
#16 kg_new_connection (exts=0x7f3259d382e0, context=<optimized out>, time_rec=0x0, ret_flags=0x7f3259d3846c, output_token=0x7f3259d38480, actual_mech_type=0x0, input_token=<optimized out>, input_chan_bindings=0x0, time_req=<optimized out>, req_flags=<optimized out>, mech_type=0x7f327629d600 <krb5_gss_oid_array>, target_name=<optimized out>, context_handle=0x7f3248009e30, cred=0x7f3248029df0, minor_status=0x7f3259d38464) at init_sec_context.c:583
        major_status = 851968
        k_cred = 0x0
        ctx = 0x7f3248008b40
        ctx_free = 0x7f3248008b40
        token = {length = 0, value = 0x0}
        keyblock = 0x7f3248000d60
        code = <optimized out>
        now = 1453817449
#17 krb5_gss_init_sec_context_ext (minor_status=minor_status@entry=0x7f3259d38464, claimant_cred_handle=0x7f3248029df0, claimant_cred_handle@entry=0x0, context_handle=context_handle@entry=0x7f3248009e30, target_name=<optimized out>, mech_type=0x7f327629d600 <krb5_gss_oid_array>, req_flags=req_flags@entry=58, time_req=time_req@entry=0, input_chan_bindings=input_chan_bindings@entry=0x0, input_token=input_token@entry=0x0, actual_mech_type=actual_mech_type@entry=0x0, output_token=output_token@entry=0x7f3259d38480, ret_flags=ret_flags@entry=0x7f3259d3846c, time_rec=time_rec@entry=0x0, exts=exts@entry=0x7f3259d382e0) at init_sec_context.c:971
        context = 0x7f3248012320
        defcred = 0x7f3248029df0
        cred = 0x7f3248029df0
        kerr = <optimized out>
        major_status = <optimized out>
        tmp_min_stat = 1207962976
#18 0x00007f327607fdb7 in krb5_gss_init_sec_context (minor_status=minor_status@entry=0x7f3259d38464, claimant_cred_handle=claimant_cred_handle@entry=0x0, context_handle=context_handle@entry=0x7f3248009e30, target_name=<optimized out>, mech_type=<optimized out>, req_flags=req_flags@entry=58, time_req=time_req@entry=0, input_chan_bindings=input_chan_bindings@entry=0x0, input_token=input_token@entry=0x0, actual_mech_type=actual_mech_type@entry=0x0, output_token=output_token@entry=0x7f3259d38480, ret_flags=ret_flags@entry=0x7f3259d3846c, time_rec=time_rec@entry=0x0) at init_sec_context.c:1073
        exts = {iakerb = {conv = 0x0, verified = 0}}
#19 0x00007f327606989b in gss_init_sec_context (minor_status=0x7f3259d38464, claimant_cred_handle=<optimized out>, context_handle=0x7f3248031f98, target_name=0x7f3248029a10, req_mech_type=<optimized out>, req_flags=58, time_req=0, input_chan_bindings=0x0, input_token=0x0, actual_mech_type=0x0, output_token=0x7f3259d38480, ret_flags=0x7f3259d3846c, time_rec=0x0) at g_init_sec_context.c:210
        status = <optimized out>
        temp_minor_status = 0
        union_name = 0x7f3248029a10
        union_cred = <optimized out>
        internal_name = 0x7f324800a050
        union_ctx_id = 0x7f3248009e20
        selected_mech = 0x7f3234009e30
        mech = <optimized out>
        input_cred_handle = 0x0
#20 0x00007f32762a2c94 in gssapi_client_mech_step (conn_context=0x7f32480092f0, params=0x7f3248031f80, serverin=<optimized out>, serverinlen=<optimized out>, prompt_need=<optimized out>, clientout=0x7f3259d38658, clientoutlen=0x7f3259d38630, oparams=0x7f32480037f0) at gssapi.c:1658
        text = 0x7f32480092f0
        input_token = <optimized out>
        output_token = 0x7f3259d38480
        real_input_token = {length = 0, value = 0x0}
        real_output_token = {length = 0, value = 0x0}
        maj_stat = <optimized out>
        min_stat = 0
        max_input = 1207959584
        name_token = {length = 36, value = 0x0}
        ret = <optimized out>
        req_flags = 58
        out_req_flags = 0
        client_creds = <optimized out>
#21 0x00007f327cb3c7f5 in sasl_client_step (conn=0x7f3248002f80, serverin=0x6360 <Address 0x6360 out of bounds>, serverin@entry=0x0, serverinlen=6, serverinlen@entry=0, prompt_need=0xffffffffffffffff, prompt_need@entry=0x7f3259d38640, clientout=0x7f3259d38658, clientoutlen=0x7f3259d38630) at client.c:1008
        result = 0
#22 0x00007f327cb3cb76 in sasl_client_start (conn=<optimized out>, mechlist=<optimized out>, prompt_need=<optimized out>, clientout=<optimized out>, clientoutlen=0x7f3259d38630, mech=0x7f3259d38638) at client.c:946
        c_conn = <optimized out>
        ordered_mechs = 0x0
        name = <optimized out>
        m = <optimized out>
        bestm = <optimized out>
        i = <optimized out>
        list_len = <optimized out>
        name_len = <optimized out>
        bestssf = <optimized out>
        minssf = <optimized out>
        result = <optimized out>
        server_can_cb = <optimized out>
        cbindingdisp = <optimized out>
        cur_cbindingdisp = <optimized out>
        best_cbindingdisp = <optimized out>
#23 0x00007f327dd2d913 in ldap_pvt_sasl_secprops (in=<optimized out>, secprops=0x7f3259d38864) at cyrus.c:870
        v = 0
        next = 0x200000002 <Address 0x200000002 out of bounds>
        i = <optimized out>
        j = 1919135557
        l = <optimized out>
        props = 0x7f327e957ef0 <ldap_sasl_interact_cb>
        sflags = <optimized out>
        got_sflags = <optimized out>
        max_ssf = <optimized out>
        got_max_ssf = <optimized out>
        min_ssf = <optimized out>
        got_min_ssf = <optimized out>
        maxbufsize = <optimized out>
        got_maxbufsize = <optimized out>
#24 0x0000000000000001 in ?? ()
No symbol table info available.
#25 0x00007f3259d38680 in ?? ()
No symbol table info available.
#26 0x00007f327c45c1a8 in ___vsprintf_chk (s=0x634c <Address 0x634c out of bounds>, flags=25440, slen=6, format=0xffffffffffffffff <Address 0xffffffffffffffff out of bounds>, args=0xfefefefefefefeff) at vsprintf_chk.c:69
        f = {_sbf = {_f = {_flags = -72515583, _IO_read_ptr = 0x7f3248029a10 "\020\232\002H2\177", _IO_read_end = 0x7f3248029a10 "\020\232\002H2\177", _IO_read_base = 0x7f3248029a10 "\020\232\002H2\177", _IO_write_base = 0x7f3248029a10 "\020\232\002H2\177", _IO_write_ptr = 0x7f3248029a3a "BOGOTA@", _IO_write_end = 0xffffffffffffffff <Address 0xffffffffffffffff out of bounds>, _IO_buf_base = 0x7f3248029a10 "\020\232\002H2\177", _IO_buf_end = 0xffffffffffffffff <Address 0xffffffffffffffff out of bounds>, _IO_save_base = 0x0, _IO_backup_base = 0x0, _IO_save_end = 0x0, _markers = 0x0, _chain = 0x0, _fileno = 1208035942, _flags2 = 4, _old_offset = 139854833821186, _cur_column = 0, _vtable_offset = 0 '\000', _shortbuf = "H", _lock = 0x0, _offset = 139853933088503, _codecvt = 0xebd6a344dc199100, _wide_data = 0x1418d0, _freeres_list = 0x7f3248028ab0, _freeres_buf = 0x7f327263af45, _freeres_size = 0, _mode = 1207997136, _unused2 = "2\177\000\000\360~\225~2\177\000\000@|\002H2\177\000"}, vtable = 0x7f327dd30f32 <ldap_ld_free+818>}, _s = {_allocate_buffer = 0x7f327e957ef0 <ldap_sasl_interact_cb>, _free_buffer = 0x7f3248027c40}}
        ret = <optimized out>
#27 0x0000000000000000 in ?? ()
No symbol table info available.
(gdb) 


Particularly frame 2 is giving a hint about what's happening:

#2  0x00007f327c37f546 in __assert_fail_base (fmt=0x7f327c4cf128 ".so.1 must be installed for pthread_cancel to work\n", assertion=0x7f327e690f45 "r == 0", file=<optimized out>, line=<optimized out>, function=<optimized out>) at assert.c:77

I any case, all the dependencies of krb5-libs rpm have been installed.


we cannot see symbols in frame 4. From the abrt report, the address corresponds to:
              , {   "address": 139854845478787
                ,   "build_id": "77b09d93c29e0d455d45790de86303c07b003035"
                ,   "build_id_offset": 149379
                ,   "function_name": "k5_mutex_unlock.part.2"
                ,   "file_name": "/lib64/libkrb5.so.3"
                }

Version-Release number of selected component (if applicable): krb5-libs-1.12.2-14.el7.x86_64


How reproducible: not easy to reproduce. I have seen only one customer having this issue.

Please, don't hesitate to contact me if needed.

Regards,

German.