Bug 1147659

A utility ldclt in the 389-ds-base package acts as a cyrus-sasl client to get authenticated to access the Directory Server.  The utility crashes if it runs with 4 or more threads.  It turned out the cyrus-sasl client library (client.c) is not thread safe (or contains a bug). 

tested cyrus-sasl packages:

    cyrus-sasl-lib-2.1.26-17.el7.x86_64
    cyrus-sasl-gssapi-2.1.26-17.el7.x86_64
    cyrus-sasl-2.1.26-17.el7.x86_64
    cyrus-sasl-scram-2.1.26-17.el7.x86_64
    cyrus-sasl-debuginfo-2.1.26-17.el7.x86_64
    cyrus-sasl-devel-2.1.26-17.el7.x86_64
    cyrus-sasl-md5-2.1.26-17.el7.x86_64
    cyrus-sasl-plain-2.1.26-17.el7.x86_64

Description:
This cmechlist is a global variable shared among threads.
static cmech_list_t *cmechlist; /* global var which holds the list */

This is the contents of cmechlist when it's still healthy
(gdb) p cmechlist->mech_list
$2 = (cmechanism_t *) 0x7ffff001bfb0
(gdb) p *cmechlist->mech_list
$3 = {m = {version = 4, plugname = 0x7ffff001b8d0 "scram", plug = 0x7fffcc5d1160 <scram_client_plugins>}, next = 0x7ffff001aaf0}
(gdb) p *cmechlist->mech_list->next
$4 = {m = {version = 4, plugname = 0x7ffff001ab20 "gssapiv2", plug = 0x7fffcf005220 <gssapi_client_plugins+96>},
  next = 0x7ffff001aaa0}
(gdb) p *cmechlist->mech_list->next->next
$5 = {m = {version = 4, plugname = 0x7ffff001aad0 "gssapiv2", plug = 0x7fffcf0051c0 <gssapi_client_plugins>}, next = 0x7ffff0016920}
(gdb) p *cmechlist->mech_list->next->next->next
$6 = {m = {version = 4, plugname = 0x7ffff0016950 "digestmd5", plug = 0x7fffcf5f81a0 <digestmd5_client_plugins>},
  next = 0x7ffff000ab70}
(gdb) p *cmechlist->mech_list->next->next->next->next
$7 = {m = {version = 4, plugname = 0x7ffff000aba0 "EXTERNAL", plug = 0x7ffff6da65c0 <external_client_plugins>}, next = 0x7ffff00157f0}
(gdb) p *cmechlist->mech_list->next->next->next->next->next
$8 = {m = {version = 4, plugname = 0x7ffff00151a0 "crammd5", plug = 0x7fffcf7fe120 <crammd5_client_plugins>}, next = 0x7ffff001b1c0}
(gdb) p *cmechlist->mech_list->next->next->next->next->next->next
$9 = {m = {version = 4, plugname = 0x7ffff001ab70 "login", plug = 0x7fffcc9db100 <login_client_plugins>}, next = 0x7ffff001b870}
(gdb) p *cmechlist->mech_list->next->next->next->next->next->next->next
$10 = {m = {version = 4, plugname = 0x7ffff001b220 "plain", plug = 0x7fffcc7d6100 <plain_client_plugins>}, next = 0x7ffff0014470}
(gdb) p *cmechlist->mech_list->next->next->next->next->next->next->next->next
$11 = {m = {version = 4, plugname = 0x7ffff00144a0 "anonymous", plug = 0x7fffec7f80e0 <anonymous_client_plugins>}, next = 0x0}

This is a mech_list which is held in a context.  The variable c_conn is passed from ldap_int_sasl_close as lc->lconn_sasl_authctx or lc->lconn_sasl_sockctx.  Note that the address of c_conn->mech_list appears as cmechlist->mech_list->next->next->next in the global mech_list.
(gdb) p c_conn->mech_list
$12 = (cmechanism_t *) 0x7ffff0016920
(gdb) p *c_conn->mech_list
$13 = {m = {version = 4, plugname = 0x7ffff0016950 "digestmd5", plug = 0x7fffcf5f81a0 <digestmd5_client_plugins>},
  next = 0x7ffff000ab70}
(gdb) p *c_conn->mech_list->next
$14 = {m = {version = 4, plugname = 0x7ffff000aba0 "EXTERNAL", plug = 0x7ffff6da65c0 <external_client_plugins>},
  next = 0x7ffff00157f0}
...
$18 = {m = {version = 4, plugname = 0x7ffff00144a0 "anonymous", plug = 0x7fffec7f80e0 <anonymous_client_plugins>}, next = 0x0}
(gdb) p *c_conn->mech_list->next->next->next->next->next->next
Cannot access memory at address 0x0

In client_dispose (client.c in cyrus-sasl lib), it has this code:
 326
 327 static void client_dispose(sasl_conn_t *pconn)
 328 {
...
 346   if (c_conn->mech_list != cmechlist->mech_list) {
 347       /* free connection-specific mech_list */
 348       cmechanism_t *m, *prevm;
 349
 350       m = c_conn->mech_list; /* m point to beginning of the list */
 351
 352       while (m) {
 353       prevm = m;
 354       m = m->next;
 355       sasl_FREE(prevm);
 356       }
 357   }
That is, c_conn->mech_list is freed even if it is part of the global cmechlist.

Once client_dispose is called, the global mach_list is corrupted and it crashes the sasl client program, in this case ldclt.

(gdb) p cmechlist->mech_list
$36 = (cmechanism_t *) 0x7ffff001bfb0
(gdb) p *cmechlist->mech_list
$37 = {m = {version = 4, plugname = 0x7ffff001b8d0 "scram", plug = 0x7fffcc5d1160 <scram_client_plugins>}, next = 0x7ffff001aaf0}
(gdb) p *cmechlist->mech_list->next
$38 = {m = {version = 4, plugname = 0x7ffff001ab20 "gssapiv2", plug = 0x7fffcf005220 <gssapi_client_plugins+96>},
  next = 0x7ffff001aaa0}
(gdb) p *cmechlist->mech_list->next->next
$39 = {m = {version = 4, plugname = 0x7ffff001aad0 "gssapiv2", plug = 0x7fffcf0051c0 <gssapi_client_plugins>}, next = 0x7ffff0016920}
(gdb) p *cmechlist->mech_list->next->next->next
$40 = {m = {version = -268352416, plugname = 0x7ffff0016950 "digestmd5", plug = 0x7fffcf5f81a0 <digestmd5_client_plugins>},  next = 0x7ffff000ab70}
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(gdb) p *cmechlist->mech_list->next->next->next->next
$41 = {m = {version = -268343024, plugname = 0x7ffff000aba0 "EXTERNAL", plug = 0x7ffff6da65c0 <external_client_plugins>},  next = 0x7ffff00157f0}

The check at the line 346 is not enough to determine whether the mech object should be freed or not.

    346   if (c_conn->mech_list != cmechlist->mech_list) {

Instead of checking the head mech_list, individual mech must be checked if it exists in the global list cmechlist->mech_list or not.  The mech object should be freed only if it is not found in the global list.