The backtrace indicates that the crash happens because setregid has succeeded for at least one thread, but then fails for some other thread. There is no way to continue safely, so glibc aborts. Some threads have already switched credentials, and there is no provision for partial setregid failures.

This is likely caused by Samba's credential switching using direct system calls. The pam_keyinit module seems not quite thread-safe, in part due to what it does. There are several such modules. In general, PAM does not interact well with multiple threads.

This means the issue is the pam_keyinit module.

It needs something like this:


 modules/pam_keyinit/pam_keyinit.c | 36 +++++++++++++++++++++++++++---------
 1 file changed, 27 insertions(+), 9 deletions(-)

diff --git a/modules/pam_keyinit/pam_keyinit.c b/modules/pam_keyinit/pam_keyinit.c
index 92e4953b..5707776c 100644
--- a/modules/pam_keyinit/pam_keyinit.c
+++ b/modules/pam_keyinit/pam_keyinit.c
@@ -64,6 +64,24 @@ static void error(pam_handle_t *pamh, const char *fmt, ...)
 	va_end(va);
 }
 
+static int pam_setreuid(uid_t ruid, uid_t euid)
+{
+#if defined(HAVE_LINUX_32BIT_SYSCALLS)
+        return syscall(SYS_setreuid32, ruid, euid);
+#else
+        return syscall(SYS_setreuid, ruid, euid);
+#endif
+}
+
+static int pam_setregid(gid_t rgid, gid_t egid)
+{
+#if defined(HAVE_LINUX_32BIT_SYSCALLS)
+        return syscall(SYS_setregid32, rgid, egid);
+#else
+        return syscall(SYS_setregid, rgid, egid);
+#endif
+}
+
 /*
  * initialise the session keyring for this process
  */
@@ -140,14 +158,14 @@ static int kill_keyrings(pam_handle_t *pamh, int error_ret)
 
 		/* switch to the real UID and GID so that we have permission to
 		 * revoke the key */
-		if (revoke_as_gid != old_gid && setregid(-1, revoke_as_gid) < 0) {
+		if (revoke_as_gid != old_gid && pam_setregid(-1, revoke_as_gid) < 0) {
 			error(pamh, "Unable to change GID to %d temporarily\n", revoke_as_gid);
 			return error_ret;
 		}
 
 		if (revoke_as_uid != old_uid && setresuid(-1, revoke_as_uid, old_uid) < 0) {
 			error(pamh, "Unable to change UID to %d temporarily\n", revoke_as_uid);
-			if (getegid() != old_gid && setregid(-1, old_gid) < 0)
+			if (getegid() != old_gid && pam_setregid(-1, old_gid) < 0)
 				error(pamh, "Unable to change GID back to %d\n", old_gid);
 			return error_ret;
 		}
@@ -157,12 +175,12 @@ static int kill_keyrings(pam_handle_t *pamh, int error_ret)
 		}
 
 		/* return to the original UID and GID (probably root) */
-		if (revoke_as_uid != old_uid && setreuid(-1, old_uid) < 0) {
+		if (revoke_as_uid != old_uid && pam_setreuid(-1, old_uid) < 0) {
 			error(pamh, "Unable to change UID back to %d\n", old_uid);
 			ret = error_ret;
 		}
 
-		if (revoke_as_gid != old_gid && setregid(-1, old_gid) < 0) {
+		if (revoke_as_gid != old_gid && pam_setregid(-1, old_gid) < 0) {
 			error(pamh, "Unable to change GID back to %d\n", old_gid);
 			ret = error_ret;
 		}
@@ -215,14 +233,14 @@ static int do_keyinit(pam_handle_t *pamh, int argc, const char **argv, int error
 
 	/* switch to the real UID and GID so that the keyring ends up owned by
 	 * the right user */
-	if (gid != old_gid && setregid(gid, -1) < 0) {
+	if (gid != old_gid && pam_setregid(gid, -1) < 0) {
 		error(pamh, "Unable to change GID to %d temporarily\n", gid);
 		return error_ret;
 	}
 
-	if (uid != old_uid && setreuid(uid, -1) < 0) {
+	if (uid != old_uid && pam_setreuid(uid, -1) < 0) {
 		error(pamh, "Unable to change UID to %d temporarily\n", uid);
-		if (setregid(old_gid, -1) < 0)
+		if (pam_setregid(old_gid, -1) < 0)
 			error(pamh, "Unable to change GID back to %d\n", old_gid);
 		return error_ret;
 	}
@@ -230,12 +248,12 @@ static int do_keyinit(pam_handle_t *pamh, int argc, const char **argv, int error
 	ret = init_keyrings(pamh, force, error_ret);
 
 	/* return to the original UID and GID (probably root) */
-	if (uid != old_uid && setreuid(old_uid, -1) < 0) {
+	if (uid != old_uid && pam_setreuid(old_uid, -1) < 0) {
 		error(pamh, "Unable to change UID back to %d\n", old_uid);
 		ret = error_ret;
 	}
 
-	if (gid != old_gid && setregid(old_gid, -1) < 0) {
+	if (gid != old_gid && pam_setregid(old_gid, -1) < 0) {
 		error(pamh, "Unable to change GID back to %d\n", old_gid);
 		ret = error_ret;
 	}


Configure check for HAVE_LINUX_32BIT_SYSCALLS should look like this:

https://git.samba.org/?p=uid_wrapper.git;a=blob;f=ConfigureChecks.cmake;h=0e2d84f90d16f9b7afeb0016dcd6f8473fbbf944;hb=refs/heads/master#l101


There are also several local variables which should be thread local or atomic.

s/local variables/global variables/

For writing tests you can use pam_wrapper and libpamtest https://cwrap.org/pam_wrapper.html
https://gitlab.com/cwrap/pam_wrapper/-/blob/master/tests/test_pam_wrapper.c <<< tests using libpamtest

Do we know why smbd is loading pam_keyinit?  It's not in /etc/pam.d/samba

I have no idea. Looking into sosreport-KYOTO2-2021-08-11-dpmuupi it isn't configured in /etc.pam.d/samba. However the backtrace above shows we end up in pam_keyinit.

*** Bug 2036107 has been marked as a duplicate of this bug. ***

From the setreuid(2) manpage:

   C library/kernel differences
       At the kernel level, user IDs and group IDs are a per-thread attribute.  However, POSIX requires that all threads in a process share the same credentials.   The
       NPTL  threading  implementation  handles  the  POSIX requirements by providing wrapper functions for the various system calls that change process UIDs and GIDs.
       These wrapper functions (including those for setreuid() and setregid()) employ a signal-based technique to ensure that when one thread changes credentials,  all
       of the other threads in the process also change their credentials.  For details, see nptl(7).

Upstream PR:
    https://github.com/linux-pam/linux-pam/pull/432

master:
    pam_keyinit: thread-safe implementation - a35e092e24ee7632346a0e1b4a203c04d4cd2c62

Since the problem described in this bug report should be
resolved in a recent advisory, it has been closed with a
resolution of ERRATA.

For information on the advisory (pam bug fix and enhancement update), and where to find the updated
files, follow the link below.

If the solution does not work for you, open a new bug report.

https://access.redhat.com/errata/RHBA-2022:7723