User has a perl script that works on Solaris.

  The script uses the perl dialect:

$) = $id;
$> = $id;

then authenticates successfully with PAM. It even prints
the proper audit message of success, but then causes the
PAM auth to fail.

  The failure is like this:

static int check_ack(int fd, int seq)
[...]
	rc = audit_get_reply(fd, &rep, GET_REPLY_NONBLOCKING, MSG_PEEK);
[...]
	else if (rc > 0 && rep.type == NLMSG_ERROR) {
		int error = rep.error->error;
		/* Eat the message */
		(void)audit_get_reply(fd, &rep, GET_REPLY_NONBLOCKING, 0);

		/* NLMSG_ERROR can indicate success, only report nonzero */ 
		if (error) {
			errno = -error;
			return error;
		}
	}
	return 0;

what happens is that "rc > 0" and "rep.type == NLMSG_ERROR";
where rc is the message length. Then it returns non zero, and
PAM fails.

  The rep.error->error is set in kernel, in:

int audit_get_reply(int fd, struct audit_reply *rep, reply_t block, int peek)
[...]
	len = recvfrom(fd, &rep->msg, sizeof(rep->msg), block|peek,
		(struct sockaddr*)&nladdr, &nladdrlen);
[...]
	len = adjust_reply(rep, len);

above, len is the message length;

  But the error flag is set at:

static int adjust_reply(struct audit_reply *rep, int len)
{
	rep->type     = rep->msg.nlh.nlmsg_type;

because rep->msg.nlh.nlmsg_type == NLMSG_ERROR

  This is a Request For Comment bug report because I can only
see that the entire PAM connection dies at that point, but
I could only track the issue following the execution under gdb.

  Since it is only the audit log that at the end causes the
entire PAM authentication to return PAM_SYSTEM_ERR, due to

int pam_authenticate(pam_handle_t *pamh, int flags)
[...]
    retval = _pam_dispatch(pamh, flags, PAM_AUTHENTICATE);

int _pam_dispatch(pam_handle_t *pamh, int flags, int choice)
[...]
#ifdef HAVE_LIBAUDIT
    if (choice != PAM_CHAUTHTOK || flags & PAM_UPDATE_AUTHTOK || retval != PAM_SUCCESS) {
	retval = _pam_auditlog(pamh, choice, retval, flags, h);
    }
#endif

int
_pam_auditlog(pam_handle_t *pamh, int action, int retval, int flags, struct handler *h)
[...]
  if (_pam_audit_writelog(pamh, audit_fd, type, message,
      grantors ? grantors : "?", retval) < 0)
    retval = PAM_SYSTEM_ERR;

  So, from my understanding, if PAM were built without
HAVE_LIBAUDIT defined, it would have worked.

  Either way, the root of the issue is euid != uid.


  As extra notes, written after above chunk, I did check a bit the
kernel side, and it looks like the bad value is returned due to:

---8<---
SYSCALL_DEFINE6(recvfrom, int, fd, void __user *, ubuf, size_t, size,
		unsigned int, flags, struct sockaddr __user *, addr,
		int __user *, addr_len)
[...]
	err = sock_recvmsg(sock, &msg, size, flags);

	if (err >= 0 && addr != NULL) {
		err2 = move_addr_to_user(&address,
					 msg.msg_namelen, addr, addr_len);
		if (err2 < 0)
			err = err2;
	}

	fput_light(sock->file, fput_needed);
out:
	return err;
---8<---

  Following up, I see:

---8<---
static inline int __sock_recvmsg(struct kiocb *iocb, struct socket *sock,
				 struct msghdr *msg, size_t size, int flags)
{
	int err = security_socket_recvmsg(sock, msg, size, flags);

	return err ?: __sock_recvmsg_nosec(iocb, sock, msg, size, flags);
}
---8<---

int security_socket_recvmsg(struct socket *sock, struct msghdr *msg,
			    int size, int flags)
{
	return security_ops->socket_recvmsg(sock, msg, size, flags);
}

and then it will depend on "backend" security. Likely
selinux could flag an error due to euid != uid, e.g.:

static int selinux_socket_recvmsg(struct socket *sock, struct msghdr *msg,
				  int size, int flags)
{
	return sock_has_perm(current, sock->sk, SOCKET__READ);
}

static int sock_has_perm(struct task_struct *task, struct sock *sk, u32 perms)
{
[...]
	return avc_has_perm(tsid, sksec->sid, sksec->sclass, perms, &ad);
}


  I will ask the user about selinux, and possible related
messages.