Note: This bug is displayed in read-only format because
the product is no longer active in Red Hat Bugzilla.
RHEL Engineering is moving the tracking of its product development work on RHEL 6 through RHEL 9 to Red Hat Jira (issues.redhat.com). If you're a Red Hat customer, please continue to file support cases via the Red Hat customer portal. If you're not, please head to the "RHEL project" in Red Hat Jira and file new tickets here. Individual Bugzilla bugs in the statuses "NEW", "ASSIGNED", and "POST" are being migrated throughout September 2023. Bugs of Red Hat partners with an assigned Engineering Partner Manager (EPM) are migrated in late September as per pre-agreed dates. Bugs against components "kernel", "kernel-rt", and "kpatch" are only migrated if still in "NEW" or "ASSIGNED". If you cannot log in to RH Jira, please consult article #7032570. That failing, please send an e-mail to the RH Jira admins at rh-issues@redhat.com to troubleshoot your issue as a user management inquiry. The email creates a ServiceNow ticket with Red Hat. Individual Bugzilla bugs that are migrated will be moved to status "CLOSED", resolution "MIGRATED", and set with "MigratedToJIRA" in "Keywords". The link to the successor Jira issue will be found under "Links", have a little "two-footprint" icon next to it, and direct you to the "RHEL project" in Red Hat Jira (issue links are of type "https://issues.redhat.com/browse/RHEL-XXXX", where "X" is a digit). This same link will be available in a blue banner at the top of the page informing you that that bug has been migrated.
DescriptionWelterlen Benoit
2018-11-29 10:33:31 UTC
Description of problem:
The customer gets segfaults in sshd:
Nov 19 09:56:59 $SERVERNAME kernel: sshd[26319]: segfault at 0 ip (null) sp 00007fffc8f705b8 error 14 in sshd[559746ef1000+c8000]
The backtrace is reporting krb:
Core was generated by `sshd: host [priv]'.
Program terminated with signal 11, Segmentation fault.
#0 0x0000000000000000 in ?? ()
(gdb) bt
#0 0x0000000000000000 in ?? ()
#1 0x00007fdf02799969 in krb5int_hmac_keyblock (hash=hash@entry=0x7fdf0299eda0 <krb5int_hash_md5>, keyblock=keyblock@entry=0x559ed62428f0, data=data@entry=0x7ffca80ea7d0, num_data=num_data@entry=1, output=output@entry=0x7ffca80ea7c0)
at hmac.c:145
#2 0x00007fdf02794518 in usage_key (hash=hash@entry=0x7fdf0299eda0 <krb5int_hash_md5>, session_keyblock=session_keyblock@entry=0x559ed62428f0, usage=usage@entry=2, out=<optimized out>, out=<optimized out>, enc=<optimized out>)
at enc_rc4.c:64
#3 0x00007fdf02794ae4 in krb5int_arcfour_decrypt (ktp=<optimized out>, key=0x559ed62428f0, usage=2, ivec=0x0, data=0x7ffca80ea920, num_data=4) at enc_rc4.c:260
#4 0x00007fdf02790b1b in krb5_k_decrypt (context=context@entry=0x559ed623bd80, key=0x559ed62428f0, usage=usage@entry=2, cipher_state=cipher_state@entry=0x0, input=input@entry=0x559ed623c4f0, output=output@entry=0x7ffca80eaa20)
at decrypt.c:78
#5 0x00007fdf02790bed in krb5_c_decrypt (context=context@entry=0x559ed623bd80, keyblock=keyblock@entry=0x7ffca80eab98, usage=2, cipher_state=cipher_state@entry=0x0, input=input@entry=0x559ed623c4f0, output=output@entry=0x7ffca80eaa20)
at decrypt.c:98
#6 0x00007fdf029ea115 in krb5_decrypt_tkt_part (context=context@entry=0x559ed623bd80, srv_key=srv_key@entry=0x7ffca80eab98, ticket=0x559ed623c4e0) at decrypt_tk.c:56
#7 0x00007fdf02a07350 in try_one_entry (context=context@entry=0x559ed623bd80, ent=ent@entry=0x7ffca80eab80, keyblock_out=keyblock_out@entry=0x7ffca80eab40, req=<optimized out>) at rd_req_dec.c:296
#8 0x00007fdf02a07c76 in decrypt_ticket (keyblock_out=0x7ffca80eab40, keytab=0x559ed623c8d0, server=0x559ed6242820, req=0x559ed623c4a0, context=0x559ed623bd80) at rd_req_dec.c:423
#9 rd_req_decoded_opt (context=0x559ed623bd80, auth_context=auth_context@entry=0x7ffca80ead88, req=0x559ed623c4a0, server=0x559ed6242820, keytab=0x559ed623c8d0, ap_req_options=ap_req_options@entry=0x7ffca80ead50,
ticket=ticket@entry=0x0, check_valid_flag=check_valid_flag@entry=1) at rd_req_dec.c:504
#10 0x00007fdf02a088ca in krb5_rd_req_decoded (context=<optimized out>, auth_context=auth_context@entry=0x7ffca80ead88, req=<optimized out>, server=<optimized out>, keytab=<optimized out>,
ap_req_options=ap_req_options@entry=0x7ffca80ead50, ticket=ticket@entry=0x0) at rd_req_dec.c:808
#11 0x00007fdf02caa35c in kg_accept_krb5 (minor_status=minor_status@entry=0x559ed6228004, context_handle=context_handle@entry=0x559ed6227ff0, verifier_cred_handle=0x559ed623c740, input_token=<optimized out>, input_chan_bindings=0x0,
src_name=0x7ffca80eb1b8, mech_type=mech_type@entry=0x7ffca80eb1c8, output_token=output_token@entry=0x7ffca80eb2c0, ret_flags=ret_flags@entry=0x7ffca80eb1a4, time_rec=time_rec@entry=0x0,
delegated_cred_handle=delegated_cred_handle@entry=0x7ffca80eb1b0, exts=exts@entry=0x7ffca80eb100) at accept_sec_context.c:644
#12 0x00007fdf02cab83a in krb5_gss_accept_sec_context_ext (minor_status=0x559ed6228004, context_handle=0x559ed6227ff0, verifier_cred_handle=<optimized out>, input_token=<optimized out>, input_chan_bindings=<optimized out>,
src_name=<optimized out>, mech_type=mech_type@entry=0x7ffca80eb1c8, output_token=output_token@entry=0x7ffca80eb2c0, ret_flags=ret_flags@entry=0x7ffca80eb1a4, time_rec=time_rec@entry=0x0,
delegated_cred_handle=delegated_cred_handle@entry=0x7ffca80eb1b0, exts=exts@entry=0x7ffca80eb100) at accept_sec_context.c:1308
#13 0x00007fdf02cab999 in krb5_gss_accept_sec_context (minor_status=<optimized out>, context_handle=<optimized out>, verifier_cred_handle=<optimized out>, input_token=<optimized out>, input_chan_bindings=<optimized out>,
src_name=<optimized out>, mech_type=0x7ffca80eb1c8, output_token=0x7ffca80eb2c0, ret_flags=0x7ffca80eb1a4, time_rec=0x0, delegated_cred_handle=0x7ffca80eb1b0) at accept_sec_context.c:1337
#14 0x00007fdf02c99056 in gss_accept_sec_context (minor_status=minor_status@entry=0x559ed6228004, context_handle=context_handle@entry=0x559ed6228008, verifier_cred_handle=<optimized out>,
input_token_buffer=input_token_buffer@entry=0x7ffca80eb2b0, input_chan_bindings=input_chan_bindings@entry=0x0, src_name=src_name@entry=0x559ed6228028, mech_type=mech_type@entry=0x7ffca80eb270,
output_token=output_token@entry=0x7ffca80eb2c0, ret_flags=ret_flags@entry=0x7ffca80eb2a8, time_rec=time_rec@entry=0x0, d_cred=d_cred@entry=0x559ed6228030) at g_accept_sec_context.c:267
#15 0x0000559ed495c3a4 in ssh_gssapi_accept_ctx (ctx=0x559ed6228000, recv_tok=recv_tok@entry=0x7ffca80eb2b0, send_tok=send_tok@entry=0x7ffca80eb2c0, flags=flags@entry=0x7ffca80eb2a8) at gss-serv.c:209
#16 0x0000559ed4955fd8 in mm_answer_gss_accept_ctx (sock=7, m=0x7ffca80eb320) at monitor.c:1899
#17 0x0000559ed4957299 in monitor_read (pmonitor=pmonitor@entry=0x559ed62274d0, ent=0x559ed4bf8390 <mon_dispatch_proto20+336>, pent=pent@entry=0x7ffca80eb3c8) at monitor.c:566
#18 0x0000559ed4957a0c in monitor_child_preauth (_authctxt=_authctxt@entry=0x559ed6223560, pmonitor=0x559ed62274d0) at monitor.c:349
#19 0x0000559ed493c1f4 in privsep_preauth (authctxt=0x559ed6223560) at sshd.c:667
#20 main (ac=<optimized out>, av=<optimized out>) at sshd.c:2179
Version-Release number of selected component (if applicable):
-RHEL 7.6 - openssh-server-7.4p1-16.el7.x86_64
-Was working previously with RHEL 7.5.
-FIPS is enabled
-a 3rd party library is linked (One Idendity vas), but not in the bt
How reproducible:
always (on customer site)
Steps to Reproduce:
1.
2.
3.
Actual results:
sshd segfault
Expected results:
no segfault
Additional info:
I tried to analyse the issue in the case:
The issue is after the call of krb5int_hmac_keyblock as we can see in the backtrace:
8<---------- 8< ---------------- 8< ---------------- 8< --------
#0 0x0000000000000000 in ?? ()
#1 0x00007f38bfcca969 in krb5int_hmac_keyblock (hash=hash@entry=0x7f38bfecfda0 <krb5int_hash_md5>, keyblock=keyblock@entry=0x55e406465760, data=data@entry=0x7ffccf074d50, num_data=num_data@entry=1, output=output@entry=0x7ffccf074d40)
at hmac.c:145
#2 0x00007f38bfcc5518 in usage_key (hash=hash@entry=0x7f38bfecfda0 <krb5int_hash_md5>, session_keyblock=session_keyblock@entry=0x55e406465760, usage=usage@entry=2, out=<optimized out>, out=<optimized out>, enc=<optimized out>)
at enc_rc4.c:64
#3 0x00007f38bfcc5ae4 in krb5int_arcfour_decrypt (ktp=<optimized out>, key=0x55e406465760, usage=2, ivec=0x0, data=0x7ffccf074ea0, num_data=4) at enc_rc4.c:260
#4 0x00007f38bfcc1b1b in krb5_k_decrypt (context=context@entry=0x55e40645ddf0, key=0x55e406465760, usage=usage@entry=2, cipher_state=cipher_state@entry=0x0, input=input@entry=0x55e40645e540, output=output@entry=0x7ffccf074fa0)
at decrypt.c:78
#5 0x00007f38bfcc1bed in krb5_c_decrypt (context=context@entry=0x55e40645ddf0, keyblock=keyblock@entry=0x7ffccf075118, usage=2, cipher_state=cipher_state@entry=0x0, input=input@entry=0x55e40645e540, output=output@entry=0x7ffccf074fa0)
at decrypt.c:98
8<---------- 8< ---------------- 8< ---------------- 8< --------
More precisely, it is in: HMAC_Update(ctx, (uint8_t *)iov->data.data, iov->data.length);
that calls then
8<---------- 8< ---------------- 8< ---------------- 8< --------
int EVP_DigestUpdate(EVP_MD_CTX *ctx, const void *data, size_t count)
{
#ifdef OPENSSL_FIPS
FIPS_selftest_check();
#endif
return ctx->update(ctx, data, count);
}
8<---------- 8< ---------------- 8< ---------------- 8< --------
- If we look in details (and in asm):
8<---------- 8< ---------------- 8< ---------------- 8< --------
(gdb) disassemble /m HMAC_Update
Dump of assembler code for function HMAC_Update:
167 {
168 if (!ctx->md)
0x00007f38c136df80 <+0>: cmpq $0x0,(%rdi)
0x00007f38c136df84 <+4>: je 0x7f38c136df90 <HMAC_Update+16>
169 return 0;
170
171 return EVP_DigestUpdate(&ctx->md_ctx, data, len);
0x00007f38c136df86 <+6>: add $0x8,%rdi
0x00007f38c136df8a <+10>: jmpq 0x7f38c1403060 <EVP_DigestUpdate>
0x00007f38c136df8f <+15>: nop
172 }
0x00007f38c136df90 <+16>: xor %eax,%eax
0x00007f38c136df92 <+18>: retq
0x00007f38c136df93: nopl (%rax)
0x00007f38c136df96: nopw %cs:0x0(%rax,%rax,1)
End of assembler dump.
(gdb) disassemble /m EVP_DigestUpdate
Dump of assembler code for function EVP_DigestUpdate:
292 {
0x00007f38c1403060 <+0>: push %r12
0x00007f38c1403062 <+2>: mov %rdx,%r12
0x00007f38c1403065 <+5>: push %rbp
0x00007f38c1403066 <+6>: mov %rsi,%rbp
0x00007f38c1403069 <+9>: push %rbx
0x00007f38c140306a <+10>: mov %rdi,%rbx
293 #ifdef OPENSSL_FIPS
294 FIPS_selftest_check();
0x00007f38c140306d <+13>: callq 0x7f38c1477ec0 <FIPS_selftest_check>
295 #endif
296 return ctx->update(ctx, data, count);
0x00007f38c1403072 <+18>: mov 0x28(%rbx),%rax
0x00007f38c1403076 <+22>: mov %rbx,%rdi
0x00007f38c1403079 <+25>: mov %rbp,%rsi
0x00007f38c140307e <+30>: mov %r12,%rdx
0x00007f38c1403083 <+35>: jmpq *%rax <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
0x00007f38c1403085: nop
0x00007f38c1403086: nopw %cs:0x0(%rax,%rax,1)
297 }
0x00007f38c140307c <+28>: pop %rbx
0x00007f38c140307d <+29>: pop %rbp
0x00007f38c1403081 <+33>: pop %r12
End of assembler dump.
8<---------- 8< ---------------- 8< ---------------- 8< --------
The issue is that the program tries to jump to the address pointed by the register %rax, but this one is null:
- rax 0x0 0
-> This is the reason of the segfault.
- Now, the question is why this value.
This comes from the variable 'HMAC_CTX c' initialized in krb5int_hmac_keyblock:
---
HMAC_CTX_init(&c);
HMAC_Init(&c, keyblock->contents, keyblock->length, map_digest(hash));
---
At this moment, this is the value of ctx:
> p *ctx
$63 = {md = 0x7f38c1721a80 <md5_md>, md_ctx = {digest = 0x0, engine = 0x0, flags = 0, md_data = 0x0, pctx = 0x0, update = 0x0}, i_ctx = {digest = 0x0, engine = 0x0, flags = 0, md_data = 0x0, pctx = 0x0, update = 0x0}, o_ctx = {
digest = 0x0, engine = 0x0, flags = 0, md_data = 0x0, pctx = 0x0, update = 0x0}, key_length = 0, key = '\000' <repeats 127 times>}
>> md_ctx is null. And it is md_ctx->update that will be used in EVP_DigestUpdate.
- The return value of HMAC_CTX_init and HMAC_Init are not checked.
The variable should be correctly initialized in HMAC_Init:
8<---------- 8< ---------------- 8< ---------------- 8< --------
int HMAC_Init(HMAC_CTX *ctx, const void *key, int len, const EVP_MD *md)
{
if (key && md)
HMAC_CTX_init(ctx);
return HMAC_Init_ex(ctx, key, len, md, NULL);
}
int HMAC_Init_ex(HMAC_CTX *ctx, const void *key, int len,
const EVP_MD *md, ENGINE *impl)
{
int i, j, reset = 0;
unsigned char pad[HMAC_MAX_MD_CBLOCK];
#ifdef OPENSSL_FIPS
/* If FIPS mode switch to approved implementation if possible */
if (FIPS_mode()) {
const EVP_MD *fipsmd;
if (md) {
fipsmd = FIPS_get_digestbynid(EVP_MD_type(md));
if (fipsmd)
md = fipsmd;
}
}
if (FIPS_mode()) {
/* If we have an ENGINE need to allow non FIPS */
if ((impl || ctx->i_ctx.engine)
&& !(ctx->i_ctx.flags & EVP_CIPH_FLAG_NON_FIPS_ALLOW)) {
EVPerr(EVP_F_HMAC_INIT_EX, EVP_R_DISABLED_FOR_FIPS);
return 0;
}
}
#endif
/* If we are changing MD then we must have a key */
if (md != NULL && md != ctx->md && (key == NULL || len < 0))
return 0;
if (md != NULL) {
reset = 1;
ctx->md = md;
} else if (ctx->md) {
md = ctx->md;
} else {
return 0;
}
if (key != NULL) {
#ifdef OPENSSL_FIPS
if (FIPS_mode() && !(md->flags & EVP_MD_FLAG_FIPS)
&& (!(ctx->md_ctx.flags & EVP_MD_CTX_FLAG_NON_FIPS_ALLOW)
|| !(ctx->i_ctx.flags & EVP_MD_CTX_FLAG_NON_FIPS_ALLOW)
|| !(ctx->o_ctx.flags & EVP_MD_CTX_FLAG_NON_FIPS_ALLOW)))
goto err;
#endif
reset = 1;
j = EVP_MD_block_size(md);
OPENSSL_assert(j <= (int)sizeof(ctx->key));
if (j < len) {
if (!EVP_DigestInit_ex(&ctx->md_ctx, md, impl))
goto err;
if (!EVP_DigestUpdate(&ctx->md_ctx, key, len))
goto err;
if (!EVP_DigestFinal_ex(&(ctx->md_ctx), ctx->key,
&ctx->key_length))
goto err;
} else {
...
8<---------- 8< ---------------- 8< ---------------- 8< --------
We can see that we can exit (goto err: return 0;) without initializing ctx->md_ctx (and in the condition, this variable is used in the case of FIPS use, which is your case).
As the return value of HMAC_Init is not used, the program continues with a partially initialized variable.
Description of problem: The customer gets segfaults in sshd: Nov 19 09:56:59 $SERVERNAME kernel: sshd[26319]: segfault at 0 ip (null) sp 00007fffc8f705b8 error 14 in sshd[559746ef1000+c8000] The backtrace is reporting krb: Core was generated by `sshd: host [priv]'. Program terminated with signal 11, Segmentation fault. #0 0x0000000000000000 in ?? () (gdb) bt #0 0x0000000000000000 in ?? () #1 0x00007fdf02799969 in krb5int_hmac_keyblock (hash=hash@entry=0x7fdf0299eda0 <krb5int_hash_md5>, keyblock=keyblock@entry=0x559ed62428f0, data=data@entry=0x7ffca80ea7d0, num_data=num_data@entry=1, output=output@entry=0x7ffca80ea7c0) at hmac.c:145 #2 0x00007fdf02794518 in usage_key (hash=hash@entry=0x7fdf0299eda0 <krb5int_hash_md5>, session_keyblock=session_keyblock@entry=0x559ed62428f0, usage=usage@entry=2, out=<optimized out>, out=<optimized out>, enc=<optimized out>) at enc_rc4.c:64 #3 0x00007fdf02794ae4 in krb5int_arcfour_decrypt (ktp=<optimized out>, key=0x559ed62428f0, usage=2, ivec=0x0, data=0x7ffca80ea920, num_data=4) at enc_rc4.c:260 #4 0x00007fdf02790b1b in krb5_k_decrypt (context=context@entry=0x559ed623bd80, key=0x559ed62428f0, usage=usage@entry=2, cipher_state=cipher_state@entry=0x0, input=input@entry=0x559ed623c4f0, output=output@entry=0x7ffca80eaa20) at decrypt.c:78 #5 0x00007fdf02790bed in krb5_c_decrypt (context=context@entry=0x559ed623bd80, keyblock=keyblock@entry=0x7ffca80eab98, usage=2, cipher_state=cipher_state@entry=0x0, input=input@entry=0x559ed623c4f0, output=output@entry=0x7ffca80eaa20) at decrypt.c:98 #6 0x00007fdf029ea115 in krb5_decrypt_tkt_part (context=context@entry=0x559ed623bd80, srv_key=srv_key@entry=0x7ffca80eab98, ticket=0x559ed623c4e0) at decrypt_tk.c:56 #7 0x00007fdf02a07350 in try_one_entry (context=context@entry=0x559ed623bd80, ent=ent@entry=0x7ffca80eab80, keyblock_out=keyblock_out@entry=0x7ffca80eab40, req=<optimized out>) at rd_req_dec.c:296 #8 0x00007fdf02a07c76 in decrypt_ticket (keyblock_out=0x7ffca80eab40, keytab=0x559ed623c8d0, server=0x559ed6242820, req=0x559ed623c4a0, context=0x559ed623bd80) at rd_req_dec.c:423 #9 rd_req_decoded_opt (context=0x559ed623bd80, auth_context=auth_context@entry=0x7ffca80ead88, req=0x559ed623c4a0, server=0x559ed6242820, keytab=0x559ed623c8d0, ap_req_options=ap_req_options@entry=0x7ffca80ead50, ticket=ticket@entry=0x0, check_valid_flag=check_valid_flag@entry=1) at rd_req_dec.c:504 #10 0x00007fdf02a088ca in krb5_rd_req_decoded (context=<optimized out>, auth_context=auth_context@entry=0x7ffca80ead88, req=<optimized out>, server=<optimized out>, keytab=<optimized out>, ap_req_options=ap_req_options@entry=0x7ffca80ead50, ticket=ticket@entry=0x0) at rd_req_dec.c:808 #11 0x00007fdf02caa35c in kg_accept_krb5 (minor_status=minor_status@entry=0x559ed6228004, context_handle=context_handle@entry=0x559ed6227ff0, verifier_cred_handle=0x559ed623c740, input_token=<optimized out>, input_chan_bindings=0x0, src_name=0x7ffca80eb1b8, mech_type=mech_type@entry=0x7ffca80eb1c8, output_token=output_token@entry=0x7ffca80eb2c0, ret_flags=ret_flags@entry=0x7ffca80eb1a4, time_rec=time_rec@entry=0x0, delegated_cred_handle=delegated_cred_handle@entry=0x7ffca80eb1b0, exts=exts@entry=0x7ffca80eb100) at accept_sec_context.c:644 #12 0x00007fdf02cab83a in krb5_gss_accept_sec_context_ext (minor_status=0x559ed6228004, context_handle=0x559ed6227ff0, verifier_cred_handle=<optimized out>, input_token=<optimized out>, input_chan_bindings=<optimized out>, src_name=<optimized out>, mech_type=mech_type@entry=0x7ffca80eb1c8, output_token=output_token@entry=0x7ffca80eb2c0, ret_flags=ret_flags@entry=0x7ffca80eb1a4, time_rec=time_rec@entry=0x0, delegated_cred_handle=delegated_cred_handle@entry=0x7ffca80eb1b0, exts=exts@entry=0x7ffca80eb100) at accept_sec_context.c:1308 #13 0x00007fdf02cab999 in krb5_gss_accept_sec_context (minor_status=<optimized out>, context_handle=<optimized out>, verifier_cred_handle=<optimized out>, input_token=<optimized out>, input_chan_bindings=<optimized out>, src_name=<optimized out>, mech_type=0x7ffca80eb1c8, output_token=0x7ffca80eb2c0, ret_flags=0x7ffca80eb1a4, time_rec=0x0, delegated_cred_handle=0x7ffca80eb1b0) at accept_sec_context.c:1337 #14 0x00007fdf02c99056 in gss_accept_sec_context (minor_status=minor_status@entry=0x559ed6228004, context_handle=context_handle@entry=0x559ed6228008, verifier_cred_handle=<optimized out>, input_token_buffer=input_token_buffer@entry=0x7ffca80eb2b0, input_chan_bindings=input_chan_bindings@entry=0x0, src_name=src_name@entry=0x559ed6228028, mech_type=mech_type@entry=0x7ffca80eb270, output_token=output_token@entry=0x7ffca80eb2c0, ret_flags=ret_flags@entry=0x7ffca80eb2a8, time_rec=time_rec@entry=0x0, d_cred=d_cred@entry=0x559ed6228030) at g_accept_sec_context.c:267 #15 0x0000559ed495c3a4 in ssh_gssapi_accept_ctx (ctx=0x559ed6228000, recv_tok=recv_tok@entry=0x7ffca80eb2b0, send_tok=send_tok@entry=0x7ffca80eb2c0, flags=flags@entry=0x7ffca80eb2a8) at gss-serv.c:209 #16 0x0000559ed4955fd8 in mm_answer_gss_accept_ctx (sock=7, m=0x7ffca80eb320) at monitor.c:1899 #17 0x0000559ed4957299 in monitor_read (pmonitor=pmonitor@entry=0x559ed62274d0, ent=0x559ed4bf8390 <mon_dispatch_proto20+336>, pent=pent@entry=0x7ffca80eb3c8) at monitor.c:566 #18 0x0000559ed4957a0c in monitor_child_preauth (_authctxt=_authctxt@entry=0x559ed6223560, pmonitor=0x559ed62274d0) at monitor.c:349 #19 0x0000559ed493c1f4 in privsep_preauth (authctxt=0x559ed6223560) at sshd.c:667 #20 main (ac=<optimized out>, av=<optimized out>) at sshd.c:2179 Version-Release number of selected component (if applicable): -RHEL 7.6 - openssh-server-7.4p1-16.el7.x86_64 -Was working previously with RHEL 7.5. -FIPS is enabled -a 3rd party library is linked (One Idendity vas), but not in the bt How reproducible: always (on customer site) Steps to Reproduce: 1. 2. 3. Actual results: sshd segfault Expected results: no segfault Additional info: I tried to analyse the issue in the case: The issue is after the call of krb5int_hmac_keyblock as we can see in the backtrace: 8<---------- 8< ---------------- 8< ---------------- 8< -------- #0 0x0000000000000000 in ?? () #1 0x00007f38bfcca969 in krb5int_hmac_keyblock (hash=hash@entry=0x7f38bfecfda0 <krb5int_hash_md5>, keyblock=keyblock@entry=0x55e406465760, data=data@entry=0x7ffccf074d50, num_data=num_data@entry=1, output=output@entry=0x7ffccf074d40) at hmac.c:145 #2 0x00007f38bfcc5518 in usage_key (hash=hash@entry=0x7f38bfecfda0 <krb5int_hash_md5>, session_keyblock=session_keyblock@entry=0x55e406465760, usage=usage@entry=2, out=<optimized out>, out=<optimized out>, enc=<optimized out>) at enc_rc4.c:64 #3 0x00007f38bfcc5ae4 in krb5int_arcfour_decrypt (ktp=<optimized out>, key=0x55e406465760, usage=2, ivec=0x0, data=0x7ffccf074ea0, num_data=4) at enc_rc4.c:260 #4 0x00007f38bfcc1b1b in krb5_k_decrypt (context=context@entry=0x55e40645ddf0, key=0x55e406465760, usage=usage@entry=2, cipher_state=cipher_state@entry=0x0, input=input@entry=0x55e40645e540, output=output@entry=0x7ffccf074fa0) at decrypt.c:78 #5 0x00007f38bfcc1bed in krb5_c_decrypt (context=context@entry=0x55e40645ddf0, keyblock=keyblock@entry=0x7ffccf075118, usage=2, cipher_state=cipher_state@entry=0x0, input=input@entry=0x55e40645e540, output=output@entry=0x7ffccf074fa0) at decrypt.c:98 8<---------- 8< ---------------- 8< ---------------- 8< -------- More precisely, it is in: HMAC_Update(ctx, (uint8_t *)iov->data.data, iov->data.length); that calls then 8<---------- 8< ---------------- 8< ---------------- 8< -------- int EVP_DigestUpdate(EVP_MD_CTX *ctx, const void *data, size_t count) { #ifdef OPENSSL_FIPS FIPS_selftest_check(); #endif return ctx->update(ctx, data, count); } 8<---------- 8< ---------------- 8< ---------------- 8< -------- - If we look in details (and in asm): 8<---------- 8< ---------------- 8< ---------------- 8< -------- (gdb) disassemble /m HMAC_Update Dump of assembler code for function HMAC_Update: 167 { 168 if (!ctx->md) 0x00007f38c136df80 <+0>: cmpq $0x0,(%rdi) 0x00007f38c136df84 <+4>: je 0x7f38c136df90 <HMAC_Update+16> 169 return 0; 170 171 return EVP_DigestUpdate(&ctx->md_ctx, data, len); 0x00007f38c136df86 <+6>: add $0x8,%rdi 0x00007f38c136df8a <+10>: jmpq 0x7f38c1403060 <EVP_DigestUpdate> 0x00007f38c136df8f <+15>: nop 172 } 0x00007f38c136df90 <+16>: xor %eax,%eax 0x00007f38c136df92 <+18>: retq 0x00007f38c136df93: nopl (%rax) 0x00007f38c136df96: nopw %cs:0x0(%rax,%rax,1) End of assembler dump. (gdb) disassemble /m EVP_DigestUpdate Dump of assembler code for function EVP_DigestUpdate: 292 { 0x00007f38c1403060 <+0>: push %r12 0x00007f38c1403062 <+2>: mov %rdx,%r12 0x00007f38c1403065 <+5>: push %rbp 0x00007f38c1403066 <+6>: mov %rsi,%rbp 0x00007f38c1403069 <+9>: push %rbx 0x00007f38c140306a <+10>: mov %rdi,%rbx 293 #ifdef OPENSSL_FIPS 294 FIPS_selftest_check(); 0x00007f38c140306d <+13>: callq 0x7f38c1477ec0 <FIPS_selftest_check> 295 #endif 296 return ctx->update(ctx, data, count); 0x00007f38c1403072 <+18>: mov 0x28(%rbx),%rax 0x00007f38c1403076 <+22>: mov %rbx,%rdi 0x00007f38c1403079 <+25>: mov %rbp,%rsi 0x00007f38c140307e <+30>: mov %r12,%rdx 0x00007f38c1403083 <+35>: jmpq *%rax <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< 0x00007f38c1403085: nop 0x00007f38c1403086: nopw %cs:0x0(%rax,%rax,1) 297 } 0x00007f38c140307c <+28>: pop %rbx 0x00007f38c140307d <+29>: pop %rbp 0x00007f38c1403081 <+33>: pop %r12 End of assembler dump. 8<---------- 8< ---------------- 8< ---------------- 8< -------- The issue is that the program tries to jump to the address pointed by the register %rax, but this one is null: - rax 0x0 0 -> This is the reason of the segfault. - Now, the question is why this value. This comes from the variable 'HMAC_CTX c' initialized in krb5int_hmac_keyblock: --- HMAC_CTX_init(&c); HMAC_Init(&c, keyblock->contents, keyblock->length, map_digest(hash)); --- At this moment, this is the value of ctx: > p *ctx $63 = {md = 0x7f38c1721a80 <md5_md>, md_ctx = {digest = 0x0, engine = 0x0, flags = 0, md_data = 0x0, pctx = 0x0, update = 0x0}, i_ctx = {digest = 0x0, engine = 0x0, flags = 0, md_data = 0x0, pctx = 0x0, update = 0x0}, o_ctx = { digest = 0x0, engine = 0x0, flags = 0, md_data = 0x0, pctx = 0x0, update = 0x0}, key_length = 0, key = '\000' <repeats 127 times>} >> md_ctx is null. And it is md_ctx->update that will be used in EVP_DigestUpdate. - The return value of HMAC_CTX_init and HMAC_Init are not checked. The variable should be correctly initialized in HMAC_Init: 8<---------- 8< ---------------- 8< ---------------- 8< -------- int HMAC_Init(HMAC_CTX *ctx, const void *key, int len, const EVP_MD *md) { if (key && md) HMAC_CTX_init(ctx); return HMAC_Init_ex(ctx, key, len, md, NULL); } int HMAC_Init_ex(HMAC_CTX *ctx, const void *key, int len, const EVP_MD *md, ENGINE *impl) { int i, j, reset = 0; unsigned char pad[HMAC_MAX_MD_CBLOCK]; #ifdef OPENSSL_FIPS /* If FIPS mode switch to approved implementation if possible */ if (FIPS_mode()) { const EVP_MD *fipsmd; if (md) { fipsmd = FIPS_get_digestbynid(EVP_MD_type(md)); if (fipsmd) md = fipsmd; } } if (FIPS_mode()) { /* If we have an ENGINE need to allow non FIPS */ if ((impl || ctx->i_ctx.engine) && !(ctx->i_ctx.flags & EVP_CIPH_FLAG_NON_FIPS_ALLOW)) { EVPerr(EVP_F_HMAC_INIT_EX, EVP_R_DISABLED_FOR_FIPS); return 0; } } #endif /* If we are changing MD then we must have a key */ if (md != NULL && md != ctx->md && (key == NULL || len < 0)) return 0; if (md != NULL) { reset = 1; ctx->md = md; } else if (ctx->md) { md = ctx->md; } else { return 0; } if (key != NULL) { #ifdef OPENSSL_FIPS if (FIPS_mode() && !(md->flags & EVP_MD_FLAG_FIPS) && (!(ctx->md_ctx.flags & EVP_MD_CTX_FLAG_NON_FIPS_ALLOW) || !(ctx->i_ctx.flags & EVP_MD_CTX_FLAG_NON_FIPS_ALLOW) || !(ctx->o_ctx.flags & EVP_MD_CTX_FLAG_NON_FIPS_ALLOW))) goto err; #endif reset = 1; j = EVP_MD_block_size(md); OPENSSL_assert(j <= (int)sizeof(ctx->key)); if (j < len) { if (!EVP_DigestInit_ex(&ctx->md_ctx, md, impl)) goto err; if (!EVP_DigestUpdate(&ctx->md_ctx, key, len)) goto err; if (!EVP_DigestFinal_ex(&(ctx->md_ctx), ctx->key, &ctx->key_length)) goto err; } else { ... 8<---------- 8< ---------------- 8< ---------------- 8< -------- We can see that we can exit (goto err: return 0;) without initializing ctx->md_ctx (and in the condition, this variable is used in the case of FIPS use, which is your case). As the return value of HMAC_Init is not used, the program continues with a partially initialized variable.