Bug 2282825

Description of problem:

nvmeof REST APIs don't work when mTLS enabled

>>> when mTLS was disabled, it works
=======================
pavangovindraj@Pavans-MacBook-Pro cephci %curl -i -k --location -X GET 'https://10.0.208.141:8443/api/nvmeof/gateway' -H 'Accept: application/vnd.ceph.api.v1.0+json' -H 'Content-Type: application/json' -H "Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJjZXBoLWRhc2hib2FyZCIsImp0aSI6IjZjN2I4MTlkLTJlN2QtNDczZS05M2RjLWZjYWM2MDMyODRmZCIsImV4cCI6MTcxNjM0NDc3NiwiaWF0IjoxNzE2MzE1OTc2LCJ1c2VybmFtZSI6ImFkbWluIn0.ZVefXDEC5MHp_hBQ6WI3jHG8J79_qb4G6rlHxI2cyJ4"
HTTP/1.1 200 OK
Content-Type: application/vnd.ceph.api.v1.0+json
Server: Ceph-Dashboard
Date: Tue, 21 May 2024 18:26:51 GMT
Content-Security-Policy: frame-ancestors 'self';
X-Content-Type-Options: nosniff
Strict-Transport-Security: max-age=63072000; includeSubDomains; preload
Vary: Accept-Encoding
Content-Length: 213

{"cli_version": "", "version": "1.2.7", "name": "client.nvmeof.rbd.ceph-pavan-ceph-2-ucpuof-node2.uclswc", "group": "", "addr": "10.0.210.196", "port": "5500", "load_balancing_group": 2, "spdk_version": "24.01.1"}%  


>>>> nvmeof related REST calls fail when mTLS enabled
======================================================
[root@ceph-pavan-ceph-2-s606ie-node4 mtls_new]# podman run --add-host=installer_node:10.0.210.111 -v /root/mtls_new/server.crt:/root/server.crt:z -v /root/mtls_new/client.crt:/root/client.crt:z -v /root/mtls_new/client.key:/root/client.key:z -it --rm registry-proxy.engineering.redhat.com/rh-osbs/ceph-nvmeof-cli:1.2.9-2 --server-address installer_node --client-key /root/client.key --client-cert /root/client.crt --server-cert /root/server.crt gw info
Enable server auth since both --client-key and --client-cert are provided
CLI's version: 1.2.9
Gateway's version: 1.2.9
Gateway's name: client.nvmeof.rbd.ceph-pavan-ceph-2-s606ie-node1-installer.ifdpzh
Gateway's host name: ceph-pavan-ceph-2-s606ie-node1-installer
Gateway's load balancing group: 1
Gateway's address: 10.0.210.111
Gateway's port: 5500
SPDK version: 24.01.1
[root@ceph-pavan-ceph-2-s606ie-node4 mtls_new]#
but REST APIs don't work (yet)
pavangovindraj@Pavans-MacBook-Pro cephci %curl -i -k --location -X POST 'https://10.0.210.111:8443/api/auth' -H 'Accept: application/vnd.ceph.api.v1.0+json' -H 'Content-Type: application/json' --data '{"password": "admin123", "username": "admin"}'
HTTP/1.1 201 Created
Content-Type: application/vnd.ceph.api.v1.0+json
Server: Ceph-Dashboard
Date: Wed, 22 May 2024 11:45:22 GMT
Content-Security-Policy: frame-ancestors 'self';
X-Content-Type-Options: nosniff
Strict-Transport-Security: max-age=63072000; includeSubDomains; preload
Vary: Accept-Encoding
Content-Length: 1288
Set-Cookie: token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJjZXBoLWRhc2hib2FyZCIsImp0aSI6IjQyM2NiNmM1LWQwOWEtNGY0Ny1iNTQwLTFlMDVjYTBiZmYzMCIsImV4cCI6MTcxNjQwNzEyMywiaWF0IjoxNzE2Mzc4MzIzLCJ1c2VybmFtZSI6ImFkbWluIn0.s3gWpQmtpwLVj-_pRihFDh88x1FiexEI76-2T9u9XUs; HttpOnly; Path=/; SameSite=Strict; Secure

{"token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJjZXBoLWRhc2hib2FyZCIsImp0aSI6IjQyM2NiNmM1LWQwOWEtNGY0Ny1iNTQwLTFlMDVjYTBiZmYzMCIsImV4cCI6MTcxNjQwNzEyMywiaWF0IjoxNzE2Mzc4MzIzLCJ1c2VybmFtZSI6ImFkbWluIn0.s3gWpQmtpwLVj-_pRihFDh88x1FiexEI76-2T9u9XUs", "username": "admin", "permissions": {"cephfs": ["create", "delete", "read", "update"], "config-opt": ["create", "delete", "read", "update"], "dashboard-settings": ["create", "delete", "read", "update"], "grafana": ["create", "delete", "read", "update"], "hosts": ["create", "delete", "read", "update"], "iscsi": ["create", "delete", "read", "update"], "log": ["create", "delete", "read", "update"], "manager": ["create", "delete", "read", "update"], "monitor": ["create", "delete", "read", "update"], "nfs-ganesha": ["create", "delete", "read", "update"], "nvme-of": ["create", "delete", "read", "update"], "osd": ["create", "delete", "read", "update"], "pool": ["create", "delete", "read", "update"], "prometheus": ["create", "delete", "read", "update"], "rbd-image": ["create", "delete", "read", "update"], "rbd-mirroring": ["create", "delete", "read", "update"], "rgw": ["create", "delete", "read", "update"], "user": ["create", "delete", "read", "update"]}, "pwdExpirationDate": null, "sso": false, "pwdUpdateRequired": false}%                                                                                                                                                                                     

pavangovindraj@Pavans-MacBook-Pro cephci %curl -i -k --location -X GET 'https://10.0.210.111:8443/api/nvmeof/gateway' -H 'Accept: application/vnd.ceph.api.v1.0+json' -H 'Content-Type: application/json' -H "Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJjZXBoLWRhc2hib2FyZCIsImp0aSI6IjQyM2NiNmM1LWQwOWEtNGY0Ny1iNTQwLTFlMDVjYTBiZmYzMCIsImV4cCI6MTcxNjQwNzEyMywiaWF0IjoxNzE2Mzc4MzIzLCJ1c2VybmFtZSI6ImFkbWluIn0.s3gWpQmtpwLVj-_pRihFDh88x1FiexEI76-2T9u9XUs"
HTTP/1.1 504 Gateway Timeout
Content-Type: application/json
Server: Ceph-Dashboard
Date: Wed, 22 May 2024 11:45:58 GMT
Content-Security-Policy: frame-ancestors 'self';
X-Content-Type-Options: nosniff
Strict-Transport-Security: max-age=63072000; includeSubDomains; preload
Vary: Accept-Encoding
Content-Length: 105

{"detail": "failed to connect to all addresses", "code": "StatusCode.UNAVAILABLE", "component": "nvmeof"}%                                                                                                        
pavangovindraj@Pavans-MacBook-Pro cephci %pwd
/Users/pavangovindraj/workspace/pavan_cephci/cephci


Version-Release number of selected component (if applicable):
>>>versions
============
[root@ceph-pavan-ceph-2-s606ie-node4 ~]# ceph versions
{
    "mon": {
        "ceph version 18.2.1-188.el9cp (b1ae9c989e2f41dcfec0e680c11d1d9465b1db0e) reef (stable)": 3
    },
    "mgr": {
        "ceph version 18.2.1-188.el9cp (b1ae9c989e2f41dcfec0e680c11d1d9465b1db0e) reef (stable)": 2
    },
    "osd": {
        "ceph version 18.2.1-188.el9cp (b1ae9c989e2f41dcfec0e680c11d1d9465b1db0e) reef (stable)": 9
    },
    "rgw": {
        "ceph version 18.2.1-188.el9cp (b1ae9c989e2f41dcfec0e680c11d1d9465b1db0e) reef (stable)": 1
    },
    "overall": {
        "ceph version 18.2.1-188.el9cp (b1ae9c989e2f41dcfec0e680c11d1d9465b1db0e) reef (stable)": 15
    }
}
[root@ceph-pavan-ceph-2-s606ie-node4 ~]# ceph orch ls
NAME                       PORTS             RUNNING  REFRESHED  AGE  PLACEMENT                                                                                               
mgr                                              2/2  2m ago     19h  label:mgr                                                                                               
mon                                              3/3  2m ago     19h  label:mon                                                                                               
node-proxy                                       0/0  -          19h  *                                                                                                       
nvmeof.rbd                 ?:4420,5500,8009      3/3  2m ago     17h  ceph-pavan-ceph-2-s606ie-node1-installer;ceph-pavan-ceph-2-s606ie-node2;ceph-pavan-ceph-2-s606ie-node3  
osd.all-available-devices                          9  2m ago     19h  *                                                                                                       
rgw.rgw.1                  ?:80                  1/1  2m ago     19h  label:rgw                                                                                               
[root@ceph-pavan-ceph-2-s606ie-node4 ~]#

How reproducible: always


Steps to Reproduce:
1. Enable mTLS as per doc https://ibmdocs-test.dcs.ibm.com/docs/en/storage-ceph/7.1?topic=gateway-configuring-mtls-authentication

2. Try REST APIs as posted above

3. Observer failures


Actual results: nvmeof REST APIs don't work when mTLS enabled


Expected results:  REST API of nvmeof should be 2XX status


Additional info: NA