Description of problem:
please include ceph fsid in the cephadm-root ca certificate
If we have two or more ceph clusters(pri and sec) and rgw deployed on both sites. and if we want to trust both sites ca certificates on a client node, currently only one site rgw request is successful without any ssl cert issues. may be because of naming conflict in the ca cert subject


log snippet:

[root@ceph-pri-hsm-cephadm-h0a759-node6 anchors]# ls
RH-IT-Root-CA.crt  ceph-qe-ca.pem  cephqe-ca.pem  pri-site-cephadm-root-ca.crt  sec-site-cephadm-root-ca.crt
[root@ceph-pri-hsm-cephadm-h0a759-node6 anchors]# 
[root@ceph-pri-hsm-cephadm-h0a759-node5 ~]# #pri
[root@ceph-pri-hsm-cephadm-h0a759-node5 ~]# curl https://10.0.65.88:443
<?xml version="1.0" encoding="UTF-8"?><ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Owner><ID>anonymous</ID></Owner><Buckets></Buckets></ListAllMyBucketsResult>[root@ceph-pri-hsm-cephadm-h0a759-node5 ~]# 
[root@ceph-pri-hsm-cephadm-h0a759-node5 ~]# 
[root@ceph-pri-hsm-cephadm-h0a759-node5 ~]# #sec
[root@ceph-pri-hsm-cephadm-h0a759-node5 ~]# curl https://10.0.67.126:443
curl: (35) error:0200008A:rsa routines::invalid padding
[root@ceph-pri-hsm-cephadm-h0a759-node5 ~]#
[root@ceph-pri-hsm-cephadm-h0a759-node6 ~]# openssl x509 -in pri-site-cephadm-root-ca.crt  -noout  -text
Certificate:
    Data:
        Version: 3 (0x2)
        Serial Number:
            64:97:43:d6:9b:ed:a5:6c:08:05:d9:3d:5a:04:25:b4:e3:2c:b2:c1
        Signature Algorithm: sha256WithRSAEncryption
        Issuer: CN = cephadm-root
        Validity
            Not Before: Oct 30 14:37:04 2024 GMT
            Not After : Oct 31 14:37:04 2034 GMT
        Subject: CN = cephadm-root
        Subject Public Key Info:
            Public Key Algorithm: rsaEncryption
                Public-Key: (4096 bit)
                Modulus:
                    00:99:89:0d:54:bd:eb:5c:06:e5:ac:ad:76:20:45:
                    dc:7a:6e:31:5b:c2:a2:dd:bc:92:18:37:31:7e:df:
                    6f:49:20:d1:34:24:c7:51:8b:7c:2c:b7:90:f8:4a:
                    0e:56:64:06:6c:3c:05:db:fc:f7:16:60:bc:f8:c7:
                    64:83:c7:41:ff:b7:34:ce:98:f0:d5:10:24:7a:03:
                    b2:81:90:b1:1f:15:a4:fe:84:f6:e7:06:5c:af:84:
                    a8:80:3c:d2:15:b4:fb:74:f5:f7:3f:71:fe:3f:13:
                    1a:fd:bc:ef:1f:bb:e0:66:39:13:81:3f:78:b1:65:
                    ee:df:e2:e5:e5:86:e2:0f:f0:cc:12:68:2d:cb:6f:
                    94:3b:e6:50:39:64:01:f6:ef:6c:f1:df:3e:b6:a1:
                    ae:69:e0:20:01:d0:7b:d0:3f:30:8b:fc:f8:93:eb:
                    bb:ea:ce:3f:eb:de:e8:57:3c:9f:17:b1:a3:f6:e0:
                    9a:70:22:54:22:cf:17:5d:49:08:92:a9:45:1b:15:
                    8c:25:af:c4:35:21:d0:75:df:d4:9f:e9:d6:0f:28:
                    8d:d0:a0:82:e5:bf:7c:10:ca:ee:96:fa:ab:95:b7:
                    66:98:eb:1f:51:01:43:20:62:3e:b3:8f:d2:f7:43:
                    d9:5e:cb:bc:92:30:a3:fa:c0:c1:f8:0d:77:ef:5b:
                    9a:79:66:e7:ea:30:b9:c2:36:fa:fa:9d:47:c5:c0:
                    a7:ab:02:54:3b:77:89:02:4e:87:02:f0:f4:38:66:
                    fa:ad:a6:95:74:de:70:98:5c:45:f2:b4:3f:1a:19:
                    ae:93:65:07:ca:ea:63:0f:98:9e:a2:dc:88:f2:c6:
                    9d:ba:07:2d:c8:62:f0:c7:cf:f9:04:54:83:ed:20:
                    f0:bc:fd:b0:a9:b2:21:18:3b:fd:ca:aa:53:50:7a:
                    59:a9:92:e2:93:54:ce:aa:fa:d1:ca:d7:73:82:d3:
                    23:d8:e2:bd:13:a6:26:39:84:da:8d:c5:d6:b2:11:
                    f5:5b:03:a8:32:38:4f:a2:11:f2:9d:70:d3:65:d9:
                    0d:1d:6b:df:a3:2e:8a:72:f1:d8:5f:d0:22:20:46:
                    17:f7:90:9b:33:93:bb:8c:e6:bd:74:c8:e9:fa:d0:
                    9b:5a:b7:a3:66:3c:95:57:9c:f0:b5:5b:22:e4:ad:
                    93:98:a3:89:b1:e5:a0:d5:09:63:a9:75:3a:e9:fb:
                    7b:82:f3:fa:f6:b1:b5:d9:ee:b8:64:28:67:3e:21:
                    a3:60:96:68:1b:e9:49:1e:06:db:f3:82:a1:85:f8:
                    35:7e:80:f5:7b:f0:7a:fb:78:1a:5a:90:a4:ce:5a:
                    6f:6a:3c:df:e0:88:51:18:36:92:46:c2:8b:36:a4:
                    15:ad:a1
                Exponent: 65537 (0x10001)
        X509v3 extensions:
            X509v3 Subject Alternative Name: 
                IP Address:10.0.64.243
            X509v3 Basic Constraints: critical
                CA:TRUE
    Signature Algorithm: sha256WithRSAEncryption
    Signature Value:
        29:76:bb:0d:6f:83:23:a6:e0:bc:d4:3c:fb:ff:d7:77:42:48:
        9a:7d:42:db:c6:fb:c4:84:88:19:75:96:5d:7d:03:d3:bf:af:
        07:2b:83:74:51:bc:3f:de:cb:79:1d:12:42:c9:e8:c0:20:2a:
        62:aa:03:27:bd:1e:aa:ec:32:e0:27:80:9d:32:db:c5:88:84:
        58:66:f1:b8:48:5e:3c:b3:9a:e2:d4:5e:66:06:e2:02:85:db:
        5d:d3:62:2c:8d:5b:7f:cd:4b:7b:67:85:9b:72:8d:85:fe:0e:
        8c:17:ab:f9:bc:9e:6c:f1:9a:0e:a2:43:61:c3:e8:7a:95:c0:
        fc:e7:1f:81:be:0d:72:b5:bb:49:18:58:ad:4e:ef:91:86:2d:
        e1:dd:26:61:cf:69:49:57:22:e4:02:3e:02:ed:1d:53:7b:fb:
        94:bc:f3:f5:02:20:15:bf:32:fd:6c:c6:3f:28:f3:e3:ab:1b:
        01:7f:a6:03:96:03:67:71:9c:91:69:7b:2e:ea:58:78:db:c9:
        6f:8b:05:25:cf:f6:25:f6:13:88:c9:85:c1:bf:65:18:bb:8d:
        47:db:67:f7:bb:61:30:73:e6:cd:1f:61:9f:58:56:db:89:c5:
        d4:4a:a2:20:e9:f1:5d:29:48:2b:5d:4f:ee:44:2f:7d:38:8c:
        55:1d:4f:cf:d0:a5:88:1c:9c:ec:09:c9:40:26:90:40:27:b2:
        8c:1b:13:a9:45:cb:4c:3e:5d:1a:46:d2:b8:c9:66:3d:db:f1:
        3c:af:13:51:e4:b0:0c:06:8b:04:d5:5e:22:09:0a:fd:42:4c:
        45:8a:ee:fa:06:9d:95:cf:7e:5f:2f:2f:f4:bf:07:f0:5f:06:
        2d:d8:4d:c3:54:a2:9e:81:c9:c8:af:7e:79:4a:34:68:04:28:
        5a:60:0b:15:2b:ca:af:4a:6b:73:41:01:0d:6d:41:02:31:42:
        95:b5:db:59:35:b1:51:bd:fa:2b:fb:db:90:7a:63:02:e6:dc:
        bd:39:f5:c2:7d:39:cb:67:fc:2d:2f:5e:cd:10:cc:b9:ff:bd:
        b8:f4:bf:f7:26:b1:30:8b:13:b2:a2:86:51:f3:36:34:c1:1e:
        de:37:d8:88:e5:71:d2:5b:5d:45:de:a0:5d:a6:ab:e9:56:e6:
        d2:af:3c:72:a1:43:c3:d9:0a:40:5b:2a:da:5e:7d:90:94:ba:
        ac:55:3a:c2:0e:3f:a1:78:c7:5a:0d:32:f8:62:f0:e5:3a:72:
        b2:6f:4f:1e:b4:10:89:28:a8:3d:d2:c2:71:92:a4:df:40:a8:
        a9:a2:04:ed:41:29:50:ac:c2:f0:18:b7:e9:7f:ae:cd:b0:9f:
        62:cd:e6:2d:32:7d:49:6d
[root@ceph-pri-hsm-cephadm-h0a759-node6 ~]# 
[root@ceph-pri-hsm-cephadm-h0a759-node6 ~]# 
[root@ceph-pri-hsm-cephadm-h0a759-node6 ~]# openssl x509 -in sec-site-cephadm-root-ca.crt  -noout  -text
Certificate:
    Data:
        Version: 3 (0x2)
        Serial Number:
            4c:1b:d7:b7:24:28:c4:d9:5b:d6:7f:fa:64:7d:ec:ce:bf:36:18:16
        Signature Algorithm: sha256WithRSAEncryption
        Issuer: CN = cephadm-root
        Validity
            Not Before: Oct 30 14:50:13 2024 GMT
            Not After : Oct 31 14:50:13 2034 GMT
        Subject: CN = cephadm-root
        Subject Public Key Info:
            Public Key Algorithm: rsaEncryption
                Public-Key: (4096 bit)
                Modulus:
                    00:91:02:00:6b:f0:af:28:b1:fc:72:a9:c2:31:48:
                    21:99:b3:63:21:95:f6:b5:49:ae:44:9b:44:cc:93:
                    2a:b9:ee:98:47:de:66:6c:24:0f:23:d5:d8:db:1a:
                    d0:d7:d9:80:8b:7d:20:c8:af:56:8a:c3:29:8e:a9:
                    5d:a4:5f:b7:a3:d5:54:80:ba:e4:59:7a:48:17:dc:
                    3a:e5:dc:8f:a4:e3:32:5c:12:6f:ec:44:ac:32:2b:
                    35:df:d2:75:e7:f7:8f:7a:2d:1b:50:9a:c8:cd:15:
                    9b:b6:e6:86:6d:a9:f9:51:9c:ed:ac:0d:cf:18:ee:
                    f4:9c:fe:d8:06:15:10:f1:36:e6:d0:0b:5c:55:51:
                    06:68:42:ad:16:6f:f0:ca:9b:93:37:c6:ae:d1:13:
                    31:8e:d4:3f:74:2c:56:37:b2:d3:10:cc:6e:3d:d2:
                    07:0f:8d:04:1e:f1:97:6c:e3:5b:95:57:64:a4:2e:
                    03:2f:8b:ab:54:d8:14:8c:cc:f5:ca:15:c8:36:39:
                    74:fe:5a:d0:9a:5d:be:f4:06:ef:66:02:6c:21:fd:
                    a1:cf:bf:68:39:d4:01:2e:44:a3:cc:0b:24:7c:4a:
                    13:7b:50:01:72:4a:0a:70:86:34:1e:23:89:87:83:
                    d7:3c:ce:e1:ae:09:aa:a1:8c:c6:07:6a:e2:29:97:
                    c6:38:f0:1b:e9:b5:80:8b:1d:35:e5:e8:0c:3c:d9:
                    2c:d1:18:68:eb:37:6b:e1:94:9f:ba:e8:06:15:f0:
                    f0:a2:ce:cd:e3:6d:50:7e:b1:1b:0f:0f:5b:2a:f1:
                    9a:8a:73:d0:9f:3e:24:5b:54:30:bc:3b:43:db:a4:
                    92:f4:00:06:89:d0:2e:bd:16:7d:d2:88:c0:fc:4c:
                    67:f2:16:13:a8:d4:56:1c:4f:0b:d4:7d:0a:57:f2:
                    fb:a4:a0:ea:bd:80:80:50:b2:67:5d:de:2d:ba:d5:
                    77:19:8f:6c:40:40:72:ff:62:3e:16:bd:aa:88:a6:
                    6e:91:81:0c:4e:73:e5:95:64:69:a1:a1:cc:cb:e0:
                    de:e1:76:e2:ba:09:26:0b:6f:c0:05:ef:55:e0:be:
                    2f:63:93:69:21:fa:d6:6d:29:fd:ae:36:af:ed:83:
                    6e:01:f6:09:91:bd:ee:bd:73:d7:12:04:00:be:d6:
                    5a:5d:0c:89:5d:41:9d:25:01:eb:e1:c7:ba:6d:96:
                    15:a7:a0:ee:cd:34:0f:12:57:76:63:d4:de:fc:b3:
                    b6:b3:81:bf:6b:db:f3:74:94:37:df:3d:bf:63:a0:
                    d8:47:8d:0b:f4:59:9d:3e:40:d2:a6:d0:55:97:61:
                    c9:4f:d0:0f:95:17:1c:bc:c4:1c:3f:a3:4c:81:76:
                    74:5e:75
                Exponent: 65537 (0x10001)
        X509v3 extensions:
            X509v3 Subject Alternative Name: 
                IP Address:10.0.67.90
            X509v3 Basic Constraints: critical
                CA:TRUE
    Signature Algorithm: sha256WithRSAEncryption
    Signature Value:
        58:35:62:89:e9:bd:2b:c2:6a:c2:f7:0a:1b:ee:7d:13:32:c4:
        e1:c8:03:35:33:4a:46:e8:e7:71:8e:99:fa:5e:36:f6:ad:cd:
        5a:f5:05:c4:3b:91:bf:51:14:ba:a1:0b:41:e6:2e:b7:08:36:
        f9:51:e8:6a:6f:6b:d9:90:63:5a:c1:01:a5:a0:ac:bb:d9:9e:
        b2:99:ba:f7:2f:a4:72:a2:12:35:59:19:a6:5d:a2:7b:c6:2a:
        e6:a7:4a:f2:9b:55:95:ab:c6:13:0e:6b:de:cb:ed:0b:c4:98:
        90:19:7b:56:3f:93:97:1b:da:db:e9:b1:fa:cc:5f:b2:2c:4b:
        57:f0:bf:2f:46:ad:c0:c7:00:da:a5:bc:47:4f:ff:af:63:17:
        bb:9e:06:4d:16:a4:e9:81:c1:c5:e5:4f:61:b0:ec:31:c3:0d:
        cd:ae:1c:e3:b4:aa:84:0c:eb:c9:a0:aa:08:4b:a8:49:5e:86:
        8a:25:fe:ce:0b:ee:d9:3c:4c:e1:9b:ad:17:09:32:25:d2:07:
        14:fc:9d:4e:08:48:73:40:b2:75:a7:84:48:c3:57:cc:96:2c:
        c9:11:ee:2e:72:00:24:e8:d4:16:c8:73:2f:9e:3a:54:9f:10:
        8b:6d:92:0e:bc:d9:2e:0d:a6:51:b5:c2:5f:96:4a:53:41:da:
        77:c6:de:3e:0b:39:1c:8b:8c:27:55:ca:61:83:04:b3:52:50:
        9c:40:57:4c:21:62:9d:3f:74:c0:4a:bb:da:93:f8:fa:22:02:
        49:41:2c:7e:f3:cf:5b:c0:a7:61:f9:5b:d6:53:65:ee:7e:a1:
        94:4b:a0:f0:50:34:88:0c:ac:c5:9e:4b:51:17:a5:8a:b8:45:
        97:20:ec:85:15:58:ab:e7:dc:44:1c:4d:0a:c7:e9:77:c4:28:
        49:af:39:15:b5:4c:da:4d:78:68:ac:9b:1b:35:07:41:d3:9b:
        44:19:0f:b1:f8:2a:92:78:c7:51:70:34:30:83:87:e8:b7:ca:
        6a:bb:66:50:17:c8:4a:c6:8d:1a:6c:f0:63:87:bb:76:48:11:
        35:d3:0b:cb:7c:51:16:d1:12:b7:27:83:94:d8:02:32:96:9e:
        50:c1:06:e5:39:6d:ed:b7:f3:31:44:af:a6:23:f3:4d:a2:0f:
        e5:f1:17:ed:d3:0a:84:1c:f8:f2:e6:80:ff:57:8e:5b:84:7a:
        d3:e1:00:1c:c1:5f:ce:08:94:b0:e9:d1:13:aa:5d:68:1f:cd:
        39:39:f9:41:45:ea:d7:02:a2:83:70:0a:42:ff:a4:01:f9:eb:
        c0:9b:20:4a:b3:55:30:58:37:f3:28:96:08:8b:29:b7:ee:5e:
        56:75:87:55:28:8f:b0:fb
[root@ceph-pri-hsm-cephadm-h0a759-node6 ~]# 

Version-Release number of selected component (if applicable):
ceph version 19.2.0-52.el9cp

How reproducible:
always

Steps to Reproduce:
1.deploy rhcs 8.0 on two clusters with rgw daemons
2.establish multisite between them by trusting pri site cephadm-root ca cert on the sec site client and vice versa
3.try to trust both sites cephadm-root ca certs from a node, and access rgw on both sites. only one request is successful, the other is failing with invalid padding.

Actual results:
not able to trust both sites rgw certs from the same node

Expected results:
expected each site cephadm-root ca cert subject is different so that we can trust both sites rgw certs from a node.

Additional info: