Bug 2188266

In OSP17.1 CI, tempest object_storage tests started failing with 401 Unauthorized (after switch to CEPH 6.0):

>  2023-04-19 17:12:04,881 172091 INFO     [tempest.lib.common.rest_client] Request (TestObjectStorageBasicOps:test_swift_basic_ops): 401 GET http://10.0.0.117:8080/swift/v1/AUTH_e235c32e898741dbaeb320a29c598e61 0.466s
>  2023-04-19 17:12:04,882 172091 DEBUG    [tempest.lib.common.rest_client] Request - Headers: {'X-Auth-Token': '<omitted>'}
>        Body: None
>    Response - Headers: {'content-length': '12', 'x-trans-id': 'tx00000bdb7c8ee62f4f03b-00644020e4-5f35-default', 'x-openstack-request-id': 'tx00000bdb7c8ee62f4f03b-00644020e4-5f35-default', 'accept-ranges': 'bytes', 'content-type': 'text/plain; charset=utf-8', 'date': 'Wed, 19 Apr 2023 17:12:04 GMT', 'connection': 'close', 'status': '401', 'content-location': 'http://10.0.0.117:8080/swift/v1/AUTH_e235c32e898741dbaeb320a29c598e61'}
>        Body: b'AccessDenied'
>  }}}
>
>   {'content-length': '12', 'x-trans-id': 'tx00000bdb7c8ee62f4f03b-00644020e4-5f35-default', 'x-openstack-request-id': 'tx00000bdb7c8ee62f4f03b-00644020e4-5f35-default', 'accept-ranges': 'bytes', 'content-type': 'text/plain; charset=utf-8', 'date': 'Wed, 19 Apr 2023 17:12:04 GMT', 'connection': 'close', 'status': '401', 'content-location': 'http://10.0.0.117:8080/swift/v1/AUTH_e235c32e898741dbaeb320a29c598e61'}
>        Body: b'AccessDenied'

Seems RGW is container is running and reachable correctly.
And to my basic understanding looks configured:
> [root@controller-0 ~]# cephadm shell
> Inferring fsid 09a79c57-b8c4-5bdf-bcb2-4cebe6dc7fd5
> Inferring config /var/lib/ceph/09a79c57-b8c4-5bdf-bcb2-4cebe6dc7fd5/mon.controller-0/config
> Using ceph image with id '35949bb370c9' and tag '6-115' created on 2023-03-13 13:39:34 +0000 UTC
> undercloud-0.ctlplane.redhat.local:8787/rh-osbs/rhceph@sha256:f87ef89a37af6703583758a0d806e54ad8b420be9ae517149cfdb81f3628f137
> [ceph: root@controller-0 /]# ceph config dump
> WHO                                 MASK  LEVEL     OPTION                                 VALUE                                                                                                                           RO
> global                                    advanced  cluster_network                        172.17.4.0/24                                                                                                                   *
> global                                    basic     container_image                        undercloud-0.ctlplane.redhat.local:8787/rh-osbs/rhceph@sha256:f87ef89a37af6703583758a0d806e54ad8b420be9ae517149cfdb81f3628f137  *
> global                                    advanced  ms_bind_ipv4                           true
> global                                    advanced  ms_bind_ipv6                           false
> global                                    advanced  public_network                         172.17.3.0/24                                                                                                                   *
> global                                    advanced  rgw_keystone_accepted_admin_roles      ResellerAdmin, swiftoperator                                                                                                    *
> global                                    advanced  rgw_keystone_accepted_reader_roles     SwiftSystemReader                                                                                                               *
> global                                    advanced  rgw_keystone_accepted_roles            member, Member, admin                                                                                                           *
> global                                    advanced  rgw_keystone_admin_domain              default                                                                                                                         *
> global                                    advanced  rgw_keystone_admin_password            MEDwPGlUCt8b9F3EmMN3oN8MD                                                                                                       *
> global                                    advanced  rgw_keystone_admin_project             service                                                                                                                         *
> global                                    advanced  rgw_keystone_admin_user                swift                                                                                                                           *
> global                                    advanced  rgw_keystone_api_version               3
> global                                    advanced  rgw_keystone_implicit_tenants          true                                                                                                                            *
> global                                    basic     rgw_keystone_url                       http://172.17.1.23:5000                                                                                                         *
> global                                    advanced  rgw_keystone_verify_ssl                false
> global                                    advanced  rgw_max_attr_name_len                  128
> global                                    advanced  rgw_max_attr_size                      1024
> global                                    advanced  rgw_max_attrs_num_in_req               90
> global                                    advanced  rgw_s3_auth_use_keystone               true
> global                                    advanced  rgw_swift_account_in_url               true
> global                                    advanced  rgw_swift_enforce_content_length       true
> global                                    advanced  rgw_swift_versioning_enabled           true
> global                                    advanced  rgw_trust_forwarded_https              true
> mon                                       advanced  auth_allow_insecure_global_id_reclaim  false
> mon                                       advanced  public_network                         172.17.3.0/24                                                                                                                   *
> mgr                                       advanced  mgr/cephadm/container_image_base       undercloud-0.ctlplane.redhat.local:8787/rh-osbs/rhceph
> mgr                                       advanced  mgr/cephadm/container_init             True                                                                                                                            *
> mgr                                       advanced  mgr/cephadm/migration_current          5                                                                                                                               *
> mgr                                       advanced  mgr/orchestrator/orchestrator          cephadm
> osd                                       advanced  osd_memory_target_autotune             true
> client.rgw.rgw                            advanced  rgw_realm                              default                                                                                                                         *
> client.rgw.rgw                            advanced  rgw_zone                               default                                                                                                                         *
> client.rgw.rgw.controller-0.lsclci        basic     rgw_frontends                          beast endpoint=172.17.3.141:8080                                                                                                *
> client.rgw.rgw.controller-1.tpqnnw        basic     rgw_frontends                          beast endpoint=172.17.3.94:8080                                                                                                 *
> client.rgw.rgw.controller-2.uleffk        basic     rgw_frontends                          beast endpoint=172.17.3.38:8080                                                                                                 *

rgw endpoints reachable:
> [ceph: root@controller-0 /]# curl 172.17.3.141:8080
> <?xml version="1.0" encoding="UTF-8"?><ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Owner><ID>anonymous</ID><DisplayName></DisplayName></Owner><Buckets></Buckets></ListAllMyBucketsResult>[ceph: root@controller-0 /]#
> [ceph: root@controller-0 /]# curl 172.17.3.94:8080
> <?xml version="1.0" encoding="UTF-8"?><ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Owner><ID>anonymous</ID><DisplayName></DisplayName></Owner><Buckets></Buckets></ListAllMyBucketsResult>[ceph: root@controller-0 /]# curl 172.17.3.38:8080
> <?xml version="1.0" encoding="UTF-8"?><ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Owner><ID>anonymous</ID><DisplayName></DisplayName></Owner><Buckets></Buckets></ListAllMyBucketsResult>[ceph: root@controller-0 /]

Keystone endpoint access info configured also looks most likely correct:
> [ceph: root@controller-0 /]# curl http://172.17.1.23:5000
> {"versions": {"values": [{"id": "v3.14", "status": "stable", "updated": "2020-04-07T00:00:00Z", "links": [{"rel": "self", "href": "http://172.17.1.23:5000/v3/"}], "media-types": [{"base": "application/json", "type": "application/vnd.openstack.identity-v3+json"}]}]}}
> 
> [ceph: root@controller-0 /]# curl -H "Content-Type: application/json" -d '{"auth":{"identity":{"methods":["password"],"password":{"user":{"name":"swift","domain":{"name":"default"},"password":"MEDwPGlUCt8b9F3EmMN3oN8MD"}}}}}' "http://172.17.1.23:5000/v3/auth/tokens"
> {"token": {"methods": ["password"], "user": {"domain": {"id": "default", "name": "Default"}, "id": "60ce80fc957b4660abe3300ee125f6e5", "name": "swift", "password_expires_at": null}, "audit_ids": ["2d-IJKkOS1io3fgA47IVOQ"], "expires_at": "2023-04-20T11:05:04.000000Z", "issued_at": "2023-04-20T10:05:04.000000Z"}}

Versions in rgw container:
> [ceph: root@controller-0 /]# rpm -qa|grep -iE ceph\|rgw | sort -h
> ceph-base-17.2.5-75.el9cp.x86_64
> ceph-common-17.2.5-75.el9cp.x86_64
> ceph-grafana-dashboards-17.2.5-75.el9cp.noarch
> ceph-immutable-object-cache-17.2.5-75.el9cp.x86_64
> ceph-mds-17.2.5-75.el9cp.x86_64
> ceph-mgr-17.2.5-75.el9cp.x86_64
> ceph-mgr-cephadm-17.2.5-75.el9cp.noarch
> ceph-mgr-dashboard-17.2.5-75.el9cp.noarch
> ceph-mgr-diskprediction-local-17.2.5-75.el9cp.noarch
> ceph-mgr-k8sevents-17.2.5-75.el9cp.noarch
> ceph-mgr-modules-core-17.2.5-75.el9cp.noarch
> ceph-mgr-rook-17.2.5-75.el9cp.noarch
> ceph-mon-17.2.5-75.el9cp.x86_64
> ceph-osd-17.2.5-75.el9cp.x86_64
> ceph-prometheus-alerts-17.2.5-75.el9cp.noarch
> ceph-radosgw-17.2.5-75.el9cp.x86_64
> ceph-selinux-17.2.5-75.el9cp.x86_64
> ceph-volume-17.2.5-75.el9cp.noarch
> cephadm-17.2.5-75.el9cp.noarch
> cephfs-mirror-17.2.5-75.el9cp.x86_64
> libcephfs2-17.2.5-75.el9cp.x86_64
> libcephsqlite-17.2.5-75.el9cp.x86_64
> librgw2-17.2.5-75.el9cp.x86_64
> nfs-ganesha-ceph-4.0.8-2.el9cp.x86_64
> nfs-ganesha-rgw-4.0.8-2.el9cp.x86_64
> python3-ceph-argparse-17.2.5-75.el9cp.x86_64
> python3-ceph-common-17.2.5-75.el9cp.x86_64
> python3-cephfs-17.2.5-75.el9cp.x86_64
> python3-rgw-17.2.5-75.el9cp.x86_64

Undercloud TripleO versions:
> [stack@undercloud-0 ~]$ rpm -qa| grep tripleo | sort -h
> ansible-role-tripleo-modify-image-1.5.1-1.20230211112201.b6eedb6.el9ost.noarch
> ansible-tripleo-ipa-0.2.3-1.20220825212007.1c47cb9.el9ost.noarch
> ansible-tripleo-ipsec-11.0.1-1.20220727105329.b5559c8.el9ost.noarch
> openstack-tripleo-common-15.4.1-1.20230407010832.e1e7eeb.el9ost.noarch
> openstack-tripleo-common-containers-15.4.1-1.20230407010832.e1e7eeb.el9ost.noarch
> openstack-tripleo-heat-templates-14.3.1-1.20230412011051.2e6d826.el9ost.noarch
> openstack-tripleo-validations-14.3.2-1.20230323020947.447bd2f.el9ost.noarch
> puppet-tripleo-14.2.3-1.20230323230826.d0c3708.el9ost.noarch
> python3-tripleo-common-15.4.1-1.20230407010832.e1e7eeb.el9ost.noarch
> python3-tripleoclient-16.5.1-1.20230407001101.210ed7c.el9ost.noarch
> tripleo-ansible-3.3.1-1.20230412012501.0a826ca.el9ost.noarch