Description of problem: ======================== After upgrading cluster from RHCS 4.0 to RHCS 4.1 on RHEL 8.2 zongroup and zoneid changed so we are unable to access existing containers data [root@f09-h17-b05-5039ms ~]# swift -A http://172.17.39.213:8080/auth/1.0 -U johndoe:swift -K LkV5LcdPkXUFCB5gau9j0ahSlBFyh2Jc1YAMKTUa list mycontainers3 Container GET failed: http://172.17.39.213:8080/swift/v1/mycontainers3?format=json 301 Moved Permanently Failed Transaction ID: tx000000000000000000002-005ef04900-3e2e0-default [root@f09-h17-b05-5039ms ~]# swift -A http://172.17.39.213:8080/auth/1.0 -U johndoe:swift -K LkV5LcdPkXUFCB5gau9j0ahSlBFyh2Jc1YAMKTUa list mycontainers1 Container GET failed: http://172.17.39.213:8080/swift/v1/mycontainers1?format=json 301 Moved Permanently Failed Transaction ID: tx000000000000000000004-005ef04aa4-3e2e0-default [root@f09-h17-b05-5039ms ~]# swift -A http://172.17.39.213:8080/auth/1.0 -U johndoe:swift -K LkV5LcdPkXUFCB5gau9j0ahSlBFyh2Jc1YAMKTUa list mycontainers2 Container GET failed: http://172.17.39.213:8080/swift/v1/mycontainers2?format=json 301 Moved Permanently Failed Transaction ID: tx000000000000000000006-005ef04aa9-3e2e0-default [root@f09-h17-b05-5039ms ~]# swift -A http://172.17.39.213:8080/auth/1.0 -U johndoe:swift -K LkV5LcdPkXUFCB5gau9j0ahSlBFyh2Jc1YAMKTUa list mycontainers4 Container GET failed: http://172.17.39.213:8080/swift/v1/mycontainers4?format=json 301 Moved Permanently Failed Transaction ID: tx000000000000000000008-005ef04aac-3e2e0-default [root@f09-h17-b05-5039ms ~]# swift -A http://172.17.39.213:8080/auth/1.0 -U johndoe:swift -K LkV5LcdPkXUFCB5gau9j0ahSlBFyh2Jc1YAMKTUa list mycontainers5 Container GET failed: http://172.17.39.213:8080/swift/v1/mycontainers5?format=json 301 Moved Permanently Failed Transaction ID: tx00000000000000000000a-005ef04aaf-3e2e0-default Version-Release number of selected component (if applicable): ============================================================= ceph version 14.2.8-59.el8cp (53387608e81e6aa2487c952a604db06faa5b2cd0) nautilus (stable) === From HAPROXY -============== [root@f09-h17-b05-5039ms ~]# swift -A http://172.17.36.200:5000/auth/1.0 -U johndoe:swift -K LkV5LcdPkXUFCB5gau9j0ahSlBFyh2Jc1YAMKTUa list mycontainers5 Container GET failed: http://172.17.36.200:5000/swift/v1/mycontainers5?format=json 301 Moved Permanently Failed Transaction ID: tx000000000000000000001-005ef04cce-3e2e9-default [root@f09-h17-b05-5039ms ~]# 2020-06-22 06:07:43.867 7f80dcad6700 1 ====== starting new request req=0x7f81386d4890 ===== 2020-06-22 06:07:43.869 7f80dcad6700 0 req 10 0.002s NOTICE: request for data in a different zonegroup (62e506a6-251d-4a67-8687-7e70309170c9 != 8efdcc4d-363e-4b6e-8a12-395e88cd1b9a) 2020-06-22 06:07:43.869 7f80dcad6700 1 ====== req done req=0x7f81386d4890 op status=0 http_status=301 latency=0.00199998s ====== [root@f10-h09-000-6049p ~]# radosgw-admin realm list { "default_info": "", "realms": [] } [root@f10-h09-000-6049p ~]# radosgw-admin zonegroup list { "default_info": "8efdcc4d-363e-4b6e-8a12-395e88cd1b9a", "zonegroups": [ "default" ] } # radosgw-admin zonegroup get { "id": "8efdcc4d-363e-4b6e-8a12-395e88cd1b9a", "name": "default", "api_name": "default", "is_master": "true", "endpoints": [], "hostnames": [], "hostnames_s3website": [], "master_zone": "9ad49faf-7079-4262-b311-189f98a578bd", "zones": [ { "id": "9ad49faf-7079-4262-b311-189f98a578bd", "name": "default", "endpoints": [], "log_meta": "false", "log_data": "false", "bucket_index_max_shards": 0, "read_only": "false", "tier_type": "", "sync_from_all": "true", "sync_from": [], "redirect_zone": "" } ], "placement_targets": [ { "name": "default-placement", "tags": [], "storage_classes": [ "STANDARD" ] } ], "default_placement": "default-placement", "realm_id": "" } [root@f10-h09-000-6049p ceph]# radosgw-admin zone get { "id": "9ad49faf-7079-4262-b311-189f98a578bd", "name": "default", "domain_root": "default.rgw.meta:root", "control_pool": "default.rgw.control", "gc_pool": "default.rgw.log:gc", "lc_pool": "default.rgw.log:lc", "log_pool": "default.rgw.log", "intent_log_pool": "default.rgw.log:intent", "usage_log_pool": "default.rgw.log:usage", "reshard_pool": "default.rgw.log:reshard", "user_keys_pool": "default.rgw.meta:users.keys", "user_email_pool": "default.rgw.meta:users.email", "user_swift_pool": "default.rgw.meta:users.swift", "user_uid_pool": "default.rgw.meta:users.uid", "otp_pool": "default.rgw.otp", "system_key": { "access_key": "", "secret_key": "" }, "placement_pools": [ { "key": "default-placement", "val": { "index_pool": "default.rgw.buckets.index", "storage_classes": { "STANDARD": { "data_pool": "default.rgw.buckets.data" } }, "data_extra_pool": "default.rgw.buckets.non-ec", "index_type": 0 } } ], "metadata_heap": "", "realm_id": "" } [root@f10-h09-000-6049p ~]# radosgw-admin metadata list bucket.instance [ "mycontainers2:9ad49faf-7079-4262-b311-189f98a578bd.147713.2", "mycontainers4:9ad49faf-7079-4262-b311-189f98a578bd.147713.1", "mycontainers1:9ad49faf-7079-4262-b311-189f98a578bd.147713.4", "mycontainers5:9ad49faf-7079-4262-b311-189f98a578bd.147713.3", "mycontainers3:9ad49faf-7079-4262-b311-189f98a578bd.147713.5" ] [root@f10-h09-000-6049p ceph]# rados -p .rgw.root ls zone_info.9ad49faf-7079-4262-b311-189f98a578bd zonegroup_info.8efdcc4d-363e-4b6e-8a12-395e88cd1b9a zone_names.default zonegroups_names.default Pre upgrade =========== ceph-rgw-f11-h09-000-6049p.rgw0.log-20200618.gz:2020-06-16 11:53:12.234 7f35ac0f5100 1 mgrc service_daemon_register rgw.f11-h09-000-6049p.rgw0 metadata {arch=x86_64,ceph_release=nautilus,ceph_version=ceph version 14.2.4-125.el8cp (db63624068590e593c47150c7574d08c1ec0d3e4) nautilus (stable),ceph_version_short=14.2.4-125.el8cp,cpu=Intel(R) Xeon(R) Gold 6140 CPU @ 2.30GHz,distro=rhel,distro_description=Red Hat Enterprise Linux 8.2 (Ootpa),distro_version=8.2,frontend_config#0=beast endpoint=172.17.38.145:8080,frontend_type#0=beast,hostname=f11-h09-000-6049p.rdu2.scalelab.redhat.com,kernel_description=#1 SMP Fri Mar 27 14:35:58 UTC 2020,kernel_version=4.18.0-193.el8.x86_64,mem_swap_kb=4189180,mem_total_kb=394610624,num_handles=1,os=Linux,pid=100110,zone_id=8fc211fa-f06f-497a-ba87-c5b34c471872,zone_name=default,zonegroup_id=62e506a6-251d-4a67-8687-7e70309170c9,zonegroup_name=default} <------ check zoneid and zonegroup id Post upgrade ============ ceph-rgw-f11-h09-000-6049p.rgw0.log-20200620.gz:2020-06-19 19:49:40.678 7f93009d5100 1 mgrc service_daemon_register rgw.f11-h09-000-6049p.rgw0 metadata {arch=x86_64,ceph_release=nautilus,ceph_version=ceph version 14.2.8-59.el8cp (53387608e81e6aa2487c952a604db06faa5b2cd0) nautilus (stable),ceph_version_short=14.2.8-59.el8cp,cpu=Intel(R) Xeon(R) Gold 6140 CPU @ 2.30GHz,distro=rhel,distro_description=Red Hat Enterprise Linux 8.2 (Ootpa),distro_version=8.2,frontend_config#0=beast endpoint=172.17.38.145:8080,frontend_type#0=beast,hostname=f11-h09-000-6049p.rdu2.scalelab.redhat.com,kernel_description=#1 SMP Fri Mar 27 14:35:58 UTC 2020,kernel_version=4.18.0-193.el8.x86_64,mem_swap_kb=4189180,mem_total_kb=394610624,num_handles=1,os=Linux,pid=1269577,zone_id=9ad49faf-7079-4262-b311-189f98a578bd,zone_name=default,zonegroup_id=8efdcc4d-363e-4b6e-8a12-395e88cd1b9a,zonegroup_name=default} <------ check zonegroup and zone id