Created attachment 1710393 [details] podman logs for ceph containers on controller-0 Description of problem: After installing OSP16.0 with TLS-Everywhere enabled and Ceph+RGW, I get the following: (overcloud) [stack@undercloud-0 ~]$ swift stat Account HEAD failed: https://overcloud.redhat.local:13808/swift/v1/AUTH_875075aede26404ba3566b3ff83d4afd 503 Service Unavailable Version-Release number of selected component (if applicable): RHOS_TRUNK-16.0-RHEL-8-20200730.n.1 How reproducible: Always Steps to Reproduce: 1. Install OSP on hybrid setup with about features enabled. 2. 3. Actual results: It is not possible to interact with swift. Expected results: Additional info: [root@controller-0 ~]# podman ps -a | grep ceph 34372dbbb02c undercloud-0.ctlplane.redhat.local:8787/ceph/ceph-4.0-rhel-8:latest 56 minutes ago Up 56 minutes ago ceph-rgw-controller-0-rgw0 aee397e5dd39 undercloud-0.ctlplane.redhat.local:8787/ceph/ceph-4.0-rhel-8:latest 59 minutes ago Up 59 minutes ago ceph-mgr-controller-0 7a86b04b3f61 undercloud-0.ctlplane.redhat.local:8787/ceph/ceph-4.0-rhel-8:latest About an hour ago Up About an hour ago ceph-mon-controller-0 [root@controller-0 ~]# podman exec -ti ceph-rgw-controller-0-rgw0 ceph -v ceph version 14.2.4-125.el8cp (db63624068590e593c47150c7574d08c1ec0d3e4) nautilus (stable) [root@controller-0 ~]# podman logs ceph-rgw-controller-0-rgw0 | head 2020-08-04 16:20:02 /opt/ceph-container/bin/entrypoint.sh: static: does not generate config HEALTH_WARN too few PGs per OSD (25 < min 30) 2020-08-04 16:20:03 /opt/ceph-container/bin/entrypoint.sh: SUCCESS exec: PID 104: spawning /usr/bin/radosgw --cluster ceph --setuser ceph --setgroup ceph --log-to-stderr=true --err-to-stderr=true --default-log-to-file=false --foreground -n client.rgw.controller-0.rgw0 -k /var/lib/ceph/radosgw/ceph-rgw.controller-0.rgw0/keyring exec: Waiting 104 to quit 2020-08-04 16:20:03.278 7f8dba12b0c0 0 framework: beast 2020-08-04 16:20:03.278 7f8dba12b0c0 0 framework conf key: ssl_endpoint, val: 172.17.3.19:8080 2020-08-04 16:20:03.278 7f8dba12b0c0 0 framework conf key: ssl_certificate, val: /etc/pki/tls/certs/ceph_rgw.pem 2020-08-04 16:20:03.279 7f8dba12b0c0 0 deferred set uid:gid to 167:167 (ceph:ceph) 2020-08-04 16:20:03.279 7f8dba12b0c0 0 ceph version 14.2.4-125.el8cp (db63624068590e593c47150c7574d08c1ec0d3e4) nautilus (stable), process radosgw, pid 104 [root@controller-0 ~]# podman logs ceph-rgw-controller-0-rgw0 | head -20 2020-08-04 16:20:02 /opt/ceph-container/bin/entrypoint.sh: static: does not generate config HEALTH_WARN too few PGs per OSD (25 < min 30) 2020-08-04 16:20:03 /opt/ceph-container/bin/entrypoint.sh: SUCCESS exec: PID 104: spawning /usr/bin/radosgw --cluster ceph --setuser ceph --setgroup ceph --log-to-stderr=true --err-to-stderr=true --default-log-to-file=false --foreground -n client.rgw.controller-0.rgw0 -k /var/lib/ceph/radosgw/ceph-rgw.controller-0.rgw0/keyring exec: Waiting 104 to quit 2020-08-04 16:20:03.278 7f8dba12b0c0 0 framework: beast 2020-08-04 16:20:03.278 7f8dba12b0c0 0 framework conf key: ssl_endpoint, val: 172.17.3.19:8080 2020-08-04 16:20:03.278 7f8dba12b0c0 0 framework conf key: ssl_certificate, val: /etc/pki/tls/certs/ceph_rgw.pem 2020-08-04 16:20:03.279 7f8dba12b0c0 0 deferred set uid:gid to 167:167 (ceph:ceph) 2020-08-04 16:20:03.279 7f8dba12b0c0 0 ceph version 14.2.4-125.el8cp (db63624068590e593c47150c7574d08c1ec0d3e4) nautilus (stable), process radosgw, pid 104 2020-08-04 16:20:14.971 7f8dba12b0c0 0 starting handler: beast 2020-08-04 16:20:14.975 7f8dba12b0c0 0 set uid:gid to 167:167 (ceph:ceph) 2020-08-04 16:20:14.992 7f8d83180700 0 RGWReshardLock::lock failed to acquire lock on reshard.0000000000 ret=-16 2020-08-04 16:20:14.996 7f8dba12b0c0 1 mgrc service_daemon_register rgw.controller-0.rgw0 metadata {arch=x86_64,ceph_release=nautilus,ceph_version=ceph version 14.2.4-125.el8cp (db63624068590e593c47150c7574d08c1ec0d3e4) nautilus (stable),ceph_version_short=14.2.4-125.el8cp,container_image=undercloud-0.ctlplane.redhat.local:8787/ceph/ceph-4.0-rhel-8:latest,cpu=Intel(R) Xeon(R) CPU E5-2630 v4 @ 2.20GHz,distro=rhel,distro_description=Red Hat Enterprise Linux 8.1 (Ootpa),distro_version=8.1,frontend_config#0=beast ssl_endpoint=172.17.3.19:8080 ssl_certificate=/etc/pki/tls/certs/ceph_rgw.pem,frontend_type#0=beast,hostname=controller-0.redhat.local,kernel_description=#1 SMP Tue Jul 21 14:11:32 UTC 2020,kernel_version=4.18.0-147.24.2.el8_1.x86_64,mem_cgroup_limit=29328670720,mem_swap_kb=0,mem_total_kb=28641600,num_handles=1,os=Linux,pid=104,zone_id=b39e77e3-425a-47a5-9010-9cf17c392a58,zone_name=default,zonegroup_id=2b562fd5-8da7-4888-a80f-71bdb7a38048,zonegroup_name=default} 2020-08-04 16:27:30.914 7f8d8097b700 1 ssl handshake failed: http request 2020-08-04 16:27:30.931 7f8d7f178700 1 ssl handshake failed: http request 2020-08-04 16:27:31.011 7f8d7f979700 1 ssl handshake failed: http request 2020-08-04 16:27:32.915 7f8d7c172700 1 ssl handshake failed: http request 2020-08-04 16:27:32.932 7f8d7a16e700 1 ssl handshake failed: http request 2020-08-04 16:27:33.013 7f8d7a16e700 1 ssl handshake failed: http request
[root@controller-0 ~]# podman exec -ti ceph-rgw-controller-0-rgw0 sh sh-4.4# cat /etc/ceph/ceph. ceph.client.admin.keyring ceph.client.radosgw.keyring ceph.mgr.controller-0.keyring ceph.mgr.controller-2.keyring ceph.client.openstack.keyring ceph.conf ceph.mgr.controller-1.keyring ceph.mon.keyring sh-4.4# cat /etc/ceph/ceph.conf [client.rgw.controller-0.rgw0] host = controller-0 keyring = /var/lib/ceph/radosgw/ceph-rgw.controller-0.rgw0/keyring log file = /var/log/ceph/ceph-rgw-controller-0.rgw0.log rgw frontends = beast ssl_endpoint=172.17.3.19:8080 ssl_certificate=/etc/pki/tls/certs/ceph_rgw.pem rgw thread pool size = 512 [client.rgw.controller-1.rgw0] host = controller-1 keyring = /var/lib/ceph/radosgw/ceph-rgw.controller-1.rgw0/keyring log file = /var/log/ceph/ceph-rgw-controller-1.rgw0.log rgw frontends = beast ssl_endpoint=172.17.3.106:8080 ssl_certificate=/etc/pki/tls/certs/ceph_rgw.pem rgw thread pool size = 512 [client.rgw.controller-2.rgw0] host = controller-2 keyring = /var/lib/ceph/radosgw/ceph-rgw.controller-2.rgw0/keyring log file = /var/log/ceph/ceph-rgw-controller-2.rgw0.log rgw frontends = beast ssl_endpoint=172.17.3.68:8080 ssl_certificate=/etc/pki/tls/certs/ceph_rgw.pem rgw thread pool size = 512 # Please do not change this file directly since it is managed by Ansible and will be overwritten [global] cluster network = 172.17.4.0/24 fsid = 238b99d7-f9a5-4ff5-863d-380b288c18f3 mon host = [v2:172.17.3.19:3300,v1:172.17.3.19:6789],[v2:172.17.3.106:3300,v1:172.17.3.106:6789],[v2:172.17.3.68:3300,v1:172.17.3.68:6789] mon initial members = controller-0,controller-1,controller-2 osd pool default crush rule = -1 osd_pool_default_pg_num = 32 osd_pool_default_pgp_num = 32 osd_pool_default_size = 3 public network = 172.17.3.0/24 rgw_keystone_accepted_admin_roles = ResellerAdmin, swiftoperator rgw_keystone_accepted_roles = member, Member, admin rgw_keystone_admin_domain = default rgw_keystone_admin_password = 86pnzX1VwskV6IekrW99OOHat rgw_keystone_admin_project = service rgw_keystone_admin_user = swift rgw_keystone_api_version = 3 rgw_keystone_implicit_tenants = true rgw_keystone_revocation_interval = 0 rgw_keystone_url = https://overcloud.internalapi.redhat.local:5000 rgw_s3_auth_use_keystone = true rgw_swift_account_in_url = true rgw_swift_versioning_enabled = true rgw_trust_forwarded_https = true
This was fixed in 16.1. Please use fixed-in version from duplicate bug 1846812. *** This bug has been marked as a duplicate of bug 1846812 ***