Bug 1669214

Description of problem:


Version-Release number of selected component (if applicable):


How reproducible:


Steps to Reproduce:
ostestr --regex tempest.api.object_storage.test_account_quotas_negative.AccountQuotasNegativeTest.test_user_modify_quota

Actual results:
tempest.api.object_storage.test_account_quotas_negative.AccountQuotasNegativeTest.test_user_modify_quota[id-d1dc5076-555e-4e6d-9697-28f1fe976324,negative]
----------------------------------------------------------------------------------------------------------------------------------------------------------

Captured traceback:
~~~~~~~~~~~~~~~~~~~
    Traceback (most recent call last):
      File "/usr/lib/python2.7/site-packages/tempest/api/object_storage/test_account_quotas_negative.py", line 56, in setUp
        "POST", url="", headers=headers, body="")
      File "/usr/lib/python2.7/site-packages/tempest/lib/common/rest_client.py", line 668, in request
        self._error_checker(resp, resp_body)
      File "/usr/lib/python2.7/site-packages/tempest/lib/common/rest_client.py", line 769, in _error_checker
        raise exceptions.Forbidden(resp_body, resp=resp)
    tempest.lib.exceptions.Forbidden: Forbidden
    Details: AccessDenied


Captured pythonlogging:
~~~~~~~~~~~~~~~~~~~~~~~
    2019-01-09 23:12:26,639 365 INFO     [tempest.lib.common.rest_client] Request (AccountQuotasNegativeTest:setUp): 403 POST http://100.82.36.190:8080/swift/v1 0.181s
    2019-01-09 23:12:26,639 365 DEBUG    [tempest.lib.common.rest_client] Request - Headers: {'X-Account-Meta-Quota-Bytes': '20', 'X-Auth-Token': '<omitted>'}
            Body:
        Response - Headers: {'status': '403', u'content-length': '12', 'content-location': 'http://100.82.36.190:8080/swift/v1', u'accept-ranges': 'bytes', u'connection': 'close', u'x-trans-id': 'tx000000000000000000d9b-005c367fda-15789-default', u'date': 'Wed, 09 Jan 2019 23:12:26 GMT', u'content-type': 'text/plain; charset=utf-8', u'x-openstack-request-id': 'tx000000000000000000d9b-005c367fda-15789-default'}
            Body: AccessDenied

----------------------------------------
civetweb logging in /var/log/messages, unsure if related.

Jan 23 20:08:33 mr-14g-controller-0 journal: 2019-01-23 20:08:33.945361 7f46d00b1700  0 NOTICE: couldn't map swift user 4eba560a393945c3a53460aac8afa515

Expected results:
Tempest test should pass

Additional info:

Tempest.conf
------------------

[DEFAULT]
debug = true
use_stderr = false
log_file = tempest.log

[network-feature-enabled]
ipv6_subnet_attributes = true
api_extensions = default-subnetpools,qos,availability_zone,network_availability_zone,auto-allocated-topology,ext-gw-mode,binding,agent,subnet_allocation,l3_agent_scheduler,tag,address-scope,external-net,standard-attr-tag,flavors,segment,net-mtu,network-ip-availability,qos-default,quotas,revision-if-match,l3-ha,provider,multi-provider,quota_details,l2_adjacency,trunk,extraroute,net-mtu-writable,subnet-service-types,standard-attr-timestamp,service-type,qos-rule-type-details,l3-flavors,port-security,extra_dhcp_opt,standard-attr-revisions,pagination,sorting,security-group,dhcp_agent_scheduler,router_availability_zone,rbac-policies,project-id,qos-bw-limit-direction,tag-ext,standard-attr-description,ip-substring-filtering,router,allowed-address-pairs,ip_allocation,qos-fip,trunk-details

[auth]
tempest_roles = _member_,Member, ResellerAdmin
admin_username = admin
admin_project_name = admin
admin_domain_name = Default
use_dynamic_credentials = true
admin_password = xxxxxxxxxxxxxxxxxx
admin_project_id = c9fb570856934e5ca84d1f3d1cd2b526

[scenario]
img_dir = etc
img_file = cirros-0.3.5-x86_64-disk.img

[object-storage]
reseller_admin_role = ResellerAdmin
region = regionOne

[oslo-concurrency]
lock_path = /tmp

[compute-feature-enabled]
live_migration = false
live_migrate_paused_instances = true
preserve_ports = true
console_output = false
resize = True
attach_encrypted_volume = False
api_extensions = NMN,OS-DCF,OS-EXT-AZ,OS-EXT-IMG-SIZE,OS-EXT-IPS,OS-EXT-IPS-MAC,OS-EXT-SRV-ATTR,OS-EXT-STS,OS-FLV-DISABLED,OS-FLV-EXT-DATA,OS-SCH-HNT,OS-SRV-USG,os-access-ips,os-admin-actions,os-admin-password,os-agents,os-aggregates,os-assisted-volume-snapshots,os-attach-interfaces,os-availability-zone,os-baremetal-ext-status,os-baremetal-nodes,os-block-device-mapping,os-block-device-mapping-v2-boot,os-cell-capacities,os-cells,os-certificates,os-cloudpipe,os-cloudpipe-update,os-config-drive,os-console-auth-tokens,os-console-output,os-consoles,os-create-backup,os-create-server-ext,os-deferred-delete,os-evacuate,os-extended-evacuate-find-host,os-extended-floating-ips,os-extended-hypervisors,os-extended-networks,os-extended-quotas,os-extended-rescue-with-image,os-extended-services,os-extended-services-delete,os-extended-status,os-extended-volumes,os-fixed-ips,os-flavor-access,os-flavor-extra-specs,os-flavor-manage,os-flavor-rxtx,os-flavor-swap,os-floating-ip-dns,os-floating-ip-pools,os-floating-ips,os-floating-ips-bulk,os-fping,os-hide-server-addresses,os-hosts,os-hypervisor-status,os-hypervisors,os-instance-actions,os-instance_usage_audit_log,os-keypairs,os-lock-server,os-migrate-server,os-migrations,os-multiple-create,os-networks,os-networks-associate,os-pause-server,os-personality,os-preserve-ephemeral-rebuild,os-quota-class-sets,os-quota-sets,os-rescue,os-security-group-default-rules,os-security-groups,os-server-diagnostics,os-server-external-events,os-server-group-quotas,os-server-groups,os-server-list-multi-status,os-server-password,os-server-sort-keys,os-server-start-stop,os-services,os-shelve,os-simple-tenant-usage,os-suspend-server,os-tenant-networks,os-used-limits,os-used-limits-for-admin,os-user-data,os-user-quotas,os-virtual-interfaces,os-volume-attachment-update,os-volumes

[identity]
username = demo
password = secrete
project_name = demo
alt_username = alt_demo
alt_password = secrete
alt_project_name = alt_demo
disable_ssl_certificate_validation = true
region = regionOne
uri = http://100.82.36.190:5000//v3
auth_version = v3
uri_v3 = http://100.82.36.190:5000/v3

[image]
image_path = http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img
region = regionOne
http_image = http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img

[compute]
region = regionOne
flavor_ref = 5ca18338-ac66-4cbd-91cf-89f870951de6
flavor_ref_alt = f0c66dfa-e435-4df1-b09c-4688eb5c00c9
image_ref = 5e6bb6f3-d212-4423-8da2-8566abda7cb8
image_ref_alt = b3128550-717b-4f7a-b8dd-749c458dfd7a

[network]
region = regionOne
public_network_id = 008b9378-c481-4c57-8199-858aa67a105d
floating_network_name = public

[orchestration]
stack_owner_role = swiftoperator
region = regionOne

[volume]
backend1_name = tripleo_iscsi
region = regionOne
min_microversion = 3.0
max_microversion = 3.50

[volume-feature-enabled]
bootable = true
backup = False
api_v2 = False
api_v3 = True
api_extensions = OS-SCH-HNT,os-hosts,os-vol-tenant-attr,os-quota-sets,os-types-manage,os-volume-encryption-metadata,os-snapshot-actions,backups,cgsnapshots,os-used-limits,os-volume-type-access,consistencygroups,os-vol-host-attr,encryption,os-availability-zone,capabilities,os-volume-actions,os-types-extra-specs,os-snapshot-manage,os-vol-mig-status-attr,os-volume-unmanage,os-volume-manage,os-image-create,os-extended-services,os-extended-snapshot-attributes,os-snapshot-unmanage,qos-specs,os-quota-class-sets,os-volume-transfer,os-vol-image-meta,os-admin-actions,os-services,scheduler-stats

[object-storage-feature-enabled]
discoverability = False
discoverable_apis =

[validation]
image_ssh_user = cirros

[service_available]
ceilometer = True
horizon = True
cinder = True
nova = True
neutron = True
trove = False
glance = True
manila = False
panko = True
ironic = False
mistral = False
heat = True
zaqar = False
swift = True
sahara = False
gnocchi = True
octavia = False
aodh = True
aodh_plugin = True

[dashboard]
dashboard_url = http://100.82.36.190/dashboard/
login_url = http://100.82.36.190/dashboard/auth/login/

[image-feature-enabled]
api_v1 = False
api_v2 = True

[identity-feature-enabled]
api_v2 = False
api_v3 = True
api_extensions = s3tokens,OS-EP-FILTER,OS-REVOKE,OS-FEDERATION,OS-INHERIT,OS-SIMPLE-CERT,OS-TRUST,OS-PKI,OS-ENDPOINT-POLICY,OS-OAUTH1,OS-EC2
------------------------------

ceph.conf on controllers

[client.rgw.mr-14g-controller-0]
host = mr-14g-controller-0
keyring = /var/lib/ceph/radosgw/ceph-rgw.mr-14g-controller-0/keyring
log file = /var/log/ceph/ceph-rgw-mr-14g-controller-0.log
rgw frontends = civetweb port=192.168.170.12:8080 num_threads=100

[client.rgw.mr-14g-controller-1]
host = mr-14g-controller-1
keyring = /var/lib/ceph/radosgw/ceph-rgw.mr-14g-controller-1/keyring
log file = /var/log/ceph/ceph-rgw-mr-14g-controller-1.log
rgw frontends = civetweb port=192.168.170.13:8080 num_threads=100

[client.rgw.mr-14g-controller-2]
host = mr-14g-controller-2
keyring = /var/lib/ceph/radosgw/ceph-rgw.mr-14g-controller-2/keyring
log file = /var/log/ceph/ceph-rgw-mr-14g-controller-2.log
rgw frontends = civetweb port=192.168.170.14:8080 num_threads=100

# Please do not change this file directly since it is managed by Ansible and will be overwritten
[global]
cluster network = 192.168.180.0/24
fsid = eb28c9a4-1b45-11e9-b81c-5254001e8ca3
journal_collocation = False
journal_size = 10000
log file = /dev/null
# log file = /var/log/ceph/ceph.log
mon cluster log file = /dev/null
mon host = 192.168.170.12,192.168.170.13,192.168.170.14
mon initial members = mr-14g-controller-0,mr-14g-controller-1,mr-14g-controller-2
osd_pool_default_pg_num = 128
osd_pool_default_pgp_num = 128
osd_pool_default_size = 3
public network = 192.168.170.0/24
raw_multi_journal = True
rgw_keystone_admin_domain = default
rgw_keystone_admin_password = FGfWyB4q6xfkM3DtG9RXteRHW
rgw_keystone_admin_project = service
rgw_keystone_admin_user = swift
rgw_keystone_api_version = 3
rgw_keystone_implicit_tenants = true
rgw_keystone_revocation_interval = 0
rgw_keystone_url = http://192.168.140.251:5000
rgw_s3_auth_use_keystone = true
rgw_keystone_accepted_roles = Member, admin, _member_, ResellerAdmin
rgw_swift_enforce_content_length = true
rgw_log_nonexistent_bucket = true
rgw_enable_ops_log = true
debug ms = 1
debug rgw = 20
# Preluminous_compat entry added - Start
mon_health_preluminous_compat=true
# Preluminous_compat entry added - End

Also: it is very difficult to know where to look for the actual exception stacktrace to find the root cause.