Created attachment 1961563 [details] Picture of the SegFault Description of problem: A total of Five segmentation faults (reporting that all are the same) since Nov 2022. Secure site, no data other than pictures. Red Hat onsite indicates that this is the only cluster of many which has seen this. *** Caught signal (Segmentation fault) ** in thread 7fe803d57700 thread_name:radosgw ceph version 16.2.0-152.el8cp ... 1: /lib64/libpthread.so.0(+0x12c20) [0x7fe964f6c20] 2: (ceph::buffer::v15_2_0::ptr::ptr(ceph::buffer::v15_2_0::ptr counst&)+0x1b [0x7fe96edb9a8b] 3: (ceph::buffer::v15_2_0::ptr_node::cloner::operator()ceph::buffer::v15_2_0::ptr_node const&)+0x2e [0x7fe96edbbbae] 4: (std::_Rb_tree_node<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> >* . . . [0x7fe96f631c63] 5: (std::_Rb_tree_node<std::ppair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> >* . . . [0x7fe96f631931] 6: (std::Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char> . . . [0x7fe96f631f13] 7: (ObjectCache::get(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, ObjectCacheInfo&, unsigned int, rgw_cache_entry_info*)+(0xe39) [0x7fe96f790089] 8: (RGWSI_SysObj_Cache::raw_stat(rgw_raw_obj const&, unsigned long*, std::chrono::time_point<ceph::real_clock, std::chrono::duration<unsigned log, std::ratio<11, . . . [0x7fe96fbe2227] 9: (RGWSI_SysObj_Core::get_system_obj_state_impl(RGWSysObjectCtxBase*, rgw_raw_obj const&, RGWSsysObjState**, RGWObjVersionTracker*, optional_yield)+0x524 [0x7fe96f706b54] 10: (RGWSI_SysObj_Core::get_system_obj_state(RGWSysObjectCtxBase*, rgw_raw_obj const&, RGWSysObjState**, RGWObjVersionTracker*, optional_yield)+0x4a) [0x7fe96f70730a] 11: (RGWSI_SysObj_Core::stat(RGWSysObjectCtxbase&, RGWSI_SysObj_Obj_GetObjState&, rgw_raw_obj const&, std::map<std::__css11::basic_string<char, std::char_traits . . . [0x7fe96f70b602] 12: (RGWSI_SysObj::Obj::ROp::stat(optional_yield)+0x5f) [0x7fe96f70391f] 13: (rgw_get_system_obj(RGWSysObjectCtx&, rgw_pool const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> const&, ceph::buffer . . . [0x7fe96faba5b0] 14: (RGWSI_MetaBackend_SObj::get_entry(RGWSI_MetaBackend::Context*, std::_cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, . . . [0x7fe96fbd1617] 15: (RGWSI_Bucket_SObj::do_read_bucket_instance_info(ptr_wrapper<RGWSI_MetaBackend::Context, 4>&, std::__cxx11::basic_string<char, std::char_traits<char> . . . [0x7fe96fbb547c] 16: (RGWSI_Bucket_SObj::read_bucket_instatance_info(ptr_wrapper<RGWSI_MetaBackend::Context, 4>&, std::__cxx11::basic_string<char, std::char_traits<char> . . . [0x7fe96fbb59d3] 17: /lib64/libradosgw.so.2(+0x67bb52) [0x7fe96f754b52] 18: (std::_Function_handler<int (RGWSI_MetaBackend_handler::Op*), RGWBucketInstanceMetadatahandler::call(std::optional<std::variant<RGWSI_MetaBackend_CtxParams_SObj> . . . [0x7fe96f76d906] 19: /lib64/libradosgw.so.2(+0xaf5ade) [0x7fe96fbceade] 20: (RGWSI_MetaBackend_SObj::call(std::optional<std::variant<RGWSI_MetaBackend_CtxParams_SObj> >, stdfunction<int(RGWSI_MetaBackend::Context*)>)+0x180) [0x7fe96fbd1fe0] 21: (RGWSI_MetaBackend_Handler::call(std::optional<std::variant<RGWSI_MetaBackend_CtxParams_SObj> >, std::function<int(RGWSI_MetaBackend_Handler::Op*)>)+0x5f) [0x7fe96fbce90f] 22: (RGWBucketCtl::read_bucket_info(rgw_bucket const&, RGWBucketInfo*, optional_yield, RGWBucketCtl::BucketInstance::GetParams const&, RGWObjVersionTracker*)+0x1fc) [0x7fe96f755c5c] 23: (rgw::sal::RGWRadosBucket::get_bucket_info(optional_yield)+0x133) [0x7fe96faaba13] 24: (rgw::sal::RGWRadosStore::get_bucket(rgw::sal::RGWUser*, rgw_bucket const&, std::unique_ptr<rgw::sal::RGWBucket, std::default_delete<rgw::sal::RGWBucket> . . . [0x7fe96faaa450] 25: (rgw_build_bucket_policies(rgw::sal::RGWRadosStore*, req_state*, optional_yield)+0xe40) [0x7fe96f9911d0] 26: (RGWHandler::do_init_permissions(optional_yield)+0x38) [0x7fe96f992948] 27: (RGWHandler_REST::init_permissions(RGWOp*, optional_yield)+0x166) [0x7fe96fa265f6] 28: (rgw_process_authenticated(RGWHandler_REST*, RGWOp*&, RGWRequest*, req_state*, optional_yield, bool)+0x1ac) [0x7fe96f626e2c] 29: (process_request(rgw::sal::RGWRadosStore*, RGWEST*, RGWRequest*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rgw::auth::StratergyRegistry const&, RGWestfulIO*, OpsLogSocket*, optional_yield, rgw::dmclock::Scheduler*, std::__cxx11::basic_string<char, std::char_traits<chr>, std::allocator<char> >*, std::chrono::duration<usigned long, std::ratio<11, 10000000001> >*, int*)+0269d) [0x7fe96f62bcbd] 30: /lib64/libradosgw.so.2(+0x4a455b) [0x7fe96f57d55b] 31: /lib64/libradosgw.so.2(+ox4a5ff4) [0x7fe96f57eff4] 32: /lib64/libradosgw.so.2(+0x4a625e) [0x7fe96f57f25e] I will attach the stack trace (that I typed in) and the pdf file showing the stack trace. I was not able to match this stack trace to another BZ. Version-Release number of selected component (if applicable): v5.0z4 How reproducible: Random Steps to Reproduce: 1. 2. 3. Actual results: Expected results: Additional info: Attachments: RGW-Error-Journalctl.pdf and rgw-error.txt