Description of problem: rgw daemon is crashing while executing the below query [cephuser@ceph-hmaheswa-rhcs7-fiisfo-node6 ~]$ aws s3api select-object-content --endpoint-url http://10.0.210.237:80 --bucket bkt1 --key example1_json --expression-type 'SQL' --input-serialization '{"JSON": {"Type": "DOCUMENT"}, "CompressionType": "NONE"}' --output-serialization '{"CSV": {}}' --expression "select phonenumbers.type from s3object[*].phonenumbers;" /dev/stdout ("Connection broken: InvalidChunkLength(got length b'', 0 bytes read)", InvalidChunkLength(got length b'', 0 bytes read)) [cephuser@ceph-hmaheswa-rhcs7-fiisfo-node6 ~]$ crash info: [root@ceph-hmaheswa-rhcs7-fiisfo-node5 crash]# ceph crash info 2023-08-09T03:28:38.019406Z_b2fa6c9f-51d6-4555-bb8b-cd5ed65e585c { "backtrace": [ "/lib64/libc.so.6(+0x54df0) [0x7f10f8faadf0]", "/lib64/libc.so.6(+0xa154c) [0x7f10f8ff754c]", "raise()", "abort()", "/usr/bin/radosgw(+0x309588) [0x56207f96c588]", "/usr/bin/radosgw(+0x6126f7) [0x56207fc756f7]", "/usr/bin/radosgw(+0xb97369) [0x5620801fa369]", "/usr/bin/radosgw(+0x634c00) [0x56207fc97c00]", "/usr/bin/radosgw(+0x61dc0d) [0x56207fc80c0d]", "/usr/bin/radosgw(+0xb144d4) [0x5620801774d4]", "/usr/bin/radosgw(+0x61fe06) [0x56207fc82e06]", "(RGWSelectObj_ObjStore_S3::run_s3select_on_json(char const*, char const*, unsigned long)+0x9de) [0x56207fca72be]", "(RGWSelectObj_ObjStore_S3::json_processing(ceph::buffer::v15_2_0::list&, long, long)+0x633) [0x56207fcaa6f3]", "(RGWRados::get_obj_iterate_cb(DoutPrefixProvider const*, rgw_raw_obj const&, long, long, long, bool, RGWObjState*, void*)+0x11a) [0x56207fd98aaa]", "/usr/bin/radosgw(+0x730dd6) [0x56207fd93dd6]", "(RGWRados::iterate_obj(DoutPrefixProvider const*, RGWObjectCtx&, RGWBucketInfo&, rgw_obj const&, long, long, unsigned long, int (*)(DoutPrefixProvider const*, rgw_raw_obj const&, long, long, long, bool, RGWObjState*, void*), void*, optional_yield)+0x428) [0x56207fd993a8]", "(RGWRados::Object::Read::iterate(DoutPrefixProvider const*, long, long, RGWGetDataCB*, optional_yield)+0x134) [0x56207fd99b64]", "(RGWGetObj::execute(optional_yield)+0xf59) [0x56207fbc4809]", "(RGWSelectObj_ObjStore_S3::execute(optional_yield)+0xc1) [0x56207fcac5b1]", "(rgw_process_authenticated(RGWHandler_REST*, RGWOp*&, RGWRequest*, req_state*, optional_yield, rgw::sal::Driver*, bool)+0xa72) [0x56207fa761a2]", "(process_request(RGWProcessEnv const&, RGWRequest*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, RGWRestfulIO*, optional_yield, rgw::dmclock::Scheduler*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >*, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> >*, int*)+0x1039) [0x56207fa79719]", "/usr/bin/radosgw(+0xb4f176) [0x5620801b2176]", "/usr/bin/radosgw(+0x371ce1) [0x56207f9d4ce1]", "make_fcontext()" ], "ceph_version": "18.1.2-1.el9cp", "crash_id": "2023-08-09T03:28:38.019406Z_b2fa6c9f-51d6-4555-bb8b-cd5ed65e585c", "entity_name": "client.rgw.rgw.1.ceph-hmaheswa-rhcs7-fiisfo-node5.vnrgwe", "os_id": "rhel", "os_name": "Red Hat Enterprise Linux", "os_version": "9.2 (Plow)", "os_version_id": "9.2", "process_name": "radosgw", "stack_sig": "124e6c557a0776129578d472f8f34663835b6315f712be574af3fa8704e90d3d", "timestamp": "2023-08-09T03:28:38.019406Z", "utsname_hostname": "ceph-hmaheswa-rhcs7-fiisfo-node5", "utsname_machine": "x86_64", "utsname_release": "5.14.0-284.25.1.el9_2.x86_64", "utsname_sysname": "Linux", "utsname_version": "#1 SMP PREEMPT_DYNAMIC Thu Jul 20 09:11:28 EDT 2023" } [root@ceph-hmaheswa-rhcs7-fiisfo-node5 crash]# json data uploaded: { "firstName": "Joe", "lastName": "Jackson", "gender": "male", "age": "twenty", "address": { "streetAddress": "101", "city": "San Diego", "state": "CA" }, "firstName": "Joe_2", "lastName": "Jackson_2", "gender": "male", "age": 21, "address": { "streetAddress": "101", "city": "San Diego", "state": "CA" }, "phoneNumbers": [ { "type": "home1", "number": "734928_1","addr": 11 }, { "type": "home2", "number": "734928_2","addr": 22 }, { "type": "home3", "number": "734928_3","addr": 33 }, { "type": "home4", "number": "734928_4","addr": 44 }, { "type": "home5", "number": "734928_5","addr": 55 }, { "type": "home6", "number": "734928_6","addr": 66 }, { "type": "home7", "number": "734928_7","addr": 77 }, { "type": "home8", "number": "734928_8","addr": 88 }, { "type": "home9", "number": "734928_9","addr": 99 }, { "type": "home10", "number": "734928_10","addr": 100 } ], "key_after_array": "XXX", "description" : { "main_desc" : "value_1", "second_desc" : "value_2" } } Version-Release number of selected component (if applicable): ceph version 18.1.2-1.el9cp How reproducible: always Steps to Reproduce: 1.deploy rhcs7.0 ceph cluster 2.upload a json object using aws-cli 3.execute the query "select phonenumbers.type from s3object[*].phonenumbers;" Actual results: rgw daemon is crashing Expected results: It should display proper error instead of crashing Additional info: rgw logs and crash logs are present here: http://magna002.ceph.redhat.com/ceph-qe-logs/HemanthSai/s3select_json_bz_logs/ceph-client.rgw.rgw.1.ceph-hmaheswa-rhcs7-fiisfo-node5.vnrgwe.log rgw node:10.0.210.237 creds: root/passwd ; cephuser/cephuser
the crash is fixed on https://github.com/ceph/ceph/pull/52651