Bug 2230230

Description of problem:
rgw daemon is crashing while executing the below query
[cephuser@ceph-hmaheswa-rhcs7-fiisfo-node6 ~]$ aws s3api select-object-content --endpoint-url http://10.0.210.237:80 --bucket bkt1 --key example1_json --expression-type 'SQL' --input-serialization '{"JSON": {"Type": "DOCUMENT"}, "CompressionType": "NONE"}' --output-serialization '{"CSV": {}}' --expression "select phonenumbers.type from s3object[*].phonenumbers;" /dev/stdout

("Connection broken: InvalidChunkLength(got length b'', 0 bytes read)", InvalidChunkLength(got length b'', 0 bytes read))
[cephuser@ceph-hmaheswa-rhcs7-fiisfo-node6 ~]$


crash info:
[root@ceph-hmaheswa-rhcs7-fiisfo-node5 crash]# ceph crash info 2023-08-09T03:28:38.019406Z_b2fa6c9f-51d6-4555-bb8b-cd5ed65e585c
{
    "backtrace": [
        "/lib64/libc.so.6(+0x54df0) [0x7f10f8faadf0]",
        "/lib64/libc.so.6(+0xa154c) [0x7f10f8ff754c]",
        "raise()",
        "abort()",
        "/usr/bin/radosgw(+0x309588) [0x56207f96c588]",
        "/usr/bin/radosgw(+0x6126f7) [0x56207fc756f7]",
        "/usr/bin/radosgw(+0xb97369) [0x5620801fa369]",
        "/usr/bin/radosgw(+0x634c00) [0x56207fc97c00]",
        "/usr/bin/radosgw(+0x61dc0d) [0x56207fc80c0d]",
        "/usr/bin/radosgw(+0xb144d4) [0x5620801774d4]",
        "/usr/bin/radosgw(+0x61fe06) [0x56207fc82e06]",
        "(RGWSelectObj_ObjStore_S3::run_s3select_on_json(char const*, char const*, unsigned long)+0x9de) [0x56207fca72be]",
        "(RGWSelectObj_ObjStore_S3::json_processing(ceph::buffer::v15_2_0::list&, long, long)+0x633) [0x56207fcaa6f3]",
        "(RGWRados::get_obj_iterate_cb(DoutPrefixProvider const*, rgw_raw_obj const&, long, long, long, bool, RGWObjState*, void*)+0x11a) [0x56207fd98aaa]",
        "/usr/bin/radosgw(+0x730dd6) [0x56207fd93dd6]",
        "(RGWRados::iterate_obj(DoutPrefixProvider const*, RGWObjectCtx&, RGWBucketInfo&, rgw_obj const&, long, long, unsigned long, int (*)(DoutPrefixProvider const*, rgw_raw_obj const&, long, long, long, bool, RGWObjState*, void*), void*, optional_yield)+0x428) [0x56207fd993a8]",
        "(RGWRados::Object::Read::iterate(DoutPrefixProvider const*, long, long, RGWGetDataCB*, optional_yield)+0x134) [0x56207fd99b64]",
        "(RGWGetObj::execute(optional_yield)+0xf59) [0x56207fbc4809]",
        "(RGWSelectObj_ObjStore_S3::execute(optional_yield)+0xc1) [0x56207fcac5b1]",
        "(rgw_process_authenticated(RGWHandler_REST*, RGWOp*&, RGWRequest*, req_state*, optional_yield, rgw::sal::Driver*, bool)+0xa72) [0x56207fa761a2]",
        "(process_request(RGWProcessEnv const&, RGWRequest*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, RGWRestfulIO*, optional_yield, rgw::dmclock::Scheduler*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >*, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> >*, int*)+0x1039) [0x56207fa79719]",
        "/usr/bin/radosgw(+0xb4f176) [0x5620801b2176]",
        "/usr/bin/radosgw(+0x371ce1) [0x56207f9d4ce1]",
        "make_fcontext()"
    ],
    "ceph_version": "18.1.2-1.el9cp",
    "crash_id": "2023-08-09T03:28:38.019406Z_b2fa6c9f-51d6-4555-bb8b-cd5ed65e585c",
    "entity_name": "client.rgw.rgw.1.ceph-hmaheswa-rhcs7-fiisfo-node5.vnrgwe",
    "os_id": "rhel",
    "os_name": "Red Hat Enterprise Linux",
    "os_version": "9.2 (Plow)",
    "os_version_id": "9.2",
    "process_name": "radosgw",
    "stack_sig": "124e6c557a0776129578d472f8f34663835b6315f712be574af3fa8704e90d3d",
    "timestamp": "2023-08-09T03:28:38.019406Z",
    "utsname_hostname": "ceph-hmaheswa-rhcs7-fiisfo-node5",
    "utsname_machine": "x86_64",
    "utsname_release": "5.14.0-284.25.1.el9_2.x86_64",
    "utsname_sysname": "Linux",
    "utsname_version": "#1 SMP PREEMPT_DYNAMIC Thu Jul 20 09:11:28 EDT 2023"
}
[root@ceph-hmaheswa-rhcs7-fiisfo-node5 crash]#


json data uploaded:
{
 "firstName": "Joe",
 "lastName": "Jackson",
 "gender": "male",
 "age": "twenty",
 "address": {
 "streetAddress": "101",
 "city": "San Diego",
 "state": "CA"
 },

 "firstName": "Joe_2",
 "lastName": "Jackson_2",
 "gender": "male",
 "age": 21,
 "address": {
 "streetAddress": "101",
 "city": "San Diego",
 "state": "CA"
 },

 "phoneNumbers": [
   { "type": "home1", "number": "734928_1","addr": 11 },
   { "type": "home2", "number": "734928_2","addr": 22 },
   { "type": "home3", "number": "734928_3","addr": 33 },
   { "type": "home4", "number": "734928_4","addr": 44 },
   { "type": "home5", "number": "734928_5","addr": 55 },
   { "type": "home6", "number": "734928_6","addr": 66 },
   { "type": "home7", "number": "734928_7","addr": 77 },
   { "type": "home8", "number": "734928_8","addr": 88 },
   { "type": "home9", "number": "734928_9","addr": 99 },
   { "type": "home10", "number": "734928_10","addr": 100 }
 ],

 "key_after_array": "XXX",

 "description" : {
   "main_desc" : "value_1",
   "second_desc" : "value_2"
 }
}


Version-Release number of selected component (if applicable):
ceph version 18.1.2-1.el9cp

How reproducible:
always

Steps to Reproduce:
1.deploy rhcs7.0 ceph cluster
2.upload a json object using aws-cli
3.execute the query "select phonenumbers.type from s3object[*].phonenumbers;"

Actual results:
rgw daemon is crashing

Expected results:
It should display proper error instead of crashing

Additional info:
rgw logs and crash logs are present here: http://magna002.ceph.redhat.com/ceph-qe-logs/HemanthSai/s3select_json_bz_logs/ceph-client.rgw.rgw.1.ceph-hmaheswa-rhcs7-fiisfo-node5.vnrgwe.log

rgw node:10.0.210.237
creds: root/passwd ; cephuser/cephuser