There is a race condition in the sg driver with outstanding commands and the driver's device removal callback, sg_remove().  When the main scsi device is deleted, sg_remove() will be called and free the memory used by the attached sg device, including the command buffers.  But sg_removal() fails to wait for any running commands to complete, resulting in the driver freeing memory which will be used by the completion handling of the running commands.  This leaves a race condition where the commands take long enough to complete that the removal callback will have freed everything first, resulting in a crash from using freed and reused memory.

This can be recreated with the scsi_debug driver.
1) set the delay option on scsi_debug to 10000 (10 seconds).
2) send a command to the device, e.g. sg_readcap /dev/sg2
3) delete the scsi_debug scsi device. e.g. echo "1" >/sys/block/sdc/device/delete

The memory for the sg device will be freed before the command completes.  If it has been reused, it will cause a crash when the command is finally completed.

Hi, 

Our Netapp testers have observed the same issue while testing with RHEL 5.8 with SAS connected array and LSI MPT Fusion SAS 2.0 Device Driver with MPP multipath driver.

Kernel : 
2.6.18-308.el5xen #1 SMP Fri Jan 27 17:26:57 EST 2012 x86_64 x86_64 x86_64 GNU/Linux

Exception Trace :

PID: 63     TASK: ffff8803bddab0c0  CPU: 20  COMMAND: "ksoftirqd/20"
 #0 [ffff8803bde0bc30] crash_kexec at ffffffff802a7dca
 #1 [ffff8803bde0bcf0] __die at ffffffff8026427d
 #2 [ffff8803bde0bd30] die at ffffffff8026cca4
 #3 [ffff8803bde0bd60] do_general_protection at ffffffff80264594
 #4 [ffff8803bde0bda0] error_exit at ffffffff8025f82b
    [exception RIP: sg_cmd_done+46]
    RIP: ffffffff880b70ad  RSP: ffff8803bde0be58  RFLAGS: 00010006
    RAX: ffffffff880b707f  RBX: 0000000000010000  RCX: 0000000000000000
    RDX: 0000000000010000  RSI: ffff88037134fe58  RDI: ffffffff880ba88a
    RBP: ffff88037134e070   R8: 00000014be097e58   R9: 00000000fffffffb
    R10: ffff88038493f398  R11: ffffffff8807af9f  R12: ffff880380a806f0
    R13: 3d48544150564544  R14: ffff88037134fe58  R15: 0000000000000000
    ORIG_RAX: ffffffffffffffff  CS: 10000e030  SS: e02b
 #5 [ffff8803bde0bea0] scsi_end_async at ffffffff8807afca [scsi_mod]
 #6 [ffff8803bde0bec0] scsi_end_request at ffffffff8807a51e [scsi_mod]
 #7 [ffff8803bde0bef0] scsi_io_completion at ffffffff8807a6b4 [scsi_mod]
 #8 [ffff8803bde0bf60] blk_done_softirq at ffffffff80238f50
 #9 [ffff8803bde0bf80] __do_softirq at ffffffff80212ebc
#10 [ffff8803bde0bfb0] call_softirq at ffffffff8025fda4
--- <IRQ stack> ---
#11 [ffff8803bddffec8] do_softirq at ffffffff8026db89
#12 [ffff8803bddffed8] ksoftirqd at ffffffff8029137e
#13 [ffff8803bddffee8] kthread at ffffffff802338cf
#14 [ffff8803bddfff48] kernel_thread at ffffffff8025fb2c

The command completion occured on device which is already removed.

Device removed from SG table ( 9:0:1:2 )

Sep 11 10:05:36 ictm-soyuzhost-Cfg05-4 kernel: sd 9:0:1:2: Attached scsi disk sdeb
Sep 11 10:05:36 ictm-soyuzhost-Cfg05-4 kernel:   Vendor: <5>sd 9:0:1:2: Attached scsi generic sg135 type 0
Sep 11 10:05:36 ictm-soyuzhost-Cfg05-4 kernel: NETAPP    Model: IN<5>  Vendor: NETAPP    Model: VirtualDisk       Rev: 8786
Sep 11 10:05:36 ictm-soyuzhost-Cfg05-4 kernel:   Type:   Direct-Access                      ANSI SCSI revision: 05

crash> sg_dev_arr
sg_dev_arr = $35 = (Sg_device **) 0xffff88038067b000
crash>

crash> p *((Sg_device*)0xffff88038067b000+135)
$34 = {
  device = 0x0,
  o_excl_wait = {
    lock = {
      raw_lock = {
        slock = 0
      }
    },
    task_list = {
      next = 0xffffffff88079161,
      prev = 0x5500000000
    }
  },
  sg_tablesize = 1,
  headfp = 0x500000000,
  detached = 0 '\000',
  exclude = 0 '\000',
  sgdebug = 0 '\000',
  disk = 0xffffffff880756b7,
  cdev = 0x200000000
}

crash> struct scsi_device 0xffff88037faa2800 | grep sdev_state
  sdev_state = SDEV_DEL,

kobj = {
      k_name = 0xffff88037faa29e0 "9:0:1:2",
      name = "9:0:1:2\000\000\000\000\000\000\000\000\000\000\000\000",
      kref = {
        refcount = {
          counter = 1
        }
      },

crash> struct scsi_io_context 0xffff88037134fe48
struct scsi_io_context {
  data = 0xffff88037134e070,
  done = 0xffffffff880b707f <sg_cmd_done>,
  sense = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"
}

In sg_cmd_done() Sg_request is corrupt. Looks like its freed.

crash> struct Sg_request 0xffff88037134e070 ( Corrupt )
struct Sg_request {
  nextrp = 0x737973207379732f,
  parentfp = 0x2030207772207366,
  data = {
    k_use_sg = 2608,
    sglist_len = 28719,
    bufflen = 795045746,
    b_malloc_len = 796095842,
    buffer = 0x75622f636f72702f,
    dio_in_use = 115 's',
    cmd_opcode = 47 '/'
  },
  header = {
    interface_id = 544433762,
    dxfer_direction = 807434098,
    cmd_len = 32 ' ',
    mx_sb_len = 48 '0',
    iovec_count = 25610,
    dxfer_len = 1953527397,
    dxferp = 0x702f7665642f2073,
    cmdp = 0x7470766564207374 <Address 0x7470766564207374 out of bounds>,
    sbp = 0x3020302077722073,
    timeout = 1701064458,
    flags = 1685270390,
    pack_id = 790638945,
    usr_ptr = 0x7772203374786520,
    status = 44 ',',
    masked_status = 100 'd',
    msg_status = 97 'a',
    sb_len_wr = 116 't',
    host_status = 15713,
    driver_status = 29295,
    resid = 1701995876,
    duration = 540024932,
    info = 1836321328
  },
  sense_b = "/dev/shm tmpfs rw 0 0\nnone /proc/sys/fs/binfmt_misc binfmt_misc rw 0 0\nsunrpc /var/lib/nfs/rpc_p",
  res_used = 105 'i',
  orphan = 112 'p',
  sg_io_owned = 101 'e',
  done = 102 'f'
}

I see major changes in RHEL 6.3 while handling device removal's and how command completions are handled in sg driver. So is there any plan to merge those changes with RHEL 5.9?.

Thanks
Shiva

This request was not resolved in time for the current release.
Red Hat invites you to ask your support representative to
propose this request, if still desired, for consideration in
the next release of Red Hat Enterprise Linux.

Thank you for submitting this request for inclusion in Red Hat Enterprise Linux 5. We've carefully evaluated the request, but are unable to include it in the  last planned RHEL5 minor release. This Bugzilla will soon be CLOSED as WONTFIX. To request that Red Hat re-consider this request, please re-open the bugzilla via  appropriate support channels and provide additional business and/or technical details about its importance to you.

Thank you for submitting this request for inclusion in Red Hat Enterprise Linux 5. We've carefully evaluated the request, but are unable to include it in RHEL5 stream. If the issue is critical for your business, please provide additional business justification through the appropriate support channels (https://access.redhat.com/site/support).

The needinfo request[s] on this closed bug have been removed as they have been unresolved for 1000 days