Bug 1379665
| Summary: | Ganesha crashes while removing files from clients. | ||
|---|---|---|---|
| Product: | [Red Hat Storage] Red Hat Gluster Storage | Reporter: | Shashank Raj <sraj> |
| Component: | nfs-ganesha | Assignee: | Soumya Koduri <skoduri> |
| Status: | CLOSED ERRATA | QA Contact: | Arthy Loganathan <aloganat> |
| Severity: | high | Docs Contact: | |
| Priority: | unspecified | ||
| Version: | rhgs-3.2 | CC: | bugs, jthottan, kkeithle, mzywusko, ndevos, rhinduja, rhs-bugs, sbhaloth, skoduri, storage-qa-internal |
| Target Milestone: | --- | Keywords: | Triaged |
| Target Release: | RHGS 3.2.0 | ||
| Hardware: | x86_64 | ||
| OS: | Linux | ||
| Whiteboard: | |||
| Fixed In Version: | nfs-ganesha-2.4.1-1 | Doc Type: | If docs needed, set a value |
| Doc Text: | Story Points: | --- | |
| Clone Of: | 1374015 | Environment: | |
| Last Closed: | 2017-03-23 06:23:25 UTC | Type: | Bug |
| Regression: | --- | Mount Type: | --- |
| Documentation: | --- | CRM: | |
| Verified Versions: | Category: | --- | |
| oVirt Team: | --- | RHEL 7.3 requirements from Atomic Host: | |
| Cloudforms Team: | --- | Target Upstream Version: | |
| Embargoed: | |||
| Bug Depends On: | 1374015, 1375564 | ||
| Bug Blocks: | 1351528 | ||
|
Description
Shashank Raj
2016-09-27 11:41:18 UTC
(gdb) bt
#0 0x00007f6bef4126ff in __inode_ctx_free (inode=0x7f6bce160da0) at inode.c:332
#1 0x00007f6bef414b55 in __inode_destroy (table=<value optimized out>)
at inode.c:353
#2 inode_table_prune (table=<value optimized out>) at inode.c:1543
#3 0x00007f6bef4153dc in inode_unref (inode=0x7f6bce160da0) at inode.c:524
#4 0x00007f6bef6e8066 in pub_glfs_h_close (object=0x7f6af4add550)
at glfs-handleops.c:1365
#5 0x00007f6bef8fa524 in handle_release (obj_hdl=0x7f6af4b62ba8)
at /usr/src/debug/nfs-ganesha-2.4.0/src/FSAL/FSAL_GLUSTER/handle.c:71
#6 0x00000000004e21e6 in mdcache_lru_clean (entry=0x7f6b5a15ab48)
at /usr/src/debug/nfs-ganesha-2.4.0/src/FSAL/Stackable_FSALs/FSAL_MDCACHE/mdcache_lru.c:421
#7 mdcache_lru_get (entry=0x7f6b5a15ab48)
at /usr/src/debug/nfs-ganesha-2.4.0/src/FSAL/Stackable_FSALs/FSAL_MDCACHE/mdcache_lru.c:1201
#8 0x00000000004ecbf1 in mdcache_alloc_handle (export=0xe035f0,
sub_handle=0x7f6af4b68fe8, attrs_in=0x7f6b5a15ac60, attrs_out=0x0,
new_directory=false, entry=0x7f6b5a15abc8, state=0x0)
at /usr/src/debug/nfs-ganesha-2.4.0/src/FSAL/Stackable_FSALs/FSAL_MDCACHE/mdcache_helpers.c:117
#9 mdcache_new_entry (export=0xe035f0, sub_handle=0x7f6af4b68fe8,
attrs_in=0x7f6b5a15ac60, attrs_out=0x0, new_directory=false,
entry=0x7f6b5a15abc8, state=0x0)
at /usr/src/debug/nfs-ganesha-2.4.0/src/FSAL/Stackable_FSALs/FSAL_MDCACHE/mdcache_helpers.c:371
#10 0x00000000004e521e in mdcache_alloc_and_check_handle (
export=<value optimized out>, sub_handle=<value optimized out>,
new_obj=0x7f6b5a15ad30, new_directory=<value optimized out>,
attrs_in=<value optimized out>, attrs_out=0x0, tag=0x51f862 "lookup ",
parent=0xe02c50, name=0x7f6addf7157c "def79973", invalidate=true, state=0x0)
at /usr/src/debug/nfs-ganesha-2.4.0/src/FSAL/Stackable_FSALs/FSAL_MDCACHE/mdcache_handle.c:93
#11 0x00000000004ec215 in mdc_lookup_uncached (mdc_parent=0xe02c50,
name=0x7f6addf7157c "def79973", new_entry=0x7f6b5a15add8,
attrs_out=<value optimized out>)
at /usr/src/debug/nfs-ganesha-2.4.0/src/FSAL/Stackable_FSALs/FSAL_MDCACHE/mdcache_helpers.c:986
#12 0x00000000004e45a6 in mdcache_readdir (dir_hdl=0xe02c88,
whence=<value optimized out>, dir_state=0x7f6b5a15ae50,
cb=0x42bca0 <populate_dirent>, attrmask=<value optimized out>,
eod_met=0x7f6b5a15b06f)
at /usr/src/debug/nfs-ganesha-2.4.0/src/FSAL/Stackable_FSALs/FSAL_MDCACHE/mdcac---Type <return> to continue, or q <return> to quit---q
Quit
(gdb) p *(xlator_t *)(long)inode->_ctx[0]
$7 = {name = 0x7f6af41a4118 "", type = 0x7f6ae4206978 "(\300\026\310j\177",
instance_name = 0x0, next = 0x7561c0, prev = 0x4e4c40, parents = 0x4e3e50,
children = 0x4e3b00, options = 0x4e36e0, dlhandle = 0x4e4bf0, fops = 0x4e4320,
cbks = 0x4e6990, dumpops = 0x4e6550, volume_options = {next = 0x4e6100,
prev = 0x4e5cc0}, fini = 0x4e3e60 <mdcache_readlink>,
init = 0x4e4300 <mdcache_test_access>,
reconfigure = 0x4e7710 <mdcache_getattrs>,
mem_acct_init = 0x4e7380 <mdcache_setattrs>, notify = 0x4e3b70 <mdcache_link>,
loglevel = 5125088, latencies = {{min = 5131344, max = 5133440,
total = 2.5431179326767319e-317, std = 2.5424618134990547e-317,
mean = 2.5416317832140414e-317, count = 5148272}, {min = 5148048,
max = 5147024, total = 2.5423669528950532e-317,
std = 2.5416792135160422e-317, mean = 2.5417266438180429e-317,
count = 5146832}, {min = 5144608, max = 5144704,
total = 2.5418610296737118e-317, std = 2.5448333285990927e-317,
mean = 2.5448807589010935e-317, count = 5151088}, {min = 5150992,
max = 5151184, total = 2.5450704801090965e-317,
std = 2.5451179104110973e-317, mean = 2.545165340713098e-317,
count = 5125184}, {min = 5125280, max = 5125376,
total = 2.5323196339212256e-317, std = 2.5323670642232264e-317,
mean = 2.5324144945252271e-317, count = 5125760}, {min = 5151568,
max = 5151664, total = 2.5453076316191003e-317,
std = 2.5453550619211011e-317, mean = 2.5436950013510745e-317,
count = 5144880}, {min = 5144976, max = 5146608,
total = 2.5433471791364022e-317, std = 2.5426515347070578e-317,
mean = 2.5422799973413851e-317, count = 5145456}, {min = 5146192,
max = 5145072, total = 2.5397899064863453e-317,
std = 2.5420507508817148e-317, mean = 0, count = 0}, {min = 0, max = 0,
total = 0, std = 0, mean = 0, count = 1}, {min = 67, max = 249090,
total = -3.1019956711109088e-195, std = 6.9216917256078698e-310,
mean = 6.9217287135458876e-310, count = 1433582}, {min = 1, max = 0,
total = 3.3102398271363518e-322, std = 1.230668117225961e-318, mean = 0,
count = 10920380356516740874}, {min = 4294967716, max = 0, total = 0,
std = 0, mean = 0, count = 1475853366}, {min = 705328000, max = 0,
total = 0, std = 7.2916844643976761e-315, mean = 4.1254480637239053e-315,
count = 1475853366}, {min = 705329000, max = 1475853366,
total = 4.1254480637239053e-315, std = 0, mean = 5.1870226870842929e-210,
count = 0}, {min = 60, max = 140097749877984,
total = 6.9217485337872854e-310, std = 6.9216851455341886e-310,
mean = 7.5076972142905743e+160, count = 140101559541008}, {
min = 140097348841264, max = 32, total = 0, std = 1.4821969375237396e-323,
mean = 7.2917110253667966e-315, count = 1475858742}, {min = 0, max = 0,
---Type <return> to continue, or q <return> to quit---q
totalQuit
(gdb) p *(xlator_t *)(long)inode->_ctx[1]
$8 = {name = 0x7f6bd4016af0 "ozone-dht",
type = 0x7f6bd401dc50 "cluster/distribute", instance_name = 0x0,
next = 0x7f6bd401c0d0, prev = 0x7f6bd401e900, parents = 0x7f6bd401f580,
children = 0x7f6bd401dcc0, options = 0x7f6be97dbddc, dlhandle = 0x7f6bd401dd10,
fops = 0x7f6be0fcd3e0, cbks = 0x7f6be0fcd780, dumpops = 0x7f6be0fcd720,
volume_options = {next = 0x7f6bd401e3b0, prev = 0x7f6bd401e3b0},
fini = 0x7f6be0db2760 <dht_fini>, init = 0x7f6be0db3160 <dht_init>,
reconfigure = 0x7f6be0db2020 <dht_reconfigure>,
mem_acct_init = 0x7f6be0db2670 <mem_acct_init>,
notify = 0x7f6be0d70110 <dht_notify>, loglevel = GF_LOG_NONE, latencies = {{
min = 0, max = 0, total = 0, std = 0, mean = 0,
count = 0} <repeats 55 times>}, history = 0x0, ctx = 0xd352e0,
graph = 0x7f6bd4003790, itable = 0x0, init_succeeded = 1 '\001',
private = 0x7f6bd4058710, mem_acct = 0x7f6bd4054fe0, winds = 0,
switched = 0 '\000', local_pool = 0x7f6bd4059170, is_autoloaded = _gf_false}
(gdb) p *(xlator_t *)(long)inode->_ctx[2]
$9 = {name = 0x7f6bd401dbe0 "ozone-io-cache",
type = 0x7f6bd40237b0 "performance/io-cache", instance_name = 0x0,
next = 0x7f6bd4021670, prev = 0x7f6bd40242a0, parents = 0x7f6bd4024f20,
children = 0x7f6bd4023820, options = 0x7f6be97dc08c, dlhandle = 0x7f6bd40238c0,
fops = 0x7f6be071f020, cbks = 0x7f6be071f3c0, dumpops = 0x7f6be071f360,
volume_options = {next = 0x7f6bd40241d0, prev = 0x7f6bd40241d0},
fini = 0x7f6be0510480 <fini>, init = 0x7f6be0516d20 <init>,
reconfigure = 0x7f6be0517210 <reconfigure>,
mem_acct_init = 0x7f6be0510ac0 <mem_acct_init>,
notify = 0x7f6bef47ffc0 <default_notify>, loglevel = GF_LOG_NONE, latencies = {{
min = 0, max = 0, total = 0, std = 0, mean = 0,
count = 0} <repeats 55 times>}, history = 0x0, ctx = 0xd352e0,
graph = 0x7f6bd4003790, itable = 0x0, init_succeeded = 1 '\001',
private = 0x7f6bd403f7f0, mem_acct = 0x7f6bd403c4d0, winds = 0,
switched = 0 '\000', local_pool = 0x7f6bd403f960, is_autoloaded = _gf_false}
(gdb) p *(xlator_t *)(long)inode->_ctx[3]
$10 = {name = 0x7f6bd4024e40 "ozone-quick-read",
type = 0x7f6bd4024eb0 "performance/quick-read", instance_name = 0x0,
next = 0x7f6bd4022c10, prev = 0x7f6bd40257b0, parents = 0x7f6bd4026bb0,
children = 0x7f6bd4025750, options = 0x7f6be97dc138, dlhandle = 0x7f6bd4024fc0,
fops = 0x7f6be050b020, cbks = 0x7f6be050b360, dumpops = 0x7f6be050b3a0,
volume_options = {next = 0x7f6bd4025680, prev = 0x7f6bd4025680},
fini = 0x7f6be0305c90 <fini>, init = 0x7f6be0309200 <init>,
reconfigure = 0x7f6be0305fc0 <reconfigure>,
mem_acct_init = 0x7f6be03061b0 <mem_acct_init>,
notify = 0x7f6bef47ffc0 <default_notify>, loglevel = GF_LOG_NONE, latencies = {{
min = 0, max = 0, total = 0, std = 0, mean = 0,
count = 0} <repeats 55 times>}, history = 0x0, ctx = 0xd352e0,
graph = 0x7f6bd4003790, itable = 0x0, init_succeeded = 1 '\001',
private = 0x7f6bd403c3f0, mem_acct = 0x7f6bd4039170, winds = 0,
switched = 0 '\000', local_pool = 0x0, is_autoloaded = _gf_false}
(gdb) p *(xlator_t *)(long)inode->_ctx[4]
$11 = {name = 0x7f6bd4022a30 "ozone-md-cache",
type = 0x7f6bd4027880 "performance/md-cache", instance_name = 0x0,
next = 0x7f6bd40257b0, prev = 0x7f6bd4028130, parents = 0x7f6bd4029460,
children = 0x7f6bd40278f0, options = 0x7f6be97dc290, dlhandle = 0x7f6bd4027990,
fops = 0x7f6bd3ffe000, cbks = 0x7f6bd3ffe340, dumpops = 0x0, volume_options = {
next = 0x7f6bd4028000, prev = 0x7f6bd4028000}, fini = 0x7f6bd3deed60 <fini>,
init = 0x7f6bd3deedb0 <init>, reconfigure = 0x7f6bd3deefc0 <reconfigure>,
mem_acct_init = 0x7f6bd3deefb0 <mem_acct_init>,
notify = 0x7f6bef47ffc0 <default_notify>, loglevel = GF_LOG_NONE, latencies = {{
min = 0, max = 0, total = 0, std = 0, mean = 0,
count = 0} <repeats 55 times>}, history = 0x0, ctx = 0xd352e0,
graph = 0x7f6bd4003790, itable = 0x0, init_succeeded = 1 '\001',
private = 0x7f6bd4036080, mem_acct = 0x7f6bd4032f90, winds = 0,
switched = 0 '\000', local_pool = 0x0, is_autoloaded = _gf_false}
(gdb) p *(xlator_t *)(long)inode->_ctx[5]
$12 = {name = 0xdc6da0 "gfapi", type = 0xdc6e00 "mount/api", instance_name = 0x0,
next = 0x7f6bd40297f0, prev = 0x0, parents = 0x0, children = 0x0,
options = 0x7f6bec3f506c, dlhandle = 0xdc6ee0, fops = 0x7f6be2cc30c0,
cbks = 0x7f6be2cc3000, dumpops = 0x7f6be2cc3060, volume_options = {
next = 0xdc8a80, prev = 0xdea930}, fini = 0x7f6be2ac1a90 <fini>,
init = 0x7f6be2ac1a80 <init>, reconfigure = 0,
mem_acct_init = 0x7f6be2ac1ad0 <mem_acct_init>,
notify = 0x7f6be2ac1c50 <notify>, loglevel = GF_LOG_NONE, latencies = {{
min = 0, max = 0, total = 0, std = 0, mean = 0,
count = 0} <repeats 55 times>}, history = 0x0, ctx = 0xd352e0, graph = 0x0,
itable = 0x0, init_succeeded = 1 '\001', private = 0xd35150,
mem_acct = 0xdc8b20, winds = 0, switched = 0 '\000', local_pool = 0x0,
is_autoloaded = _gf_false}
(gdb) p *(xlator_t *)(long)inode->_ctx[6]
Cannot access memory at address 0x0
Except for inode->ctx[0], every other ctx set is still intact.
Since there have been quite some fixes which went in wrt to refcount, could you please check if this issue still exists with the latest builds available (gluster and nfs-ganesha) . Tried creating and removing ~200000 files from 2 clients. Issue is not seen in the latest build, glusterfs-ganesha-3.8.4-5.el7rhgs.x86_64 nfs-ganesha-gluster-2.4.1-1.el7rhgs.x86_64 Issue not seen with the latest build, hence moving the bug to verified state. Since the problem described in this bug report should be resolved in a recent advisory, it has been closed with a resolution of ERRATA. For information on the advisory, and where to find the updated files, follow the link below. If the solution does not work for you, open a new bug report. https://rhn.redhat.com/errata/RHEA-2017-0493.html |