Login
[x]
Log in using an account from:
Fedora Account System
Red Hat Associate
Red Hat Customer
Or login using a Red Hat Bugzilla account
Forgot Password
Login:
Hide Forgot
Create an Account
Red Hat Bugzilla – Attachment 314761 Details for
Bug 458684
GFS2: glock deadlock in page fault path
[?]
New
Simple Search
Advanced Search
My Links
Browse
Requests
Reports
Current State
Search
Tabular reports
Graphical reports
Duplicates
Other Reports
User Changes
Plotly Reports
Bug Status
Bug Severity
Non-Defaults
|
Product Dashboard
Help
Page Help!
Bug Writing Guidelines
What's new
Browser Support Policy
5.0.4.rh83 Release notes
FAQ
Guides index
User guide
Web Services
Contact
Legal
This site requires JavaScript to be enabled to function correctly, please enable it.
Latest hang analyzer
gfs2_hangalyzer.c (text/plain), 13.01 KB, created by
Robert Peterson
on 2008-08-21 22:42:28 UTC
(
hide
)
Description:
Latest hang analyzer
Filename:
MIME Type:
Creator:
Robert Peterson
Created:
2008-08-21 22:42:28 UTC
Size:
13.01 KB
patch
obsolete
>/* gfs2_hangalyzer - a tool to figure out why gfs2 is hung - by Bob Peterson */ >/* Copyright 2008, Red Hat, Inc. */ >#include <stdio.h> >#include <string.h> >#include <stdlib.h> >#include <sys/types.h> >#include <sys/stat.h> >#include <fcntl.h> >#include <unistd.h> >#include <time.h> >#include <limits.h> >#include <sys/wait.h> > >#define CONF "/etc/cluster/cluster.conf" >#define MAX_NODES 64 >#define MAX_MOUNTS 12 >#define MAX_HUNG_GLOCKS 20 > >struct nodeinfo { > int node_id; > char nodename[80]; > int mount_count; > char debugfs[PATH_MAX]; > char *mounts[MAX_MOUNTS]; >}; > >int verbose = 0; >int use_qarsh = 0; >const char *shell_prg[2] = {"/usr/bin/ssh", "/usr/bin/qarsh"}; >const char *shell_prg2[2] = {"ssh", "qarsh"}; >char onenode[256]; >struct nodeinfo *node[MAX_NODES]; >int node_count; >extern char *optarg; >char hung_glocks[MAX_HUNG_GLOCKS][40]; >int hung_glock_cnt; > >/* ------------------------------------------------------------------------ */ >/* usage */ >/* ------------------------------------------------------------------------ */ >void usage(char **argv) >{ > printf("Format is: \n\t%s -n anynode [-q][-v][-a]\n", argv[0]); > printf("-n any node name in the cluster.\n"); > printf("-q use qarsh instead of ssh\n"); > printf("-v increase verbose setting\n"); > printf("If -q is specified, qarsh will be used to fetch info, " > "otherwise ssh.\n"); >} > >/* ------------------------------------------------------------------------ */ >/* init */ >/* ------------------------------------------------------------------------ */ >int init(int argc, char *argv[]) >{ > int optchar; > > memset(onenode, 0, sizeof(onenode)); > > while (1) { > optchar = getopt(argc, argv, "qvn:"); > if (optchar <= 0) > break; > switch (optchar) { > case 'n': > strcpy(onenode, optarg); > break; > case 'v': > verbose++; > break; > case 'q': > use_qarsh = 1; > break; > default: > fprintf(stderr, "Unknown option: %c\n", optchar); > usage(argv); > exit(0); > } > } > if (onenode[0] == '\0') { > fprintf(stderr, "%s error: node not specified.\n", argv[0]); > usage(argv); > exit(-1); > } > return 0; >} > >/* ------------------------------------------------------------------------ */ >/* temp_node_file - concoct a temporary filename we can use */ >/* ------------------------------------------------------------------------ */ >const char *temp_node_file(const char *nodename) >{ > static char fname[256]; > > sprintf(fname,"/tmp/gfs2_hangalyzer.%s", nodename); > return fname; >} > >/* ------------------------------------------------------------------------ */ >/* temp_glock_file - concoct a glock filename we can use */ >/* ------------------------------------------------------------------------ */ >const char *temp_glock_file(int n, int mnt, const char *mntpt) >{ > static char fname[256]; > > sprintf(fname, "%s.%s.%d", temp_node_file(node[n]->nodename), > mntpt, mnt + 1); > return fname; >} > >/* ------------------------------------------------------------------------ */ >/* do_node_cmd - execute a qarsh/ssh command and save the output in tmp */ >/* ------------------------------------------------------------------------ */ >void do_node_cmd(const char *nodename, const char *cmd) >{ > pid_t qarshpid; > int fd; > > unlink(temp_node_file(nodename)); > if ((qarshpid = fork()) == 0) { > close(1); > fd = creat(temp_node_file(nodename), S_IRUSR | S_IWUSR | > S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); > if (fd) { > if (!verbose) > fprintf(stderr, "%120.120s\r", " "); > fprintf(stderr, "exec: %s %s %s %s %s \"%s\"%c", > shell_prg[use_qarsh], shell_prg2[use_qarsh], > "-l", "root", nodename, cmd, > verbose ? '\n' : '\r'); > if ((execl(shell_prg[use_qarsh], shell_prg2[use_qarsh], > "-l", "root", nodename, cmd, NULL)) == -1) { > perror(cmd); > exit(-1); > } > close(fd); > } else > perror(temp_node_file(nodename)); > exit(0); > } > waitpid(qarshpid, NULL, 0); > if (!verbose) > fprintf(stderr, "%120.120s\r", " "); > return; >} > >/* ------------------------------------------------------------------------ */ >/* analyze_cluster_conf */ >/* ------------------------------------------------------------------------ */ >int analyze_cluster_conf(void) >{ > FILE *fp; > char line[1024]; > int nid; > > do_node_cmd(onenode, > "/bin/grep 'clusternode name' " > "/etc/cluster/cluster.conf"); > fp = fopen(temp_node_file(onenode), "rt"); > if (!fp) { > perror(temp_node_file(onenode)); > return -1; > } > node_count = 0; > while (fgets(line, sizeof(line) - 1, fp)) { > char *p, *p2, *p3, *p4; > > if (verbose >= 2) > printf("%s", line); > p = strstr(line,"<clusternode name="); > if (p) { > p += 19; > p2 = strchr(p,'"'); > node[node_count] = malloc(sizeof(struct nodeinfo)); > memset(node[node_count], 0, sizeof(struct nodeinfo)); > strncpy(node[node_count]->nodename, p, p2 - p); > > p3 = strstr(line,"nodeid="); > if (p3) > p3 += 8; > p4 = strchr(p3,'"'); > if (p4) > *p4 = '\0'; > sscanf(p3, "%d", &nid); > node[node_count]->node_id = nid; > if (verbose >= 1) > printf("%d:nodeid %d: %s\n", node_count + 1, > node[node_count]->node_id, > node[node_count]->nodename); > node_count++; > } > } > if (verbose) > printf("node_count = %d\n", node_count); > fclose(fp); > return 0; >} > >/* ------------------------------------------------------------------------ */ >/* find_debugfs */ >/* ------------------------------------------------------------------------ */ >void find_debugfs(int n) >{ > char line[PATH_MAX], tmp[PATH_MAX]; > ssize_t count = 0; > int fd; > > do_node_cmd(node[n]->nodename, "/bin/grep debugfs /proc/mounts"); > fd = open(temp_node_file(node[n]->nodename), O_RDONLY); > count = read(fd, line, PATH_MAX); > line[count] = '\0'; > if (count) > sscanf(line, "%s %s", tmp, node[n]->debugfs); > close(fd); > return; >} > >/* ------------------------------------------------------------------------ */ >/* find_mounts */ >/* figure out all the gfs2 mounts on a node, fill in mounts, and return # */ >/* ------------------------------------------------------------------------ */ >int find_mounts(int n) >{ > char line[PATH_MAX]; > FILE *file; > char cmd[256]; > int fd; > > sprintf(cmd, "/bin/ls %s/gfs2/", node[n]->debugfs); > do_node_cmd(node[n]->nodename, cmd); > > node[n]->mount_count = 0; > file = fopen(temp_node_file(node[n]->nodename), "rt"); > if (!file) > return -1; > while (fgets(line, PATH_MAX, file)) { > char *p; > > while ((p = strchr(line, '\n'))) > *p = '\0'; > while ((p = strchr(line, '\r'))) > *p = '\0'; > node[n]->mounts[node[n]->mount_count] = > malloc(PATH_MAX); > memset(node[n]->mounts[node[n]->mount_count], 0, > PATH_MAX); > strcpy(node[n]->mounts[node[n]->mount_count], line); > node[n]->mount_count++; > } > close(fd); > return 0; >} > >/* ------------------------------------------------------------------------ */ >/* fetch_glock_dumps */ >/* ------------------------------------------------------------------------ */ >int fetch_glock_dumps(int n) >{ > int mntpt; > char cmd[256]; > char new_fn[PATH_MAX]; > > for (mntpt = 0; mntpt < node[n]->mount_count; mntpt++) { > sprintf(cmd, "/bin/cat %s/gfs2/%s/glocks", > node[n]->debugfs, node[n]->mounts[mntpt]); > if (verbose) > fprintf(stderr, "fetching glocks for %s: %s\n", > node[n]->nodename, node[n]->mounts[mntpt]); > do_node_cmd(node[n]->nodename, cmd); > memset(new_fn, 0, sizeof(new_fn)); > rename(temp_node_file(node[n]->nodename), > temp_glock_file(n, mntpt, node[n]->mounts[mntpt])); > } > return 0; >} > >/* ------------------------------------------------------------------------ */ >/* decode_glock_flags - print a user-friendly version of the glock flags */ >/* ------------------------------------------------------------------------ */ >void decode_glock_flags(const char *nodename, const char *gflags) >{ > const char *p; > int flagi, count = 0; > const char *flags = "lsDdpyfir?"; > const char *flagdesc[10] = {"locked", "sticky", > "demote", "pending demote", > "demote in prog", "dirty", > "lflush", "invalidate in prog", > "reply pending", "unknown flag" }; > > p = strchr(gflags, ' '); > if (p == gflags || p == gflags + 1) > return; > printf("%-10.10s: ", nodename); > p = gflags; > while (p && *p != ' ') { > for (flagi = 0; flagi < 9; flagi++) > if (*p == flags[flagi]) > break; > printf("%s%s", count ? ", " : "(", flagdesc[flagi]); > count++; > p++; > } > printf(")\n"); >} > >/* ------------------------------------------------------------------------ */ >/* find_dump_glock - find a given glock in all the glock dumps and dump it */ >/* ------------------------------------------------------------------------ */ >void find_dump_glock(const char *glock_id, const char *mnt_id) >{ > int n; > int mntpt; > FILE *waiter; > char line[1024], *p; > int found_it, lines_printed = 0; > char *gf; > > if (verbose) > printf("Dumping all occurrences of glock %s\n", glock_id); > for (n = 0; n < node_count; n++) { > /* printf("%-10.10s: --------------------------------------" > "--------------------- %s\n", > node[n]->nodename, glock_id); */ > for (mntpt = 0; mntpt < node[n]->mount_count; mntpt++) { > if (strcmp(node[n]->mounts[mntpt], mnt_id)) > continue; > waiter = fopen(temp_glock_file(n, mntpt, > node[n]->mounts[mntpt]), > "rt"); > found_it = 0; > while (fgets(line, PATH_MAX, waiter)) { > while ((p = strchr(line, '\n'))) > *p = '\0'; > while ((p = strchr(line, '\r'))) > *p = '\0'; > if (line[0] == 'G') { > if (strstr(line, glock_id)) { > found_it = 1; > printf("%-10.10s: %s: %s\n", > node[n]->nodename, > node[n]->mounts[mntpt], > line); > gf = strstr(line, "f:"); > if (gf) { > gf += 2; > decode_glock_flags( > node[n]->nodename, > gf); > } > lines_printed++; > } else if (found_it) { > break; > } > } else if (found_it) { > printf("%-10.10s: %s: %s\n", > node[n]->nodename, > node[n]->mounts[mntpt], line); > lines_printed++; > } > } > fclose(waiter); > if (!found_it) > printf("%-10.10s: %s: No refs found to %s\n", > node[n]->nodename, mnt_id, glock_id); > } > } > if (lines_printed) > printf("\n\n\n"); >} > >/* ------------------------------------------------------------------------ */ >/* analyze_glocks */ >/* ------------------------------------------------------------------------ */ >int analyze_glocks(void) >{ > int n, g; > int mntpt; > FILE *waiter; > char line[1024]; > int holders_waiting = 0; > char last_glock[PATH_MAX]; > char holder[PATH_MAX]; > char glock_id[80], *p; > int glock_already_done; > > memset(last_glock, 0, sizeof(last_glock)); > memset(hung_glocks, 0, sizeof(hung_glocks)); > hung_glock_cnt = 0; > /* find all the glocks that are in the wait state */ > for (n = 0; n < node_count; n++) { > for (mntpt = 0; mntpt < node[n]->mount_count; mntpt++) { > waiter = fopen(temp_glock_file(n, mntpt, > node[n]->mounts[mntpt]), > "rt"); > while (fgets(line, PATH_MAX, waiter)) { > while ((p = strchr(line, '\n'))) > *p = '\0'; > while ((p = strchr(line, '\r'))) > *p = '\0'; > if (line[0] == 'G') { > strcpy(last_glock, line); > continue; > } > if (line[1] == 'H' && strchr(line, 'W')) { > strcpy(holder, line); > /* We have a waiter */ > holders_waiting++; > if (verbose >= 2) { > printf("Glock waited on: \n"); > printf("%s\n", last_glock); > printf("%s\n", holder); > } > memset(glock_id, 0, sizeof(glock_id)); > strncpy(glock_id, &last_glock[11], 20); > p = strchr(glock_id, ' '); > if (p) > *p = '\0'; > /* See if we've already processed it */ > glock_already_done = 0; > for (g = 0; g < hung_glock_cnt; g++) { > if (!strcmp(hung_glocks[g], > glock_id)) { > glock_already_done = 1; > break; > } > } > if (!glock_already_done) { > strcpy(hung_glocks[ > hung_glock_cnt], > glock_id); > hung_glock_cnt++; > find_dump_glock(glock_id, > node[n]->mounts[mntpt]); > } > } > } > fclose(waiter); > } > } > if (!hung_glock_cnt) > printf("No waiting glocks found on any node.\n"); > return 0; >} > >/* ------------------------------------------------------------------------ */ >/* main */ >/* ------------------------------------------------------------------------ */ >int main(int argc, char **argv) >{ > int error, i; > > /* step 1 - analyze the cluster.conf file to determine the nodes.*/ > init(argc, argv); > memset(node, 0, sizeof(node)); > error = analyze_cluster_conf(); > > for (i = 0; i < node_count; i++) > find_debugfs(i); > for (i = 0; i < node_count; i++) > find_mounts(i); > for (i = 0; i < node_count; i++) > fetch_glock_dumps(i); > analyze_glocks(); > /* free up the memory we allocated for node names. */ > for (i = 0; i<node_count; i++) > if (node[i]) > free(node[i]); > exit(0); >}
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Raw
Actions:
View
Attachments on
bug 458684
:
314028
|
314549
|
314680
|
314745
|
314760
|
314761
|
314762
|
314981
|
315157
|
315399
|
315400
|
315660
|
316226
|
316480
|
316579
|
316802