Attachment 314761 Details for Bug 458684 – Latest hang analyzer

Latest hang analyzer

gfs2_hangalyzer.c (text/plain), 13.01 KB, created by Robert Peterson on 2008-08-21 22:42:28 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Robert Peterson

Created: 2008-08-21 22:42:28 UTC

Size: 13.01 KB

patch

obsolete

>/* gfs2_hangalyzer - a tool to figure out why gfs2 is hung - by Bob Peterson */
>/* Copyright 2008, Red Hat, Inc.                           */
>#include <stdio.h>
>#include <string.h>
>#include <stdlib.h>
>#include <sys/types.h>
>#include <sys/stat.h>
>#include <fcntl.h>
>#include <unistd.h>
>#include <time.h>
>#include <limits.h>
>#include <sys/wait.h>
>
>#define CONF "/etc/cluster/cluster.conf"
>#define MAX_NODES 64
>#define MAX_MOUNTS 12
>#define MAX_HUNG_GLOCKS 20
>
>struct nodeinfo {
>	int node_id;
>	char nodename[80];
>	int mount_count;
>	char debugfs[PATH_MAX];
>	char *mounts[MAX_MOUNTS];
>};
>
>int verbose = 0;
>int use_qarsh = 0;
>const char *shell_prg[2] = {"/usr/bin/ssh", "/usr/bin/qarsh"};
>const char *shell_prg2[2] = {"ssh", "qarsh"};
>char onenode[256];
>struct nodeinfo *node[MAX_NODES];
>int node_count;
>extern char *optarg;
>char hung_glocks[MAX_HUNG_GLOCKS][40];
>int hung_glock_cnt;
>
>/* ------------------------------------------------------------------------ */
>/* usage                                                                    */
>/* ------------------------------------------------------------------------ */
>void usage(char **argv)
>{
>	printf("Format is: \n\t%s -n anynode [-q][-v][-a]\n", argv[0]);
>	printf("-n any node name in the cluster.\n");
>	printf("-q use qarsh instead of ssh\n");
>	printf("-v increase verbose setting\n");
>	printf("If -q is specified, qarsh will be used to fetch info, "
>	       "otherwise ssh.\n");
>}
>
>/* ------------------------------------------------------------------------ */
>/* init                                                                     */
>/* ------------------------------------------------------------------------ */
>int init(int argc, char *argv[])
>{
>	int optchar;
>
>	memset(onenode, 0, sizeof(onenode));
>
>	while (1) {
>		optchar = getopt(argc, argv, "qvn:");
>		if (optchar <= 0)
>			break;
>		switch (optchar) {
>		case 'n':
>			strcpy(onenode, optarg);
>			break;
>		case 'v':
>			verbose++;
>			break;
>		case 'q':
>			use_qarsh = 1;
>			break;
>		default:
>			fprintf(stderr, "Unknown option: %c\n", optchar);
>			usage(argv);
>			exit(0);
>		}
>	}
>	if (onenode[0] == '\0') {
>		fprintf(stderr, "%s error: node not specified.\n", argv[0]);
>		usage(argv);
>		exit(-1);
>	}
>	return 0;
>}
>
>/* ------------------------------------------------------------------------ */
>/* temp_node_file - concoct a temporary filename we can use                 */
>/* ------------------------------------------------------------------------ */
>const char *temp_node_file(const char *nodename)
>{
>	static char fname[256];
>
>	sprintf(fname,"/tmp/gfs2_hangalyzer.%s", nodename);
>	return fname;
>}
>
>/* ------------------------------------------------------------------------ */
>/* temp_glock_file - concoct a glock filename we can use                    */
>/* ------------------------------------------------------------------------ */
>const char *temp_glock_file(int n, int mnt, const char *mntpt)
>{
>	static char fname[256];
>
>	sprintf(fname, "%s.%s.%d", temp_node_file(node[n]->nodename),
>		mntpt, mnt + 1);
>	return fname;
>}
>
>/* ------------------------------------------------------------------------ */
>/* do_node_cmd - execute a qarsh/ssh command and save the output in tmp     */
>/* ------------------------------------------------------------------------ */
>void do_node_cmd(const char *nodename, const char *cmd)
>{
>	pid_t qarshpid;
>	int fd;
>
>	unlink(temp_node_file(nodename));
>	if ((qarshpid = fork()) == 0) {
>		close(1);
>		fd = creat(temp_node_file(nodename), S_IRUSR | S_IWUSR |
>			   S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
>		if (fd) {
>			if (!verbose)
>				fprintf(stderr, "%120.120s\r", " ");
>			fprintf(stderr, "exec: %s %s %s %s %s \"%s\"%c", 
>				shell_prg[use_qarsh], shell_prg2[use_qarsh],
>				"-l", "root", nodename, cmd,
>				verbose ? '\n' : '\r');
>			if ((execl(shell_prg[use_qarsh], shell_prg2[use_qarsh],
>				   "-l", "root", nodename, cmd, NULL)) == -1) {
>				perror(cmd);
>				exit(-1);
>			}
>			close(fd);
>		} else
>			perror(temp_node_file(nodename));
>		exit(0);
>	}
>	waitpid(qarshpid, NULL, 0);
>	if (!verbose)
>		fprintf(stderr, "%120.120s\r", " ");
>	return;
>}
>
>/* ------------------------------------------------------------------------ */
>/* analyze_cluster_conf                                                     */
>/* ------------------------------------------------------------------------ */
>int analyze_cluster_conf(void)
>{
>	FILE *fp;
>	char line[1024];
>	int nid;
>
>	do_node_cmd(onenode,
>		    "/bin/grep 'clusternode name' "
>		    "/etc/cluster/cluster.conf");
>	fp = fopen(temp_node_file(onenode), "rt");
>	if (!fp) {
>		perror(temp_node_file(onenode));
>		return -1;
>	}
>	node_count = 0;
>	while (fgets(line, sizeof(line) - 1, fp)) {
>		char *p, *p2, *p3, *p4;
>
>		if (verbose >= 2)
>			printf("%s", line);
>		p = strstr(line,"<clusternode name=");
>		if (p) {
>			p += 19;
>			p2 = strchr(p,'"');
>			node[node_count] = malloc(sizeof(struct nodeinfo));
>			memset(node[node_count], 0, sizeof(struct nodeinfo));
>			strncpy(node[node_count]->nodename, p, p2 - p);
>
>			p3 = strstr(line,"nodeid=");
>			if (p3)
>				p3 += 8;
>			p4 = strchr(p3,'"');
>			if (p4)
>				*p4 = '\0';
>			sscanf(p3, "%d", &nid);
>			node[node_count]->node_id = nid;
>			if (verbose >= 1)
>				printf("%d:nodeid %d: %s\n", node_count + 1,
>				       node[node_count]->node_id,
>				       node[node_count]->nodename);
>			node_count++;
>		}
>	}
>	if (verbose)
>		printf("node_count = %d\n", node_count);
>	fclose(fp);
>	return 0;
>}
>
>/* ------------------------------------------------------------------------ */
>/* find_debugfs                                                             */
>/* ------------------------------------------------------------------------ */
>void find_debugfs(int n)
>{
>	char line[PATH_MAX], tmp[PATH_MAX];
>	ssize_t count = 0;
>	int fd;
>
>	do_node_cmd(node[n]->nodename, "/bin/grep debugfs /proc/mounts");
>	fd = open(temp_node_file(node[n]->nodename), O_RDONLY);
>	count = read(fd, line, PATH_MAX);
>	line[count] = '\0';
>	if (count)
>		sscanf(line, "%s %s", tmp, node[n]->debugfs);
>	close(fd);
>	return;
>}
>
>/* ------------------------------------------------------------------------ */
>/* find_mounts                                                              */
>/* figure out all the gfs2 mounts on a node, fill in mounts, and return #   */
>/* ------------------------------------------------------------------------ */
>int find_mounts(int n)
>{
>	char line[PATH_MAX];
>	FILE *file;
>	char cmd[256];
>	int fd;
>
>	sprintf(cmd, "/bin/ls %s/gfs2/", node[n]->debugfs);
>	do_node_cmd(node[n]->nodename, cmd);
>
>	node[n]->mount_count = 0;
>	file = fopen(temp_node_file(node[n]->nodename), "rt");
>	if (!file)
>		return -1;
>	while (fgets(line, PATH_MAX, file)) {
>		char *p;
>
>		while ((p = strchr(line, '\n')))
>			*p = '\0';
>		while ((p = strchr(line, '\r')))
>			*p = '\0';
>		node[n]->mounts[node[n]->mount_count] =
>			malloc(PATH_MAX);
>		memset(node[n]->mounts[node[n]->mount_count], 0,
>		       PATH_MAX);
>		strcpy(node[n]->mounts[node[n]->mount_count], line);
>		node[n]->mount_count++;
>	}
>	close(fd);
>	return 0;
>}
>
>/* ------------------------------------------------------------------------ */
>/* fetch_glock_dumps                                                        */
>/* ------------------------------------------------------------------------ */
>int fetch_glock_dumps(int n)
>{
>	int mntpt;
>	char cmd[256];
>	char new_fn[PATH_MAX];
>
>	for (mntpt = 0; mntpt < node[n]->mount_count; mntpt++) {
>		sprintf(cmd, "/bin/cat %s/gfs2/%s/glocks",
>			node[n]->debugfs, node[n]->mounts[mntpt]);
>		if (verbose)
>			fprintf(stderr, "fetching glocks for %s: %s\n",
>				node[n]->nodename, node[n]->mounts[mntpt]);
>		do_node_cmd(node[n]->nodename, cmd);
>		memset(new_fn, 0, sizeof(new_fn));
>		rename(temp_node_file(node[n]->nodename),
>		       temp_glock_file(n, mntpt, node[n]->mounts[mntpt]));
>	}
>	return 0;
>}
>
>/* ------------------------------------------------------------------------ */
>/* decode_glock_flags - print a user-friendly version of the glock flags    */
>/* ------------------------------------------------------------------------ */
>void decode_glock_flags(const char *nodename, const char *gflags)
>{
>	const char *p;
>	int flagi, count = 0;
>	const char *flags = "lsDdpyfir?";
>	const char *flagdesc[10] = {"locked",         "sticky",
>				    "demote",         "pending demote",
>				    "demote in prog", "dirty",
>				    "lflush",         "invalidate in prog",
>				    "reply pending",  "unknown flag" };
>
>	p = strchr(gflags, ' ');
>	if (p == gflags || p == gflags + 1)
>		return;
>	printf("%-10.10s:                         ", nodename);
>	p = gflags;
>	while (p && *p != ' ') {
>		for (flagi = 0; flagi < 9; flagi++)
>			if (*p == flags[flagi])
>				break;
>		printf("%s%s", count ? ", " : "(", flagdesc[flagi]);
>		count++;
>		p++;
>	}
>	printf(")\n");
>}
>
>/* ------------------------------------------------------------------------ */
>/* find_dump_glock - find a given glock in all the glock dumps and dump it  */
>/* ------------------------------------------------------------------------ */
>void find_dump_glock(const char *glock_id, const char *mnt_id)
>{
>	int n;
>	int mntpt;
>	FILE *waiter;
>	char line[1024], *p;
>	int found_it, lines_printed = 0;
>	char *gf;
>
>	if (verbose)
>		printf("Dumping all occurrences of glock %s\n", glock_id);
>	for (n = 0; n < node_count; n++) {
>		/* printf("%-10.10s: --------------------------------------"
>		       "--------------------- %s\n",
>		       node[n]->nodename, glock_id); */
>		for (mntpt = 0; mntpt < node[n]->mount_count; mntpt++) {
>			if (strcmp(node[n]->mounts[mntpt], mnt_id))
>				continue;
>			waiter = fopen(temp_glock_file(n, mntpt,
>						       node[n]->mounts[mntpt]),
>				       "rt");
>			found_it = 0;
>			while (fgets(line, PATH_MAX, waiter)) {
>				while ((p = strchr(line, '\n')))
>					*p = '\0';
>				while ((p = strchr(line, '\r')))
>					*p = '\0';
>				if (line[0] == 'G') {
>					if (strstr(line, glock_id)) {
>						found_it = 1;
>						printf("%-10.10s: %s: %s\n",
>						       node[n]->nodename,
>						       node[n]->mounts[mntpt],
>						       line);
>						gf = strstr(line, "f:");
>						if (gf) {
>							gf += 2;
>							decode_glock_flags(
>							     node[n]->nodename,
>							     gf);
>						}
>						lines_printed++;
>					} else if (found_it) {
>						break;
>					}
>				} else if (found_it) {
>					printf("%-10.10s: %s: %s\n",
>					       node[n]->nodename,
>					       node[n]->mounts[mntpt], line);
>					lines_printed++;
>				}
>			}
>			fclose(waiter);
>			if (!found_it)
>				printf("%-10.10s: %s: No refs found to %s\n",
>				       node[n]->nodename, mnt_id, glock_id);
>		}
>	}
>	if (lines_printed)
>		printf("\n\n\n");
>}
>
>/* ------------------------------------------------------------------------ */
>/* analyze_glocks                                                           */
>/* ------------------------------------------------------------------------ */
>int analyze_glocks(void)
>{
>	int n, g;
>	int mntpt;
>	FILE *waiter;
>	char line[1024];
>	int holders_waiting = 0;
>	char last_glock[PATH_MAX];
>	char holder[PATH_MAX];
>	char glock_id[80], *p;
>	int glock_already_done;
>
>	memset(last_glock, 0, sizeof(last_glock));
>	memset(hung_glocks, 0, sizeof(hung_glocks));
>	hung_glock_cnt = 0;
>	/* find all the glocks that are in the wait state */
>	for (n = 0; n < node_count; n++) {
>		for (mntpt = 0; mntpt < node[n]->mount_count; mntpt++) {
>			waiter = fopen(temp_glock_file(n, mntpt,
>						       node[n]->mounts[mntpt]),
>				       "rt");
>			while (fgets(line, PATH_MAX, waiter)) {
>				while ((p = strchr(line, '\n')))
>					*p = '\0';
>				while ((p = strchr(line, '\r')))
>					*p = '\0';
>				if (line[0] == 'G') {
>					strcpy(last_glock, line);
>					continue;
>				}
>				if (line[1] == 'H' && strchr(line, 'W')) {
>					strcpy(holder, line);
>					/* We have a waiter */
>					holders_waiting++;
>					if (verbose >= 2) {
>						printf("Glock waited on: \n");
>						printf("%s\n", last_glock);
>						printf("%s\n", holder);
>					}
>					memset(glock_id, 0, sizeof(glock_id));
>					strncpy(glock_id, &last_glock[11], 20);
>					p = strchr(glock_id, ' ');
>					if (p)
>						*p = '\0';
>					/* See if we've already processed it */
>					glock_already_done = 0;
>					for (g = 0; g < hung_glock_cnt; g++) {
>						if (!strcmp(hung_glocks[g],
>							    glock_id)) {
>							glock_already_done = 1;
>							break;
>						}
>					}
>					if (!glock_already_done) {
>						strcpy(hung_glocks[
>							       hung_glock_cnt],
>						       glock_id);
>						hung_glock_cnt++;
>						find_dump_glock(glock_id,
>							node[n]->mounts[mntpt]);
>					}
>				}
>			}
>			fclose(waiter);
>		}
>	}
>	if (!hung_glock_cnt)
>		printf("No waiting glocks found on any node.\n");
>	return 0;
>}
>
>/* ------------------------------------------------------------------------ */
>/* main                                                                     */
>/* ------------------------------------------------------------------------ */
>int main(int argc, char **argv)
>{
>	int error, i;
>
>	/* step 1 - analyze the cluster.conf file to determine the nodes.*/
>	init(argc, argv);
>	memset(node, 0, sizeof(node));
>	error = analyze_cluster_conf();
>
>	for (i = 0; i < node_count; i++)
>		find_debugfs(i);
>	for (i = 0; i < node_count; i++)
>		find_mounts(i);
>	for (i = 0; i < node_count; i++)
>		fetch_glock_dumps(i);
>	analyze_glocks();
>	/* free up the memory we allocated for node names. */
>	for (i = 0; i<node_count; i++)
>		if (node[i])
>			free(node[i]);
>	exit(0);
>}

Actions: View

Attachments on bug 458684: 314028 | 314549 | 314680 | 314745 | 314760 | 314761 | 314762 | 314981 | 315157 | 315399 | 315400 | 315660 | 316226 | 316480 | 316579 | 316802