Attachment 314680 Details for Bug 458684 – Crude start of a hang analyzer tool

Crude start of a hang analyzer tool

gfs2_hangalyzer.c (text/plain), 10.34 KB, created by Robert Peterson on 2008-08-21 04:40:24 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Robert Peterson

Created: 2008-08-21 04:40:24 UTC

Size: 10.34 KB

patch

obsolete

>/* gfs2_hangalyzer - a tool to figure out why gfs2 is hung - by Bob Peterson */
>/* Copyright 2008, Red Hat, Inc.                           */
>#include <stdio.h>
>#include <string.h>
>#include <stdlib.h>
>#include <sys/types.h>
>#include <sys/stat.h>
>#include <fcntl.h>
>#include <unistd.h>
>#include <time.h>
>#include <limits.h>
>#include <sys/wait.h>
>
>#define CONF "/etc/cluster/cluster.conf"
>#define MAX_NODES 64
>#define MAX_MOUNTS 12
>
>struct nodeinfo {
>	int node_id;
>	char nodename[80];
>	int mount_count;
>	char debugfs[PATH_MAX];
>	char *mounts[MAX_MOUNTS];
>};
>
>int verbose = 0;
>int no_fetch = 0;
>int use_qarsh = 0;
>const char *shell_prg[2] = {"/usr/bin/ssh", "/usr/bin/qarsh"};
>const char *shell_prg2[2] = {"ssh", "qarsh"};
>char onenode[256];
>struct nodeinfo *node[MAX_NODES];
>int node_count;
>extern char *optarg;
>
>/* ------------------------------------------------------------------------ */
>/* usage                                                                    */
>/* ------------------------------------------------------------------------ */
>void usage(char **argv)
>{
>	printf("Format is: \n\t%s -n any node[-q][-v]\n", argv[0]);
>	printf("If -q is specified, qarsh will be used to fetch info, "
>	       "otherwise ssh.\n");
>}
>
>/* ------------------------------------------------------------------------ */
>/* init                                                                     */
>/* ------------------------------------------------------------------------ */
>int init(int argc, char *argv[])
>{
>	int optchar;
>
>	memset(onenode, 0, sizeof(onenode));
>
>	while (1) {
>		optchar = getopt(argc, argv, "qvn:");
>		if (optchar <= 0)
>			break;
>		switch (optchar) {
>		case 'n':
>			strcpy(onenode, optarg);
>			break;
>		case 'v':
>			verbose++;
>			break;
>		case 'q':
>			use_qarsh = 1;
>			break;
>		default:
>			fprintf(stderr, "Unknown option: %c\n", optchar);
>			usage(argv);
>			exit(0);
>		}
>	}
>	if (onenode[0] == '\0') {
>		fprintf(stderr, "%s error: node not specified.\n", argv[0]);
>		usage(argv);
>		exit(-1);
>	}
>	return 0;
>}
>
>/* ------------------------------------------------------------------------ */
>/* temp_node_file - concoct a temporary filename we can use                 */
>/* ------------------------------------------------------------------------ */
>const char *temp_node_file(const char *nodename)
>{
>	static char fname[256];
>
>	sprintf(fname,"/tmp/gfs2_hangalyzer.%s", nodename);
>	return fname;
>}
>
>/* ------------------------------------------------------------------------ */
>/* temp_glock_file - concoct a glock filename we can use                    */
>/* ------------------------------------------------------------------------ */
>const char *temp_glock_file(int n, int mnt)
>{
>	static char fname[256];
>
>	sprintf(fname, "%s.%d", temp_node_file(node[n]->nodename), mnt + 1);
>	return fname;
>}
>
>/* ------------------------------------------------------------------------ */
>/* do_node_cmd - execute a qarsh/ssh command and save the output in tmp     */
>/* ------------------------------------------------------------------------ */
>void do_node_cmd(const char *nodename, const char *cmd)
>{
>	pid_t qarshpid;
>	int fd;
>
>	unlink(temp_node_file(nodename));
>	if ((qarshpid = fork()) == 0) {
>		close(1);
>		fd = creat(temp_node_file(nodename), S_IRUSR | S_IWUSR |
>			   S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
>		if (fd) {
>			if (verbose)
>				fprintf(stderr, "exec: %s %s %s %s %s "
>					"\"%s\"\n", shell_prg[use_qarsh],
>					shell_prg2[use_qarsh],
>					"-l", "root", nodename, cmd);
>			if ((execl(shell_prg[use_qarsh], shell_prg2[use_qarsh],
>				   "-l", "root", nodename, cmd, NULL)) == -1) {
>				perror(cmd);
>				exit(-1);
>			}
>			close(fd);
>		} else
>			perror(temp_node_file(nodename));
>		exit(0);
>	}
>	waitpid(qarshpid, NULL, 0);
>	return;
>}
>
>/* ------------------------------------------------------------------------ */
>/* analyze_cluster_conf                                                     */
>/* ------------------------------------------------------------------------ */
>int analyze_cluster_conf(void)
>{
>	FILE *fp;
>	char line[1024];
>	int nid;
>
>	do_node_cmd(onenode,
>		    "/bin/grep 'clusternode name' "
>		    "/etc/cluster/cluster.conf");
>	fp = fopen(temp_node_file(onenode), "rt");
>	if (!fp) {
>		perror(temp_node_file(onenode));
>		return -1;
>	}
>	node_count = 0;
>	while (fgets(line, sizeof(line) - 1, fp)) {
>		char *p, *p2, *p3, *p4;
>
>		if (verbose >= 2)
>			printf("%s", line);
>		p = strstr(line,"<clusternode name=");
>		if (p) {
>			p += 19;
>			p2 = strchr(p,'"');
>			node[node_count] = malloc(sizeof(struct nodeinfo));
>			memset(node[node_count], 0, sizeof(struct nodeinfo));
>			strncpy(node[node_count]->nodename, p, p2 - p);
>
>			p3 = strstr(line,"nodeid=");
>			if (p3)
>				p3 += 8;
>			p4 = strchr(p3,'"');
>			if (p4)
>				*p4 = '\0';
>			sscanf(p3, "%d", &nid);
>			node[node_count]->node_id = nid;
>			if (verbose >= 1)
>				printf("%d:nodeid %d: %s\n", node_count + 1,
>				       node[node_count]->node_id,
>				       node[node_count]->nodename);
>			node_count++;
>		}
>	}
>	if (verbose)
>		printf("node_count = %d\n", node_count);
>	fclose(fp);
>	return 0;
>}
>
>/* ------------------------------------------------------------------------ */
>/* find_debugfs                                                             */
>/* ------------------------------------------------------------------------ */
>void find_debugfs(int n)
>{
>	char line[PATH_MAX], tmp[PATH_MAX];
>	ssize_t count = 0;
>	int fd;
>
>	do_node_cmd(node[n]->nodename, "/bin/grep debugfs /proc/mounts");
>	fd = open(temp_node_file(node[n]->nodename), O_RDONLY);
>	count = read(fd, line, PATH_MAX);
>	line[count] = '\0';
>	if (count)
>		sscanf(line, "%s %s", tmp, node[n]->debugfs);
>	close(fd);
>	return;
>}
>
>/* ------------------------------------------------------------------------ */
>/* find_mounts                                                              */
>/* figure out all the gfs2 mounts on a node, fill in mounts, and return #   */
>/* ------------------------------------------------------------------------ */
>int find_mounts(int n)
>{
>	char line[PATH_MAX];
>	FILE *file;
>	char cmd[256];
>	int fd;
>
>	sprintf(cmd, "/bin/ls %s/gfs2/", node[n]->debugfs);
>	do_node_cmd(node[n]->nodename, cmd);
>
>	node[n]->mount_count = 0;
>	file = fopen(temp_node_file(node[n]->nodename), "rt");
>	if (!file)
>		return -1;
>	while (fgets(line, PATH_MAX, file)) {
>		char *p;
>
>		while ((p = strchr(line, '\n')))
>			*p = '\0';
>		while ((p = strchr(line, '\r')))
>			*p = '\0';
>		node[n]->mounts[node[n]->mount_count] =
>			malloc(PATH_MAX);
>		memset(node[n]->mounts[node[n]->mount_count], 0,
>		       PATH_MAX);
>		strcpy(node[n]->mounts[node[n]->mount_count], line);
>		node[n]->mount_count++;
>	}
>	close(fd);
>	return 0;
>}
>
>/* ------------------------------------------------------------------------ */
>/* fetch_glock_dumps                                                        */
>/* ------------------------------------------------------------------------ */
>int fetch_glock_dumps(int n)
>{
>	int mntpt;
>	char cmd[256];
>	char new_fn[PATH_MAX];
>
>	for (mntpt = 0; mntpt < node[n]->mount_count; mntpt++) {
>		sprintf(cmd, "/bin/cat %s/gfs2/%s/glocks",
>			node[n]->debugfs, node[n]->mounts[mntpt]);
>		if (verbose)
>			fprintf(stderr, "fetching glocks for %s: %s\n",
>				node[n]->nodename, node[n]->mounts[mntpt]);
>		do_node_cmd(node[n]->nodename, cmd);
>		memset(new_fn, 0, sizeof(new_fn));
>		rename(temp_node_file(node[n]->nodename),
>		       temp_glock_file(n, mntpt));
>	}
>	return 0;
>}
>
>/* ------------------------------------------------------------------------ */
>/* find_dump_glock - find a given glock in all the glock dumps and dump it  */
>/* ------------------------------------------------------------------------ */
>void find_dump_glock(const char *glock_id)
>{
>	int n;
>	int mntpt;
>	FILE *waiter;
>	char line[1024], *p;
>	int found_it;
>
>	printf("Dumping all occurrences of glock %s\n", glock_id);
>	for (n = 0; n < node_count; n++) {
>		printf("%s: -------------------------\n", node[n]->nodename);
>		for (mntpt = 0; mntpt < node[n]->mount_count; mntpt++) {
>			waiter = fopen(temp_glock_file(n, mntpt), "rt");
>			found_it = 0;
>			while (fgets(line, PATH_MAX, waiter)) {
>				while ((p = strchr(line, '\n')))
>					*p = '\0';
>				while ((p = strchr(line, '\r')))
>					*p = '\0';
>				if (line[0] == 'G') {
>					if (strstr(line, glock_id)) {
>						found_it = 1;
>						printf("%s\n", line);
>					} else if (found_it) {
>						break;
>					}
>				} else if (found_it)
>					printf("%s\n", line);
>			}
>			fclose(waiter);
>		}
>	}
>}
>
>/* ------------------------------------------------------------------------ */
>/* analyze_glocks                                                           */
>/* ------------------------------------------------------------------------ */
>int analyze_glocks(void)
>{
>	int n;
>	int mntpt;
>	FILE *waiter;
>	char line[1024];
>	int holders_waiting = 0;
>	char last_glock[PATH_MAX];
>	char holder[PATH_MAX];
>	char glock_id[80], *p;
>
>	memset(last_glock, 0, sizeof(last_glock));
>	/* find all the glocks that are in the wait state */
>	for (n = 0; n < node_count; n++) {
>		for (mntpt = 0; mntpt < node[n]->mount_count; mntpt++) {
>			waiter = fopen(temp_glock_file(n, mntpt), "rt");
>			while (fgets(line, PATH_MAX, waiter)) {
>				while ((p = strchr(line, '\n')))
>					*p = '\0';
>				while ((p = strchr(line, '\r')))
>					*p = '\0';
>				if (line[0] == 'G') {
>					strcpy(last_glock, line);
>					continue;
>				}
>				if (line[1] == 'H' && strchr(line, 'W')) {
>					strcpy(holder, line);
>					/* We have a waiter */
>					holders_waiting++;
>					if (verbose) {
>						printf("Glock waited on: \n");
>						printf("%s\n", last_glock);
>						printf("%s\n", holder);
>					}
>					memset(glock_id, 0, sizeof(glock_id));
>					strncpy(glock_id, &last_glock[11], 20);
>					p = strchr(glock_id, ' ');
>					if (p)
>						*p = '\0';
>					find_dump_glock(glock_id);
>				}
>			}
>			fclose(waiter);
>		}
>	}
>	return 0;
>}
>
>/* ------------------------------------------------------------------------ */
>/* main                                                                     */
>/* ------------------------------------------------------------------------ */
>int main(int argc, char **argv)
>{
>	int error, i;
>
>	/* step 1 - analyze the cluster.conf file to determine the nodes.*/
>	init(argc, argv);
>	memset(node, 0, sizeof(node));
>	error = analyze_cluster_conf();
>
>	for (i = 0; i < node_count; i++)
>		find_debugfs(i);
>	for (i = 0; i < node_count; i++)
>		find_mounts(i);
>	for (i = 0; i < node_count; i++)
>		fetch_glock_dumps(i);
>	analyze_glocks();
>	/* free up the memory we allocated for node names. */
>	for (i = 0; i<node_count; i++)
>		if (node[i])
>			free(node[i]);
>	exit(0);
>}

Actions: View

Attachments on bug 458684: 314028 | 314549 | 314680 | 314745 | 314760 | 314761 | 314762 | 314981 | 315157 | 315399 | 315400 | 315660 | 316226 | 316480 | 316579 | 316802