Attachment 895478 Details for Bug 1001210 – totemiba: Add multicast recovery - upstream commit

[patch] totemiba: Add multicast recovery - upstream commit

2014-05-14-0001-flatiron-totemiba-Add-multicast-recovery.patch (text/plain), 4.34 KB, created by Jan Friesse on 2014-05-14 12:59:01 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Jan Friesse

Created: 2014-05-14 12:59:01 UTC

Size: 4.34 KB

patch

obsolete

>From 0d4bea9be05f1697571d2064e05aed3bcab61a08 Mon Sep 17 00:00:00 2001
>From: Yevheniy Demchenko <zheka@uvt.cz>
>Date: Wed, 14 May 2014 14:32:12 +0200
>Subject: [PATCH] totemiba: Add multicast recovery
>
>Totemiba wasn't able to survive SubnetManager handover or
>restart. If SM was migrated to another node, corosync logged
>"multicast error" and losses connectivity.
>
>Commit should solve this situation.
>
>Signed-off-by: Yevheniy Demchenko <zheka@uvt.cz>
>---
> exec/totemiba.c |   62 +++++++++++++++++++++++++++++++++++++++++++++++-------
> 1 files changed, 54 insertions(+), 8 deletions(-)
>
>diff --git a/exec/totemiba.c b/exec/totemiba.c
>index 104024c..73cc9be 100644
>--- a/exec/totemiba.c
>+++ b/exec/totemiba.c
>@@ -82,6 +82,8 @@
> 
> #define MAX_MTU_SIZE 4096
> 
>+#define MCAST_REJOIN_MSEC	100
>+
> struct totemiba_instance {
> 	struct sockaddr bind_addr;
> 
>@@ -206,6 +208,10 @@ struct totemiba_instance {
> 	struct list_head token_send_buf_head;
> 
> 	struct list_head recv_token_recv_buf_head;
>+
>+	int mcast_seen_joined;
>+
>+	poll_timer_handle mcast_rejoin;
> };
> union u {
> 	uint64_t wr_id;
>@@ -517,6 +523,31 @@ static int mcast_cq_recv_event_fn (hdb_handle_t poll_handle, int fd, int events,
> 	return (0);
> }
> 
>+static void mcast_rejoin (void *data)
>+{
>+	int res;
>+	struct totemiba_instance *instance = (struct totemiba_instance *)data;
>+
>+	res = rdma_leave_multicast (instance->mcast_cma_id, &instance->mcast_addr);
>+	if (instance->mcast_ah) {
>+		ibv_destroy_ah (instance->mcast_ah);
>+		instance->mcast_ah = 0;
>+	}
>+
>+	res = rdma_join_multicast (instance->mcast_cma_id, &instance->mcast_addr, instance);
>+	if (res != 0) {
>+		log_printf (LOGSYS_LEVEL_DEBUG,
>+		    "rdma_join_multicast failed, errno=%d, rejoining in %u ms\n",
>+		    MCAST_REJOIN_MSEC,
>+		    errno);
>+		poll_timer_add (instance->totemiba_poll_handle,
>+			MCAST_REJOIN_MSEC,
>+			(void *)instance,
>+			mcast_rejoin,
>+			&instance->mcast_rejoin);
>+	}
>+}
>+
> static int mcast_rdma_event_fn (hdb_handle_t poll_handle, int fd, int events, void *context)
> {
> 	struct totemiba_instance *instance = (struct totemiba_instance *)context;
>@@ -534,8 +565,16 @@ static int mcast_rdma_event_fn (hdb_handle_t poll_handle, int fd, int events, vo
> 	 * occurs when we resolve the multicast address
> 	 */
> 	case RDMA_CM_EVENT_ADDR_RESOLVED:
>-		rdma_join_multicast (instance->mcast_cma_id, &instance->mcast_addr, instance);
>+		res = rdma_join_multicast (instance->mcast_cma_id, &instance->mcast_addr, instance);
> 		usleep(1000);
>+		if (res == 0) break;
>+	case RDMA_CM_EVENT_MULTICAST_ERROR:
>+		log_printf (LOGSYS_LEVEL_ERROR, "multicast error, trying to rejoin in %u ms\n", MCAST_REJOIN_MSEC);
>+		poll_timer_add (instance->totemiba_poll_handle,
>+			MCAST_REJOIN_MSEC,
>+			(void *)instance,
>+			mcast_rejoin,
>+			&instance->mcast_rejoin);
> 		break;
> 	/*
> 	 * occurs when the CM joins the multicast group
>@@ -544,14 +583,15 @@ static int mcast_rdma_event_fn (hdb_handle_t poll_handle, int fd, int events, vo
> 		instance->mcast_qpn = event->param.ud.qp_num;
> 		instance->mcast_qkey = event->param.ud.qkey;
> 		instance->mcast_ah = ibv_create_ah (instance->mcast_pd, &event->param.ud.ah_attr);
>-
>-		instance->totemiba_iface_change_fn (instance->rrp_context, &instance->my_id);
>+		if (instance->mcast_seen_joined == 0) {
>+			log_printf (LOGSYS_LEVEL_DEBUG, "joining mcast 1st time, running callbacks\n");
>+			instance->totemiba_iface_change_fn (instance->rrp_context, &instance->my_id);
>+			instance->mcast_seen_joined=1;
>+		}
>+		log_printf (LOGSYS_LEVEL_DEBUG, "Joined multicast!\n");
> 		break;
> 	case RDMA_CM_EVENT_ADDR_ERROR:
> 	case RDMA_CM_EVENT_ROUTE_ERROR:
>-	case RDMA_CM_EVENT_MULTICAST_ERROR:
>-		log_printf (LOGSYS_LEVEL_ERROR, "multicast error\n");
>-		break;
> 	case RDMA_CM_EVENT_DEVICE_REMOVAL:
> 		break;
> 	default:
>@@ -1454,7 +1494,10 @@ int totemiba_mcast_flush_send (
> 	sge.lkey = send_buf->mr->lkey;
> 	sge.addr = (uintptr_t)msg;
> 
>-	res = ibv_post_send (instance->mcast_cma_id->qp, &send_wr, &failed_send_wr);
>+	if (instance->mcast_ah != 0) {
>+		res = ibv_post_send (instance->mcast_cma_id->qp, &send_wr, &failed_send_wr);
>+	}
>+
> 	return (res);
> }
> 
>@@ -1492,7 +1535,10 @@ int totemiba_mcast_noflush_send (
> 	sge.lkey = send_buf->mr->lkey;
> 	sge.addr = (uintptr_t)msg;
> 
>-	res = ibv_post_send (instance->mcast_cma_id->qp, &send_wr, &failed_send_wr);
>+	if (instance->mcast_ah != 0) {
>+		res = ibv_post_send (instance->mcast_cma_id->qp, &send_wr, &failed_send_wr);
>+	}
>+
> 	return (res);
> }
> 
>-- 
>1.7.1
>

Actions: View | Diff

Attachments on bug 1001210: 790643 | 790644 | 790645 | 790646 | 790647 | 793906 | 797602 | 797603 | 797604 | 797605 | 797606 | 797607 | 797619 | 797620 | 800375 | 800376 | 800377 | 892304 | 895478 | 895956 | 899961