Attachment 908003 Details for Bug 1108525 – Move ringid store and load from totem library

[patch] Move ringid store and load from totem library

tmp.vkaCceatQq (text/plain), 11.05 KB, created by Jan Friesse on 2014-06-12 08:03:15 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Jan Friesse

Created: 2014-06-12 08:03:15 UTC

Size: 11.05 KB

patch

obsolete

>From da46ecfc3087de97ad9a76fe6a156f10170503a2 Mon Sep 17 00:00:00 2001
>From: Jan Friesse <jfriesse@redhat.com>
>Date: Fri, 30 May 2014 12:24:58 +0200
>Subject: [PATCH] Move ringid store and load from totem library
>
>Functions for storing and loading ring id was in the totem library. This
>causes problem, what to do when it's impossible to load or store ring
>id. Easy solution seemed to be assert, but sadly this makes hard for
>user to find out what happened (because corosync was just aborted and
>logsys didn't flush)
>
>Solution is to move these functions to main.c, where is much easier to
>handle error. This also makes libtotem free of any file system
>operations.
>
>Signed-off-by: Jan Friesse <jfriesse@redhat.com>
>Reviewed-by: Christine Caulfield <ccaulfie@redhat.com>
>---
> exec/main.c                    |   84 +++++++++++++++++++++++++++++++++++
> exec/totemsrp.c                |   95 +++++++++-------------------------------
> exec/util.h                    |    1 +
> include/corosync/totem/totem.h |   20 ++++++---
> 4 files changed, 120 insertions(+), 80 deletions(-)
>
>diff --git a/exec/main.c b/exec/main.c
>index d630058..7b5d73f 100644
>--- a/exec/main.c
>+++ b/exec/main.c
>@@ -703,6 +703,87 @@ int main_mcast (
> 	return (totempg_groups_mcast_joined (corosync_group_handle, iovec, iov_len, guarantee));
> }
> 
>+static void corosync_ring_id_create_or_load (
>+	struct memb_ring_id *memb_ring_id,
>+	const struct totem_ip_address *addr)
>+{
>+	int fd;
>+	int res = 0;
>+	char filename[PATH_MAX];
>+
>+	snprintf (filename, sizeof(filename), "%s/ringid_%s",
>+		get_run_dir(), totemip_print (addr));
>+	fd = open (filename, O_RDONLY, 0700);
>+	/*
>+	 * If file can be opened and read, read the ring id
>+	 */
>+	if (fd != -1) {
>+		res = read (fd, &memb_ring_id->seq, sizeof (uint64_t));
>+		close (fd);
>+	}
>+	/*
>+	 * If file could not be opened or read, create a new ring id
>+	 */
>+	if ((fd == -1) || (res != sizeof (uint64_t))) {
>+		memb_ring_id->seq = 0;
>+		umask(0);
>+		fd = open (filename, O_CREAT|O_RDWR, 0700);
>+		if (fd != -1) {
>+			res = write (fd, &memb_ring_id->seq, sizeof (uint64_t));
>+			close (fd);
>+			if (res == -1) {
>+				LOGSYS_PERROR (errno, LOGSYS_LEVEL_ERROR,
>+					"Couldn't write ringid file '%s'", filename);
>+
>+				corosync_exit_error (COROSYNC_DONE_STORE_RINGID);
>+			}
>+		} else {
>+			LOGSYS_PERROR (errno, LOGSYS_LEVEL_ERROR,
>+				"Couldn't create ringid file '%s'", filename);
>+
>+			corosync_exit_error (COROSYNC_DONE_STORE_RINGID);
>+		}
>+	}
>+
>+	totemip_copy(&memb_ring_id->rep, addr);
>+	assert (!totemip_zero_check(&memb_ring_id->rep));
>+}
>+
>+static void corosync_ring_id_store (
>+	const struct memb_ring_id *memb_ring_id,
>+	const struct totem_ip_address *addr)
>+{
>+	char filename[PATH_MAX];
>+	int fd;
>+	int res;
>+
>+	snprintf (filename, sizeof(filename), "%s/ringid_%s",
>+		get_run_dir(), totemip_print (addr));
>+
>+	fd = open (filename, O_WRONLY, 0777);
>+	if (fd == -1) {
>+		fd = open (filename, O_CREAT|O_RDWR, 0777);
>+	}
>+	if (fd == -1) {
>+		LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR,
>+			"Couldn't store new ring id %llx to stable storage",
>+			memb_ring_id->seq);
>+
>+		corosync_exit_error (COROSYNC_DONE_STORE_RINGID);
>+	}
>+	log_printf (LOGSYS_LEVEL_DEBUG,
>+		"Storing new sequence id for ring %llx", memb_ring_id->seq);
>+	res = write (fd, &memb_ring_id->seq, sizeof(memb_ring_id->seq));
>+	close (fd);
>+	if (res != sizeof(memb_ring_id->seq)) {
>+		LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR,
>+			"Couldn't store new ring id %llx to stable storage",
>+			memb_ring_id->seq);
>+
>+		corosync_exit_error (COROSYNC_DONE_STORE_RINGID);
>+	}
>+}
>+
> static qb_loop_timer_handle recheck_the_q_level_timer;
> void corosync_recheck_the_q_level(void *data)
> {
>@@ -1231,6 +1312,9 @@ int main (int argc, char **argv, char **envp)
> 
> 	ip_version = totem_config.ip_version;
> 
>+	totem_config.totem_memb_ring_id_create_or_load = corosync_ring_id_create_or_load;
>+	totem_config.totem_memb_ring_id_store = corosync_ring_id_store;
>+
> 	totem_config.totem_logging_configuration = totem_logging_configuration;
> 	totem_config.totem_logging_configuration.log_subsys_id = _logsys_subsys_create("TOTEM", "totem,"
> 			"totemmrp.c,totemrrp.c,totemip.c,totemconfig.c,totemcrypto.c,totemsrp.c,"
>diff --git a/exec/totemsrp.c b/exec/totemsrp.c
>index dc6b209..dcda8d1 100644
>--- a/exec/totemsrp.c
>+++ b/exec/totemsrp.c
>@@ -91,7 +91,6 @@
> #include "totemnet.h"
> 
> #include "cs_queue.h"
>-#include "util.h"
> 
> #define LOCALHOST_IP				inet_addr("127.0.0.1")
> #define QUEUE_RTR_ITEMS_SIZE_MAX		16384 /* allow 16384 retransmit items */
>@@ -467,6 +466,14 @@ struct totemsrp_instance {
> 	void (*totemsrp_waiting_trans_ack_cb_fn) (
> 		int waiting_trans_ack);
> 
>+	void (*memb_ring_id_create_or_load) (
>+		struct memb_ring_id *memb_ring_id,
>+		const struct totem_ip_address *addr);
>+
>+	void (*memb_ring_id_store) (
>+		const struct memb_ring_id *memb_ring_id,
>+		const struct totem_ip_address *addr);
>+
> 	int global_seqno;
> 
> 	int my_token_held;
>@@ -623,8 +630,6 @@ static int srp_addr_equal (const struct srp_addr *a, const struct srp_addr *b);
> 
> static void memb_leave_message_send (struct totemsrp_instance *instance);
> 
>-static void memb_ring_id_create_or_load (struct totemsrp_instance *, struct memb_ring_id *);
>-
> static void token_callbacks_execute (struct totemsrp_instance *instance, enum totem_callback_token_type type);
> static void memb_state_gather_enter (struct totemsrp_instance *instance, enum gather_state_from gather_from);
> static void messages_deliver_to_app (struct totemsrp_instance *instance, int skip, unsigned int end_point);
>@@ -632,7 +637,7 @@ static int orf_token_mcast (struct totemsrp_instance *instance, struct orf_token
> 	int fcc_mcasts_allowed);
> static void messages_free (struct totemsrp_instance *instance, unsigned int token_aru);
> 
>-static void memb_ring_id_set_and_store (struct totemsrp_instance *instance,
>+static void memb_ring_id_set (struct totemsrp_instance *instance,
> 	const struct memb_ring_id *ring_id);
> static void target_set_completed (void *context);
> static void memb_state_commit_token_update (struct totemsrp_instance *instance);
>@@ -872,6 +877,12 @@ int totemsrp_initialize (
> 	instance->totemsrp_log_printf = totem_config->totem_logging_configuration.log_printf;
> 
> 	/*
>+	 * Configure totem store and load functions
>+	 */
>+	instance->memb_ring_id_create_or_load = totem_config->totem_memb_ring_id_create_or_load;
>+	instance->memb_ring_id_store = totem_config->totem_memb_ring_id_store;
>+
>+	/*
> 	 * Initialize local variables for totemsrp
> 	 */
> 	totemip_copy (&instance->mcast_address, &totem_config->interfaces[0].mcast_addr);
>@@ -2107,7 +2118,8 @@ static void memb_state_commit_enter (
> 
> 	instance->memb_timer_state_gather_consensus_timeout = 0;
> 
>-	memb_ring_id_set_and_store (instance, &instance->commit_token->ring_id);
>+	memb_ring_id_set (instance, &instance->commit_token->ring_id);
>+	instance->memb_ring_id_store (&instance->my_ring_id, &instance->my_id.addr[0]);
> 
> 	instance->token_ring_id_seq = instance->my_ring_id.seq;
> 
>@@ -3292,79 +3304,12 @@ static void memb_merge_detect_transmit (struct totemsrp_instance *instance)
> 		sizeof (struct memb_merge_detect));
> }
> 
>-static void memb_ring_id_create_or_load (
>-	struct totemsrp_instance *instance,
>-	struct memb_ring_id *memb_ring_id)
>-{
>-	int fd;
>-	int res = 0;
>-	char filename[PATH_MAX];
>-
>-	snprintf (filename, sizeof(filename), "%s/ringid_%s",
>-		get_run_dir(), totemip_print (&instance->my_id.addr[0]));
>-	fd = open (filename, O_RDONLY, 0700);
>-	/*
>-	 * If file can be opened and read, read the ring id
>-	 */
>-	if (fd != -1) {
>-		res = read (fd, &memb_ring_id->seq, sizeof (uint64_t));
>-		close (fd);
>-	}
>-	/*
>- 	 * If file could not be opened or read, create a new ring id
>- 	 */
>-	if ((fd == -1) || (res != sizeof (uint64_t))) {
>-		memb_ring_id->seq = 0;
>-		umask(0);
>-		fd = open (filename, O_CREAT|O_RDWR, 0700);
>-		if (fd != -1) {
>-			res = write (fd, &memb_ring_id->seq, sizeof (uint64_t));
>-			close (fd);
>-			if (res == -1) {
>-				LOGSYS_PERROR (errno, instance->totemsrp_log_level_warning,
>-					"Couldn't write ringid file '%s'", filename);
>-			}
>-		} else {
>-			LOGSYS_PERROR (errno, instance->totemsrp_log_level_warning,
>-				"Couldn't create ringid file '%s'", filename);
>-		}
>-	}
>-
>-	totemip_copy(&memb_ring_id->rep, &instance->my_id.addr[0]);
>-	assert (!totemip_zero_check(&memb_ring_id->rep));
>-	instance->token_ring_id_seq = memb_ring_id->seq;
>-}
>-
>-static void memb_ring_id_set_and_store (
>+static void memb_ring_id_set (
> 	struct totemsrp_instance *instance,
> 	const struct memb_ring_id *ring_id)
> {
>-	char filename[256];
>-	int fd;
>-	int res;
> 
> 	memcpy (&instance->my_ring_id, ring_id, sizeof (struct memb_ring_id));
>-
>-	snprintf (filename, sizeof(filename), "%s/ringid_%s",
>-		get_run_dir(), totemip_print (&instance->my_id.addr[0]));
>-
>-	fd = open (filename, O_WRONLY, 0777);
>-	if (fd == -1) {
>-		fd = open (filename, O_CREAT|O_RDWR, 0777);
>-	}
>-	if (fd == -1) {
>-		LOGSYS_PERROR(errno, instance->totemsrp_log_level_warning,
>-			"Couldn't store new ring id %llx to stable storage",
>-			instance->my_ring_id.seq);
>-		assert (0);
>-		return;
>-	}
>-	log_printf (instance->totemsrp_log_level_debug,
>-		"Storing new sequence id for ring %llx", instance->my_ring_id.seq);
>-	//assert (fd > 0);
>-	res = write (fd, &instance->my_ring_id.seq, sizeof (unsigned long long));
>-	assert (res == sizeof (unsigned long long));
>-	close (fd);
> }
> 
> int totemsrp_callback_token_create (
>@@ -4641,7 +4586,9 @@ void main_iface_change_fn (
> 	totemip_copy (&instance->my_memb_list[0].addr[iface_no], iface_addr);
> 
> 	if (instance->iface_changes++ == 0) {
>-		memb_ring_id_create_or_load (instance, &instance->my_ring_id);
>+		instance->memb_ring_id_create_or_load (&instance->my_ring_id,
>+		    &instance->my_id.addr[0]);
>+		instance->token_ring_id_seq = instance->my_ring_id.seq;
> 		log_printf (
> 			instance->totemsrp_log_level_debug,
> 			"Created or loaded sequence id %llx.%s for this ring.",
>diff --git a/exec/util.h b/exec/util.h
>index 63caf74..9efd35e 100644
>--- a/exec/util.h
>+++ b/exec/util.h
>@@ -58,6 +58,7 @@ enum e_corosync_done {
> 	COROSYNC_DONE_ALREADY_RUNNING = 18,
> 	COROSYNC_DONE_STD_TO_NULL_REDIR = 19,
> 	COROSYNC_DONE_SERVICE_ENGINE_INIT = 20,
>+	COROSYNC_DONE_STORE_RINGID = 21,
> 	COROSYNC_DONE_PLOAD = 99
> };
> 
>diff --git a/include/corosync/totem/totem.h b/include/corosync/totem/totem.h
>index 72a3e1a..7f9fb0f 100644
>--- a/include/corosync/totem/totem.h
>+++ b/include/corosync/totem/totem.h
>@@ -99,6 +99,12 @@ typedef enum {
> 	TOTEM_TRANSPORT_RDMA = 2
> } totem_transport_t;
> 
>+#define MEMB_RING_ID
>+struct memb_ring_id {
>+	struct totem_ip_address rep;
>+	unsigned long long seq;
>+} __attribute__((packed));
>+
> struct totem_config {
> 	int version;
> 
>@@ -181,6 +187,14 @@ struct totem_config {
> 	unsigned int miss_count_const;
> 
> 	int ip_version;
>+
>+	void (*totem_memb_ring_id_create_or_load) (
>+	    struct memb_ring_id *memb_ring_id,
>+	    const struct totem_ip_address *addr);
>+
>+	void (*totem_memb_ring_id_store) (
>+	    const struct memb_ring_id *memb_ring_id,
>+	    const struct totem_ip_address *addr);
> };
> 
> #define TOTEM_CONFIGURATION_TYPE
>@@ -200,12 +214,6 @@ enum totem_event_type {
> 	TOTEM_EVENT_NEW_MSG,
> };
> 
>-#define MEMB_RING_ID
>-struct memb_ring_id {
>-	struct totem_ip_address rep;
>-	unsigned long long seq;
>-} __attribute__((packed));
>-
> typedef struct {
> 	int is_dirty;
> 	time_t last_updated;
>-- 
>1.7.1
>

Actions: View | Diff

Attachments on bug 1108525: 908000 | 908001 | 908002 | 908003