Attachment 318124 Details for Bug 238498 – RHEL5 (0.8.4) patch

[patch] RHEL5 (0.8.4) patch

piranha-0.8.4-nanny-monitor.patch (text/plain), 33.41 KB, created by Lon Hohberger on 2008-09-30 20:56:16 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Lon Hohberger

Created: 2008-09-30 20:56:16 UTC

Size: 33.41 KB

patch

obsolete

>diff -urN piranha.old/globals.h piranha.new/globals.h
>--- piranha.old/globals.h	2002-01-18 15:21:06.000000000 -0500
>+++ piranha.new/globals.h	2008-09-30 15:12:00.000000000 -0400
>@@ -5,6 +5,10 @@
> #define PULSE	"pulse 1.56"
> #define NANNY	"nanny 1.46"
> 
>+#define NANNY_STATUS_SIZE 8
>+#define NANNY_ACTIVE_STATUS "[ active ]"
>+#define NANNY_INACTIVE_STATUS "[inactive]"
>+
> #define CFGFILE	"/etc/sysconfig/ha/lvs.cf"
> #define IPVSADM	"/sbin/ipvsadm"
> #define RESERVE	"/usr/sbin/scsi_reserve"
>diff -urN piranha.old/ipvs_exec.c piranha.new/ipvs_exec.c
>--- piranha.old/ipvs_exec.c	1969-12-31 19:00:00.000000000 -0500
>+++ piranha.new/ipvs_exec.c	2008-09-30 16:15:05.000000000 -0400
>@@ -0,0 +1,202 @@
>+/*
>+** ipvs_exec.c -- Run IPVS commands
>+**
>+** Red Hat Clustering Tools 
>+** Copyright 1999 Red Hat Inc.
>+**
>+** Author: Erik Troan <ewt@redhat.com>
>+**
>+** 5-01-2007 - split from nanny.c by lhh@redhat.com
>+*/
>+
>+#include <arpa/inet.h>
>+#include <ctype.h>
>+#include <errno.h>
>+#include <fcntl.h>
>+#include <netinet/ip.h>
>+#include <netinet/ip_icmp.h>
>+#include <popt.h>
>+#include <setjmp.h>
>+#include <stdlib.h>
>+#include <string.h>
>+#include <sys/poll.h>
>+#include <sys/signal.h>
>+#include <sys/socket.h>
>+#include <sys/time.h>
>+#include <sys/wait.h>
>+#include <unistd.h>
>+#include <math.h>		/* for pow() */
>+#include <time.h>
>+#include <sys/types.h>
>+#include <regex.h>
>+
>+#include "globals.h"
>+#include "util.h"
>+#include "nanny.h"
>+
>+#define _(a) (a)
>+#define N_(a) (a)
>+
>+
>+int
>+runCommand (char *cmd, int flags, char **argv, int log_flag)
>+{
>+	int child;
>+	int status;
>+	int rc;
>+	int log_it;
>+
>+	log_it = log_flag;
>+	if (flags & NANNY_FLAG_VERBOSE)
>+		log_it = -1;
>+
>+	if (log_it)
>+		logArgv (flags, argv);
>+
>+	if (flags & NANNY_FLAG_NORUN)
>+		return 0;
>+
>+	if (!(child = fork ())) {
>+		execv (argv[0], argv);
>+		exit (1);
>+	}
>+
>+	rc = waitpid (child, &status, 0);
>+
>+	if (!WIFEXITED (status) || WEXITSTATUS (status)) {
>+		piranha_log (flags, (char *) "%s command failed!", cmd);
>+		return 1;
>+	}
>+
>+	return 0;
>+}
>+
>+int
>+shutdownDev (int flags, char *ipvsadm, char *virtualAddress, int port,
>+	     struct in_addr *remoteAddr, int service_type, int fwmark,
>+	     int use_udp)
>+{
>+
>+	char *argv[40];
>+	char **argp = argv;
>+	char virtName[80];
>+	char fwmStr[20];
>+	int rc = 0;
>+
>+	if (service_type == SERV_LVS) {
>+		/* Virtual Server */
>+
>+		*argp++ = ipvsadm;
>+		*argp++ = (char *) "-d";
>+
>+		if (fwmark) {
>+			sprintf (fwmStr, "%d", fwmark);
>+			*argp++ = (char *) "-f";
>+			*argp++ = fwmStr;
>+		} else {
>+			sprintf (virtName, "%s:%d", virtualAddress, port);
>+			if (use_udp)
>+				*argp++ = (char *) "-u";
>+			else
>+				*argp++ = (char *) "-t";
>+			*argp++ = virtName;
>+		}
>+
>+		*argp++ = (char *) "-r";
>+		*argp++ = inet_ntoa (*remoteAddr);
>+		*argp = NULL;
>+
>+		rc = runCommand (ipvsadm, flags, argv, -1);
>+
>+	} 
>+	/* Nothing needs to be done for FOS */
>+	return rc;
>+}
>+
>+int
>+adjustDevice (int flags, char *ipvsadm, char *virtualAddress, int port,
>+	      struct in_addr *remoteAddr, char *routingMethod, int weight,
>+	      int fwmark, int use_udp)
>+{
>+	char *argv[40];
>+	char **argp = argv;
>+	char virtName[80];
>+	char weightStr[20];
>+	char fwmStr[20];
>+
>+	sprintf (weightStr, "%d", weight);
>+
>+	*argp++ = ipvsadm;
>+	*argp++ = (char *) "-e";
>+
>+	if (fwmark) {
>+		sprintf (fwmStr, "%d", fwmark);
>+		*argp++ = (char *) "-f";
>+		*argp++ = fwmStr;
>+	} else {
>+		sprintf (virtName, "%s:%d", virtualAddress, port);
>+		if (use_udp)
>+			*argp++ = (char *) "-u";
>+		else
>+			*argp++ = (char *) "-t";
>+		*argp++ = virtName;
>+	}
>+
>+	*argp++ = (char *) "-r";
>+	*argp++ = inet_ntoa (*remoteAddr);
>+	*argp++ = routingMethod;
>+	*argp++ = (char *) "-w";
>+	*argp++ = weightStr;
>+	*argp = NULL;
>+
>+	return runCommand (ipvsadm, flags, argv, 0);
>+}
>+
>+int
>+bringUpDev (int flags, char *ipvsadm, char *virtualAddress, int port,
>+	    struct in_addr *remoteAddr, char *routingMethod, int weight,
>+	    int service_type, int fwmark, int use_udp)
>+{
>+
>+	char *argv[40];
>+	char **argp = argv;
>+	char virtName[80];
>+	char weightStr[20];
>+	char fwmStr[20];
>+	int rc = 0;
>+
>+	if (service_type == SERV_LVS) {
>+		/* Virtual Server */
>+
>+		sprintf (weightStr, "%d", weight);
>+
>+		*argp++ = ipvsadm;
>+		*argp++ = (char *) "-a";
>+
>+		if (fwmark) {
>+			sprintf (fwmStr, "%d", fwmark);
>+			*argp++ = (char *) "-f";
>+			*argp++ = fwmStr;
>+		} else {
>+			sprintf (virtName, "%s:%d", virtualAddress, port);
>+			if (use_udp)
>+				*argp++ = (char *) "-u";
>+			else
>+				*argp++ = (char *) "-t";
>+			*argp++ = virtName;
>+		}
>+
>+		*argp++ = (char *) "-r";
>+		*argp++ = inet_ntoa (*remoteAddr);
>+		*argp++ = routingMethod;
>+		*argp++ = (char *) "-w";
>+		*argp++ = weightStr;
>+		*argp = NULL;
>+
>+		rc = runCommand (ipvsadm, flags, argv, -1);
>+	}
>+	/* Nothing needs to be done for FOS */
>+	return rc;
>+}
>+
>+
>diff -urN piranha.old/lvs.cf.5 piranha.new/lvs.cf.5
>--- piranha.old/lvs.cf.5	2006-05-15 17:58:30.000000000 -0400
>+++ piranha.new/lvs.cf.5	2008-09-30 15:18:18.000000000 -0400
>@@ -115,6 +115,11 @@
> .BI "address = a.b.c.d"
> This is the address to be used for the virtual server.
> .TP
>+.BI "sorry_server = a.b.c.d"
>+This is the address to be used for the 'sorry' server.  If specified, requests
>+for this virtual server will be redirected to this IP address in the event
>+that no real servers are available to handle the request.
>+.TP
> .BI "vip_nmask = a.b.c.d"
> Optional. This is the subnet mask to apply to the address of the virtual
> server.
>diff -urN piranha.old/lvsconfig.c piranha.new/lvsconfig.c
>--- piranha.old/lvsconfig.c	2008-09-30 15:04:04.000000000 -0400
>+++ piranha.new/lvsconfig.c	2008-09-30 15:18:18.000000000 -0400
>@@ -151,6 +151,7 @@
> #define LVS_SAW_VS_FWMARK           (1 << 20)
> #define LVS_SAW_VS_QUIESCESERVER    (1 << 21)
> #define LVS_SAW_VS_USEREGEX         (1 << 22)
>+#define LVS_SAW_VS_SORRYSERVER	    (1 << 23)
> 
> #define LVS_SAW_AS_ADDRESS	    (1 << 0)
> #define LVS_SAW_AS_ISACTIVE	    (1 << 1)
>@@ -853,7 +854,6 @@
> 			inet_ntoa (eserver->pmask));
>     }
> 
>-
>   if (!(saw & LVS_SAW_VS_PROTOCOL) && eserver->protocol != IPPROTO_TCP)
>     outputFormat (ob, (char *) "\tprotocol = udp\n");
> 
>@@ -913,6 +913,7 @@
> 		(char *) "\treentry = %d\n")
> 	OUTPUT_MISSING_IPADDR (LVS_SAW_VS_MASTERSERVER, eserver->masterServer,
> 				     (char *) "\tmaster = %s\n")
>+
>     if (outputRealServers) {
> 	i = -1;
> 	while (++i < eserver->numServers)
>@@ -964,6 +965,9 @@
>   if (rc)
>     return rc;
> 
>+  /* Zap the sorry server... */
>+  memset(&vserver->sorry_server, 0, sizeof(vserver->sorry_server));
>+
>   while (*toks.argv && strcmp (toks.argv[0], "}"))
>     {
>       dumpToks (fs, &toks, "	virtual server: ");
>@@ -998,6 +1002,16 @@
> 	    }
> 
> 	}
>+      else if (TOKEN ("sorry_server"))
>+	{
>+	  CHECK_ARGS (1) if (inet_aton (toks.argv[2], &vserver->sorry_server))
>+	    {
>+	      OUTPUT_IPADDR (eserver, sorry_server, 2, LVS_SAW_VS_SORRYSERVER);
>+	    }
>+	  else
>+	    return LVS_ERROR_BADHOSTNAME;
>+
>+	}
>       else if (TOKEN ("vip_nmask"))
> 	{
> 	  CHECK_ARGS (1) if (inet_aton (toks.argv[2], &vserver->vip_nmask))
>diff -urN piranha.old/lvsconfig.h piranha.new/lvsconfig.h
>--- piranha.old/lvsconfig.h	2005-07-28 13:16:03.000000000 -0400
>+++ piranha.new/lvsconfig.h	2008-09-30 15:18:18.000000000 -0400
>@@ -119,10 +119,12 @@
> {
>   char *name;
>   int isActive;
>+  int runstate;
>   char *virtualDevice;
>   struct in_addr virtualAddress;	/* for Virtual IP address */
>   struct in_addr vip_nmask;
>   struct in_addr broadcast;
>+  struct in_addr sorry_server;
> 
>   unsigned int fwmark;		/* for fwmark-based virtual server */
>   int protocol;			/* IPPROTO_TCP or IPPROTO_UDP */
>diff -urN piranha.old/lvsd.c piranha.new/lvsd.c
>--- piranha.old/lvsd.c	2006-05-15 18:37:52.000000000 -0400
>+++ piranha.new/lvsd.c	2008-09-30 15:18:18.000000000 -0400
>@@ -112,10 +112,12 @@
> #include <sys/signal.h>
> #include <sys/wait.h>
> #include <unistd.h>
>+#include <assert.h>
> 
> #include "globals.h"
> #include "lvsconfig.h"
> #include "util.h"
>+#include "nanny.h"
> 
> #define _PROGRAM_VERSION LVS	/* From globals.h */
> 
>@@ -129,12 +131,40 @@
> 
> struct clientInfo
> {
>+  struct lvsVirtualServer *vserver;
>   struct in_addr vip;
>   struct in_addr address;
>   int port, protocol;
>+  int clientPipe;
>+  int clientActive;
>   pid_t clientMonitor;		/* -1 if died */
> };
> 
>+
>+int
>+shutdownDev (int flags, char *ipvsadm, char *virtualAddress, int port,
>+	     struct in_addr *remoteAddr, int service_type, int fm, int udp);
>+
>+int
>+bringUpDev (int flags, char *ipvsadm, char *virtualAddress, int port,
>+	    struct in_addr *remoteAddr, char *routingMethod, int weight,
>+	    int service_type, int fm, int udp);
>+
>+char *
>+rm_to_str(int redirectType) {
>+  switch (redirectType)
>+    {
>+    case LVS_REDIRECT_NAT:
>+      return (char *) "-m";
>+    case LVS_REDIRECT_DIRECT:
>+      return (char *) "-g";
>+    case LVS_REDIRECT_TUNNEL:
>+      return (char *) "-i";
>+    }
>+   return NULL;
>+}
>+
>+
> /* returns 0 if clients match, 1 otherwise */
> 
> int cmpRealServers(struct clientInfo *client1, struct clientInfo *client2)
>@@ -180,6 +210,11 @@
> }
> 
> 
>+static void
>+handleAlarmSignal (int signal)
>+{
>+	/* nothing */
>+}
> 
> 
> int
>@@ -304,18 +339,26 @@
> {
> 
>   struct clientInfo clientInfo;
>-  int rc;
>+  int rc, cld_pipe[2] = {-1, -1}, cfl, have_sorry;
>   char *realAddress;
>   char *virtAddress;
>   char *argv[42];
>   char **arg = argv;
>   char portNum[20], timeoutNum[20], weightNum[20], reentryNum[20];
>   char fwmNum[20];
>+  char buf[64];
>   pid_t child;
> 
>   if (!service->isActive)
>     return 0;
> 
>+  memset(buf, 0, sizeof(buf));
>+  have_sorry = memcmp(&(vserver->sorry_server), buf,
>+		      sizeof(vserver->sorry_server));
>+
>+  /* assume we're alive */
>+  vserver->runstate = 1;
>+  clientInfo.vserver = vserver;
>   clientInfo.vip = vserver->virtualAddress;
>   clientInfo.address = service->address;
>   clientInfo.protocol = vserver->protocol;
>@@ -419,8 +462,8 @@
>         *arg++ = config->rshCommand;
>     }
>     
>-
>-  if (!(flags & LVS_FLAG_ASDAEMON))
>+  /* send this to nanny if we don't have a sorry server */
>+  if (!(flags & LVS_FLAG_ASDAEMON) || have_sorry)
>     {
>       *arg++ = (char *) "--nodaemon";	/* Pass along --nodaemon */
>     }
>@@ -438,14 +481,46 @@
> 
>   if (!(flags & LVS_FLAG_TESTSTARTS))
>     {
>+      if (have_sorry && pipe(cld_pipe) < 0) {
>+        /* do exactly what here? */
>+	assert(0);
>+      }
>+
>       child = fork ();
> 
>       if (!child)
> 	{
>+	  /* Smoke a pipe, yo */
>+	  if (have_sorry) {
>+	    dup2(cld_pipe[0], STDIN_FILENO);
>+	    dup2(cld_pipe[1], STDOUT_FILENO);
>+	    dup2(cld_pipe[1], STDERR_FILENO);
>+
>+	    if (cld_pipe[0] > 2)
>+		close(cld_pipe[0]);
>+	    if (cld_pipe[1] > 2)
>+	  	close(cld_pipe[1]);
>+
>+	    /* Don't block on writes.  Or anything. */
>+	    cfl = fcntl(STDIN_FILENO, F_GETFL);
>+	    fcntl(STDIN_FILENO, F_SETFL, cfl | O_NONBLOCK);
>+	    cfl = fcntl(STDOUT_FILENO, F_GETFL);
>+	    fcntl(STDOUT_FILENO, F_SETFL, cfl | O_NONBLOCK);
>+	    cfl = fcntl(STDERR_FILENO, F_GETFL);
>+	    fcntl(STDERR_FILENO, F_SETFL, cfl | O_NONBLOCK);
>+	  }
>+
> 	  execv (argv[0], argv);
> 	  exit (1);
> 	}
> 
>+      /* don't need this half of it */
>+      /* nonblocking I/o */
>+      if (have_sorry) {
>+        close(cld_pipe[1]);
>+        cfl = fcntl(cld_pipe[0], F_GETFL);
>+        fcntl(cld_pipe[0], F_SETFL, cfl | O_NONBLOCK);
>+      }
>       piranha_log (flags,
> 		   (char *) "create_monitor for %s/%s running as pid %d",
> 		   vserver->name, service->name, child);
>@@ -459,7 +534,10 @@
>   free (virtAddress);
>   free (realAddress);
> 
>+  clientInfo.vserver = vserver;
>   clientInfo.clientMonitor = child;
>+  clientInfo.clientPipe = cld_pipe[0];
>+  clientInfo.clientActive = 0;
>   clients[*numClients] = clientInfo;
>   (*numClients)++;
> 
>@@ -467,8 +545,6 @@
> }
> 
> 
>-
>-
> int
> shutdownClientMonitor (struct lvsConfig *config,
> 		       struct lvsVirtualServer *vserver,
>@@ -495,6 +571,10 @@
>     {
>       kill (clients[which].clientMonitor, SIGTERM);
>       waitpid (clients[which].clientMonitor, NULL, 0);
>+      clients[which].vserver = NULL;
>+      clients[which].clientActive = 0;
>+      close(clients[which].clientPipe);
>+      clients[which].clientPipe = -1;
>     }
> 
>   if (which != (*numClientsPtr - 1))
>@@ -956,6 +1036,147 @@
>   return 0;
> }
> 
>+int
>+activeClientCount(struct lvsVirtualServer *vs, struct clientInfo *clients,
>+		  int numClients)
>+{
>+  int count = 0, i;
>+
>+  for (i = 0; i < numClients; i++) {
>+    if (clients[i].vserver == vs && clients[i].clientActive)
>+      count++;
>+  }
>+
>+  return count;
>+}
>+
>+
>+void
>+updateClientStatus(struct clientInfo *client)
>+{
>+  char buf[80];
>+  int n;
>+
>+  memset(buf, 0, sizeof(buf));
>+  while ((n = read(client->clientPipe, buf, sizeof(buf))) >= 0) {
>+    if (n >= NANNY_STATUS_SIZE) {
>+      if (strstr(buf, NANNY_ACTIVE_STATUS) && !client->clientActive) {
>+        //fprintf(stderr, "nanny pid %d activated\n", client->clientMonitor );
>+        client->clientActive = 1;
>+      } else if (strstr(buf, NANNY_INACTIVE_STATUS) && client->clientActive) {
>+        //fprintf(stderr, "nanny pid %d inactive\n", client->clientMonitor );
>+        client->clientActive = 0;
>+      }
>+    }
>+
>+    if (n == 0)
>+      break;
>+  }
>+}
>+
>+
>+int
>+checkClientMonitor(struct lvsConfig *config, struct lvsVirtualServer *vs,
>+		   struct clientInfo *clients,
>+		   int numClients, int flags)
>+{
>+  fd_set rfds;
>+  int max = -1, i, have_sorry;
>+  int nready = 0;
>+  struct timeval tv = {0, 0};
>+  char buf[64];
>+
>+  /* XXX */
>+  assert(sizeof(buf) > sizeof(config->virtServers[i].sorry_server) );
>+
>+  /* This is a no-op if we don't have a sorry_server */
>+  memset(buf, 0, sizeof(buf));
>+  have_sorry = memcmp(&(vs->sorry_server), buf,
>+		sizeof(vs->sorry_server));
>+  if (!have_sorry)
>+      return 0;
>+
>+  /* We've got a sorry server - run our list looking for clients */
>+  /* Find all clientPipes ... */
>+  FD_ZERO(&rfds);
>+  for (i = 0; i < numClients; i++) {
>+    if (clients[i].vserver == vs && clients[i].clientPipe >= 0) { 
>+      FD_SET(clients[i].clientPipe, &rfds);
>+      if (clients[i].clientPipe > max)
>+        max = clients[i].clientPipe;
>+    }
>+  }
>+
>+  nready = select(max + 1, &rfds, NULL, NULL, &tv);
>+  if (nready < 0) {
>+    piranha_log (flags,
>+	         (char *) "select(): %s", strerror(errno));
>+    return -1;
>+  } else if (nready > 0) {
>+
>+    /* Ok, update our status... thingies. */
>+    for (i = 0; i < numClients; i++) {
>+      if (clients[i].vserver == vs &&
>+          FD_ISSET(clients[i].clientPipe,&rfds)) {
>+        
>+	FD_CLR(clients[i].clientPipe, &rfds);
>+        updateClientStatus(&clients[i]);
>+      }
>+    }
>+  }
>+
>+  /* count active clients and see if we need to activate the
>+     sorry_server */
>+  max = activeClientCount(vs, clients, numClients);
>+
>+  if (!max && vs->runstate) {
>+    vs->runstate = 0;
>+
>+    inet_ntop(AF_INET, &vs->sorry_server, buf, sizeof(buf));
>+    printf("Virtual Server %s: No real servers; activate sorry_server %s\n",
>+	   vs->name, buf);
>+    inet_ntop(AF_INET, &vs->virtualAddress, buf, sizeof(buf));
>+    bringUpDev(flags,
>+	       config->vsadm,
>+	       buf, vs->port,
>+	       &vs->sorry_server,
>+	       rm_to_str(config->redirectType),
>+	       1, SERV_LVS,
>+	       vs->fwmark, (vs->protocol == IPPROTO_UDP));
>+
>+
>+  } else if (max && !vs->runstate) {
>+    vs->runstate = 1;
>+
>+    inet_ntop(AF_INET, &vs->sorry_server, buf, sizeof(buf));
>+    printf("Virtual Server %s: %d real servers; deactivate sorry_server %s\n",
>+	   vs->name, max, buf);
>+    inet_ntop(AF_INET, &vs->virtualAddress, buf, sizeof(buf));
>+    shutdownDev(flags,
>+	        config->vsadm,
>+	        buf, vs->port,
>+	        &vs->sorry_server,
>+	        SERV_LVS,
>+	        vs->fwmark, (vs->protocol == IPPROTO_UDP));
>+  }
>+
>+  return 0;
>+}
>+
>+
>+int
>+checkClientMonitors(struct lvsConfig *config, struct clientInfo *clients,
>+		    int numClients, int flags)
>+{
>+  int i, ret = 0;
>+
>+  for (i = 0; i < config->numVirtServers; i++) {
>+    ret += checkClientMonitor(config, &config->virtServers[i], clients,
>+			      numClients, flags);
>+  }
>+
>+  return ret;
>+}
> 
> 
> int
>@@ -990,6 +1211,7 @@
>   signal (SIGINT, handleTermSignal);
>   signal (SIGTERM, handleTermSignal);
>   signal (SIGHUP, handleHupSignal);
>+  signal (SIGALRM, handleAlarmSignal);
> 
>   for (i = 0; i < config->numVirtServers; i++)
>     {
>@@ -1005,6 +1227,7 @@
>     }
> 
>   sigfillset (&sigs);
>+  sigdelset (&sigs, SIGALRM);
>   sigdelset (&sigs, SIGINT);
>   sigdelset (&sigs, SIGCHLD);
>   sigdelset (&sigs, SIGTERM);
>@@ -1013,7 +1236,9 @@
> 
>   while (!doShutdown)
>     {
>+      alarm(5);
>       sigsuspend (&sigs);
>+      alarm(0);
> 
>       while ((pid = waitpid (-1, &status, WNOHANG)) > 0)
> 	{
>@@ -1038,10 +1263,17 @@
> 			     config->virtServers[vsn].servers[sn].name);
> 	      
> 	      doShutdown = 1;
>+
>+	      /* Clean up client pipes */
> 	      clients[i].clientMonitor = -1;
>+	      clients[i].clientActive = 0;
>+	      close(clients[i].clientPipe);
>+	      clients[i].clientPipe = -1;
> 	    }
> 	}
> 
>+      checkClientMonitors(config, clients, numClients, flags);
>+
>       if (termSignal)
> 	{
> 	  piranha_log (flags,
>diff -urN piranha.old/Makefile piranha.new/Makefile
>--- piranha.old/Makefile	2008-09-30 15:04:04.000000000 -0400
>+++ piranha.new/Makefile	2008-09-30 16:35:20.000000000 -0400
>@@ -130,16 +130,16 @@
> pkglists:
> 
> 
>-lvsd:		lvsd.o lvsconfig.o util.o
>+lvsd:		lvsd.o lvsconfig.o util.o ipvs_exec.o
> 
> fos:		fos.o lvsconfig.o util.o
> 
>-nanny:		nanny.o util.o
>+nanny:		nanny.o util.o ipvs_exec.o
> 
> send_arp:	send_arp.c
> 		$(CC) $(CFLAGS) -o send_arp send_arp.c
> 
>-pulse:		pulse.o lvsconfig.o util.o linkstate.o
>+pulse:		pulse.o lvsconfig.o util.o linkstate.o ipvs_exec.o
> 
> lvsconfig:	lvsconfig.o main.o 
> 
>@@ -167,11 +167,11 @@
> 			$(LOGROTATEDIR)
> 
> 		$(INSTALL) -m 0700 piranha-passwd $(SBIN)/
>-		$(INSTALL) -m 0755 -s lvsd $(SBIN)/
>-		$(INSTALL) -m 0755 -s fos $(SBIN)/
>-		$(INSTALL) -m 0755 -s pulse $(SBIN)/
>-		$(INSTALL) -m 0755 -s nanny $(SBIN)/
>-		$(INSTALL) -m 0755 -s send_arp $(SBIN)/
>+		$(INSTALL) -m 0755 lvsd $(SBIN)/
>+		$(INSTALL) -m 0755 fos $(SBIN)/
>+		$(INSTALL) -m 0755 pulse $(SBIN)/
>+		$(INSTALL) -m 0755 nanny $(SBIN)/
>+		$(INSTALL) -m 0755 send_arp $(SBIN)/
> 		$(INSTALL) -m 0644 lvsd.8 $(MANDIR)/man8/
> 		$(INSTALL) -m 0644 fos.8 $(MANDIR)/man8/
> 		$(INSTALL) -m 0644 lvs.cf.5 $(MANDIR)/man5/
>diff -urN piranha.old/nanny.c piranha.new/nanny.c
>--- piranha.old/nanny.c	2008-09-30 15:04:04.000000000 -0400
>+++ piranha.new/nanny.c	2008-09-30 16:16:18.000000000 -0400
>@@ -177,6 +177,110 @@
> /* static const char *http_recv_str = "HTTP"; */
> /* Default send/expect for port 80 */
> 
>+
>+int
>+shutdownDev (int flags, char *ipvsadm, char *virtualAddress, int port,
>+	     struct in_addr *remoteAddr, int service_type, int fm, int udp);
>+
>+int
>+adjustDevice (int flags, char *ipvsadm, char *virtualAddress, int port,
>+	      struct in_addr *remoteAddr, char *routingMethod, int weight,
>+	      int fm, int udp);
>+
>+int
>+bringUpDev (int flags, char *ipvsadm, char *virtualAddress, int port,
>+	    struct in_addr *remoteAddr, char *routingMethod, int weight,
>+	    int service_type, int fm, int udp);
>+
>+char *
>+getExecOutput (int flags, char **argv, int timeout)
>+{
>+
>+	pid_t child;
>+	int status;
>+	sigset_t sigs, oldSigs;
>+	int p[2];
>+	int rc;
>+	char buf[4096];
>+
>+	sigemptyset (&sigs);
>+	sigaddset (&sigs, SIGTERM);
>+	sigaddset (&sigs, SIGINT);
>+	sigaddset (&sigs, SIGALRM);
>+
>+	pipe (p);
>+
>+	if (!(child = fork ())) {
>+		int stdin, stderr;
>+
>+		close (p[0]);
>+		if (p[1] != STDOUT_FILENO) {
>+		        close (STDOUT_FILENO);
>+			dup2 (p[1], STDOUT_FILENO);
>+			close (p[1]);
>+		}
>+
>+		stdin = open ("/dev/null", O_RDONLY);
>+		if (stdin != STDIN_FILENO) {
>+			close (STDIN_FILENO);
>+			dup2 (stdin, STDIN_FILENO);
>+		}
>+		stderr = open ("/dev/null", O_WRONLY);
>+		if (stderr != STDERR_FILENO) {
>+			close (STDERR_FILENO);
>+			dup2 (stderr, STDERR_FILENO);
>+		}
>+
>+		execvp (argv[0], argv);
>+		exit (1);
>+	}
>+
>+	close (p[1]);
>+
>+	alarm (timeout);
>+
>+	needsKilling = child;
>+	sigprocmask (SIG_UNBLOCK, &sigs, &oldSigs);
>+
>+	rc = waitpid (child, &status, 0);
>+	sigprocmask (SIG_SETMASK, &oldSigs, NULL);
>+	needsKilling = 0;
>+
>+	if (rc < 0) {
>+		if (rc == EINTR)
>+			piranha_log (flags, (char *)
>+				     "unexpected return \"%s\" running the following:");
>+		else
>+			piranha_log (flags,
>+				     (char *) "timeout running the following:");
>+
>+		logArgv (flags, argv);
>+		kill (child, SIGKILL);
>+		waitpid (child, &status, 0);
>+		close (p[0]);	/* Per bugzilla #20974 */
>+		return NULL;
>+	}
>+
>+	if (!WIFEXITED (status) || WEXITSTATUS (status)) {
>+		piranha_log (flags,
>+			     (char *) "The following exited abnormally:");
>+		logArgv (flags, argv);
>+		close (p[0]);	/* Per bugzilla #20974 */
>+		return NULL;
>+	}
>+
>+	rc = read (p[0], buf, sizeof (buf) - 1);
>+	if (rc >= 1)
>+	  buf[rc - 1] = '\0';	/* chop */
>+	else
>+	  buf[0] = '\0';
>+	
>+	close (p[0]);
>+
>+	return strdup (buf);
>+}
>+
>+
> static void
> handleTermSignal (int signal)
> {
>@@ -617,252 +721,6 @@
> 	return return_status;
> }
> 
>-int
>-runCommand (char *cmd, int flags, char **argv, int log_flag)
>-{
>-	int child;
>-	int status;
>-	int rc;
>-	int log_it;
>-
>-	log_it = log_flag;
>-	if (flags & NANNY_FLAG_VERBOSE)
>-		log_it = -1;
>-
>-	if (log_it)
>-		logArgv (flags, argv);
>-
>-	if (flags & NANNY_FLAG_NORUN)
>-		return 0;
>-
>-	if (!(child = fork ())) {
>-		execv (argv[0], argv);
>-		exit (1);
>-	}
>-
>-	rc = waitpid (child, &status, 0);
>-
>-	if (!WIFEXITED (status) || WEXITSTATUS (status)) {
>-		piranha_log (flags, (char *) "%s command failed!", cmd);
>-		return 1;
>-	}
>-
>-	return 0;
>-}
>-
>-int
>-shutdownDev (int flags, char *ipvsadm, char *virtualAddress, int port,
>-	     struct in_addr *remoteAddr, int service_type)
>-{
>-
>-	char *argv[40];
>-	char **argp = argv;
>-	char virtName[80];
>-	char fwmStr[20];
>-	int rc = 0;
>-
>-	if (service_type == SERV_LVS) {
>-		/* Virtual Server */
>-
>-		*argp++ = ipvsadm;
>-		*argp++ = (char *) "-d";
>-
>-		if (fwmark) {
>-			sprintf (fwmStr, "%d", fwmark);
>-			*argp++ = (char *) "-f";
>-			*argp++ = fwmStr;
>-		} else {
>-			sprintf (virtName, "%s:%d", virtualAddress, port);
>-			if (use_udp)
>-				*argp++ = (char *) "-u";
>-			else
>-				*argp++ = (char *) "-t";
>-			*argp++ = virtName;
>-		}
>-
>-		*argp++ = (char *) "-r";
>-		*argp++ = inet_ntoa (*remoteAddr);
>-		*argp = NULL;
>-
>-		rc = runCommand (ipvsadm, flags, argv, -1);
>-
>-	} 
>-	/* Nothing needs to be done for FOS */
>-	return rc;
>-}
>-
>-int
>-adjustDevice (int flags, char *ipvsadm, char *virtualAddress, int port,
>-	      struct in_addr *remoteAddr, char *routingMethod, int weight)
>-{
>-	char *argv[40];
>-	char **argp = argv;
>-	char virtName[80];
>-	char weightStr[20];
>-	char fwmStr[20];
>-
>-	sprintf (weightStr, "%d", weight);
>-
>-	*argp++ = ipvsadm;
>-	*argp++ = (char *) "-e";
>-
>-	if (fwmark) {
>-		sprintf (fwmStr, "%d", fwmark);
>-		*argp++ = (char *) "-f";
>-		*argp++ = fwmStr;
>-	} else {
>-		sprintf (virtName, "%s:%d", virtualAddress, port);
>-		if (use_udp)
>-			*argp++ = (char *) "-u";
>-		else
>-			*argp++ = (char *) "-t";
>-		*argp++ = virtName;
>-	}
>-
>-	*argp++ = (char *) "-r";
>-	*argp++ = inet_ntoa (*remoteAddr);
>-	*argp++ = routingMethod;
>-	*argp++ = (char *) "-w";
>-	*argp++ = weightStr;
>-	*argp = NULL;
>-
>-	return runCommand (ipvsadm, flags, argv, 0);
>-}
>-
>-int
>-bringUpDev (int flags, char *ipvsadm, char *virtualAddress, int port,
>-	    struct in_addr *remoteAddr, char *routingMethod, int weight,
>-	    int service_type)
>-{
>-
>-	char *argv[40];
>-	char **argp = argv;
>-	char virtName[80];
>-	char weightStr[20];
>-	char fwmStr[20];
>-	int rc = 0;
>-
>-	if (service_type == SERV_LVS) {
>-		/* Virtual Server */
>-
>-		sprintf (weightStr, "%d", weight);
>-
>-		*argp++ = ipvsadm;
>-		*argp++ = (char *) "-a";
>-
>-		if (fwmark) {
>-			sprintf (fwmStr, "%d", fwmark);
>-			*argp++ = (char *) "-f";
>-			*argp++ = fwmStr;
>-		} else {
>-			sprintf (virtName, "%s:%d", virtualAddress, port);
>-			if (use_udp)
>-				*argp++ = (char *) "-u";
>-			else
>-				*argp++ = (char *) "-t";
>-			*argp++ = virtName;
>-		}
>-
>-		*argp++ = (char *) "-r";
>-		*argp++ = inet_ntoa (*remoteAddr);
>-		*argp++ = routingMethod;
>-		*argp++ = (char *) "-w";
>-		*argp++ = weightStr;
>-		*argp = NULL;
>-
>-		rc = runCommand (ipvsadm, flags, argv, -1);
>-	}
>-	/* Nothing needs to be done for FOS */
>-	return rc;
>-}
>-
>-char *
>-getExecOutput (int flags, char **argv, int timeout)
>-{
>-
>-	pid_t child;
>-	int status;
>-	sigset_t sigs, oldSigs;
>-	int p[2];
>-	int rc;
>-	char buf[4096];
>-
>-	sigemptyset (&sigs);
>-	sigaddset (&sigs, SIGTERM);
>-	sigaddset (&sigs, SIGINT);
>-	sigaddset (&sigs, SIGALRM);
>-
>-	pipe (p);
>-
>-	if (!(child = fork ())) {
>-		int stdin, stderr;
>-
>-		close (p[0]);
>-		if (p[1] != STDOUT_FILENO) {
>-		        close (STDOUT_FILENO);
>-			dup2 (p[1], STDOUT_FILENO);
>-			close (p[1]);
>-		}
>-
>-		stdin = open ("/dev/null", O_RDONLY);
>-		if (stdin != STDIN_FILENO) {
>-			close (STDIN_FILENO);
>-			dup2 (stdin, STDIN_FILENO);
>-		}
>-		stderr = open ("/dev/null", O_WRONLY);
>-		if (stderr != STDERR_FILENO) {
>-			close (STDERR_FILENO);
>-			dup2 (stderr, STDERR_FILENO);
>-		}
>-
>-		execvp (argv[0], argv);
>-		exit (1);
>-	}
>-
>-	close (p[1]);
>-
>-	alarm (timeout);
>-
>-	needsKilling = child;
>-	sigprocmask (SIG_UNBLOCK, &sigs, &oldSigs);
>-
>-	rc = waitpid (child, &status, 0);
>-	sigprocmask (SIG_SETMASK, &oldSigs, NULL);
>-	needsKilling = 0;
>-
>-	if (rc < 0) {
>-		if (errno == EINTR)
>-			piranha_log (flags, (char *)
>-				     "unexpected return \"%s\" running the following:");
>-		else
>-			piranha_log (flags,
>-				     (char *) "timeout running the following:");
>-
>-		logArgv (flags, argv);
>-		kill (child, SIGKILL);
>-		waitpid (child, &status, 0);
>-		close (p[0]);	/* Per bugzilla #20974 */
>-		return NULL;
>-	}
>-
>-	if (!WIFEXITED (status) || WEXITSTATUS (status)) {
>-		piranha_log (flags,
>-			     (char *) "The following exited abnormally:");
>-		logArgv (flags, argv);
>-		close (p[0]);	/* Per bugzilla #20974 */
>-		return NULL;
>-	}
>-
>-	rc = read (p[0], buf, sizeof (buf) - 1);
>-	if (rc >= 1)
>-	  buf[rc - 1] = '\0';	/* chop */
>-	else
>-	  buf[0] = '\0';
>-	
>-	close (p[0]);
>-
>-	return strdup (buf);
>-}
> 
> int
> external_check (int flags, char *send_program, char *expect_str,
>@@ -1029,7 +887,8 @@
> 				     port, newWeight);
> 
> 		if (adjustDevice (flags, ipvsadm, virtualAddress,
>-				  port, remoteAddr, routingMethod, newWeight))
>+				  port, remoteAddr, routingMethod,
>+				  newWeight, fwmark, use_udp))
> 			return -1;
> 	} else {
> 		/* Querying the load information and adjusting the
>@@ -1076,7 +935,7 @@
> 				if (adjustDevice (flags, ipvsadm,
> 						  virtualAddress, port,
> 						  remoteAddr, routingMethod,
>-						  newWeight))
>+						  newWeight, fwmark, use_udp))
> 					return -1;
> 
> 				return newWeight;
>@@ -1111,7 +970,8 @@
> 							virtualAddress, port,
> 							remoteAddr,
> 							routingMethod,
>-							weight, service_type))
>+							weight, service_type,
>+							fwmark, use_udp))
> 						return -1;
> 				}
> 				*isActive = 1;
>@@ -1126,7 +986,8 @@
> 					/* Virtual Server */
> 					if (shutdownDev (flags, ipvsadm,
> 							 virtualAddress, port,
>-							 remoteAddr, service_type))
>+							 remoteAddr, service_type, fwmark,
>+							 use_udp))
> 						return -1;
> 				}
> 				*isActive = 0;
>@@ -1159,7 +1020,7 @@
> 
> 	int isActive = 0;
> 	int currCount = countThresh;
>-	int isAvail;
>+	int isAvail = 0;
> 	int isSrvUp = 0;
> 	int pingSocket;
> 	sigset_t sigs;
>@@ -1220,7 +1081,7 @@
> 
> 		if (isActive && service_type == SERV_LVS) {
> 			if (shutdownDev (flags, ipvsadm, virtualAddress, port,
>-					 remoteAddr, service_type))
>+					 remoteAddr, service_type, fwmark, use_udp))
> 				exit (1);
> 		}
> 
>@@ -1316,7 +1177,8 @@
> 				int newWeight;
> 				
> 				piranha_log (flags,
>-					     (char *) "making %s:%d available",
>+					     (char *) "%s making %s:%d available",
>+					     NANNY_ACTIVE_STATUS,
> 					     inet_ntoa (*remoteAddr), port);
> 
> 				/* compute the initial weight */
>@@ -1329,14 +1191,15 @@
> 					    (flags, ipvsadm,
> 					     virtualAddress, port,
> 					     remoteAddr, routingMethod,
>-					     newWeight))
>+					     newWeight, fwmark, use_udp))
> 						return -1;
> 				} else {
> 					if (bringUpDev
> 					    (flags, ipvsadm,
> 					     virtualAddress, port,
> 					     remoteAddr, routingMethod,
>-					     newWeight, service_type))
>+					     newWeight, service_type,
>+					     fwmark, use_udp))
> 						return -1;
> 					isSrvUp = 1;
> 				}
>@@ -1369,20 +1232,21 @@
> 			} else if (service_type == SERV_LVS) {
> 				/* Virtual Services */
> 				piranha_log (flags, (char *)
>-					     "shutting down %s:%d due to connection failure",
>+					     "%s shutting down %s:%d due to connection failure",
>+					     NANNY_INACTIVE_STATUS,
> 					     inet_ntoa (*remoteAddr), port);
> 
> 				if (quiesce_srv && isSrvUp) {
> 					if (adjustDevice
> 					    (flags, ipvsadm,
> 					     virtualAddress, port,
>-					     remoteAddr, routingMethod, 0))
>+					     remoteAddr, routingMethod, 0, fwmark, use_udp))
> 						return -1;
> 				} else {
> 					if (shutdownDev
> 					    (flags, ipvsadm,
> 					     virtualAddress, port,
>-					     remoteAddr, service_type))
>+					     remoteAddr, service_type, fwmark, use_udp))
> 						return -1;
> 					isSrvUp = 0;
> 				}
>diff -urN piranha.old/sample.cf piranha.new/sample.cf
>--- piranha.old/sample.cf	2001-08-28 15:07:55.000000000 -0400
>+++ piranha.new/sample.cf	2008-09-30 15:18:18.000000000 -0400
>@@ -127,6 +127,7 @@
> 	persistent = 60
> 	pmask = 255.255.255.255
>     	protocol = tcp
>+	# sorry_server = 127.0.0.1
> 
> 	server Real1 {
> 		address = 192.168.10.2
>diff -urN piranha.old/util.c piranha.new/util.c
>--- piranha.old/util.c	2008-09-30 15:04:04.000000000 -0400
>+++ piranha.new/util.c	2008-09-30 15:18:18.000000000 -0400
>@@ -75,6 +75,8 @@
> void
> piranha_log (int flags, char *format, ...)
> {
>+  char logmsg[256];
>+  char totalmsg[320];
> 
>   va_list args;
> 
>@@ -82,9 +84,9 @@
> 
>   if (flags & LVS_FLAG_PRINTF)
>     {
>-      printf ("%s: ", name);
>-      vprintf (format, args);
>-      printf ("\n");
>+      vsnprintf (logmsg, sizeof(logmsg), format, args);
>+      snprintf(totalmsg, sizeof(totalmsg), "%s: %s\n", name, logmsg);
>+      write(STDOUT_FILENO, totalmsg, strlen(totalmsg));
>     }
>   else if (flags & LVS_FLAG_SYSLOG)
>     {
>diff -urN piranha.old/web/web/secure/parse.php piranha.new/web/web/secure/parse.php
>--- piranha.old/web/web/secure/parse.php	2008-09-30 15:04:04.000000000 -0400
>+++ piranha.new/web/web/secure/parse.php	2008-09-30 15:18:56.000000000 -0400
>@@ -85,7 +85,8 @@
> 		"deadtime"			=> "",
> 		"reservation_conflict_action"	=> "",
> 		"debug_level"			=> "",
>-		"monitor_links"			=> ""
>+		"monitor_links"			=> "",
>+		"syncdaemon"			=> ""
> 	);
> 
> /* Global file descriptor for use as a pointer to the lvs.cf file */
>@@ -202,6 +203,9 @@
> 			case "monitor_links"			:	$prim['monitor_links']			= $datum;
> 									break;
> 
>+			case "syncdaemon"			:	$prim['syncdaemon']			= $datum;
>+									break;
>+
> 			case ""					:	break;
> 			default					:	if ($debug) { echo "<FONT COLOR=\"BLUE\">Level $level - garbage [$name] (ignored line [$buffer])</FONT><BR>"; }
> 									break;
>@@ -288,6 +292,8 @@
> 							if ($debug) { echo "<FONT COLOR=\"yellow\"><I>Asked for virtual service </I><B>\$virt[$virt_count]</B></FONT><BR>"; };
> 							if ($service == "lvs") $virt[$virt_count]['virtual']	= $datum;
> 							break;
>+			case "sorry_server"	:	if ($service == "lvs") $virt[$virt_count]['sorry_server']	= $datum;
>+							break;
> 			case "fwmark"		:	if ($service == "lvs") $virt[$virt_count]['fwmark']	= $datum;
> 							break;
> 			case "load_monitor"	:	if ($service == "lvs") $virt[$virt_count]['load_monitor']	= $datum;
>@@ -716,6 +722,11 @@
> 		if ($debug) { echo "monitor_links = "			. $prim['monitor_links']		. "<BR>"; };
> 	}
> 
>+
>+	if ($prim['syncdaemon'] != "" ){
>+		fputs ($fd, "syncdaemon = "				. $prim['syncdaemon']		. "\n", 80);
>+		if ($debug) { echo "syncdaemon = "			. $prim['syncdaemon']		. "<BR>"; };
>+	}
> 	
> 	while ( $fail[$loop1]['failover'] != "" ) {
> 		if ((($loop1 == $delete_item ) && ($level == "1")) && ($prim['service'] == "fos")) {  $loop1++; $loop2 = 1; } else {
>@@ -829,6 +840,12 @@
> 				if ($debug) { echo "$egap1 address "		. $virt[$loop3]['address']	. "<BR>"; };
> 			}
> 
>+			if (isset($virt[$loop3]['sorry_server']) &&
>+			    $virt[$loop3]['sorry_server'] != "") {
>+				fputs ($fd, "$gap1 sorry_server = "			. $virt[$loop3]['sorry_server']	. "\n", 80);
>+				if ($debug) { echo "$egap1 sorry_server "		. $virt[$loop3]['sorry_server']	. "<BR>"; };
>+			}
>+
> 			if (isset($virt[$loop3]['vip_nmask']) &&
> 			    $virt[$loop3]['vip_nmask'] != "") {
> 				fputs ($fd, "$gap1 vip_nmask = "		. $virt[$loop3]['vip_nmask']	. "\n", 80);
>diff -urN piranha.old/web/web/secure/redundancy.php piranha.new/web/web/secure/redundancy.php
>--- piranha.old/web/web/secure/redundancy.php	2005-07-28 13:16:04.000000000 -0400
>+++ piranha.new/web/web/secure/redundancy.php	2008-09-30 15:19:04.000000000 -0400
>@@ -92,6 +92,15 @@
> 		} else { 
> 			$prim['monitor_links'] = "0";
> 		}
>+		if (isset($_GET['syncdaemon'])) {
>+			if ($_GET['syncdaemon'] == "on") {
>+				$prim['syncdaemon'] = "1";
>+			} else { 
>+				$prim['syncdaemon'] = "0";
>+			}
>+		} else { 
>+			$prim['syncdaemon'] = "0";
>+		}
> 		
> 		/*
> 		$prim['backup_private']		= $_GET['redundant_private'];
>@@ -266,6 +275,17 @@
> 			?>
> 			</TD>
> 		</TR>
>+		<TR>
>+			<TD>Syncdaemon:</TD>
>+			<TD><INPUT TYPE="checkbox" NAME="syncdaemon" 
>+			<?php
>+			if ($prim['syncdaemon'] == "1") {
>+				echo "CHECKED";
>+			};
>+			echo ">";
>+			?>
>+			</TD>
>+		</TR>
> 	</TABLE>
> 	<HR>
> <?php } ?>
>diff -urN piranha.old/web/web/secure/virtual_edit_virt.php piranha.new/web/web/secure/virtual_edit_virt.php
>--- piranha.old/web/web/secure/virtual_edit_virt.php	2005-02-08 03:04:20.000000000 -0500
>+++ piranha.new/web/web/secure/virtual_edit_virt.php	2008-09-30 15:19:11.000000000 -0400
>@@ -84,6 +84,7 @@
> 
>                 $temp[0] = $_GET['address'];
>                 $temp[1] = $_GET['device'];
>+                $virt[$selected_host]['sorry_server']       =       $_GET['sorry_server'];
>                 $virt[$selected_host]['address']      		=       $_GET['address'] . " " . $_GET['device'];
> 	}
> 
>@@ -257,6 +258,10 @@
> 		</TD>
> 	</TR>
> 	<TR>
>+		<TD>Sorry Server: </TD>
>+		<TD><INPUT TYPE="TEXT" NAME="sorry_server" VALUE=<?php echo $virt[$selected_host]['sorry_server'] ?>></TD>
>+	</TR>
>+	<TR>
> 		<TD>Firewall Mark: </TD>
> 		<TD> <INPUT TYPE="TEXT" NAME="fwmark" VALUE="<?php if (isset($virt[$selected_host]['fwmark'])) {
> 								      echo $virt[$selected_host]['fwmark']; } ?>"></TD>

Actions: View | Diff

Attachments on bug 238498: 153903 | 153907 | 153914 | 153954 | 154052 | 154065 | 154291 | 318124