From: Matthew Dillon <dillon@apollo.backplane.com>
Date: Wed, 5 Dec 2012 07:32:30 +0000 (-0800)
Subject: cluster - Stabilization
X-Git-Tag: v3.4.0rc~732
X-Git-Url: https://gitweb.dragonflybsd.org/~tuxillo/dragonfly.git/commitdiff_plain/981268696baabad943c6b0db147f8ea7239cf5c2

cluster - Stabilization

* Fix disconnect/reconnect sequence for autoconn (/etc/hammer2/autoconn).
  The pipe used to signal termination of the iocom_core() was not supposed
  to be closed by iocom_done().

* The shutdown code now simply sets DMSG_IOCOMF_EOF instead of trying to
  shutdown() the socket.

* Fix double mutex lock in dmsg_msg_alloc().
---

diff --git a/lib/libdmsg/dmsg.h b/lib/libdmsg/dmsg.h
index 2426861305..c02b34d4c6 100644
--- a/lib/libdmsg/dmsg.h
+++ b/lib/libdmsg/dmsg.h
@@ -307,6 +307,7 @@ typedef struct dmsg_iocom dmsg_iocom_t;
 #define DMSG_IOCOMF_AWWORK	0x00000080	/* immediate work pending */
 #define DMSG_IOCOMF_SWORK	0x00000100	/* immediate work pending */
 #define DMSG_IOCOMF_CRYPTED	0x00000200	/* encrypt enabled */
+#define DMSG_IOCOMF_CLOSEALT	0x00000400	/* close alt_fd */
 
 /*
  * Crypto algorithm table and related typedefs.
@@ -330,6 +331,7 @@ struct crypto_algo {
 struct dmsg_master_service_info {
 	int	fd;
 	int	altfd;
+	int	noclosealt;
 	int	detachme;
 	char	*label;
 	void	*handle;
diff --git a/lib/libdmsg/msg.c b/lib/libdmsg/msg.c
index 616cc00f57..e11c7df9db 100644
--- a/lib/libdmsg/msg.c
+++ b/lib/libdmsg/msg.c
@@ -134,7 +134,7 @@ dmsg_iocom_init(dmsg_iocom_t *iocom, int sock_fd, int alt_fd,
 	TAILQ_INIT(&iocom->txmsgq);
 	iocom->sock_fd = sock_fd;
 	iocom->alt_fd = alt_fd;
-	iocom->flags = DMSG_IOCOMF_RREQ;
+	iocom->flags = DMSG_IOCOMF_RREQ | DMSG_IOCOMF_CLOSEALT;
 	if (signal_func)
 		iocom->flags |= DMSG_IOCOMF_SWORK;
 	dmsg_ioq_init(iocom, &iocom->ioq_rx);
@@ -230,7 +230,7 @@ dmsg_iocom_done(dmsg_iocom_t *iocom)
 		close(iocom->sock_fd);
 		iocom->sock_fd = -1;
 	}
-	if (iocom->alt_fd >= 0) {
+	if (iocom->alt_fd >= 0 && (iocom->flags & DMSG_IOCOMF_CLOSEALT)) {
 		close(iocom->alt_fd);
 		iocom->alt_fd = -1;
 	}
@@ -324,9 +324,7 @@ dmsg_msg_alloc(dmsg_circuit_t *circuit,
 		state->icmd = state->txcmd & DMSGF_BASECMDMASK;
 		state->func = func;
 		state->any.any = data;
-		pthread_mutex_lock(&iocom->mtx);
 		RB_INSERT(dmsg_state_tree, &circuit->statewr_tree, state);
-		pthread_mutex_unlock(&iocom->mtx);
 		state->flags |= DMSG_STATE_INSERTED;
 	}
 	/* XXX SMP race for state */
diff --git a/lib/libdmsg/msg_lnk.c b/lib/libdmsg/msg_lnk.c
index 2c67e4c210..b7ea6dbdaa 100644
--- a/lib/libdmsg/msg_lnk.c
+++ b/lib/libdmsg/msg_lnk.c
@@ -1512,7 +1512,7 @@ static
 void
 dmsg_volconf_signal(dmsg_iocom_t *iocom)
 {
-	shutdown(iocom->sock_fd, SHUT_RDWR);
+	atomic_set_int(&iocom->flags, DMSG_IOCOMF_EOF);
 }
 
 /************************************************************************
diff --git a/lib/libdmsg/service.c b/lib/libdmsg/service.c
index a267060a94..1a741f703a 100644
--- a/lib/libdmsg/service.c
+++ b/lib/libdmsg/service.c
@@ -61,6 +61,8 @@ dmsg_master_service(void *data)
 			master_auth_rxmsg,
 			info->dbgmsg_callback,
 			info->altmsg_callback);
+	if (info->noclosealt)
+		iocom.flags &= ~DMSG_IOCOMF_CLOSEALT;
 	if (info->label) {
 		dmsg_iocom_label(&iocom, "%s", info->label);
 		free(info->label);
diff --git a/sbin/hammer2/cmd_service.c b/sbin/hammer2/cmd_service.c
index 512448a1fe..ad64f66eb5 100644
--- a/sbin/hammer2/cmd_service.c
+++ b/sbin/hammer2/cmd_service.c
@@ -36,6 +36,7 @@
 #include "hammer2.h"
 
 #include <sys/xdiskioctl.h>
+#include <machine/atomic.h>
 
 struct diskcon {
 	TAILQ_ENTRY(diskcon) entry;
@@ -449,12 +450,16 @@ autoconn_thread(void *data __unused)
 
 			/*
 			 * Unstaging, stop active connection.
+			 *
+			 * We write to the pipe which causes the iocom_core
+			 * to call autoconn_disconnect_signal().
 			 */
 			if (ac->stage == 0 &&
 			    ac->state == AUTOCONN_ACTIVE) {
 				if (ac->stopme == 0) {
+					char dummy = 0;
 					ac->stopme = 1;
-					close(ac->pipefd[1]); /* signal */
+					write(ac->pipefd[1], &dummy, 1);
 				}
 			}
 
@@ -502,6 +507,7 @@ autoconn_connect_thread(void *data)
 		info->altfd = ac->pipefd[0];
 		info->altmsg_callback = autoconn_disconnect_signal;
 		info->detachme = 0;
+		info->noclosealt = 1;
 		pthread_create(&ac->thread, NULL, dmsg_master_service, info);
 		pthread_join(ac->thread, &res);
 	}
@@ -516,7 +522,7 @@ void
 autoconn_disconnect_signal(dmsg_iocom_t *iocom)
 {
 	fprintf(stderr, "autoconn: Shutting down socket\n");
-	shutdown(iocom->sock_fd, SHUT_RDWR);
+	atomic_set_int(&iocom->flags, DMSG_IOCOMF_EOF);
 }
 
 /*