cluster - Stabilization

* Fix disconnect/reconnect sequence for autoconn (/etc/hammer2/autoconn).
  The pipe used to signal termination of the iocom_core() was not supposed
  to be closed by iocom_done().

* The shutdown code now simply sets DMSG_IOCOMF_EOF instead of trying to
  shutdown() the socket.

* Fix double mutex lock in dmsg_msg_alloc().

diff --git a/lib/libdmsg/dmsg.h b/lib/libdmsg/dmsg.h
index 2426861..c02b34d 100644 (file)
--- a/lib/libdmsg/dmsg.h
+++ b/lib/libdmsg/dmsg.h
@@ -307,6 +307,7 @@ typedef struct dmsg_iocom dmsg_iocom_t;
 #define DMSG_IOCOMF_AWWORK     0x00000080      /* immediate work pending */
 #define DMSG_IOCOMF_SWORK      0x00000100      /* immediate work pending */
 #define DMSG_IOCOMF_CRYPTED    0x00000200      /* encrypt enabled */
+#define DMSG_IOCOMF_CLOSEALT   0x00000400      /* close alt_fd */
 
 /*
  * Crypto algorithm table and related typedefs.
@@ -330,6 +331,7 @@ struct crypto_algo {
 struct dmsg_master_service_info {
        int     fd;
        int     altfd;
+       int     noclosealt;
        int     detachme;
        char    *label;
        void    *handle;

diff --git a/lib/libdmsg/msg.c b/lib/libdmsg/msg.c
index 616cc00..e11c7df 100644 (file)
--- a/lib/libdmsg/msg.c
+++ b/lib/libdmsg/msg.c
@@ -134,7 +134,7 @@ dmsg_iocom_init(dmsg_iocom_t *iocom, int sock_fd, int alt_fd,
        TAILQ_INIT(&iocom->txmsgq);
        iocom->sock_fd = sock_fd;
        iocom->alt_fd = alt_fd;
-       iocom->flags = DMSG_IOCOMF_RREQ;
+       iocom->flags = DMSG_IOCOMF_RREQ | DMSG_IOCOMF_CLOSEALT;
        if (signal_func)
                iocom->flags |= DMSG_IOCOMF_SWORK;
        dmsg_ioq_init(iocom, &iocom->ioq_rx);
@@ -230,7 +230,7 @@ dmsg_iocom_done(dmsg_iocom_t *iocom)
                close(iocom->sock_fd);
                iocom->sock_fd = -1;
        }
-       if (iocom->alt_fd >= 0) {
+       if (iocom->alt_fd >= 0 && (iocom->flags & DMSG_IOCOMF_CLOSEALT)) {
                close(iocom->alt_fd);
                iocom->alt_fd = -1;
        }
@@ -324,9 +324,7 @@ dmsg_msg_alloc(dmsg_circuit_t *circuit,
                state->icmd = state->txcmd & DMSGF_BASECMDMASK;
                state->func = func;
                state->any.any = data;
-               pthread_mutex_lock(&iocom->mtx);
                RB_INSERT(dmsg_state_tree, &circuit->statewr_tree, state);
-               pthread_mutex_unlock(&iocom->mtx);
                state->flags |= DMSG_STATE_INSERTED;
        }
        /* XXX SMP race for state */

diff --git a/lib/libdmsg/msg_lnk.c b/lib/libdmsg/msg_lnk.c
index 2c67e4c..b7ea6db 100644 (file)
--- a/lib/libdmsg/msg_lnk.c
+++ b/lib/libdmsg/msg_lnk.c
@@ -1512,7 +1512,7 @@ static
 void
 dmsg_volconf_signal(dmsg_iocom_t *iocom)
 {
-       shutdown(iocom->sock_fd, SHUT_RDWR);
+       atomic_set_int(&iocom->flags, DMSG_IOCOMF_EOF);
 }
 
 /************************************************************************

diff --git a/lib/libdmsg/service.c b/lib/libdmsg/service.c
index a267060..1a741f7 100644 (file)
--- a/lib/libdmsg/service.c
+++ b/lib/libdmsg/service.c
@@ -61,6 +61,8 @@ dmsg_master_service(void *data)
                        master_auth_rxmsg,
                        info->dbgmsg_callback,
                        info->altmsg_callback);
+       if (info->noclosealt)
+               iocom.flags &= ~DMSG_IOCOMF_CLOSEALT;
        if (info->label) {
                dmsg_iocom_label(&iocom, "%s", info->label);
                free(info->label);

diff --git a/sbin/hammer2/cmd_service.c b/sbin/hammer2/cmd_service.c
index 512448a..ad64f66 100644 (file)
--- a/sbin/hammer2/cmd_service.c
+++ b/sbin/hammer2/cmd_service.c
@@ -36,6 +36,7 @@
 #include "hammer2.h"
 
 #include <sys/xdiskioctl.h>
+#include <machine/atomic.h>
 
 struct diskcon {
        TAILQ_ENTRY(diskcon) entry;
@@ -449,12 +450,16 @@ autoconn_thread(void *data __unused)
 
                        /*
                         * Unstaging, stop active connection.
+                        *
+                        * We write to the pipe which causes the iocom_core
+                        * to call autoconn_disconnect_signal().
                         */
                        if (ac->stage == 0 &&
                            ac->state == AUTOCONN_ACTIVE) {
                                if (ac->stopme == 0) {
+                                       char dummy = 0;
                                        ac->stopme = 1;
-                                       close(ac->pipefd[1]); /* signal */
+                                       write(ac->pipefd[1], &dummy, 1);
                                }
                        }
 
@@ -502,6 +507,7 @@ autoconn_connect_thread(void *data)
                info->altfd = ac->pipefd[0];
                info->altmsg_callback = autoconn_disconnect_signal;
                info->detachme = 0;
+               info->noclosealt = 1;
                pthread_create(&ac->thread, NULL, dmsg_master_service, info);
                pthread_join(ac->thread, &res);
        }
@@ -516,7 +522,7 @@ void
 autoconn_disconnect_signal(dmsg_iocom_t *iocom)
 {
        fprintf(stderr, "autoconn: Shutting down socket\n");
-       shutdown(iocom->sock_fd, SHUT_RDWR);
+       atomic_set_int(&iocom->flags, DMSG_IOCOMF_EOF);
 }
 
 /*