hammer2 - Add peer_type field to LNK_CONN and LNK_SPAN
[dragonfly.git] / sbin / hammer2 / cmd_service.c
CommitLineData
9ab15106
MD
1/*
2 * Copyright (c) 2011-2012 The DragonFly Project. All rights reserved.
3 *
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@dragonflybsd.org>
6 * by Venkatesh Srinivas <vsrinivas@dragonflybsd.org>
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 *
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in
16 * the documentation and/or other materials provided with the
17 * distribution.
18 * 3. Neither the name of The DragonFly Project nor the names of its
19 * contributors may be used to endorse or promote products derived
20 * from this software without specific, prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
25 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
26 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
27 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
28 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
29 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
30 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
31 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
32 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
33 * SUCH DAMAGE.
34 */
35
36#include "hammer2.h"
37
62efe6ec 38static void *master_accept(void *data);
29ead430
MD
39static void master_auth_signal(hammer2_router_t *router);
40static void master_auth_rxmsg(hammer2_msg_t *msg);
41static void master_link_signal(hammer2_router_t *router);
42static void master_link_rxmsg(hammer2_msg_t *msg);
e1648a68 43static void master_reconnect(const char *mntpt);
9ab15106
MD
44
45/*
46 * Start-up the master listener daemon for the machine.
47 *
48 * The master listener serves as a rendezvous point in the cluster, accepting
49 * connections, performing registrations and authentications, maintaining
50 * the spanning tree, and keeping track of message state so disconnects can
51 * be handled properly.
52 *
53 * Once authenticated only low-level messaging protocols (which includes
54 * tracking persistent messages) are handled by this daemon. This daemon
55 * does not run the higher level quorum or locking protocols.
56 *
57 * This daemon can also be told to maintain connections to other nodes,
58 * forming a messaging backbone, which in turn allows PFS's (if desired) to
59 * simply connect to the master daemon via localhost if desired.
60 * Backbones are specified via /etc/hammer2.conf.
61 */
62int
62efe6ec 63cmd_service(void)
9ab15106
MD
64{
65 struct sockaddr_in lsin;
66 int on;
67 int lfd;
68
69 /*
70 * Acquire socket and set options
71 */
72 if ((lfd = socket(AF_INET, SOCK_STREAM, 0)) < 0) {
62efe6ec 73 fprintf(stderr, "master_listen: socket(): %s\n",
9ab15106
MD
74 strerror(errno));
75 return 1;
76 }
77 on = 1;
78 setsockopt(lfd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
79
80 /*
81 * Setup listen port and try to bind. If the bind fails we assume
82 * that a master listener process is already running and silently
83 * fail.
84 */
85 bzero(&lsin, sizeof(lsin));
86 lsin.sin_family = AF_INET;
87 lsin.sin_addr.s_addr = INADDR_ANY;
88 lsin.sin_port = htons(HAMMER2_LISTEN_PORT);
89 if (bind(lfd, (struct sockaddr *)&lsin, sizeof(lsin)) < 0) {
90 close(lfd);
9b8b748f
MD
91 if (QuietOpt == 0) {
92 fprintf(stderr,
93 "master listen: daemon already running\n");
94 }
9ab15106
MD
95 return 0;
96 }
9b8b748f
MD
97 if (QuietOpt == 0)
98 fprintf(stderr, "master listen: startup\n");
9ab15106
MD
99 listen(lfd, 50);
100
101 /*
102 * Fork and disconnect the controlling terminal and parent process,
103 * executing the specified function as a pthread.
104 *
105 * Returns to the original process which can then continue running.
106 * In debug mode this call will create the pthread without forking
107 * and set NormalExit to 0, instead of fork.
108 */
62efe6ec 109 hammer2_demon(master_accept, (void *)(intptr_t)lfd);
9ab15106
MD
110 if (NormalExit)
111 close(lfd);
112 return 0;
113}
114
115/*
116 * Master listen/accept thread. Accept connections on the master socket,
117 * starting a pthread for each one.
118 */
119static
120void *
62efe6ec 121master_accept(void *data)
9ab15106
MD
122{
123 struct sockaddr_in asin;
124 socklen_t alen;
125 pthread_t thread;
e1648a68 126 hammer2_master_service_info_t *info;
9ab15106
MD
127 int lfd = (int)(intptr_t)data;
128 int fd;
e1648a68
MD
129 int i;
130 int count;
131 struct statfs *mntbuf = NULL;
132 struct statvfs *mntvbuf = NULL;
9ab15106
MD
133
134 /*
135 * Nobody waits for us
136 */
137 setproctitle("hammer2 master listen");
138 pthread_detach(pthread_self());
139
140 /*
e1648a68
MD
141 * Scan existing hammer2 mounts and reconnect to them using
142 * HAMMER2IOC_RECLUSTER.
143 */
144 count = getmntvinfo(&mntbuf, &mntvbuf, MNT_NOWAIT);
145 for (i = 0; i < count; ++i) {
146 if (strcmp(mntbuf[i].f_fstypename, "hammer2") == 0)
147 master_reconnect(mntbuf[i].f_mntonname);
148 }
149
150 /*
9ab15106
MD
151 * Accept connections and create pthreads to handle them after
152 * validating the IP.
153 */
154 for (;;) {
155 alen = sizeof(asin);
156 fd = accept(lfd, (struct sockaddr *)&asin, &alen);
157 if (fd < 0) {
158 if (errno == EINTR)
159 continue;
160 break;
161 }
162 thread = NULL;
62efe6ec 163 fprintf(stderr, "master_accept: accept fd %d\n", fd);
e1648a68
MD
164 info = malloc(sizeof(*info));
165 bzero(info, sizeof(*info));
166 info->fd = fd;
167 info->detachme = 1;
168 pthread_create(&thread, NULL, master_service, info);
9ab15106
MD
169 }
170 return (NULL);
171}
172
173/*
e1648a68
MD
174 * Normally the mount program supplies a cluster communications
175 * descriptor to the hammer2 vfs on mount, but if you kill the service
176 * daemon and restart it that link will be lost.
177 *
178 * This procedure attempts to [re]connect to existing mounts when
179 * the service daemon is started up before going into its accept
180 * loop.
eae0d690
MD
181 *
182 * NOTE: A hammer2 mount point can only accomodate one connection at a time
183 * so this will disconnect any existing connection during the
184 * reconnect.
e1648a68
MD
185 */
186static
187void
188master_reconnect(const char *mntpt)
189{
190 struct hammer2_ioc_recluster recls;
191 hammer2_master_service_info_t *info;
192 pthread_t thread;
193 int fd;
194 int pipefds[2];
195
196 fd = open(mntpt, O_RDONLY);
197 if (fd < 0) {
198 fprintf(stderr, "reconnect %s: no access to mount\n", mntpt);
199 return;
200 }
201 if (pipe(pipefds) < 0) {
202 fprintf(stderr, "reconnect %s: pipe() failed\n", mntpt);
2063f4d7 203 close(fd);
e1648a68
MD
204 return;
205 }
206 bzero(&recls, sizeof(recls));
207 recls.fd = pipefds[0];
208 if (ioctl(fd, HAMMER2IOC_RECLUSTER, &recls) < 0) {
209 fprintf(stderr, "reconnect %s: ioctl failed\n", mntpt);
210 close(pipefds[0]);
211 close(pipefds[1]);
212 close(fd);
213 return;
214 }
215 close(pipefds[0]);
2063f4d7 216 close(fd);
e1648a68
MD
217
218 info = malloc(sizeof(*info));
219 bzero(info, sizeof(*info));
220 info->fd = pipefds[1];
221 info->detachme = 1;
222 pthread_create(&thread, NULL, master_service, info);
223}
224
225/*
9ab15106 226 * Service an accepted connection (runs as a pthread)
02454b3e
MD
227 *
228 * (also called from a couple of other places)
9ab15106 229 */
9ab15106 230void *
62efe6ec 231master_service(void *data)
9ab15106 232{
e1648a68 233 hammer2_master_service_info_t *info = data;
9ab15106 234 hammer2_iocom_t iocom;
9ab15106 235
e1648a68
MD
236 if (info->detachme)
237 pthread_detach(pthread_self());
238
239 hammer2_iocom_init(&iocom, info->fd, -1,
29ead430
MD
240 master_auth_signal,
241 master_auth_rxmsg,
242 NULL);
5903497c 243 hammer2_iocom_core(&iocom);
9ab15106
MD
244
245 fprintf(stderr,
246 "iocom on fd %d terminated error rx=%d, tx=%d\n",
e1648a68
MD
247 info->fd, iocom.ioq_rx.error, iocom.ioq_tx.error);
248 close(info->fd);
249 info->fd = -1; /* safety */
250 free(info);
9ab15106
MD
251
252 return (NULL);
253}
254
62efe6ec
MD
255/************************************************************************
256 * AUTHENTICATION *
257 ************************************************************************
258 *
1b195a98
MD
259 * Callback via hammer2_iocom_core().
260 *
62efe6ec
MD
261 * Additional messaging-based authentication must occur before normal
262 * message operation. The connection has already been encrypted at
263 * this point.
264 */
29ead430 265static void master_auth_conn_rx(hammer2_msg_t *msg);
5903497c
MD
266
267static
268void
29ead430 269master_auth_signal(hammer2_router_t *router)
5903497c
MD
270{
271 hammer2_msg_t *msg;
272
273 /*
274 * Transmit LNK_CONN, enabling the SPAN protocol if both sides
275 * agree.
276 *
2063f4d7 277 * XXX put additional authentication states here?
5903497c 278 */
29ead430 279 msg = hammer2_msg_alloc(router, 0, HAMMER2_LNK_CONN |
2063f4d7 280 HAMMER2_MSGF_CREATE,
29ead430 281 master_auth_conn_rx, NULL);
2063f4d7
MD
282 msg->any.lnk_conn.peer_mask = (uint64_t)-1;
283 msg->any.lnk_conn.peer_type = HAMMER2_PEER_CLUSTER;
284
29ead430 285 hammer2_msg_write(msg);
5903497c 286
29ead430
MD
287 hammer2_router_restate(router,
288 master_link_signal,
289 master_link_rxmsg,
290 NULL);
5903497c
MD
291}
292
62efe6ec
MD
293static
294void
29ead430 295master_auth_conn_rx(hammer2_msg_t *msg)
62efe6ec 296{
5903497c 297 if (msg->any.head.cmd & HAMMER2_MSGF_DELETE)
29ead430 298 hammer2_msg_reply(msg, 0);
62efe6ec
MD
299}
300
301static
302void
29ead430 303master_auth_rxmsg(hammer2_msg_t *msg __unused)
62efe6ec 304{
62efe6ec
MD
305}
306
1b195a98
MD
307/************************************************************************
308 * POST-AUTHENTICATION SERVICE MSGS *
309 ************************************************************************
310 *
311 * Callback via hammer2_iocom_core().
9ab15106
MD
312 */
313static
314void
29ead430 315master_link_signal(hammer2_router_t *router)
9ab15106 316{
29ead430 317 hammer2_msg_lnk_signal(router);
9ab15106
MD
318}
319
9ab15106
MD
320static
321void
29ead430 322master_link_rxmsg(hammer2_msg_t *msg)
9ab15106 323{
5903497c
MD
324 hammer2_state_t *state;
325 uint32_t cmd;
326
327 /*
328 * If the message state has a function established we just
329 * call the function, otherwise we call the appropriate
330 * link-level protocol related to the original command and
331 * let it sort it out.
332 *
333 * Non-transactional one-off messages, on the otherhand,
334 * might have REPLY set.
335 */
336 state = msg->state;
81666e1b
MD
337 cmd = state ? state->msg->any.head.cmd : msg->any.head.cmd;
338
339 fprintf(stderr, "service-receive: %s\n", hammer2_msg_str(msg));
340
5903497c
MD
341 if (state && state->func) {
342 assert(state->func != NULL);
29ead430 343 state->func(msg);
5903497c
MD
344 } else {
345 switch(cmd & HAMMER2_MSGF_PROTOS) {
346 case HAMMER2_MSG_PROTO_LNK:
29ead430 347 hammer2_msg_lnk(msg);
5903497c
MD
348 break;
349 case HAMMER2_MSG_PROTO_DBG:
29ead430 350 hammer2_msg_dbg(msg);
5903497c
MD
351 break;
352 default:
29ead430 353 hammer2_msg_reply(msg, HAMMER2_MSG_ERR_NOSUPP);
5903497c
MD
354 break;
355 }
356 }
9ab15106 357}