cluster - Major kernel component work (diskiocom, xdisk, kdmsg)
[dragonfly.git] / sys / kern / kern_dmsg.c
1 /*-
2  * Copyright (c) 2012 The DragonFly Project.  All rights reserved.
3  *
4  * This code is derived from software contributed to The DragonFly Project
5  * by Matthew Dillon <dillon@backplane.com>
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  *
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in
15  *    the documentation and/or other materials provided with the
16  *    distribution.
17  * 3. Neither the name of The DragonFly Project nor the names of its
18  *    contributors may be used to endorse or promote products derived
19  *    from this software without specific, prior written permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32  * SUCH DAMAGE.
33  */
34 /*
35  * TODO: txcmd CREATE state is deferred by txmsgq, need to calculate
36  *       a streaming response.  See subr_diskiocom()'s diskiodone().
37  */
38 #include <sys/param.h>
39 #include <sys/types.h>
40 #include <sys/kernel.h>
41 #include <sys/conf.h>
42 #include <sys/systm.h>
43 #include <sys/queue.h>
44 #include <sys/tree.h>
45 #include <sys/malloc.h>
46 #include <sys/mount.h>
47 #include <sys/socket.h>
48 #include <sys/vnode.h>
49 #include <sys/file.h>
50 #include <sys/proc.h>
51 #include <sys/priv.h>
52 #include <sys/thread.h>
53 #include <sys/globaldata.h>
54 #include <sys/limits.h>
55
56 #include <sys/dmsg.h>
57
58 RB_GENERATE(kdmsg_state_tree, kdmsg_state, rbnode, kdmsg_state_cmp);
59 RB_GENERATE(kdmsg_circuit_tree, kdmsg_circuit, rbnode, kdmsg_circuit_cmp);
60
61 static int kdmsg_msg_receive_handling(kdmsg_msg_t *msg);
62 static int kdmsg_circ_msgrx(kdmsg_msg_t *msg);
63 static int kdmsg_state_msgrx(kdmsg_msg_t *msg);
64 static int kdmsg_state_msgtx(kdmsg_msg_t *msg);
65 static void kdmsg_state_cleanuprx(kdmsg_msg_t *msg);
66 static void kdmsg_state_cleanuptx(kdmsg_msg_t *msg);
67 static void kdmsg_state_abort(kdmsg_state_t *state);
68 static void kdmsg_state_free(kdmsg_state_t *state);
69
70 static void kdmsg_iocom_thread_rd(void *arg);
71 static void kdmsg_iocom_thread_wr(void *arg);
72 static int kdmsg_autorxmsg(kdmsg_msg_t *msg);
73 static void kdmsg_autocirc(kdmsg_msg_t *msg);
74 static int kdmsg_autocirc_reply(kdmsg_state_t *state, kdmsg_msg_t *msg);
75
76 static struct lwkt_token kdmsg_token = LWKT_TOKEN_INITIALIZER(kdmsg_token);
77
78 void
79 kdmsg_circ_hold(kdmsg_circuit_t *circ)
80 {
81         atomic_add_int(&circ->refs, 1);
82 }
83
84 void
85 kdmsg_circ_drop(kdmsg_circuit_t *circ)
86 {
87         kdmsg_iocom_t *iocom;
88
89         if (atomic_fetchadd_int(&circ->refs, -1) == 1) {
90                 KKASSERT(circ->span_state == NULL &&
91                          circ->circ_state == NULL &&
92                          circ->rcirc_state == NULL &&
93                          circ->recorded == 0);
94                 iocom = circ->iocom;
95                 circ->iocom = NULL;
96                 kfree(circ, iocom->mmsg);
97         }
98 }
99
100
101 /*
102  * Initialize the roll-up communications structure for a network
103  * messaging session.  This function does not install the socket.
104  */
105 void
106 kdmsg_iocom_init(kdmsg_iocom_t *iocom, void *handle, uint32_t flags,
107                  struct malloc_type *mmsg,
108                  int (*rcvmsg)(kdmsg_msg_t *msg))
109 {
110         bzero(iocom, sizeof(*iocom));
111         iocom->handle = handle;
112         iocom->mmsg = mmsg;
113         iocom->rcvmsg = rcvmsg;
114         iocom->flags = flags;
115         lockinit(&iocom->msglk, "h2msg", 0, 0);
116         TAILQ_INIT(&iocom->msgq);
117         RB_INIT(&iocom->circ_tree);
118         RB_INIT(&iocom->staterd_tree);
119         RB_INIT(&iocom->statewr_tree);
120 }
121
122 /*
123  * [Re]connect using the passed file pointer.  The caller must ref the
124  * fp for us.  We own that ref now.
125  */
126 void
127 kdmsg_iocom_reconnect(kdmsg_iocom_t *iocom, struct file *fp,
128                       const char *subsysname)
129 {
130         /*
131          * Destroy the current connection
132          */
133         lockmgr(&iocom->msglk, LK_EXCLUSIVE);
134         atomic_set_int(&iocom->msg_ctl, KDMSG_CLUSTERCTL_KILL);
135         while (iocom->msgrd_td || iocom->msgwr_td) {
136                 wakeup(&iocom->msg_ctl);
137                 lksleep(iocom, &iocom->msglk, 0, "clstrkl", hz);
138         }
139
140         /*
141          * Drop communications descriptor
142          */
143         if (iocom->msg_fp) {
144                 fdrop(iocom->msg_fp);
145                 iocom->msg_fp = NULL;
146         }
147
148         /*
149          * Setup new communications descriptor
150          */
151         iocom->msg_ctl = 0;
152         iocom->msg_fp = fp;
153         iocom->msg_seq = 0;
154         iocom->flags &= ~KDMSG_IOCOMF_EXITNOACC;
155
156         lwkt_create(kdmsg_iocom_thread_rd, iocom, &iocom->msgrd_td,
157                     NULL, 0, -1, "%s-msgrd", subsysname);
158         lwkt_create(kdmsg_iocom_thread_wr, iocom, &iocom->msgwr_td,
159                     NULL, 0, -1, "%s-msgwr", subsysname);
160         lockmgr(&iocom->msglk, LK_RELEASE);
161 }
162
163 /*
164  * Caller sets up iocom->auto_lnk_conn and iocom->auto_lnk_span, then calls
165  * this function to handle the state machine for LNK_CONN and LNK_SPAN.
166  *
167  * NOTE: Caller typically also sets the IOCOMF_AUTOCONN, IOCOMF_AUTOSPAN,
168  *       and IOCOMF_AUTOCIRC in the kdmsg_iocom_init() call.  Clients
169  *       typically set IOCOMF_AUTOFORGE to automatically forged circuits
170  *       for received SPANs.
171  */
172 static int kdmsg_lnk_conn_reply(kdmsg_state_t *state, kdmsg_msg_t *msg);
173 static int kdmsg_lnk_span_reply(kdmsg_state_t *state, kdmsg_msg_t *msg);
174
175 void
176 kdmsg_iocom_autoinitiate(kdmsg_iocom_t *iocom,
177                          void (*auto_callback)(kdmsg_msg_t *msg))
178 {
179         kdmsg_msg_t *msg;
180
181         iocom->auto_callback = auto_callback;
182
183         msg = kdmsg_msg_alloc(iocom, NULL,
184                               DMSG_LNK_CONN | DMSGF_CREATE,
185                               kdmsg_lnk_conn_reply, NULL);
186         iocom->auto_lnk_conn.head = msg->any.head;
187         msg->any.lnk_conn = iocom->auto_lnk_conn;
188         iocom->conn_state = msg->state;
189         kdmsg_msg_write(msg);
190 }
191
192 static
193 int
194 kdmsg_lnk_conn_reply(kdmsg_state_t *state, kdmsg_msg_t *msg)
195 {
196         kdmsg_iocom_t *iocom = state->iocom;
197         kdmsg_msg_t *rmsg;
198
199         if (msg->any.head.cmd & DMSGF_CREATE) {
200                 rmsg = kdmsg_msg_alloc(iocom, NULL,
201                                        DMSG_LNK_SPAN | DMSGF_CREATE,
202                                        kdmsg_lnk_span_reply, NULL);
203                 iocom->auto_lnk_span.head = rmsg->any.head;
204                 rmsg->any.lnk_span = iocom->auto_lnk_span;
205                 kdmsg_msg_write(rmsg);
206         }
207
208         /*
209          * Process shim after the CONN is acknowledged and before the CONN
210          * transaction is deleted.  For deletions this gives device drivers
211          * the ability to interlock new operations on the circuit before
212          * it becomes illegal and panics.
213          */
214         if (iocom->auto_callback)
215                 iocom->auto_callback(msg);
216
217         if ((state->txcmd & DMSGF_DELETE) == 0 &&
218             (msg->any.head.cmd & DMSGF_DELETE)) {
219                 iocom->conn_state = NULL;
220                 kdmsg_msg_reply(msg, 0);
221         }
222
223         return (0);
224 }
225
226 static
227 int
228 kdmsg_lnk_span_reply(kdmsg_state_t *state, kdmsg_msg_t *msg)
229 {
230         /*
231          * Be sure to process shim before terminating the SPAN
232          * transaction.  Gives device drivers the ability to
233          * interlock new operations on the circuit before it
234          * becomes illegal and panics.
235          */
236         if (state->iocom->auto_callback)
237                 state->iocom->auto_callback(msg);
238
239         if ((state->txcmd & DMSGF_DELETE) == 0 &&
240             (msg->any.head.cmd & DMSGF_DELETE)) {
241                 kdmsg_msg_reply(msg, 0);
242         }
243         return (0);
244 }
245
246 /*
247  * Disconnect and clean up
248  */
249 void
250 kdmsg_iocom_uninit(kdmsg_iocom_t *iocom)
251 {
252         /*
253          * Ask the cluster controller to go away
254          */
255         lockmgr(&iocom->msglk, LK_EXCLUSIVE);
256         atomic_set_int(&iocom->msg_ctl, KDMSG_CLUSTERCTL_KILL);
257
258         while (iocom->msgrd_td || iocom->msgwr_td) {
259                 wakeup(&iocom->msg_ctl);
260                 lksleep(iocom, &iocom->msglk, 0, "clstrkl", hz);
261         }
262
263         /*
264          * Drop communications descriptor
265          */
266         if (iocom->msg_fp) {
267                 fdrop(iocom->msg_fp);
268                 iocom->msg_fp = NULL;
269         }
270         lockmgr(&iocom->msglk, LK_RELEASE);
271 }
272
273 /*
274  * Cluster controller thread.  Perform messaging functions.  We have one
275  * thread for the reader and one for the writer.  The writer handles
276  * shutdown requests (which should break the reader thread).
277  */
278 static
279 void
280 kdmsg_iocom_thread_rd(void *arg)
281 {
282         kdmsg_iocom_t *iocom = arg;
283         dmsg_hdr_t hdr;
284         kdmsg_msg_t *msg = NULL;
285         kdmsg_state_t *state;
286         size_t hbytes;
287         size_t abytes;
288         int error = 0;
289
290         while ((iocom->msg_ctl & KDMSG_CLUSTERCTL_KILL) == 0) {
291                 /*
292                  * Retrieve the message from the pipe or socket.
293                  */
294                 error = fp_read(iocom->msg_fp, &hdr, sizeof(hdr),
295                                 NULL, 1, UIO_SYSSPACE);
296                 if (error)
297                         break;
298                 if (hdr.magic != DMSG_HDR_MAGIC) {
299                         kprintf("kdmsg: bad magic: %04x\n", hdr.magic);
300                         error = EINVAL;
301                         break;
302                 }
303                 hbytes = (hdr.cmd & DMSGF_SIZE) * DMSG_ALIGN;
304                 if (hbytes < sizeof(hdr) || hbytes > DMSG_AUX_MAX) {
305                         kprintf("kdmsg: bad header size %zd\n", hbytes);
306                         error = EINVAL;
307                         break;
308                 }
309                 /* XXX messy: mask cmd to avoid allocating state */
310                 msg = kdmsg_msg_alloc(iocom, NULL,
311                                       hdr.cmd & DMSGF_BASECMDMASK,
312                                       NULL, NULL);
313                 msg->any.head = hdr;
314                 msg->hdr_size = hbytes;
315                 if (hbytes > sizeof(hdr)) {
316                         error = fp_read(iocom->msg_fp, &msg->any.head + 1,
317                                         hbytes - sizeof(hdr),
318                                         NULL, 1, UIO_SYSSPACE);
319                         if (error) {
320                                 kprintf("kdmsg: short msg received\n");
321                                 error = EINVAL;
322                                 break;
323                         }
324                 }
325                 msg->aux_size = hdr.aux_bytes;
326                 if (msg->aux_size > DMSG_AUX_MAX) {
327                         kprintf("kdmsg: illegal msg payload size %zd\n",
328                                 msg->aux_size);
329                         error = EINVAL;
330                         break;
331                 }
332                 if (msg->aux_size) {
333                         abytes = DMSG_DOALIGN(msg->aux_size);
334                         msg->aux_data = kmalloc(abytes, iocom->mmsg, M_WAITOK);
335                         msg->flags |= KDMSG_FLAG_AUXALLOC;
336                         error = fp_read(iocom->msg_fp, msg->aux_data,
337                                         abytes, NULL, 1, UIO_SYSSPACE);
338                         if (error) {
339                                 kprintf("kdmsg: short msg payload received\n");
340                                 break;
341                         }
342                 }
343
344                 (void)kdmsg_circ_msgrx(msg);
345                 error = kdmsg_msg_receive_handling(msg);
346                 msg = NULL;
347         }
348
349         if (error)
350                 kprintf("kdmsg: read failed error %d\n", error);
351
352         lockmgr(&iocom->msglk, LK_EXCLUSIVE);
353         if (msg)
354                 kdmsg_msg_free(msg);
355
356         if ((state = iocom->freerd_state) != NULL) {
357                 iocom->freerd_state = NULL;
358                 kdmsg_state_free(state);
359         }
360
361         /*
362          * Shutdown the socket before waiting for the transmit side.
363          *
364          * If we are dying due to e.g. a socket disconnect verses being
365          * killed explicity we have to set KILL in order to kick the tx
366          * side when it might not have any other work to do.  KILL might
367          * already be set if we are in an unmount or reconnect.
368          */
369         fp_shutdown(iocom->msg_fp, SHUT_RDWR);
370
371         atomic_set_int(&iocom->msg_ctl, KDMSG_CLUSTERCTL_KILL);
372         wakeup(&iocom->msg_ctl);
373
374         /*
375          * Wait for the transmit side to drain remaining messages
376          * before cleaning up the rx state.  The transmit side will
377          * set KILLTX and wait for the rx side to completely finish
378          * (set msgrd_td to NULL) before cleaning up any remaining
379          * tx states.
380          */
381         lockmgr(&iocom->msglk, LK_RELEASE);
382         atomic_set_int(&iocom->msg_ctl, KDMSG_CLUSTERCTL_KILLRX);
383         wakeup(&iocom->msg_ctl);
384         while ((iocom->msg_ctl & KDMSG_CLUSTERCTL_KILLTX) == 0) {
385                 wakeup(&iocom->msg_ctl);
386                 tsleep(iocom, 0, "clstrkw", hz);
387         }
388
389         iocom->msgrd_td = NULL;
390
391         /*
392          * iocom can be ripped out from under us at this point but
393          * wakeup() is safe.
394          */
395         wakeup(iocom);
396         lwkt_exit();
397 }
398
399 static
400 void
401 kdmsg_iocom_thread_wr(void *arg)
402 {
403         kdmsg_iocom_t *iocom = arg;
404         kdmsg_msg_t *msg;
405         kdmsg_state_t *state;
406         ssize_t res;
407         size_t abytes;
408         int error = 0;
409         int retries = 20;
410
411         /*
412          * Transmit loop
413          */
414         msg = NULL;
415         lockmgr(&iocom->msglk, LK_EXCLUSIVE);
416
417         while ((iocom->msg_ctl & KDMSG_CLUSTERCTL_KILL) == 0 && error == 0) {
418                 /*
419                  * Sleep if no messages pending.  Interlock with flag while
420                  * holding msglk.
421                  */
422                 if (TAILQ_EMPTY(&iocom->msgq)) {
423                         atomic_set_int(&iocom->msg_ctl,
424                                        KDMSG_CLUSTERCTL_SLEEPING);
425                         lksleep(&iocom->msg_ctl, &iocom->msglk, 0, "msgwr", hz);
426                         atomic_clear_int(&iocom->msg_ctl,
427                                          KDMSG_CLUSTERCTL_SLEEPING);
428                 }
429
430                 while ((msg = TAILQ_FIRST(&iocom->msgq)) != NULL) {
431                         /*
432                          * Remove msg from the transmit queue and do
433                          * persist and half-closed state handling.
434                          */
435                         TAILQ_REMOVE(&iocom->msgq, msg, qentry);
436                         lockmgr(&iocom->msglk, LK_RELEASE);
437
438                         error = kdmsg_state_msgtx(msg);
439                         if (error == EALREADY) {
440                                 error = 0;
441                                 kdmsg_msg_free(msg);
442                                 lockmgr(&iocom->msglk, LK_EXCLUSIVE);
443                                 continue;
444                         }
445                         if (error) {
446                                 kdmsg_msg_free(msg);
447                                 lockmgr(&iocom->msglk, LK_EXCLUSIVE);
448                                 break;
449                         }
450
451                         /*
452                          * Dump the message to the pipe or socket.
453                          *
454                          * We have to clean up the message as if the transmit
455                          * succeeded even if it failed.
456                          */
457                         error = fp_write(iocom->msg_fp, &msg->any,
458                                          msg->hdr_size, &res, UIO_SYSSPACE);
459                         if (error || res != msg->hdr_size) {
460                                 if (error == 0)
461                                         error = EINVAL;
462                                 kdmsg_state_cleanuptx(msg);
463                                 lockmgr(&iocom->msglk, LK_EXCLUSIVE);
464                                 break;
465                         }
466                         if (msg->aux_size) {
467                                 abytes = DMSG_DOALIGN(msg->aux_size);
468                                 error = fp_write(iocom->msg_fp,
469                                                  msg->aux_data, abytes,
470                                                  &res, UIO_SYSSPACE);
471                                 if (error || res != abytes) {
472                                         if (error == 0)
473                                                 error = EINVAL;
474                                         kdmsg_state_cleanuptx(msg);
475                                         lockmgr(&iocom->msglk, LK_EXCLUSIVE);
476                                         break;
477                                 }
478                         }
479                         kdmsg_state_cleanuptx(msg);
480                         lockmgr(&iocom->msglk, LK_EXCLUSIVE);
481                 }
482         }
483
484         /*
485          * Cleanup messages pending transmission and release msgq lock.
486          */
487         if (error)
488                 kprintf("kdmsg: write failed error %d\n", error);
489         kprintf("thread_wr: Terminating iocom\n");
490
491         /*
492          * Shutdown the socket.  This will cause the rx thread to get an
493          * EOF and ensure that both threads get to a termination state.
494          */
495         fp_shutdown(iocom->msg_fp, SHUT_RDWR);
496
497         /*
498          * Set KILLTX (which the rx side waits for), then wait for the RX
499          * side to completely finish before we clean out any remaining
500          * command states.
501          */
502         lockmgr(&iocom->msglk, LK_RELEASE);
503         atomic_set_int(&iocom->msg_ctl, KDMSG_CLUSTERCTL_KILLTX);
504         wakeup(&iocom->msg_ctl);
505         while (iocom->msgrd_td) {
506                 wakeup(&iocom->msg_ctl);
507                 tsleep(iocom, 0, "clstrkw", hz);
508         }
509         lockmgr(&iocom->msglk, LK_EXCLUSIVE);
510
511         /*
512          * Simulate received MSGF_DELETE's for any remaining states.
513          * (For remote masters).
514          *
515          * Drain the message queue to handle any device initiated writes
516          * due to state callbacks.
517          */
518 cleanuprd:
519         kdmsg_drain_msgq(iocom);
520         RB_FOREACH(state, kdmsg_state_tree, &iocom->staterd_tree) {
521                 if ((state->rxcmd & DMSGF_DELETE) == 0) {
522                         lockmgr(&iocom->msglk, LK_RELEASE);
523                         kdmsg_state_abort(state);
524                         lockmgr(&iocom->msglk, LK_EXCLUSIVE);
525                         goto cleanuprd;
526                 }
527         }
528
529         /*
530          * Simulate received MSGF_DELETE's for any remaining states.
531          * (For local masters).
532          */
533 cleanupwr:
534         kdmsg_drain_msgq(iocom);
535         RB_FOREACH(state, kdmsg_state_tree, &iocom->statewr_tree) {
536                 if ((state->rxcmd & DMSGF_DELETE) == 0) {
537                         lockmgr(&iocom->msglk, LK_RELEASE);
538                         kdmsg_state_abort(state);
539                         lockmgr(&iocom->msglk, LK_EXCLUSIVE);
540                         goto cleanupwr;
541                 }
542         }
543
544         /*
545          * Retry until all work is done
546          */
547         if (--retries == 0)
548                 panic("kdmsg: comm thread shutdown couldn't drain");
549         if (TAILQ_FIRST(&iocom->msgq) ||
550             RB_ROOT(&iocom->staterd_tree) ||
551             RB_ROOT(&iocom->statewr_tree)) {
552                 goto cleanuprd;
553         }
554         iocom->flags |= KDMSG_IOCOMF_EXITNOACC;
555
556         if ((state = iocom->freewr_state) != NULL) {
557                 iocom->freewr_state = NULL;
558                 kdmsg_state_free(state);
559         }
560
561         lockmgr(&iocom->msglk, LK_RELEASE);
562
563         /*
564          * The state trees had better be empty now
565          */
566         KKASSERT(RB_EMPTY(&iocom->staterd_tree));
567         KKASSERT(RB_EMPTY(&iocom->statewr_tree));
568         KKASSERT(iocom->conn_state == NULL);
569
570         if (iocom->exit_func) {
571                 /*
572                  * iocom is invalid after we call the exit function.
573                  */
574                 iocom->msgwr_td = NULL;
575                 iocom->exit_func(iocom);
576         } else {
577                 /*
578                  * iocom can be ripped out from under us once msgwr_td is
579                  * set to NULL.  The wakeup is safe.
580                  */
581                 iocom->msgwr_td = NULL;
582                 wakeup(iocom);
583         }
584         lwkt_exit();
585 }
586
587 /*
588  * This cleans out the pending transmit message queue, adjusting any
589  * persistent states properly in the process.
590  *
591  * Caller must hold pmp->iocom.msglk
592  */
593 void
594 kdmsg_drain_msgq(kdmsg_iocom_t *iocom)
595 {
596         kdmsg_msg_t *msg;
597
598         /*
599          * Clean out our pending transmit queue, executing the
600          * appropriate state adjustments.  If this tries to open
601          * any new outgoing transactions we have to loop up and
602          * clean them out.
603          */
604         while ((msg = TAILQ_FIRST(&iocom->msgq)) != NULL) {
605                 TAILQ_REMOVE(&iocom->msgq, msg, qentry);
606                 lockmgr(&iocom->msglk, LK_RELEASE);
607                 if (kdmsg_state_msgtx(msg))
608                         kdmsg_msg_free(msg);
609                 else
610                         kdmsg_state_cleanuptx(msg);
611                 lockmgr(&iocom->msglk, LK_EXCLUSIVE);
612         }
613 }
614
615 /*
616  * Do all processing required to handle a freshly received message
617  * after its low level header has been validated.
618  */
619 static
620 int
621 kdmsg_msg_receive_handling(kdmsg_msg_t *msg)
622 {
623         kdmsg_iocom_t *iocom = msg->iocom;
624         int error;
625
626         /*
627          * State machine tracking, state assignment for msg,
628          * returns error and discard status.  Errors are fatal
629          * to the connection except for EALREADY which forces
630          * a discard without execution.
631          */
632         error = kdmsg_state_msgrx(msg);
633         if (error) {
634                 /*
635                  * Raw protocol or connection error
636                  */
637                 kdmsg_msg_free(msg);
638                 if (error == EALREADY)
639                         error = 0;
640         } else if (msg->state && msg->state->func) {
641                 /*
642                  * Message related to state which already has a
643                  * handling function installed for it.
644                  */
645                 error = msg->state->func(msg->state, msg);
646                 kdmsg_state_cleanuprx(msg);
647         } else if (iocom->flags & KDMSG_IOCOMF_AUTOANY) {
648                 error = kdmsg_autorxmsg(msg);
649                 kdmsg_state_cleanuprx(msg);
650         } else {
651                 error = iocom->rcvmsg(msg);
652                 kdmsg_state_cleanuprx(msg);
653         }
654         return error;
655 }
656
657 /*
658  * Process circuit tracking (NEEDS WORK)
659  *
660  * Called with msglk held and the msg dequeued.
661  */
662 static
663 int
664 kdmsg_circ_msgrx(kdmsg_msg_t *msg)
665 {
666         kdmsg_circuit_t dummy;
667         kdmsg_circuit_t *circ;
668         int error = 0;
669
670         if (msg->any.head.circuit) {
671                 dummy.msgid = msg->any.head.circuit;
672                 lwkt_gettoken(&kdmsg_token);
673                 circ = RB_FIND(kdmsg_circuit_tree, &msg->iocom->circ_tree,
674                                &dummy);
675                 if (circ) {
676                         msg->circ = circ;
677                         kdmsg_circ_hold(circ);
678                 }
679                 if (circ == NULL) {
680                         kprintf("KDMSG_CIRC_MSGRX CMD %08x: IOCOM %p "
681                                 "Bad circuit %016jx\n",
682                                 msg->any.head.cmd,
683                                 msg->iocom,
684                                 (intmax_t)msg->any.head.circuit);
685                         kprintf("KDMSG_CIRC_MSGRX: Avail circuits: ");
686                         RB_FOREACH(circ, kdmsg_circuit_tree,
687                                    &msg->iocom->circ_tree) {
688                                 kprintf(" %016jx", (intmax_t)circ->msgid);
689                         }
690                         kprintf("\n");
691                         error = EINVAL;
692                 }
693                 lwkt_reltoken(&kdmsg_token);
694         }
695         return (error);
696 }
697
698 /*
699  * Process state tracking for a message after reception, prior to
700  * execution.
701  *
702  * Called with msglk held and the msg dequeued.
703  *
704  * All messages are called with dummy state and return actual state.
705  * (One-off messages often just return the same dummy state).
706  *
707  * May request that caller discard the message by setting *discardp to 1.
708  * The returned state is not used in this case and is allowed to be NULL.
709  *
710  * --
711  *
712  * These routines handle persistent and command/reply message state via the
713  * CREATE and DELETE flags.  The first message in a command or reply sequence
714  * sets CREATE, the last message in a command or reply sequence sets DELETE.
715  *
716  * There can be any number of intermediate messages belonging to the same
717  * sequence sent inbetween the CREATE message and the DELETE message,
718  * which set neither flag.  This represents a streaming command or reply.
719  *
720  * Any command message received with CREATE set expects a reply sequence to
721  * be returned.  Reply sequences work the same as command sequences except the
722  * REPLY bit is also sent.  Both the command side and reply side can
723  * degenerate into a single message with both CREATE and DELETE set.  Note
724  * that one side can be streaming and the other side not, or neither, or both.
725  *
726  * The msgid is unique for the initiator.  That is, two sides sending a new
727  * message can use the same msgid without colliding.
728  *
729  * --
730  *
731  * ABORT sequences work by setting the ABORT flag along with normal message
732  * state.  However, ABORTs can also be sent on half-closed messages, that is
733  * even if the command or reply side has already sent a DELETE, as long as
734  * the message has not been fully closed it can still send an ABORT+DELETE
735  * to terminate the half-closed message state.
736  *
737  * Since ABORT+DELETEs can race we silently discard ABORT's for message
738  * state which has already been fully closed.  REPLY+ABORT+DELETEs can
739  * also race, and in this situation the other side might have already
740  * initiated a new unrelated command with the same message id.  Since
741  * the abort has not set the CREATE flag the situation can be detected
742  * and the message will also be discarded.
743  *
744  * Non-blocking requests can be initiated with ABORT+CREATE[+DELETE].
745  * The ABORT request is essentially integrated into the command instead
746  * of being sent later on.  In this situation the command implementation
747  * detects that CREATE and ABORT are both set (vs ABORT alone) and can
748  * special-case non-blocking operation for the command.
749  *
750  * NOTE!  Messages with ABORT set without CREATE or DELETE are considered
751  *        to be mid-stream aborts for command/reply sequences.  ABORTs on
752  *        one-way messages are not supported.
753  *
754  * NOTE!  If a command sequence does not support aborts the ABORT flag is
755  *        simply ignored.
756  *
757  * --
758  *
759  * One-off messages (no reply expected) are sent with neither CREATE or DELETE
760  * set.  One-off messages cannot be aborted and typically aren't processed
761  * by these routines.  The REPLY bit can be used to distinguish whether a
762  * one-off message is a command or reply.  For example, one-off replies
763  * will typically just contain status updates.
764  */
765 static
766 int
767 kdmsg_state_msgrx(kdmsg_msg_t *msg)
768 {
769         kdmsg_iocom_t *iocom = msg->iocom;
770         kdmsg_state_t *state;
771         int error;
772
773         /*
774          * Make sure a state structure is ready to go in case we need a new
775          * one.  This is the only routine which uses freerd_state so no
776          * races are possible.
777          */
778         if ((state = iocom->freerd_state) == NULL) {
779                 state = kmalloc(sizeof(*state), iocom->mmsg, M_WAITOK | M_ZERO);
780                 state->flags = KDMSG_STATE_DYNAMIC;
781                 iocom->freerd_state = state;
782         }
783
784         /*
785          * Lock RB tree and locate existing persistent state, if any.
786          *
787          * If received msg is a command state is on staterd_tree.
788          * If received msg is a reply state is on statewr_tree.
789          */
790         lockmgr(&iocom->msglk, LK_EXCLUSIVE);
791
792         state->msgid = msg->any.head.msgid;
793         state->circ = msg->circ;
794         state->iocom = iocom;
795         if (msg->any.head.cmd & DMSGF_REPLY)
796                 state = RB_FIND(kdmsg_state_tree, &iocom->statewr_tree, state);
797         else
798                 state = RB_FIND(kdmsg_state_tree, &iocom->staterd_tree, state);
799         msg->state = state;
800
801         /*
802          * Short-cut one-off or mid-stream messages (state may be NULL).
803          */
804         if ((msg->any.head.cmd & (DMSGF_CREATE | DMSGF_DELETE |
805                                   DMSGF_ABORT)) == 0) {
806                 lockmgr(&iocom->msglk, LK_RELEASE);
807                 return(0);
808         }
809
810         /*
811          * Switch on CREATE, DELETE, REPLY, and also handle ABORT from
812          * inside the case statements.
813          */
814         switch(msg->any.head.cmd & (DMSGF_CREATE|DMSGF_DELETE|DMSGF_REPLY)) {
815         case DMSGF_CREATE:
816         case DMSGF_CREATE | DMSGF_DELETE:
817                 /*
818                  * New persistant command received.
819                  */
820                 if (state) {
821                         kprintf("kdmsg_state_msgrx: duplicate transaction\n");
822                         error = EINVAL;
823                         break;
824                 }
825                 state = iocom->freerd_state;
826                 iocom->freerd_state = NULL;
827                 msg->state = state;
828                 state->msg = msg;
829                 state->icmd = msg->any.head.cmd & DMSGF_BASECMDMASK;
830                 state->rxcmd = msg->any.head.cmd & ~DMSGF_DELETE;
831                 state->txcmd = DMSGF_REPLY;
832                 state->msgid = msg->any.head.msgid;
833                 if ((state->circ = msg->circ) != NULL)
834                         kdmsg_circ_hold(state->circ);
835                 RB_INSERT(kdmsg_state_tree, &iocom->staterd_tree, state);
836                 state->flags |= KDMSG_STATE_INSERTED;
837                 error = 0;
838                 break;
839         case DMSGF_DELETE:
840                 /*
841                  * Persistent state is expected but might not exist if an
842                  * ABORT+DELETE races the close.
843                  */
844                 if (state == NULL) {
845                         if (msg->any.head.cmd & DMSGF_ABORT) {
846                                 error = EALREADY;
847                         } else {
848                                 kprintf("kdmsg_state_msgrx: "
849                                         "no state for DELETE\n");
850                                 error = EINVAL;
851                         }
852                         break;
853                 }
854
855                 /*
856                  * Handle another ABORT+DELETE case if the msgid has already
857                  * been reused.
858                  */
859                 if ((state->rxcmd & DMSGF_CREATE) == 0) {
860                         if (msg->any.head.cmd & DMSGF_ABORT) {
861                                 error = EALREADY;
862                         } else {
863                                 kprintf("kdmsg_state_msgrx: "
864                                         "state reused for DELETE\n");
865                                 error = EINVAL;
866                         }
867                         break;
868                 }
869                 error = 0;
870                 break;
871         default:
872                 /*
873                  * Check for mid-stream ABORT command received, otherwise
874                  * allow.
875                  */
876                 if (msg->any.head.cmd & DMSGF_ABORT) {
877                         if (state == NULL ||
878                             (state->rxcmd & DMSGF_CREATE) == 0) {
879                                 error = EALREADY;
880                                 break;
881                         }
882                 }
883                 error = 0;
884                 break;
885         case DMSGF_REPLY | DMSGF_CREATE:
886         case DMSGF_REPLY | DMSGF_CREATE | DMSGF_DELETE:
887                 /*
888                  * When receiving a reply with CREATE set the original
889                  * persistent state message should already exist.
890                  */
891                 if (state == NULL) {
892                         kprintf("kdmsg_state_msgrx: no state match for "
893                                 "REPLY cmd=%08x msgid=%016jx\n",
894                                 msg->any.head.cmd,
895                                 (intmax_t)msg->any.head.msgid);
896                         error = EINVAL;
897                         break;
898                 }
899                 state->rxcmd = msg->any.head.cmd & ~DMSGF_DELETE;
900                 error = 0;
901                 break;
902         case DMSGF_REPLY | DMSGF_DELETE:
903                 /*
904                  * Received REPLY+ABORT+DELETE in case where msgid has
905                  * already been fully closed, ignore the message.
906                  */
907                 if (state == NULL) {
908                         if (msg->any.head.cmd & DMSGF_ABORT) {
909                                 error = EALREADY;
910                         } else {
911                                 kprintf("kdmsg_state_msgrx: no state match "
912                                         "for REPLY|DELETE\n");
913                                 error = EINVAL;
914                         }
915                         break;
916                 }
917
918                 /*
919                  * Received REPLY+ABORT+DELETE in case where msgid has
920                  * already been reused for an unrelated message,
921                  * ignore the message.
922                  */
923                 if ((state->rxcmd & DMSGF_CREATE) == 0) {
924                         if (msg->any.head.cmd & DMSGF_ABORT) {
925                                 error = EALREADY;
926                         } else {
927                                 kprintf("kdmsg_state_msgrx: state reused "
928                                         "for REPLY|DELETE\n");
929                                 error = EINVAL;
930                         }
931                         break;
932                 }
933                 error = 0;
934                 break;
935         case DMSGF_REPLY:
936                 /*
937                  * Check for mid-stream ABORT reply received to sent command.
938                  */
939                 if (msg->any.head.cmd & DMSGF_ABORT) {
940                         if (state == NULL ||
941                             (state->rxcmd & DMSGF_CREATE) == 0) {
942                                 error = EALREADY;
943                                 break;
944                         }
945                 }
946                 error = 0;
947                 break;
948         }
949         lockmgr(&iocom->msglk, LK_RELEASE);
950         return (error);
951 }
952
953 /*
954  * Called instead of iocom->rcvmsg() if any of the AUTO flags are set.
955  * This routine must call iocom->rcvmsg() for anything not automatically
956  * handled.
957  */
958 static int
959 kdmsg_autorxmsg(kdmsg_msg_t *msg)
960 {
961         kdmsg_iocom_t *iocom = msg->iocom;
962         kdmsg_circuit_t *circ;
963         int error = 0;
964         uint32_t cmd;
965
966         /*
967          * Process a combination of the transaction command and the message
968          * flags.  For the purposes of this routine, the message command is
969          * only relevant when it initiates a transaction (where it is
970          * recorded in icmd).
971          */
972         cmd = (msg->state ? msg->state->icmd : msg->any.head.cmd) &
973               DMSGF_BASECMDMASK;
974         cmd |= msg->any.head.cmd & (DMSGF_CREATE | DMSGF_DELETE | DMSGF_REPLY);
975
976         switch(cmd) {
977         case DMSG_LNK_CONN | DMSGF_CREATE:
978         case DMSG_LNK_CONN | DMSGF_CREATE | DMSGF_DELETE:
979                 /*
980                  * Received LNK_CONN transaction.  Transmit response and
981                  * leave transaction open, which allows the other end to
982                  * start to the SPAN protocol.
983                  *
984                  * Handle shim after acknowledging the CONN.
985                  */
986                 if ((msg->any.head.cmd & DMSGF_DELETE) == 0) {
987                         if (iocom->flags & KDMSG_IOCOMF_AUTOCONN) {
988                                 kdmsg_msg_result(msg, 0);
989                                 if (iocom->auto_callback)
990                                         iocom->auto_callback(msg);
991                         } else {
992                                 error = iocom->rcvmsg(msg);
993                         }
994                         break;
995                 }
996                 /* fall through */
997         case DMSG_LNK_CONN | DMSGF_DELETE:
998                 /*
999                  * This message is usually simulated after a link is lost
1000                  * to clean up the transaction.
1001                  */
1002                 if (iocom->flags & KDMSG_IOCOMF_AUTOCONN) {
1003                         if (iocom->auto_callback)
1004                                 iocom->auto_callback(msg);
1005                         kdmsg_msg_reply(msg, 0);
1006                 } else {
1007                         error = iocom->rcvmsg(msg);
1008                 }
1009                 break;
1010         case DMSG_LNK_SPAN | DMSGF_CREATE:
1011         case DMSG_LNK_SPAN | DMSGF_CREATE | DMSGF_DELETE:
1012                 /*
1013                  * Received LNK_SPAN transaction.  We do not have to respond
1014                  * but we must leave the transaction open.
1015                  *
1016                  * If AUTOCIRC is set automatically initiate a virtual circuit
1017                  * to the received span.  This will attach a kdmsg_circuit
1018                  * to the SPAN state.  The circuit is lost when the span is
1019                  * lost.
1020                  *
1021                  * Handle shim after acknowledging the SPAN.
1022                  */
1023                 if (iocom->flags & KDMSG_IOCOMF_AUTOSPAN) {
1024                         if ((msg->any.head.cmd & DMSGF_DELETE) == 0) {
1025                                 if (iocom->flags & KDMSG_IOCOMF_AUTOFORGE)
1026                                         kdmsg_autocirc(msg);
1027                                 if (iocom->auto_callback)
1028                                         iocom->auto_callback(msg);
1029                                 break;
1030                         }
1031                         /* fall through */
1032                 } else {
1033                         error = iocom->rcvmsg(msg);
1034                         break;
1035                 }
1036                 /* fall through */
1037         case DMSG_LNK_SPAN | DMSGF_DELETE:
1038                 /*
1039                  * Process shims (auto_callback) before cleaning up the
1040                  * circuit structure and closing the transactions.  Device
1041                  * driver should ensure that the circuit is not used after
1042                  * the auto_callback() returns.
1043                  *
1044                  * Handle shim before closing the SPAN transaction.
1045                  */
1046                 if (iocom->flags & KDMSG_IOCOMF_AUTOSPAN) {
1047                         if (iocom->auto_callback)
1048                                 iocom->auto_callback(msg);
1049                         if (iocom->flags & KDMSG_IOCOMF_AUTOFORGE)
1050                                 kdmsg_autocirc(msg);
1051                         kdmsg_msg_reply(msg, 0);
1052                 } else {
1053                         error = iocom->rcvmsg(msg);
1054                 }
1055                 break;
1056         case DMSG_LNK_CIRC | DMSGF_CREATE:
1057         case DMSG_LNK_CIRC | DMSGF_CREATE | DMSGF_DELETE:
1058                 /*
1059                  * Received LNK_CIRC transaction.  We must respond and should
1060                  * leave the transaction open, allowing the circuit.  The
1061                  * remote can start issuing commands to us over the circuit
1062                  * even before we respond.
1063                  */
1064                 if (iocom->flags & KDMSG_IOCOMF_AUTOCIRC) {
1065                         if ((msg->any.head.cmd & DMSGF_DELETE) == 0) {
1066                                 circ = kmalloc(sizeof(*circ), iocom->mmsg,
1067                                                M_WAITOK | M_ZERO);
1068                                 msg->state->any.circ = circ;
1069                                 circ->iocom = iocom;
1070                                 circ->rcirc_state = msg->state;
1071                                 kdmsg_circ_hold(circ);  /* for rcirc_state */
1072                                 circ->weight = 0;
1073                                 circ->msgid = circ->rcirc_state->msgid;
1074                                 /* XXX no span link for received circuits */
1075                                 kdmsg_circ_hold(circ);  /* for circ_state */
1076 #if 0
1077                                 kprintf("KDMSG VC: RECEIVE CIRC CREATE "
1078                                         "IOCOM %p MSGID %016jx\n",
1079                                         msg->iocom, circ->msgid);
1080 #endif
1081
1082                                 if (RB_INSERT(kdmsg_circuit_tree,
1083                                               &iocom->circ_tree, circ)) {
1084                                         panic("duplicate circuitid allocated");
1085                                 }
1086                                 kdmsg_msg_result(msg, 0);
1087
1088                                 /*
1089                                  * Handle shim after adding the circuit and
1090                                  * after acknowledging the CIRC.
1091                                  */
1092                                 if (iocom->auto_callback)
1093                                         iocom->auto_callback(msg);
1094                                 break;
1095                         }
1096                         /* fall through */
1097                 } else {
1098                         error = iocom->rcvmsg(msg);
1099                         break;
1100                 }
1101                 /* fall through */
1102         case DMSG_LNK_CIRC | DMSGF_DELETE:
1103                 if (iocom->flags & KDMSG_IOCOMF_AUTOCIRC) {
1104                         circ = msg->state->any.circ;
1105                         if (circ == NULL)
1106                                 break;
1107
1108                         /*
1109                          * Handle shim before terminating the circuit.
1110                          */
1111 #if 0
1112                         kprintf("KDMSG VC: RECEIVE CIRC DELETE "
1113                                 "IOCOM %p MSGID %016jx\n",
1114                                 msg->iocom, circ->msgid);
1115 #endif
1116                         if (iocom->auto_callback)
1117                                 iocom->auto_callback(msg);
1118
1119                         KKASSERT(circ->rcirc_state == msg->state);
1120                         circ->rcirc_state = NULL;
1121                         msg->state->any.circ = NULL;
1122                         lwkt_gettoken(&kdmsg_token);
1123                         RB_REMOVE(kdmsg_circuit_tree, &iocom->circ_tree, circ);
1124                         lwkt_reltoken(&kdmsg_token);
1125                         kdmsg_circ_drop(circ);  /* for rcirc_state */
1126                         kdmsg_msg_reply(msg, 0);
1127                 } else {
1128                         error = iocom->rcvmsg(msg);
1129                 }
1130                 break;
1131         default:
1132                 /*
1133                  * Anything unhandled goes into rcvmsg.
1134                  *
1135                  * NOTE: Replies to link-level messages initiated by our side
1136                  *       are handled by the state callback, they are NOT
1137                  *       handled here.
1138                  */
1139                 error = iocom->rcvmsg(msg);
1140                 break;
1141         }
1142         return (error);
1143 }
1144
1145 /*
1146  * Handle automatic forging of virtual circuits based on received SPANs.
1147  * (AUTOFORGE).  Note that other code handles tracking received circuit
1148  * transactions (AUTOCIRC).
1149  *
1150  * We can ignore non-transactions here.  Use trans->icmd to test the
1151  * transactional command (once past the CREATE the individual message
1152  * commands are not usually the icmd).
1153  *
1154  * XXX locks
1155  */
1156 static
1157 void
1158 kdmsg_autocirc(kdmsg_msg_t *msg)
1159 {
1160         kdmsg_iocom_t *iocom = msg->iocom;
1161         kdmsg_circuit_t *circ;
1162         kdmsg_msg_t *xmsg;      /* CIRC */
1163
1164         if (msg->state == NULL)
1165                 return;
1166
1167         /*
1168          * Gaining the SPAN, automatically forge a circuit to the target.
1169          *
1170          * NOTE!! The shim is not executed until we receive an acknowlegement
1171          *        to our forged LNK_CIRC (see kdmsg_autocirc_reply()).
1172          */
1173         if (msg->state->icmd == DMSG_LNK_SPAN &&
1174             (msg->any.head.cmd & DMSGF_CREATE)) {
1175                 circ = kmalloc(sizeof(*circ), iocom->mmsg, M_WAITOK | M_ZERO);
1176                 msg->state->any.circ = circ;
1177                 circ->iocom = iocom;
1178                 circ->span_state = msg->state;
1179                 kdmsg_circ_hold(circ);  /* for span_state */
1180                 xmsg = kdmsg_msg_alloc(iocom, NULL,
1181                                        DMSG_LNK_CIRC | DMSGF_CREATE,
1182                                        kdmsg_autocirc_reply, circ);
1183                 circ->circ_state = xmsg->state;
1184                 circ->weight = msg->any.lnk_span.dist;
1185                 circ->msgid = circ->circ_state->msgid;
1186                 kdmsg_circ_hold(circ);  /* for circ_state */
1187 #if 0
1188                 kprintf("KDMSG VC: CREATE SPAN->CIRC IOCOM %p MSGID %016jx\n",
1189                         msg->iocom, circ->msgid);
1190 #endif
1191
1192                 if (RB_INSERT(kdmsg_circuit_tree, &iocom->circ_tree, circ))
1193                         panic("duplicate circuitid allocated");
1194
1195                 xmsg->any.lnk_circ.target = msg->any.head.msgid;
1196                 kdmsg_msg_write(xmsg);
1197         }
1198
1199         /*
1200          * Losing the SPAN
1201          *
1202          * NOTE: When losing a SPAN, any circuits using the span should be
1203          *       deleted by the remote end first.  XXX might not be ordered
1204          *       on actual loss of connection.
1205          */
1206         if (msg->state->icmd == DMSG_LNK_SPAN &&
1207             (msg->any.head.cmd & DMSGF_DELETE) &&
1208             msg->state->any.circ) {
1209                 circ = msg->state->any.circ;
1210                 lwkt_gettoken(&kdmsg_token);
1211                 circ->span_state = NULL;
1212                 msg->state->any.circ = NULL;
1213                 RB_REMOVE(kdmsg_circuit_tree, &iocom->circ_tree, circ);
1214 #if 0
1215                 kprintf("KDMSG VC: DELETE SPAN->CIRC IOCOM %p MSGID %016jx\n",
1216                         msg->iocom, (intmax_t)circ->msgid);
1217 #endif
1218                 kdmsg_circ_drop(circ);  /* for span_state */
1219                 lwkt_reltoken(&kdmsg_token);
1220         }
1221 }
1222
1223 static
1224 int
1225 kdmsg_autocirc_reply(kdmsg_state_t *state, kdmsg_msg_t *msg)
1226 {
1227         kdmsg_iocom_t *iocom = state->iocom;
1228         kdmsg_circuit_t *circ = state->any.circ;
1229
1230         /*
1231          * Call shim after receiving an acknowlegement to our forged
1232          * circuit and before processing a received termination.
1233          */
1234         if (iocom->auto_callback)
1235                 iocom->auto_callback(msg);
1236
1237         /*
1238          * If the remote is terminating the VC we terminate our side
1239          */
1240         if ((state->txcmd & DMSGF_DELETE) == 0 &&
1241             (msg->any.head.cmd & DMSGF_DELETE)) {
1242 #if 0
1243                 kprintf("KDMSG VC: DELETE CIRC FROM REMOTE\n");
1244 #endif
1245                 lwkt_gettoken(&kdmsg_token);
1246                 circ->circ_state = NULL;
1247                 state->any.circ = NULL;
1248                 kdmsg_circ_drop(circ);          /* for circ_state */
1249                 lwkt_reltoken(&kdmsg_token);
1250                 kdmsg_msg_reply(msg, 0);
1251         }
1252         return (0);
1253 }
1254
1255 /*
1256  * Post-receive-handling message and state cleanup.  This routine is called
1257  * after the state function handling/callback to properly dispose of the
1258  * message and update or dispose of the state.
1259  */
1260 static
1261 void
1262 kdmsg_state_cleanuprx(kdmsg_msg_t *msg)
1263 {
1264         kdmsg_iocom_t *iocom = msg->iocom;
1265         kdmsg_state_t *state;
1266
1267         if ((state = msg->state) == NULL) {
1268                 kdmsg_msg_free(msg);
1269         } else if (msg->any.head.cmd & DMSGF_DELETE) {
1270                 lockmgr(&iocom->msglk, LK_EXCLUSIVE);
1271                 state->rxcmd |= DMSGF_DELETE;
1272                 if (state->txcmd & DMSGF_DELETE) {
1273                         KKASSERT(state->flags & KDMSG_STATE_INSERTED);
1274                         if (state->rxcmd & DMSGF_REPLY) {
1275                                 KKASSERT(msg->any.head.cmd &
1276                                          DMSGF_REPLY);
1277                                 RB_REMOVE(kdmsg_state_tree,
1278                                           &iocom->statewr_tree, state);
1279                         } else {
1280                                 KKASSERT((msg->any.head.cmd &
1281                                           DMSGF_REPLY) == 0);
1282                                 RB_REMOVE(kdmsg_state_tree,
1283                                           &iocom->staterd_tree, state);
1284                         }
1285                         state->flags &= ~KDMSG_STATE_INSERTED;
1286                         if (msg != state->msg)
1287                                 kdmsg_msg_free(msg);
1288                         lockmgr(&iocom->msglk, LK_RELEASE);
1289                         kdmsg_state_free(state);
1290                 } else {
1291                         if (msg != state->msg)
1292                                 kdmsg_msg_free(msg);
1293                         lockmgr(&iocom->msglk, LK_RELEASE);
1294                 }
1295         } else if (msg != state->msg) {
1296                 kdmsg_msg_free(msg);
1297         }
1298 }
1299
1300 /*
1301  * Simulate receiving a message which terminates an activate transaction
1302  * state.  Our simulated received message must set DELETE and may also
1303  * have to set CREATE.  It must also ensure that all fields are set such
1304  * that the receive handling code can find the state (kdmsg_state_msgrx())
1305  * or an endless loop will ensue.
1306  *
1307  * This is used when the other end of the link or virtual circuit is dead
1308  * so the device driver gets a completed transaction for all pending states.
1309  */
1310 static
1311 void
1312 kdmsg_state_abort(kdmsg_state_t *state)
1313 {
1314         kdmsg_iocom_t *iocom = state->iocom;
1315         kdmsg_msg_t *msg;
1316
1317         msg = kdmsg_msg_alloc(iocom, state->circ,
1318                               DMSG_LNK_ERROR,
1319                               NULL, NULL);
1320         if ((state->rxcmd & DMSGF_CREATE) == 0)
1321                 msg->any.head.cmd |= DMSGF_CREATE;
1322         msg->any.head.cmd |= DMSGF_DELETE | (state->rxcmd & DMSGF_REPLY);
1323         msg->any.head.error = DMSG_ERR_LOSTLINK;
1324         msg->any.head.msgid = state->msgid;
1325         msg->state = state;
1326         if ((msg->circ = state->circ) != NULL)
1327                 kdmsg_circ_hold(msg->circ);
1328         kdmsg_msg_receive_handling(msg);
1329 }
1330
1331 /*
1332  * Process state tracking for a message prior to transmission.
1333  *
1334  * Called with msglk held and the msg dequeued.  Returns non-zero if
1335  * the message is bad and should be deleted by the caller.
1336  *
1337  * One-off messages are usually with dummy state and msg->state may be NULL
1338  * in this situation.
1339  *
1340  * New transactions (when CREATE is set) will insert the state.
1341  *
1342  * May request that caller discard the message by setting *discardp to 1.
1343  * A NULL state may be returned in this case.
1344  */
1345 static
1346 int
1347 kdmsg_state_msgtx(kdmsg_msg_t *msg)
1348 {
1349         kdmsg_iocom_t *iocom = msg->iocom;
1350         kdmsg_state_t *state;
1351         int error;
1352
1353         /*
1354          * Make sure a state structure is ready to go in case we need a new
1355          * one.  This is the only routine which uses freewr_state so no
1356          * races are possible.
1357          */
1358         if ((state = iocom->freewr_state) == NULL) {
1359                 state = kmalloc(sizeof(*state), iocom->mmsg, M_WAITOK | M_ZERO);
1360                 state->flags = KDMSG_STATE_DYNAMIC;
1361                 state->iocom = iocom;
1362                 iocom->freewr_state = state;
1363         }
1364
1365         /*
1366          * Lock RB tree.  If persistent state is present it will have already
1367          * been assigned to msg.
1368          */
1369         lockmgr(&iocom->msglk, LK_EXCLUSIVE);
1370         state = msg->state;
1371
1372         /*
1373          * Short-cut one-off or mid-stream messages (state may be NULL).
1374          */
1375         if ((msg->any.head.cmd & (DMSGF_CREATE | DMSGF_DELETE |
1376                                   DMSGF_ABORT)) == 0) {
1377                 lockmgr(&iocom->msglk, LK_RELEASE);
1378                 return(0);
1379         }
1380
1381
1382         /*
1383          * Switch on CREATE, DELETE, REPLY, and also handle ABORT from
1384          * inside the case statements.
1385          */
1386         switch(msg->any.head.cmd & (DMSGF_CREATE | DMSGF_DELETE |
1387                                     DMSGF_REPLY)) {
1388         case DMSGF_CREATE:
1389         case DMSGF_CREATE | DMSGF_DELETE:
1390                 /*
1391                  * Insert the new persistent message state and mark
1392                  * half-closed if DELETE is set.  Since this is a new
1393                  * message it isn't possible to transition into the fully
1394                  * closed state here.
1395                  *
1396                  * XXX state must be assigned and inserted by
1397                  *     kdmsg_msg_write().  txcmd is assigned by us
1398                  *     on-transmit.
1399                  */
1400                 KKASSERT(state != NULL);
1401                 state->icmd = msg->any.head.cmd & DMSGF_BASECMDMASK;
1402                 state->txcmd = msg->any.head.cmd & ~DMSGF_DELETE;
1403                 state->rxcmd = DMSGF_REPLY;
1404                 error = 0;
1405                 break;
1406         case DMSGF_DELETE:
1407                 /*
1408                  * Sent ABORT+DELETE in case where msgid has already
1409                  * been fully closed, ignore the message.
1410                  */
1411                 if (state == NULL) {
1412                         if (msg->any.head.cmd & DMSGF_ABORT) {
1413                                 error = EALREADY;
1414                         } else {
1415                                 kprintf("kdmsg_state_msgtx: no state match "
1416                                         "for DELETE cmd=%08x msgid=%016jx\n",
1417                                         msg->any.head.cmd,
1418                                         (intmax_t)msg->any.head.msgid);
1419                                 error = EINVAL;
1420                         }
1421                         break;
1422                 }
1423
1424                 /*
1425                  * Sent ABORT+DELETE in case where msgid has
1426                  * already been reused for an unrelated message,
1427                  * ignore the message.
1428                  */
1429                 if ((state->txcmd & DMSGF_CREATE) == 0) {
1430                         if (msg->any.head.cmd & DMSGF_ABORT) {
1431                                 error = EALREADY;
1432                         } else {
1433                                 kprintf("kdmsg_state_msgtx: state reused "
1434                                         "for DELETE\n");
1435                                 error = EINVAL;
1436                         }
1437                         break;
1438                 }
1439                 error = 0;
1440                 break;
1441         default:
1442                 /*
1443                  * Check for mid-stream ABORT command sent
1444                  */
1445                 if (msg->any.head.cmd & DMSGF_ABORT) {
1446                         if (state == NULL ||
1447                             (state->txcmd & DMSGF_CREATE) == 0) {
1448                                 error = EALREADY;
1449                                 break;
1450                         }
1451                 }
1452                 error = 0;
1453                 break;
1454         case DMSGF_REPLY | DMSGF_CREATE:
1455         case DMSGF_REPLY | DMSGF_CREATE | DMSGF_DELETE:
1456                 /*
1457                  * When transmitting a reply with CREATE set the original
1458                  * persistent state message should already exist.
1459                  */
1460                 if (state == NULL) {
1461                         kprintf("kdmsg_state_msgtx: no state match "
1462                                 "for REPLY | CREATE\n");
1463                         error = EINVAL;
1464                         break;
1465                 }
1466                 state->txcmd = msg->any.head.cmd & ~DMSGF_DELETE;
1467                 error = 0;
1468                 break;
1469         case DMSGF_REPLY | DMSGF_DELETE:
1470                 /*
1471                  * When transmitting a reply with DELETE set the original
1472                  * persistent state message should already exist.
1473                  *
1474                  * This is very similar to the REPLY|CREATE|* case except
1475                  * txcmd is already stored, so we just add the DELETE flag.
1476                  *
1477                  * Sent REPLY+ABORT+DELETE in case where msgid has
1478                  * already been fully closed, ignore the message.
1479                  */
1480                 if (state == NULL) {
1481                         if (msg->any.head.cmd & DMSGF_ABORT) {
1482                                 error = EALREADY;
1483                         } else {
1484                                 kprintf("kdmsg_state_msgtx: no state match "
1485                                         "for REPLY | DELETE\n");
1486                                 error = EINVAL;
1487                         }
1488                         break;
1489                 }
1490
1491                 /*
1492                  * Sent REPLY+ABORT+DELETE in case where msgid has already
1493                  * been reused for an unrelated message, ignore the message.
1494                  */
1495                 if ((state->txcmd & DMSGF_CREATE) == 0) {
1496                         if (msg->any.head.cmd & DMSGF_ABORT) {
1497                                 error = EALREADY;
1498                         } else {
1499                                 kprintf("kdmsg_state_msgtx: state reused "
1500                                         "for REPLY | DELETE\n");
1501                                 error = EINVAL;
1502                         }
1503                         break;
1504                 }
1505                 error = 0;
1506                 break;
1507         case DMSGF_REPLY:
1508                 /*
1509                  * Check for mid-stream ABORT reply sent.
1510                  *
1511                  * One-off REPLY messages are allowed for e.g. status updates.
1512                  */
1513                 if (msg->any.head.cmd & DMSGF_ABORT) {
1514                         if (state == NULL ||
1515                             (state->txcmd & DMSGF_CREATE) == 0) {
1516                                 error = EALREADY;
1517                                 break;
1518                         }
1519                 }
1520                 error = 0;
1521                 break;
1522         }
1523         lockmgr(&iocom->msglk, LK_RELEASE);
1524         return (error);
1525 }
1526
1527 static
1528 void
1529 kdmsg_state_cleanuptx(kdmsg_msg_t *msg)
1530 {
1531         kdmsg_iocom_t *iocom = msg->iocom;
1532         kdmsg_state_t *state;
1533
1534         if ((state = msg->state) == NULL) {
1535                 kdmsg_msg_free(msg);
1536         } else if (msg->any.head.cmd & DMSGF_DELETE) {
1537                 lockmgr(&iocom->msglk, LK_EXCLUSIVE);
1538                 state->txcmd |= DMSGF_DELETE;
1539                 if (state->rxcmd & DMSGF_DELETE) {
1540                         KKASSERT(state->flags & KDMSG_STATE_INSERTED);
1541                         if (state->txcmd & DMSGF_REPLY) {
1542                                 KKASSERT(msg->any.head.cmd &
1543                                          DMSGF_REPLY);
1544                                 RB_REMOVE(kdmsg_state_tree,
1545                                           &iocom->staterd_tree, state);
1546                         } else {
1547                                 KKASSERT((msg->any.head.cmd &
1548                                           DMSGF_REPLY) == 0);
1549                                 RB_REMOVE(kdmsg_state_tree,
1550                                           &iocom->statewr_tree, state);
1551                         }
1552                         state->flags &= ~KDMSG_STATE_INSERTED;
1553                         if (msg != state->msg)
1554                                 kdmsg_msg_free(msg);
1555                         lockmgr(&iocom->msglk, LK_RELEASE);
1556                         kdmsg_state_free(state);
1557                 } else {
1558                         if (msg != state->msg)
1559                                 kdmsg_msg_free(msg);
1560                         lockmgr(&iocom->msglk, LK_RELEASE);
1561                 }
1562         } else if (msg != state->msg) {
1563                 kdmsg_msg_free(msg);
1564         }
1565 }
1566
1567 static
1568 void
1569 kdmsg_state_free(kdmsg_state_t *state)
1570 {
1571         kdmsg_iocom_t *iocom = state->iocom;
1572         kdmsg_msg_t *msg;
1573
1574         KKASSERT((state->flags & KDMSG_STATE_INSERTED) == 0);
1575         msg = state->msg;
1576         state->msg = NULL;
1577         kfree(state, iocom->mmsg);
1578         if (msg) {
1579                 msg->state = NULL;
1580                 kdmsg_msg_free(msg);
1581         }
1582 }
1583
1584 kdmsg_msg_t *
1585 kdmsg_msg_alloc(kdmsg_iocom_t *iocom, kdmsg_circuit_t *circ, uint32_t cmd,
1586                 int (*func)(kdmsg_state_t *, kdmsg_msg_t *), void *data)
1587 {
1588         kdmsg_msg_t *msg;
1589         kdmsg_state_t *state;
1590         size_t hbytes;
1591
1592         KKASSERT(iocom != NULL);
1593         hbytes = (cmd & DMSGF_SIZE) * DMSG_ALIGN;
1594         msg = kmalloc(offsetof(struct kdmsg_msg, any) + hbytes,
1595                       iocom->mmsg, M_WAITOK | M_ZERO);
1596         msg->hdr_size = hbytes;
1597         msg->iocom = iocom;
1598         msg->any.head.magic = DMSG_HDR_MAGIC;
1599         msg->any.head.cmd = cmd;
1600         if (circ) {
1601                 kdmsg_circ_hold(circ);
1602                 msg->circ = circ;
1603                 msg->any.head.circuit = circ->msgid;
1604         }
1605
1606         if (cmd & DMSGF_CREATE) {
1607                 /*
1608                  * New transaction, requires tracking state and a unique
1609                  * msgid to be allocated.
1610                  */
1611                 KKASSERT(msg->state == NULL);
1612                 state = kmalloc(sizeof(*state), iocom->mmsg, M_WAITOK | M_ZERO);
1613                 state->flags = KDMSG_STATE_DYNAMIC;
1614                 state->func = func;
1615                 state->any.any = data;
1616                 state->msg = msg;
1617                 state->msgid = (uint64_t)(uintptr_t)state;
1618                 state->circ = circ;
1619                 state->iocom = iocom;
1620                 msg->state = state;
1621                 if (circ)
1622                         kdmsg_circ_hold(circ);
1623                 /*msg->any.head.msgid = state->msgid;XXX*/
1624
1625                 lockmgr(&iocom->msglk, LK_EXCLUSIVE);
1626                 if (RB_INSERT(kdmsg_state_tree, &iocom->statewr_tree, state))
1627                         panic("duplicate msgid allocated");
1628                 state->flags |= KDMSG_STATE_INSERTED;
1629                 msg->any.head.msgid = state->msgid;
1630                 lockmgr(&iocom->msglk, LK_RELEASE);
1631         }
1632         return (msg);
1633 }
1634
1635 kdmsg_msg_t *
1636 kdmsg_msg_alloc_state(kdmsg_state_t *state, uint32_t cmd,
1637                       int (*func)(kdmsg_state_t *, kdmsg_msg_t *), void *data)
1638 {
1639         kdmsg_iocom_t *iocom = state->iocom;
1640         kdmsg_msg_t *msg;
1641         size_t hbytes;
1642
1643         KKASSERT(iocom != NULL);
1644         hbytes = (cmd & DMSGF_SIZE) * DMSG_ALIGN;
1645         msg = kmalloc(offsetof(struct kdmsg_msg, any) + hbytes,
1646                       iocom->mmsg, M_WAITOK | M_ZERO);
1647         msg->hdr_size = hbytes;
1648         msg->iocom = iocom;
1649         msg->any.head.magic = DMSG_HDR_MAGIC;
1650         msg->any.head.cmd = cmd;
1651         msg->state = state;
1652         if (state->circ) {
1653                 kdmsg_circ_hold(state->circ);
1654                 msg->circ = state->circ;
1655                 msg->any.head.circuit = state->circ->msgid;
1656         }
1657         return(msg);
1658 }
1659
1660 void
1661 kdmsg_msg_free(kdmsg_msg_t *msg)
1662 {
1663         kdmsg_iocom_t *iocom = msg->iocom;
1664
1665         if ((msg->flags & KDMSG_FLAG_AUXALLOC) &&
1666             msg->aux_data && msg->aux_size) {
1667                 kfree(msg->aux_data, iocom->mmsg);
1668                 msg->flags &= ~KDMSG_FLAG_AUXALLOC;
1669         }
1670         if (msg->circ) {
1671                 kdmsg_circ_drop(msg->circ);
1672                 msg->circ = NULL;
1673         }
1674         if (msg->state) {
1675                 if (msg->state->msg == msg)
1676                         msg->state->msg = NULL;
1677                 msg->state = NULL;
1678         }
1679         msg->aux_data = NULL;
1680         msg->aux_size = 0;
1681         msg->iocom = NULL;
1682         kfree(msg, iocom->mmsg);
1683 }
1684
1685 /*
1686  * Circuits are tracked in a red-black tree by their circuit id (msgid).
1687  */
1688 int
1689 kdmsg_circuit_cmp(kdmsg_circuit_t *circ1, kdmsg_circuit_t *circ2)
1690 {
1691         if (circ1->msgid < circ2->msgid)
1692                 return(-1);
1693         if (circ1->msgid > circ2->msgid)
1694                 return(1);
1695         return (0);
1696 }
1697
1698 /*
1699  * Indexed messages are stored in a red-black tree indexed by their
1700  * msgid.  Only persistent messages are indexed.
1701  */
1702 int
1703 kdmsg_state_cmp(kdmsg_state_t *state1, kdmsg_state_t *state2)
1704 {
1705         if (state1->iocom < state2->iocom)
1706                 return(-1);
1707         if (state1->iocom > state2->iocom)
1708                 return(1);
1709         if (state1->circ < state2->circ)
1710                 return(-1);
1711         if (state1->circ > state2->circ)
1712                 return(1);
1713         if (state1->msgid < state2->msgid)
1714                 return(-1);
1715         if (state1->msgid > state2->msgid)
1716                 return(1);
1717         return(0);
1718 }
1719
1720 /*
1721  * Write a message.  All requisit command flags have been set.
1722  *
1723  * If msg->state is non-NULL the message is written to the existing
1724  * transaction.  msgid will be set accordingly.
1725  *
1726  * If msg->state is NULL and CREATE is set new state is allocated and
1727  * (func, data) is installed.  A msgid is assigned.
1728  *
1729  * If msg->state is NULL and CREATE is not set the message is assumed
1730  * to be a one-way message.  The originator must assign the msgid
1731  * (or leave it 0, which is typical.
1732  *
1733  * This function merely queues the message to the management thread, it
1734  * does not write to the message socket/pipe.
1735  */
1736 void
1737 kdmsg_msg_write(kdmsg_msg_t *msg)
1738 {
1739         kdmsg_iocom_t *iocom = msg->iocom;
1740         kdmsg_state_t *state;
1741
1742         if (msg->state) {
1743                 /*
1744                  * Continuance or termination of existing transaction.
1745                  * The transaction could have been initiated by either end.
1746                  *
1747                  * (Function callback and aux data for the receive side can
1748                  * be replaced or left alone).
1749                  */
1750                 state = msg->state;
1751                 msg->any.head.msgid = state->msgid;
1752                 lockmgr(&iocom->msglk, LK_EXCLUSIVE);
1753         } else {
1754                 /*
1755                  * One-off message (always uses msgid 0 to distinguish
1756                  * between a possibly lost in-transaction message due to
1757                  * competing aborts and a real one-off message?)
1758                  */
1759                 state = NULL;
1760                 msg->any.head.msgid = 0;
1761                 lockmgr(&iocom->msglk, LK_EXCLUSIVE);
1762         }
1763
1764         /*
1765          * With AUTOCIRC and AUTOFORGE it is possible for the circuit to
1766          * get ripped out in the rxthread while some other thread is
1767          * holding a ref on it inbetween allocating and sending a dmsg.
1768          */
1769         if (msg->circ && msg->circ->rcirc_state == NULL &&
1770             (msg->circ->span_state == NULL || msg->circ->circ_state == NULL)) {
1771                 kprintf("kdmsg_msg_write: Attempt to write message to "
1772                         "terminated circuit: msg %08x\n", msg->any.head.cmd);
1773                 lockmgr(&iocom->msglk, LK_RELEASE);
1774                 if (kdmsg_state_msgtx(msg)) {
1775                         if (state == NULL || msg != state->msg)
1776                                 kdmsg_msg_free(msg);
1777                 } else if ((msg->state->rxcmd & DMSGF_DELETE) == 0) {
1778                         /* XXX SMP races simulating a response here */
1779                         kdmsg_state_t *state = msg->state;
1780                         kdmsg_state_cleanuptx(msg);
1781                         kdmsg_state_abort(state);
1782                 } else {
1783                         kdmsg_state_cleanuptx(msg);
1784                 }
1785                 return;
1786         }
1787
1788         /*
1789          * This flag is not set until after the tx thread has drained
1790          * the txmsgq and simulated responses.  After that point the
1791          * txthread is dead and can no longer simulate responses.
1792          *
1793          * Device drivers should never try to send a message once this
1794          * flag is set.  They should have detected (through the state
1795          * closures) that the link is in trouble.
1796          */
1797         if (iocom->flags & KDMSG_IOCOMF_EXITNOACC) {
1798                 lockmgr(&iocom->msglk, LK_RELEASE);
1799                 panic("kdmsg_msg_write: Attempt to write message to "
1800                       "terminated iocom\n");
1801         }
1802
1803         /*
1804          * Finish up the msg fields.  Note that msg->aux_size and the
1805          * aux_bytes stored in the message header represent the unaligned
1806          * (actual) bytes of data, but the buffer is sized to an aligned
1807          * size and the CRC is generated over the aligned length.
1808          */
1809         msg->any.head.salt = /* (random << 8) | */ (iocom->msg_seq & 255);
1810         ++iocom->msg_seq;
1811
1812         if (msg->aux_data && msg->aux_size) {
1813                 uint32_t abytes = DMSG_DOALIGN(msg->aux_size);
1814
1815                 msg->any.head.aux_bytes = msg->aux_size;
1816                 msg->any.head.aux_crc = iscsi_crc32(msg->aux_data, abytes);
1817         }
1818         msg->any.head.hdr_crc = 0;
1819         msg->any.head.hdr_crc = iscsi_crc32(msg->any.buf, msg->hdr_size);
1820
1821         TAILQ_INSERT_TAIL(&iocom->msgq, msg, qentry);
1822
1823         if (iocom->msg_ctl & KDMSG_CLUSTERCTL_SLEEPING) {
1824                 atomic_clear_int(&iocom->msg_ctl,
1825                                  KDMSG_CLUSTERCTL_SLEEPING);
1826                 wakeup(&iocom->msg_ctl);
1827         }
1828
1829         lockmgr(&iocom->msglk, LK_RELEASE);
1830 }
1831
1832 /*
1833  * Reply to a message and terminate our side of the transaction.
1834  *
1835  * If msg->state is non-NULL we are replying to a one-way message.
1836  */
1837 void
1838 kdmsg_msg_reply(kdmsg_msg_t *msg, uint32_t error)
1839 {
1840         kdmsg_state_t *state = msg->state;
1841         kdmsg_msg_t *nmsg;
1842         uint32_t cmd;
1843
1844         /*
1845          * Reply with a simple error code and terminate the transaction.
1846          */
1847         cmd = DMSG_LNK_ERROR;
1848
1849         /*
1850          * Check if our direction has even been initiated yet, set CREATE.
1851          *
1852          * Check what direction this is (command or reply direction).  Note
1853          * that txcmd might not have been initiated yet.
1854          *
1855          * If our direction has already been closed we just return without
1856          * doing anything.
1857          */
1858         if (state) {
1859                 if (state->txcmd & DMSGF_DELETE)
1860                         return;
1861                 if ((state->txcmd & DMSGF_CREATE) == 0)
1862                         cmd |= DMSGF_CREATE;
1863                 if (state->txcmd & DMSGF_REPLY)
1864                         cmd |= DMSGF_REPLY;
1865                 cmd |= DMSGF_DELETE;
1866         } else {
1867                 if ((msg->any.head.cmd & DMSGF_REPLY) == 0)
1868                         cmd |= DMSGF_REPLY;
1869         }
1870
1871         /* XXX messy mask cmd to avoid allocating state */
1872         nmsg = kdmsg_msg_alloc_state(state, cmd, NULL, NULL);
1873         nmsg->any.head.error = error;
1874         kdmsg_msg_write(nmsg);
1875 }
1876
1877 /*
1878  * Reply to a message and continue our side of the transaction.
1879  *
1880  * If msg->state is non-NULL we are replying to a one-way message and this
1881  * function degenerates into the same as kdmsg_msg_reply().
1882  */
1883 void
1884 kdmsg_msg_result(kdmsg_msg_t *msg, uint32_t error)
1885 {
1886         kdmsg_state_t *state = msg->state;
1887         kdmsg_msg_t *nmsg;
1888         uint32_t cmd;
1889
1890         /*
1891          * Return a simple result code, do NOT terminate the transaction.
1892          */
1893         cmd = DMSG_LNK_ERROR;
1894
1895         /*
1896          * Check if our direction has even been initiated yet, set CREATE.
1897          *
1898          * Check what direction this is (command or reply direction).  Note
1899          * that txcmd might not have been initiated yet.
1900          *
1901          * If our direction has already been closed we just return without
1902          * doing anything.
1903          */
1904         if (state) {
1905                 if (state->txcmd & DMSGF_DELETE)
1906                         return;
1907                 if ((state->txcmd & DMSGF_CREATE) == 0)
1908                         cmd |= DMSGF_CREATE;
1909                 if (state->txcmd & DMSGF_REPLY)
1910                         cmd |= DMSGF_REPLY;
1911                 /* continuing transaction, do not set MSGF_DELETE */
1912         } else {
1913                 if ((msg->any.head.cmd & DMSGF_REPLY) == 0)
1914                         cmd |= DMSGF_REPLY;
1915         }
1916
1917         /* XXX messy mask cmd to avoid allocating state */
1918         nmsg = kdmsg_msg_alloc_state(state, cmd, NULL, NULL);
1919         nmsg->any.head.error = error;
1920         kdmsg_msg_write(nmsg);
1921 }
1922
1923 /*
1924  * Reply to a message and terminate our side of the transaction.
1925  *
1926  * If msg->state is non-NULL we are replying to a one-way message.
1927  */
1928 void
1929 kdmsg_state_reply(kdmsg_state_t *state, uint32_t error)
1930 {
1931         kdmsg_msg_t *nmsg;
1932         uint32_t cmd;
1933
1934         /*
1935          * Reply with a simple error code and terminate the transaction.
1936          */
1937         cmd = DMSG_LNK_ERROR;
1938
1939         /*
1940          * Check if our direction has even been initiated yet, set CREATE.
1941          *
1942          * Check what direction this is (command or reply direction).  Note
1943          * that txcmd might not have been initiated yet.
1944          *
1945          * If our direction has already been closed we just return without
1946          * doing anything.
1947          */
1948         if (state) {
1949                 if (state->txcmd & DMSGF_DELETE)
1950                         return;
1951                 if ((state->txcmd & DMSGF_CREATE) == 0)
1952                         cmd |= DMSGF_CREATE;
1953                 if (state->txcmd & DMSGF_REPLY)
1954                         cmd |= DMSGF_REPLY;
1955                 cmd |= DMSGF_DELETE;
1956         } else {
1957                 if ((state->txcmd & DMSGF_REPLY) == 0)
1958                         cmd |= DMSGF_REPLY;
1959         }
1960
1961         /* XXX messy mask cmd to avoid allocating state */
1962         nmsg = kdmsg_msg_alloc_state(state, cmd, NULL, NULL);
1963         nmsg->any.head.error = error;
1964         kdmsg_msg_write(nmsg);
1965 }
1966
1967 /*
1968  * Reply to a message and continue our side of the transaction.
1969  *
1970  * If msg->state is non-NULL we are replying to a one-way message and this
1971  * function degenerates into the same as kdmsg_msg_reply().
1972  */
1973 void
1974 kdmsg_state_result(kdmsg_state_t *state, uint32_t error)
1975 {
1976         kdmsg_msg_t *nmsg;
1977         uint32_t cmd;
1978
1979         /*
1980          * Return a simple result code, do NOT terminate the transaction.
1981          */
1982         cmd = DMSG_LNK_ERROR;
1983
1984         /*
1985          * Check if our direction has even been initiated yet, set CREATE.
1986          *
1987          * Check what direction this is (command or reply direction).  Note
1988          * that txcmd might not have been initiated yet.
1989          *
1990          * If our direction has already been closed we just return without
1991          * doing anything.
1992          */
1993         if (state) {
1994                 if (state->txcmd & DMSGF_DELETE)
1995                         return;
1996                 if ((state->txcmd & DMSGF_CREATE) == 0)
1997                         cmd |= DMSGF_CREATE;
1998                 if (state->txcmd & DMSGF_REPLY)
1999                         cmd |= DMSGF_REPLY;
2000                 /* continuing transaction, do not set MSGF_DELETE */
2001         } else {
2002                 if ((state->txcmd & DMSGF_REPLY) == 0)
2003                         cmd |= DMSGF_REPLY;
2004         }
2005
2006         /* XXX messy mask cmd to avoid allocating state */
2007         nmsg = kdmsg_msg_alloc_state(state, cmd, NULL, NULL);
2008         nmsg->any.head.error = error;
2009         kdmsg_msg_write(nmsg);
2010 }