2 * Copyright (c) 2003 Matthew Dillon <dillon@backplane.com>
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * NOTE! This file may be compiled for userland libraries as well as for
29 * $DragonFly: src/sys/kern/lwkt_msgport.c,v 1.24 2004/06/10 22:11:35 dillon Exp $
34 #include <sys/param.h>
35 #include <sys/systm.h>
36 #include <sys/kernel.h>
38 #include <sys/rtprio.h>
39 #include <sys/queue.h>
40 #include <sys/sysctl.h>
41 #include <sys/kthread.h>
42 #include <sys/signalvar.h>
43 #include <machine/cpu.h>
47 #include <vm/vm_param.h>
48 #include <vm/vm_kern.h>
49 #include <vm/vm_object.h>
50 #include <vm/vm_page.h>
51 #include <vm/vm_map.h>
52 #include <vm/vm_pager.h>
53 #include <vm/vm_extern.h>
54 #include <vm/vm_zone.h>
56 #include <sys/thread2.h>
57 #include <sys/msgport2.h>
59 #include <machine/stdarg.h>
60 #include <machine/ipl.h>
61 #include <machine/cpufunc.h>
63 #include <machine/smp.h>
66 #include <sys/malloc.h>
67 MALLOC_DEFINE(M_LWKTMSG, "lwkt message", "lwkt message");
71 #include <sys/stdint.h>
72 #include <libcaps/thread.h>
73 #include <sys/thread.h>
74 #include <sys/msgport.h>
75 #include <sys/errno.h>
76 #include <libcaps/globaldata.h>
77 #include <sys/thread2.h>
78 #include <sys/msgport2.h>
80 #include <machine/cpufunc.h>
85 /************************************************************************
87 ************************************************************************/
89 static void lwkt_replyport_remote(lwkt_msg_t msg);
90 static void lwkt_putport_remote(lwkt_msg_t msg);
95 * Send a message asynchronously. This function requests asynchronous
96 * completion and calls lwkt_beginmsg(). If the target port decides to
97 * run the message synchronously this function will automatically queue
98 * the message to the current thread's message queue to present a
99 * consistent interface to the caller.
101 * The message's ms_cmd must be initialized and its ms_flags must
102 * be zero'd out. lwkt_sendmsg() will initialize the ms_abort_port
103 * (abort chasing port). If abort is supported, ms_abort must also be
106 * NOTE: you cannot safely request an abort until lwkt_sendmsg() returns
109 * NOTE: MSGF_DONE is left set. The target port must clear it if the
110 * message is to be handled asynchronously, while the synchronous case
111 * can just ignore it.
114 lwkt_sendmsg(lwkt_port_t port, lwkt_msg_t msg)
118 msg->ms_flags |= MSGF_ASYNC;
119 msg->ms_flags &= ~(MSGF_REPLY1 | MSGF_REPLY2 | MSGF_QUEUED | \
120 MSGF_ABORTED | MSGF_RETRIEVED);
121 KKASSERT(msg->ms_reply_port != NULL);
122 msg->ms_abort_port = msg->ms_reply_port;
123 if ((error = lwkt_beginmsg(port, msg)) != EASYNC) {
124 lwkt_replymsg(msg, error);
131 * Send a message synchronously. This function requests synchronous
132 * completion and calls lwkt_beginmsg(). If the target port decides to
133 * run the message asynchronously this function will block waiting for
134 * the message to complete. Since MSGF_ASYNC is not set the target
135 * will not attempt to queue the reply to a reply port but will simply
136 * wake up anyone waiting on the message.
138 * A synchronous error code is always returned.
140 * The message's ms_cmd must be initialized, and its ms_flags must be
141 * at least zero'd out. lwkt_domsg() will initialize the message's
142 * ms_abort_port (abort chasing port). If abort is supported, ms_abort
143 * must also be initialized.
145 * NOTE: you cannot safely request an abort until lwkt_domsg() blocks.
146 * XXX this probably needs some work.
148 * NOTE: MSGF_DONE is left set. The target port must clear it if the
149 * message is to be handled asynchronously, while the synchronous case
150 * can just ignore it.
153 lwkt_domsg(lwkt_port_t port, lwkt_msg_t msg)
157 msg->ms_flags &= ~(MSGF_ASYNC | MSGF_REPLY1 | MSGF_REPLY2 | \
158 MSGF_QUEUED | MSGF_ABORTED | MSGF_RETRIEVED);
159 KKASSERT(msg->ms_reply_port != NULL);
160 msg->ms_abort_port = msg->ms_reply_port;
161 if ((error = lwkt_beginmsg(port, msg)) == EASYNC) {
162 error = lwkt_waitmsg(msg);
167 /************************************************************************
169 ************************************************************************/
174 * Initialize a port for use and assign it to the specified thread.
177 lwkt_initport(lwkt_port_t port, thread_t td)
179 bzero(port, sizeof(*port));
180 TAILQ_INIT(&port->mp_msgq);
182 port->mp_putport = lwkt_default_putport;
183 port->mp_waitport = lwkt_default_waitport;
184 port->mp_replyport = lwkt_default_replyport;
185 port->mp_abortport = lwkt_default_abortport;
191 * Retrieve the next message from the port's message queue, return NULL
192 * if no messages are pending. Note that callers CANNOT use the
193 * MSGF_ABORTED flag as a litmus test to determine if a message
194 * was aborted. The flag only indicates that an abort was requested.
195 * The message's error code will indicate whether an abort occured
196 * (typically by returning EINTR).
198 * Note that once a message has been dequeued it is subject to being
199 * requeued via an IPI based abort request if it is not marked MSGF_DONE.
201 * If the message has been aborted we have to guarentee that abort
202 * semantics are properly followed. The target port will always see
203 * the original message at least once, and if it does not reply the
204 * message before looping on its message port again it will then see
205 * the message again with ms_cmd set to ms_abort.
207 * The calling thread MUST own the port.
212 _lwkt_pullmsg(lwkt_port_t port, lwkt_msg_t msg)
214 if ((msg->ms_flags & MSGF_ABORTED) == 0) {
216 * normal case, remove and return the message.
218 TAILQ_REMOVE(&port->mp_msgq, msg, ms_node);
219 msg->ms_flags = (msg->ms_flags & ~MSGF_QUEUED) | MSGF_RETRIEVED;
221 if (msg->ms_flags & MSGF_RETRIEVED) {
223 * abort case, message already returned once, remvoe and
224 * return the aborted message a second time after setting
225 * ms_cmd to ms_abort.
227 TAILQ_REMOVE(&port->mp_msgq, msg, ms_node);
228 msg->ms_flags &= ~MSGF_QUEUED;
229 msg->ms_cmd = msg->ms_abort;
232 * abort case, abort races initial message retrieval. The
233 * message is returned normally but not removed from the
234 * queue. On the next loop the 'aborted' message will be
235 * dequeued and returned. Note that if the caller replies
236 * to the message it will be dequeued (the abort becomes a
239 msg->ms_flags |= MSGF_RETRIEVED;
245 lwkt_getport(lwkt_port_t port)
249 KKASSERT(port->mp_td == curthread);
251 crit_enter_quick(port->mp_td);
252 if ((msg = TAILQ_FIRST(&port->mp_msgq)) != NULL)
253 _lwkt_pullmsg(port, msg);
254 crit_exit_quick(port->mp_td);
259 * This inline helper function completes processing of a reply from an
260 * unknown cpu context.
262 * The message is being returned to the specified port. The port is
263 * owned by the mp_td thread. If we are on the same cpu as the mp_td
264 * thread we can trivially queue the message to the reply port and schedule
265 * the target thread, otherwise we have to send an ipi message to the
268 * This inline must be entered with a critical section already held.
269 * Note that the IPIQ callback function (*_remote) is entered with a
270 * critical section already held, and we obtain one in lwkt_replyport().
274 _lwkt_replyport(lwkt_port_t port, lwkt_msg_t msg, int force)
276 thread_t td = port->mp_td;
278 if (force || td->td_gd == mycpu) {
280 * We can only reply the message if the abort has caught up with us,
281 * or if no abort was issued (same case).
283 if (msg->ms_abort_port == port) {
284 KKASSERT((msg->ms_flags & MSGF_QUEUED) == 0);
285 TAILQ_INSERT_TAIL(&port->mp_msgq, msg, ms_node);
286 msg->ms_flags |= MSGF_DONE | MSGF_QUEUED | MSGF_REPLY2;
287 if (port->mp_flags & MSGPORTF_WAITING)
291 lwkt_send_ipiq(td->td_gd, (ipifunc_t)lwkt_replyport_remote, msg);
296 * This function completes reply processing for the default case in the
297 * context of the originating cpu.
301 lwkt_replyport_remote(lwkt_msg_t msg)
303 _lwkt_replyport(msg->ms_reply_port, msg, 1);
307 * This function is called in the context of the target to reply a message.
308 * Note that the lwkt_replymsg() inline has already set MSGF_REPLY1 and
309 * entered a critical section for us.
313 lwkt_default_replyport(lwkt_port_t port, lwkt_msg_t msg)
316 msg->ms_flags |= MSGF_REPLY1;
317 if (msg->ms_flags & MSGF_ASYNC) {
319 * An abort may have caught up to us while we were processing the
320 * message. If this occured we have to dequeue the message from the
321 * target port in the context of our current cpu before we can
322 * finish replying it.
324 * If an abort occurs after we reply the MSGF_REPLY1 flag will
325 * prevent it from being requeued to the target port.
327 if (msg->ms_flags & MSGF_QUEUED) {
328 KKASSERT(msg->ms_flags & MSGF_ABORTED);
329 TAILQ_REMOVE(&msg->ms_target_port->mp_msgq, msg, ms_node);
330 msg->ms_flags &= ~MSGF_QUEUED;
332 _lwkt_replyport(port, msg, 0);
335 * Synchronously executed messages cannot be aborted and are just
336 * marked done. YYY MSGF_DONE should already be set, change flag set
339 msg->ms_flags |= MSGF_DONE;
340 if (port->mp_flags & MSGPORTF_WAITING)
341 lwkt_schedule(port->mp_td);
347 * lwkt_default_putport()
349 * This function is typically assigned to the mp_putport port vector.
351 * Queue a message to the target port and wakeup the thread owning it.
352 * This function always returns EASYNC and may be assigned to a
353 * message port's mp_putport function vector. Note that we must set
354 * MSGF_QUEUED prior to sending any IPIs in order to interlock against
355 * ABORT requests and other tests that might be performed.
357 * Note that messages start out as synchronous entities, and as an
358 * optimization MSGF_DONE is usually left set (so in the synchronous path
359 * no modifications to ms_flags are ever required). If a message becomes
360 * async, i.e. you return EASYNC, then MSGF_DONE must be cleared or
361 * lwkt_replymsg() will wind up being a NOP.
363 * The inline must be called from a critical section (the remote function
364 * is called from an IPI and will be in a critical section).
369 _lwkt_putport(lwkt_port_t port, lwkt_msg_t msg, int force)
371 thread_t td = port->mp_td;
373 if (force || td->td_gd == mycpu) {
374 TAILQ_INSERT_TAIL(&port->mp_msgq, msg, ms_node);
375 if (port->mp_flags & MSGPORTF_WAITING)
378 lwkt_send_ipiq(td->td_gd, (ipifunc_t)lwkt_putport_remote, msg);
384 lwkt_putport_remote(lwkt_msg_t msg)
386 _lwkt_putport(msg->ms_target_port, msg, 1);
390 lwkt_default_putport(lwkt_port_t port, lwkt_msg_t msg)
393 msg->ms_flags |= MSGF_QUEUED; /* abort interlock */
394 msg->ms_flags &= ~MSGF_DONE;
395 msg->ms_target_port = port;
396 _lwkt_putport(port, msg, 0);
404 * Forward a message received on one port to another port. The forwarding
405 * function must deal with a pending abort but othewise essentially just
406 * issues a putport to the target port.
408 * An abort may have two side effects: First, the message may have been
409 * requeued to the current target port. If so, we must dequeue it before
413 lwkt_forwardmsg(lwkt_port_t port, lwkt_msg_t msg)
418 if (msg->ms_flags & MSGF_QUEUED) {
419 KKASSERT(msg->ms_flags & MSGF_ABORTED);
420 TAILQ_REMOVE(&msg->ms_target_port->mp_msgq, msg, ms_node);
421 msg->ms_flags &= ~MSGF_QUEUED;
423 msg->ms_flags &= ~MSGF_RETRIEVED;
424 if ((error = port->mp_putport(port, msg)) != EASYNC)
425 lwkt_replymsg(msg, error);
433 * Aborting a message is a fairly complex task. The first order of
434 * business is to get the message to the cpu that owns the target
435 * port, during which we may have to do some port chasing due to
436 * message forwarding operations.
438 * NOTE! Since an aborted message is requeued all message processing
439 * loops should check the MSGF_ABORTED flag.
441 static void lwkt_abortmsg_remote(lwkt_msg_t msg);
444 lwkt_abortmsg(lwkt_msg_t msg)
450 * A critical section protects us from reply IPIs on this cpu. We
451 * can only abort messages that have not yet completed (DONE), are not
452 * in the midst of being replied (REPLY1), and which support the
453 * abort function (ABORTABLE).
456 if ((msg->ms_flags & (MSGF_DONE|MSGF_REPLY1|MSGF_ABORTABLE)) == MSGF_ABORTABLE) {
458 * Chase the message. If REPLY1 is set the message has been replied
459 * all the way back to the originator, otherwise it is sitting on
460 * ms_target_port (but we can only complete processing if we are
461 * on the same cpu as the selected port in order to avoid
462 * SMP cache synchronization issues).
464 * When chasing through multiple ports ms_flags may not be
465 * synchronized to the current cpu, but it WILL be synchronized
466 * with regards to testing the MSGF_REPLY1 bit once we reach the
467 * target port that made the reply and since the cpu owning
468 * some port X stores the new port in ms_target_port if the message
469 * is forwarded, the current port will only ever equal the target
470 * port when we are on the correct cpu.
472 if (msg->ms_flags & MSGF_REPLY1)
473 port = msg->ms_reply_port;
475 port = msg->ms_target_port;
479 * The chase call must run on the cpu owning the port. Fully
480 * synchronous ports (mp_td == NULL) can run the call on any cpu.
483 if (td && td->td_gd != mycpu) {
484 lwkt_send_ipiq(td->td_gd, (ipifunc_t)lwkt_abortmsg_remote, msg);
486 port->mp_abortport(port, msg);
494 lwkt_abortmsg_remote(lwkt_msg_t msg)
499 if (msg->ms_flags & MSGF_REPLY1)
500 port = msg->ms_reply_port;
502 port = msg->ms_target_port;
505 if (td->td_gd != mycpu) {
506 lwkt_send_ipiq(td->td_gd, (ipifunc_t)lwkt_abortmsg_remote, msg);
508 port->mp_abortport(port, msg);
513 * The mp_abortport function is called when the abort has finally caught up
514 * to the target port or (if the message has been replied) the reply port.
517 lwkt_default_abortport(lwkt_port_t port, lwkt_msg_t msg)
520 * Set ms_abort_port to ms_reply_port to indicate the completion of
521 * the messaging chasing portion of the abort request. Note that
522 * the passed port is the port that we finally caught up to, not
523 * necessarily the reply port.
525 msg->ms_abort_port = msg->ms_reply_port;
527 if (msg->ms_flags & MSGF_REPLY2) {
529 * If REPLY2 is set we must have chased it all the way back to
530 * the reply port, but the replyport code has not queued the message
531 * (because it was waiting for the abort to catch up). We become
532 * responsible for queueing the message to the reply port.
534 KKASSERT((msg->ms_flags & MSGF_QUEUED) == 0);
535 KKASSERT(port == msg->ms_reply_port);
536 TAILQ_INSERT_TAIL(&port->mp_msgq, msg, ms_node);
537 msg->ms_flags |= MSGF_DONE | MSGF_QUEUED;
538 if (port->mp_flags & MSGPORTF_WAITING)
539 lwkt_schedule(port->mp_td);
540 } else if ((msg->ms_flags & (MSGF_QUEUED|MSGF_REPLY1)) == 0) {
542 * Abort on the target port. The message has not yet been replied
543 * and must be requeued to the target port.
545 msg->ms_flags |= MSGF_ABORTED | MSGF_QUEUED;
546 TAILQ_INSERT_TAIL(&port->mp_msgq, msg, ms_node);
547 if (port->mp_flags & MSGPORTF_WAITING)
548 lwkt_schedule(port->mp_td);
549 } else if ((msg->ms_flags & MSGF_REPLY1) == 0) {
551 * The message has not yet been retrieved by the target port, set
552 * MSGF_ABORTED so the target port can requeue the message abort after
555 msg->ms_flags |= MSGF_ABORTED;
560 * lwkt_default_waitport()
562 * If msg is NULL, dequeue the next message from the port's message
563 * queue, block until a message is ready. This function never
566 * If msg is non-NULL, block until the requested message has been returned
567 * to the port then dequeue and return it. DO NOT USE THIS TO WAIT FOR
568 * INCOMING REQUESTS, ONLY USE THIS TO WAIT FOR REPLIES.
570 * Note that the API does not currently support multiple threads waiting
571 * on a port. By virtue of owning the port it is controlled by our
572 * cpu and we can safely manipulate it's contents.
575 lwkt_default_waitport(lwkt_port_t port, lwkt_msg_t msg)
577 thread_t td = curthread;
580 KKASSERT(port->mp_td == td);
581 crit_enter_quick(td);
583 if ((msg = TAILQ_FIRST(&port->mp_msgq)) == NULL) {
584 port->mp_flags |= MSGPORTF_WAITING;
586 lwkt_deschedule_self(td);
588 } while ((msg = TAILQ_FIRST(&port->mp_msgq)) == NULL);
589 port->mp_flags &= ~MSGPORTF_WAITING;
591 _lwkt_pullmsg(port, msg);
594 * If a message is not marked done, or if it is queued, we have work
595 * to do. Note that MSGF_DONE is always set in the context of the
598 if ((msg->ms_flags & (MSGF_DONE|MSGF_QUEUED)) != MSGF_DONE) {
600 * We must own the reply port to safely mess with it's contents.
602 port = msg->ms_reply_port;
603 KKASSERT(port->mp_td == td);
605 if ((msg->ms_flags & MSGF_DONE) == 0) {
606 port->mp_flags |= MSGPORTF_WAITING; /* saved by the BGL */
611 * MSGF_PCATCH is only set by processes which wish to
612 * abort the message they are blocked on when a signal
613 * occurs. Note that we still must wait for message
614 * completion after sending an abort request.
616 if (msg->ms_flags & MSGF_PCATCH) {
617 if (sentabort == 0 && CURSIG(port->mp_td->td_proc)) {
625 * XXX set TDF_SINTR so 'ps' knows the difference between
626 * an interruptable wait and a disk wait. YYY eventually
627 * move P_SINTR to TDF_SINTR to reduce duplication.
629 td->td_flags |= TDF_SINTR;
630 lwkt_deschedule_self(td);
632 td->td_flags &= ~TDF_SINTR;
633 } while ((msg->ms_flags & MSGF_DONE) == 0);
634 port->mp_flags &= ~MSGPORTF_WAITING; /* saved by the BGL */
637 * We own the message now.
639 if (msg->ms_flags & MSGF_QUEUED) {
640 msg->ms_flags &= ~MSGF_QUEUED;
641 TAILQ_REMOVE(&port->mp_msgq, msg, ms_node);