2 * Copyright (c) 2006-2007 The DragonFly Project. All rights reserved.
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * $DragonFly: src/sys/kern/kern_syslink.c,v 1.16 2008/10/26 04:29:19 sephe Exp $
37 * This module implements the core syslink() system call and provides
38 * glue for kernel syslink frontends and backends, creating a intra-host
39 * communications infrastructure and DMA transport abstraction.
42 #include <sys/param.h>
43 #include <sys/systm.h>
44 #include <sys/kernel.h>
45 #include <sys/endian.h>
46 #include <sys/malloc.h>
47 #include <sys/alist.h>
53 #include <sys/objcache.h>
54 #include <sys/queue.h>
55 #include <sys/thread.h>
57 #include <sys/sysctl.h>
58 #include <sys/sysproto.h>
60 #include <sys/socket.h>
61 #include <sys/socketvar.h>
62 #include <sys/socketops.h>
63 #include <sys/sysref.h>
64 #include <sys/syslink.h>
65 #include <sys/syslink_msg.h>
66 #include <netinet/in.h>
68 #include <sys/thread2.h>
69 #include <sys/spinlock2.h>
72 #include "opt_syslink.h"
75 * Syslink Connection abstraction
84 struct slmsg_rb_tree reply_rb_root; /* replies to requests */
86 struct sldesc *peer; /* peer syslink, if any */
87 struct file *xfp; /* external file pointer */
88 struct slcommon *common;
90 int rwaiters; /* number of threads waiting */
91 int wblocked; /* blocked waiting for us to drain */
92 size_t cmdbytes; /* unreplied commands pending */
93 size_t repbytes; /* undrained replies pending */
94 int (*backend_wblocked)(struct sldesc *, int, sl_proto_t);
95 int (*backend_write)(struct sldesc *, struct slmsg *);
96 void (*backend_reply)(struct sldesc *,struct slmsg *,struct slmsg *);
97 void (*backend_dispose)(struct sldesc *, struct slmsg *);
100 #define SLF_RSHUTDOWN 0x0001
101 #define SLF_WSHUTDOWN 0x0002
103 static int syslink_cmd_new(struct syslink_info_new *info, int *result);
104 static struct sldesc *allocsldesc(struct slcommon *common);
105 static void setsldescfp(struct sldesc *sl, struct file *fp);
106 static void shutdownsldesc(struct sldesc *sl, int how);
107 static void shutdownsldesc2(struct sldesc *sl, int how);
108 static void sldrop(struct sldesc *sl);
109 static int syslink_validate_msg(struct syslink_msg *msg, int bytes);
110 static int syslink_validate_elm(struct syslink_elm *elm, sl_reclen_t bytes,
111 int swapit, int depth);
113 static int sl_local_mmap(struct slmsg *slmsg, char *base, size_t len);
114 static void sl_local_munmap(struct slmsg *slmsg);
116 static int backend_wblocked_user(struct sldesc *sl, int nbio, sl_proto_t proto);
117 static int backend_write_user(struct sldesc *sl, struct slmsg *slmsg);
118 static void backend_reply_user(struct sldesc *sl, struct slmsg *slcmd,
119 struct slmsg *slrep);
120 static void backend_dispose_user(struct sldesc *sl, struct slmsg *slmsg);
122 static int backend_wblocked_kern(struct sldesc *sl, int nbio, sl_proto_t proto);
123 static int backend_write_kern(struct sldesc *sl, struct slmsg *slmsg);
124 static void backend_reply_kern(struct sldesc *sl, struct slmsg *slcmd,
125 struct slmsg *slrep);
126 static void backend_dispose_kern(struct sldesc *sl, struct slmsg *slmsg);
127 static void slmsg_put(struct slmsg *slmsg);
130 * Objcache memory backend
132 * All three object caches return slmsg structures but each is optimized
133 * for syslink message buffers of varying sizes. We use the slightly
134 * more complex ctor/dtor API in order to provide ready-to-go slmsg's.
137 static struct objcache *sl_objcache_big;
138 static struct objcache *sl_objcache_small;
139 static struct objcache *sl_objcache_none;
141 MALLOC_DEFINE(M_SYSLINK, "syslink", "syslink manager");
143 static boolean_t slmsg_ctor(void *data, void *private, int ocflags);
144 static void slmsg_dtor(void *data, void *private);
148 syslinkinit(void *dummy __unused)
150 size_t n = sizeof(struct slmsg);
152 sl_objcache_none = objcache_create_mbacked(M_SYSLINK, n, NULL, 64,
153 slmsg_ctor, slmsg_dtor,
155 sl_objcache_small= objcache_create_mbacked(M_SYSLINK, n, NULL, 64,
156 slmsg_ctor, slmsg_dtor,
158 sl_objcache_big = objcache_create_mbacked(M_SYSLINK, n, NULL, 16,
159 slmsg_ctor, slmsg_dtor,
165 slmsg_ctor(void *data, void *private, int ocflags)
167 struct slmsg *slmsg = data;
169 bzero(slmsg, sizeof(*slmsg));
171 slmsg->oc = *(struct objcache **)private;
172 if (slmsg->oc == sl_objcache_none) {
174 } else if (slmsg->oc == sl_objcache_small) {
175 slmsg->maxsize = SLMSG_SMALL;
176 } else if (slmsg->oc == sl_objcache_big) {
177 slmsg->maxsize = SLMSG_BIG;
179 panic("slmsg_ctor: bad objcache?\n");
181 if (slmsg->maxsize) {
182 slmsg->msg = kmalloc(slmsg->maxsize,
183 M_SYSLINK, M_WAITOK|M_ZERO);
185 xio_init(&slmsg->xio);
191 slmsg_dtor(void *data, void *private)
193 struct slmsg *slmsg = data;
195 if (slmsg->maxsize && slmsg->msg) {
196 kfree(slmsg->msg, M_SYSLINK);
202 SYSINIT(syslink, SI_BOOT2_MACHDEP, SI_ORDER_ANY, syslinkinit, NULL)
204 static int rb_slmsg_compare(struct slmsg *msg1, struct slmsg *msg2);
205 RB_GENERATE2(slmsg_rb_tree, slmsg, rbnode, rb_slmsg_compare,
206 sysid_t, msg->sm_msgid);
211 static int syslink_enabled;
212 SYSCTL_NODE(_kern, OID_AUTO, syslink, CTLFLAG_RW, 0, "Pipe operation");
213 SYSCTL_INT(_kern_syslink, OID_AUTO, enabled,
214 CTLFLAG_RW, &syslink_enabled, 0, "Enable SYSLINK");
215 static size_t syslink_bufsize = 65536;
216 SYSCTL_UINT(_kern_syslink, OID_AUTO, bufsize,
217 CTLFLAG_RW, &syslink_bufsize, 0, "Maximum buffer size");
220 * Fileops API - typically used to glue a userland frontend with a
224 static int slfileop_read(struct file *fp, struct uio *uio,
225 struct ucred *cred, int flags);
226 static int slfileop_write(struct file *fp, struct uio *uio,
227 struct ucred *cred, int flags);
228 static int slfileop_close(struct file *fp);
229 static int slfileop_stat(struct file *fp, struct stat *sb, struct ucred *cred);
230 static int slfileop_shutdown(struct file *fp, int how);
231 static int slfileop_ioctl(struct file *fp, u_long cmd, caddr_t data,
233 static int slfileop_poll(struct file *fp, int events, struct ucred *cred);
234 static int slfileop_kqfilter(struct file *fp, struct knote *kn);
236 static struct fileops syslinkops = {
237 .fo_read = slfileop_read,
238 .fo_write = slfileop_write,
239 .fo_ioctl = slfileop_ioctl,
240 .fo_poll = slfileop_poll,
241 .fo_kqfilter = slfileop_kqfilter,
242 .fo_stat = slfileop_stat,
243 .fo_close = slfileop_close,
244 .fo_shutdown = slfileop_shutdown
247 /************************************************************************
248 * PRIMARY SYSTEM CALL INTERFACE *
249 ************************************************************************
251 * syslink(int cmd, struct syslink_info *info, size_t bytes)
254 sys_syslink(struct syslink_args *uap)
256 union syslink_info_all info;
260 * System call is under construction and disabled by default.
261 * Superuser access is also required for now, but eventually
262 * will not be needed.
264 if (syslink_enabled == 0)
266 error = priv_check(curthread, PRIV_ROOT);
271 * Load and validate the info structure. Unloaded bytes are zerod
272 * out. The label field must always be 0-filled, even if not used
275 bzero(&info, sizeof(info));
276 if ((unsigned)uap->bytes <= sizeof(info)) {
278 error = copyin(uap->info, &info, uap->bytes);
286 * Process the command
289 case SYSLINK_CMD_NEW:
290 error = syslink_cmd_new(&info.cmd_new, &uap->sysmsg_result);
296 if (error == 0 && info.head.wbflag)
297 copyout(&info, uap->info, uap->bytes);
302 * Create a linked pair of descriptors, like a pipe.
306 syslink_cmd_new(struct syslink_info_new *info, int *result)
308 struct proc *p = curproc;
312 struct sldesc *slpeer;
316 error = falloc(p, &fp1, &fd1);
319 error = falloc(p, &fp2, &fd2);
321 fsetfd(p, NULL, fd1);
325 slpeer = allocsldesc(NULL);
326 slpeer->backend_wblocked = backend_wblocked_user;
327 slpeer->backend_write = backend_write_user;
328 slpeer->backend_reply = backend_reply_user;
329 slpeer->backend_dispose = backend_dispose_user;
330 sl = allocsldesc(slpeer->common);
332 sl->backend_wblocked = backend_wblocked_user;
333 sl->backend_write = backend_write_user;
334 sl->backend_reply = backend_reply_user;
335 sl->backend_dispose = backend_dispose_user;
338 setsldescfp(sl, fp1);
339 setsldescfp(slpeer, fp2);
346 info->head.wbflag = 1; /* write back */
353 /************************************************************************
354 * LOW LEVEL SLDESC SUPPORT *
355 ************************************************************************
361 allocsldesc(struct slcommon *common)
365 sl = kmalloc(sizeof(struct sldesc), M_SYSLINK, M_WAITOK|M_ZERO);
367 common = kmalloc(sizeof(*common), M_SYSLINK, M_WAITOK|M_ZERO);
368 TAILQ_INIT(&sl->inq); /* incoming requests */
369 RB_INIT(&sl->reply_rb_root); /* match incoming replies */
370 spin_init(&sl->spin);
378 setsldescfp(struct sldesc *sl, struct file *fp)
381 fp->f_type = DTYPE_SYSLINK;
382 fp->f_flag = FREAD | FWRITE;
383 fp->f_ops = &syslinkops;
388 * Red-black tree compare function
392 rb_slmsg_compare(struct slmsg *msg1, struct slmsg *msg2)
394 if (msg1->msg->sm_msgid < msg2->msg->sm_msgid)
396 if (msg1->msg->sm_msgid == msg2->msg->sm_msgid)
403 shutdownsldesc(struct sldesc *sl, int how)
408 shutdownsldesc2(sl, how);
411 * Return unread and unreplied messages
413 spin_lock_wr(&sl->spin);
414 while ((slmsg = TAILQ_FIRST(&sl->inq)) != NULL) {
415 TAILQ_REMOVE(&sl->inq, slmsg, tqnode);
416 spin_unlock_wr(&sl->spin);
417 if (slmsg->msg->sm_proto & SM_PROTO_REPLY) {
418 sl->repbytes -= slmsg->maxsize;
419 slmsg->flags &= ~SLMSGF_ONINQ;
420 sl->peer->backend_dispose(sl->peer, slmsg);
422 /* leave ONINQ set for commands, it will cleared below */
423 spin_lock_wr(&sl->spin);
425 while ((slmsg = RB_ROOT(&sl->reply_rb_root)) != NULL) {
426 RB_REMOVE(slmsg_rb_tree, &sl->reply_rb_root, slmsg);
427 sl->cmdbytes -= slmsg->maxsize;
428 spin_unlock_wr(&sl->spin);
429 slmsg->flags &= ~SLMSGF_ONINQ;
430 sl->peer->backend_reply(sl->peer, slmsg, NULL);
431 spin_lock_wr(&sl->spin);
433 spin_unlock_wr(&sl->spin);
436 * Call shutdown on the peer with the opposite flags
450 shutdownsldesc2(sl->peer, rhow);
455 shutdownsldesc2(struct sldesc *sl, int how)
457 spin_lock_wr(&sl->spin);
460 sl->flags |= SLF_RSHUTDOWN;
463 sl->flags |= SLF_WSHUTDOWN;
466 sl->flags |= SLF_RSHUTDOWN | SLF_WSHUTDOWN;
469 spin_unlock_wr(&sl->spin);
472 * Handle signaling on the user side
476 wakeup(&sl->rwaiters);
480 sl->wblocked = 0; /* race ok */
481 wakeup(&sl->wblocked);
488 sldrop(struct sldesc *sl)
490 struct sldesc *slpeer;
492 spin_lock_wr(&sl->common->spin);
493 if (--sl->common->refs == 0) {
494 spin_unlock_wr(&sl->common->spin);
495 if ((slpeer = sl->peer) != NULL) {
498 slpeer->common = NULL;
499 KKASSERT(slpeer->xfp == NULL);
500 KKASSERT(TAILQ_EMPTY(&slpeer->inq));
501 KKASSERT(RB_EMPTY(&slpeer->reply_rb_root));
502 kfree(slpeer, M_SYSLINK);
504 KKASSERT(sl->xfp == NULL);
505 KKASSERT(TAILQ_EMPTY(&sl->inq));
506 KKASSERT(RB_EMPTY(&sl->reply_rb_root));
507 kfree(sl->common, M_SYSLINK);
509 kfree(sl, M_SYSLINK);
511 spin_unlock_wr(&sl->common->spin);
517 slmsg_put(struct slmsg *slmsg)
519 if (slmsg->flags & SLMSGF_HASXIO) {
520 slmsg->flags &= ~SLMSGF_HASXIO;
522 xio_release(&slmsg->xio);
525 slmsg->flags &= ~SLMSGF_LINMAP;
526 objcache_put(slmsg->oc, slmsg);
529 /************************************************************************
531 ************************************************************************
533 * Implement userland fileops.
539 slfileop_read(struct file *fp, struct uio *uio, struct ucred *cred, int flags)
541 struct sldesc *sl = fp->f_data; /* fp refed on call */
545 struct syslink_msg *wmsg;
550 * Kinda messy. Figure out the non-blocking state
552 if (flags & O_FBLOCKING)
554 else if (flags & O_FNONBLOCKING)
556 else if (fp->f_flag & O_NONBLOCK)
564 * iov0 - message buffer
565 * iov1 - DMA buffer or backup buffer
567 if (uio->uio_iovcnt < 1) {
571 iov0 = &uio->uio_iov[0];
572 if (uio->uio_iovcnt > 2) {
578 * Get a message, blocking if necessary.
580 spin_lock_wr(&sl->spin);
581 while ((slmsg = TAILQ_FIRST(&sl->inq)) == NULL) {
582 if (sl->flags & SLF_RSHUTDOWN) {
591 error = ssleep(&sl->rwaiters, &sl->spin, PCATCH, "slrmsg", 0);
599 * We have a message and still hold the spinlock. Make sure the
600 * uio has enough room to hold the message.
602 * Note that replies do not have XIOs.
604 if (slmsg->msgsize > iov0->iov_len) {
608 if (slmsg->xio.xio_bytes) {
609 if (uio->uio_iovcnt != 2) {
613 iov1 = &uio->uio_iov[1];
614 if (slmsg->xio.xio_bytes > iov1->iov_len) {
623 * Dequeue the message. Adjust repbytes immediately. cmdbytes
624 * are adjusted when the command is replied to, not here.
626 TAILQ_REMOVE(&sl->inq, slmsg, tqnode);
627 if (slmsg->msg->sm_proto & SM_PROTO_REPLY)
628 sl->repbytes -= slmsg->maxsize;
629 spin_unlock_wr(&sl->spin);
632 * Load the message data into the user buffer.
634 * If receiving a command an XIO may exist specifying a DMA buffer.
635 * For commands, if DMAW is set we have to copy or map the buffer
636 * so the caller can access the data being written. If DMAR is set
637 * we do not have to copy but we still must map the buffer so the
638 * caller can directly fill in the data being requested.
640 error = uiomove((void *)slmsg->msg, slmsg->msgsize, uio);
641 if (error == 0 && slmsg->xio.xio_bytes &&
642 (wmsg->sm_head.se_cmd & SE_CMDF_REPLY) == 0) {
643 if (wmsg->sm_head.se_cmd & SE_CMDF_DMAW) {
645 * Data being passed to caller or being passed in both
646 * directions, copy or map.
649 if ((flags & O_MAPONREAD) &&
650 (slmsg->xio.xio_flags & XIOF_VMLINEAR)) {
651 error = sl_local_mmap(slmsg,
655 error = xio_copy_xtou(&slmsg->xio, 0,
657 slmsg->xio.xio_bytes);
659 error = xio_copy_xtou(&slmsg->xio, 0,
661 slmsg->xio.xio_bytes);
664 } else if (wmsg->sm_head.se_cmd & SE_CMDF_DMAR) {
666 * Data will be passed back to originator, map
667 * the buffer if we can, else use the backup
668 * buffer at the same VA supplied by the caller.
671 if ((flags & O_MAPONREAD) &&
672 (slmsg->xio.xio_flags & XIOF_VMLINEAR)) {
673 error = sl_local_mmap(slmsg,
676 error = 0; /* ignore errors */
687 * Requeue the message if we could not read it successfully
689 spin_lock_wr(&sl->spin);
690 TAILQ_INSERT_HEAD(&sl->inq, slmsg, tqnode);
691 slmsg->flags |= SLMSGF_ONINQ;
692 spin_unlock_wr(&sl->spin);
693 } else if (slmsg->msg->sm_proto & SM_PROTO_REPLY) {
695 * Dispose of any received reply after we've copied it
696 * to userland. We don't need the slmsg any more.
698 slmsg->flags &= ~SLMSGF_ONINQ;
699 sl->peer->backend_dispose(sl->peer, slmsg);
700 if (sl->wblocked && sl->repbytes < syslink_bufsize) {
701 sl->wblocked = 0; /* MP race ok here */
702 wakeup(&sl->wblocked);
706 * Leave the command in the RB tree but clear ONINQ now
707 * that we have returned it to userland so userland can
710 slmsg->flags &= ~SLMSGF_ONINQ;
714 spin_unlock_wr(&sl->spin);
720 * Userland writes syslink message (optionally with DMA buffer in iov[1]).
724 slfileop_write(struct file *fp, struct uio *uio, struct ucred *cred, int flags)
726 struct sldesc *sl = fp->f_data;
729 struct syslink_msg sltmp;
730 struct syslink_msg *wmsg; /* wire message */
739 * Kinda messy. Figure out the non-blocking state
741 if (flags & O_FBLOCKING)
743 else if (flags & O_FNONBLOCKING)
745 else if (fp->f_flag & O_NONBLOCK)
753 if (uio->uio_iovcnt < 1) {
757 iov0 = &uio->uio_iov[0];
758 if (iov0->iov_len > SLMSG_BIG) {
762 if (uio->uio_iovcnt > 2) {
766 if (uio->uio_iovcnt > 1) {
767 iov1 = &uio->uio_iov[1];
768 if (iov1->iov_len > XIO_INTERNAL_SIZE) {
772 if ((intptr_t)iov1->iov_base & PAGE_MASK) {
781 * Handle the buffer-full case. slpeer cmdbytes is managed
782 * by the backend function, not us so if the callback just
783 * directly implements the message and never adjusts cmdbytes,
784 * we will never sleep here.
786 if (sl->flags & SLF_WSHUTDOWN) {
792 * Only commands can block the pipe, not replies. Otherwise a
793 * deadlock is possible.
795 error = copyin(iov0->iov_base, &sltmp, sizeof(sltmp));
798 if ((proto = sltmp.sm_proto) & SM_PROTO_ENDIAN_REV)
799 proto = bswap16(proto);
800 error = sl->peer->backend_wblocked(sl->peer, nbio, proto);
805 * Allocate a slmsg and load the message. Note that the bytes
806 * returned to userland only reflects the primary syslink message
807 * and does not include any DMA buffers.
809 if (iov0->iov_len <= SLMSG_SMALL)
810 slmsg = objcache_get(sl_objcache_small, M_WAITOK);
812 slmsg = objcache_get(sl_objcache_big, M_WAITOK);
813 slmsg->msgsize = iov0->iov_len;
816 error = uiomove((void *)wmsg, iov0->iov_len, uio);
819 error = syslink_validate_msg(wmsg, slmsg->msgsize);
823 if ((wmsg->sm_head.se_cmd & SE_CMDF_REPLY) == 0) {
825 * Install the XIO for commands if any DMA flags are set.
827 * XIOF_VMLINEAR requires that the XIO represent a
828 * contiguous set of pages associated with a single VM
829 * object (so the reader side can mmap it easily).
831 * XIOF_VMLINEAR might not be set when the kernel sends
832 * commands to userland so the reader side backs off to
833 * a backup buffer if it isn't set, but we require it
834 * for userland writes.
836 xflags = XIOF_VMLINEAR;
837 if (wmsg->sm_head.se_cmd & SE_CMDF_DMAR)
838 xflags |= XIOF_READ | XIOF_WRITE;
839 else if (wmsg->sm_head.se_cmd & SE_CMDF_DMAW)
841 if (xflags && iov1) {
843 error = xio_init_ubuf(&slmsg->xio, iov1->iov_base,
844 iov1->iov_len, xflags);
848 slmsg->flags |= SLMSGF_HASXIO;
850 error = sl->peer->backend_write(sl->peer, slmsg);
853 * Replies have to be matched up against received commands.
855 spin_lock_wr(&sl->spin);
856 slcmd = slmsg_rb_tree_RB_LOOKUP(&sl->reply_rb_root,
857 slmsg->msg->sm_msgid);
858 if (slcmd == NULL || (slcmd->flags & SLMSGF_ONINQ)) {
860 spin_unlock_wr(&sl->spin);
863 RB_REMOVE(slmsg_rb_tree, &sl->reply_rb_root, slcmd);
864 sl->cmdbytes -= slcmd->maxsize;
865 spin_unlock_wr(&sl->spin);
868 * If the original command specified DMAR, has an xio, and
869 * our write specifies a DMA buffer, then we can do a
870 * copyback. But if we are linearly mapped and the caller
871 * is using the map base address, then the caller filled in
872 * the data via the direct memory map and no copyback is
875 if ((slcmd->msg->sm_head.se_cmd & SE_CMDF_DMAR) && iov1 &&
876 (slcmd->flags & SLMSGF_HASXIO) &&
877 ((slcmd->flags & SLMSGF_LINMAP) == 0 ||
878 iov1->iov_base != slcmd->vmbase)
881 if (iov1->iov_len > slcmd->xio.xio_bytes)
882 count = slcmd->xio.xio_bytes;
884 count = iov1->iov_len;
886 error = xio_copy_utox(&slcmd->xio, 0, iov1->iov_base,
892 * If we had mapped a DMA buffer, remove it
894 if (slcmd->flags & SLMSGF_LINMAP) {
896 sl_local_munmap(slcmd);
901 * Reply and handle unblocking
903 sl->peer->backend_reply(sl->peer, slcmd, slmsg);
904 if (sl->wblocked && sl->cmdbytes < syslink_bufsize) {
905 sl->wblocked = 0; /* MP race ok here */
906 wakeup(&sl->wblocked);
910 * slmsg has already been dealt with, make sure error is
911 * 0 so we do not double-free it.
925 * Close a syslink descriptor.
927 * Disassociate the syslink from the file descriptor and disconnect from
932 slfileop_close(struct file *fp)
937 * Disassociate the file pointer. Take ownership of the ref on the
942 fp->f_ops = &badfileops;
946 * Shutdown both directions. The other side will not issue API
947 * calls to us after we've shutdown both directions.
949 shutdownsldesc(sl, SHUT_RDWR);
954 KKASSERT(sl->cmdbytes == 0);
955 KKASSERT(sl->repbytes == 0);
965 slfileop_stat (struct file *fp, struct stat *sb, struct ucred *cred)
972 slfileop_shutdown (struct file *fp, int how)
974 shutdownsldesc((struct sldesc *)fp->f_data, how);
980 slfileop_ioctl (struct file *fp, u_long cmd, caddr_t data, struct ucred *cred)
987 slfileop_poll (struct file *fp, int events, struct ucred *cred)
994 slfileop_kqfilter(struct file *fp, struct knote *kn)
999 /************************************************************************
1000 * LOCAL MEMORY MAPPING *
1001 ************************************************************************
1003 * This feature is currently not implemented
1009 sl_local_mmap(struct slmsg *slmsg, char *base, size_t len)
1011 return (EOPNOTSUPP);
1016 sl_local_munmap(struct slmsg *slmsg)
1025 sl_local_mmap(struct slmsg *slmsg, char *base, size_t len)
1027 struct vmspace *vms = curproc->p_vmspace;
1028 vm_offset_t addr = (vm_offset_t)base;
1030 /* XXX check user address range */
1031 error = vm_map_replace(
1033 (vm_offset_t)base, (vm_offset_t)base + len,
1034 slmsg->xio.xio_pages[0]->object,
1035 slmsg->xio.xio_pages[0]->pindex << PAGE_SHIFT,
1036 VM_PROT_READ|VM_PROT_WRITE,
1037 VM_PROT_READ|VM_PROT_WRITE,
1038 MAP_DISABLE_SYNCER);
1041 slmsg->flags |= SLMSGF_LINMAP;
1042 slmsg->vmbase = base;
1043 slmsg->vmsize = len;
1050 sl_local_munmap(struct slmsg *slmsg)
1052 if (slmsg->flags & SLMSGF_LINMAP) {
1053 vm_map_remove(&curproc->p_vmspace->vm_map,
1055 slmsg->vmbase + slcmd->vmsize);
1056 slmsg->flags &= ~SLMSGF_LINMAP;
1062 /************************************************************************
1063 * MESSAGE VALIDATION *
1064 ************************************************************************
1066 * Validate that the syslink message. Check that all headers and elements
1067 * conform. Correct the endian if necessary.
1069 * NOTE: If reverse endian needs to be corrected, SE_CMDF_UNTRANSLATED
1070 * is recursively flipped on all syslink_elm's in the message. As the
1071 * message traverses the mesh, multiple flips may occur. It is
1072 * up to the RPC protocol layer to correct opaque data payloads and
1073 * SE_CMDF_UNTRANSLATED prevents the protocol layer from misinterpreting
1074 * a command or reply element which has not been endian-corrected.
1078 syslink_validate_msg(struct syslink_msg *msg, int bytes)
1085 * The raw message must be properly-aligned.
1087 if (bytes & SL_ALIGNMASK)
1092 * The message must at least contain the msgid, bytes, and
1095 if (bytes < SL_MIN_PAD_SIZE)
1099 * Fix the endian if it is reversed.
1101 if (msg->sm_proto & SM_PROTO_ENDIAN_REV) {
1102 msg->sm_msgid = bswap64(msg->sm_msgid);
1103 msg->sm_sessid = bswap64(msg->sm_sessid);
1104 msg->sm_bytes = bswap16(msg->sm_bytes);
1105 msg->sm_proto = bswap16(msg->sm_proto);
1106 msg->sm_rlabel = bswap32(msg->sm_rlabel);
1107 if (msg->sm_proto & SM_PROTO_ENDIAN_REV)
1115 * Validate the contents. For PADs, the entire payload is
1116 * ignored and the minimum message size can be as small as
1119 if (msg->sm_proto == SMPROTO_PAD) {
1120 if (msg->sm_bytes < SL_MIN_PAD_SIZE ||
1121 msg->sm_bytes > bytes) {
1124 /* ignore the entire payload, it can be garbage */
1126 if (msg->sm_bytes < SL_MIN_MSG_SIZE ||
1127 msg->sm_bytes > bytes) {
1130 error = syslink_validate_elm(
1133 offsetof(struct syslink_msg,
1135 swapit, SL_MAXDEPTH);
1141 * The aligned payload size must be used to locate the
1142 * next syslink_msg in the buffer.
1144 aligned_reclen = SL_MSG_ALIGN(msg->sm_bytes);
1145 bytes -= aligned_reclen;
1146 msg = (void *)((char *)msg + aligned_reclen);
1153 syslink_validate_elm(struct syslink_elm *elm, sl_reclen_t bytes,
1154 int swapit, int depth)
1159 * If the buffer isn't big enough to fit the header, stop now!
1161 if (bytes < SL_MIN_ELM_SIZE)
1164 * All syslink_elm headers are recursively endian-adjusted. Opaque
1165 * data payloads are not.
1168 elm->se_cmd = bswap16(elm->se_cmd) ^ SE_CMDF_UNTRANSLATED;
1169 elm->se_bytes = bswap16(elm->se_bytes);
1170 elm->se_aux = bswap32(elm->se_aux);
1174 * Check element size requirements.
1176 if (elm->se_bytes < SL_MIN_ELM_SIZE || elm->se_bytes > bytes)
1180 * Recursively check structured payloads. A structured payload may
1181 * contain as few as 0 recursive elements.
1183 if (elm->se_cmd & SE_CMDF_STRUCTURED) {
1186 bytes -= SL_MIN_ELM_SIZE;
1189 if (syslink_validate_elm(elm, bytes, swapit, depth - 1))
1191 aligned_reclen = SL_MSG_ALIGN(elm->se_bytes);
1192 elm = (void *)((char *)elm + aligned_reclen);
1193 bytes -= aligned_reclen;
1199 /************************************************************************
1200 * BACKEND FUNCTIONS - USER DESCRIPTOR *
1201 ************************************************************************
1203 * Peer backend links are primarily used when userland creates a pair
1204 * of linked descriptors.
1208 * Do any required blocking / nbio handling for attempts to write to
1209 * a sldesc associated with a user descriptor.
1213 backend_wblocked_user(struct sldesc *sl, int nbio, sl_proto_t proto)
1216 int *bytesp = (proto & SM_PROTO_REPLY) ? &sl->repbytes : &sl->cmdbytes;
1219 * Block until sufficient data is drained by the target. It is
1220 * ok to have a MP race against cmdbytes.
1222 if (*bytesp >= syslink_bufsize) {
1223 spin_lock_wr(&sl->spin);
1224 while (*bytesp >= syslink_bufsize) {
1225 if (sl->flags & SLF_WSHUTDOWN) {
1234 error = ssleep(&sl->wblocked, &sl->spin,
1235 PCATCH, "slwmsg", 0);
1239 spin_unlock_wr(&sl->spin);
1245 * Unconditionally write a syslink message to the sldesc associated with
1246 * a user descriptor. Command messages are also placed in a red-black
1247 * tree so their DMA tag (if any) can be accessed and so they can be
1248 * linked to any reply message.
1252 backend_write_user(struct sldesc *sl, struct slmsg *slmsg)
1256 spin_lock_wr(&sl->spin);
1257 if (sl->flags & SLF_RSHUTDOWN) {
1259 * Not accepting new messages
1262 } else if (slmsg->msg->sm_proto & SM_PROTO_REPLY) {
1266 TAILQ_INSERT_TAIL(&sl->inq, slmsg, tqnode);
1267 sl->repbytes += slmsg->maxsize;
1268 slmsg->flags |= SLMSGF_ONINQ;
1270 } else if (RB_INSERT(slmsg_rb_tree, &sl->reply_rb_root, slmsg)) {
1272 * Write a command, but there was a msgid collision when
1273 * we tried to insert it into the RB tree.
1278 * Write a command, successful insertion into the RB tree.
1280 TAILQ_INSERT_TAIL(&sl->inq, slmsg, tqnode);
1281 sl->cmdbytes += slmsg->maxsize;
1282 slmsg->flags |= SLMSGF_ONINQ;
1285 spin_unlock_wr(&sl->spin);
1287 wakeup(&sl->rwaiters);
1292 * Our peer is replying a command we previously sent it back to us, along
1293 * with the reply message (if not NULL). We just queue the reply to
1294 * userland and free of the command.
1298 backend_reply_user(struct sldesc *sl, struct slmsg *slcmd, struct slmsg *slrep)
1304 spin_lock_wr(&sl->spin);
1305 if ((sl->flags & SLF_RSHUTDOWN) == 0) {
1306 TAILQ_INSERT_TAIL(&sl->inq, slrep, tqnode);
1307 sl->repbytes += slrep->maxsize;
1312 spin_unlock_wr(&sl->spin);
1314 sl->peer->backend_dispose(sl->peer, slrep);
1315 else if (sl->rwaiters)
1316 wakeup(&sl->rwaiters);
1322 backend_dispose_user(struct sldesc *sl, struct slmsg *slmsg)
1327 /************************************************************************
1328 * KERNEL DRIVER OR FILESYSTEM API *
1329 ************************************************************************
1334 * Create a user<->kernel link, returning the user descriptor in *fdp
1335 * and the kernel descriptor in *kslp. 0 is returned on success, and an
1336 * error code is returned on failure.
1339 syslink_ukbackend(int *fdp, struct sldesc **kslp)
1341 struct proc *p = curproc;
1351 error = falloc(p, &fp, &fd);
1354 usl = allocsldesc(NULL);
1355 usl->backend_wblocked = backend_wblocked_user;
1356 usl->backend_write = backend_write_user;
1357 usl->backend_reply = backend_reply_user;
1358 usl->backend_dispose = backend_dispose_user;
1360 ksl = allocsldesc(usl->common);
1362 ksl->backend_wblocked = backend_wblocked_kern;
1363 ksl->backend_write = backend_write_kern;
1364 ksl->backend_reply = backend_reply_kern;
1365 ksl->backend_dispose = backend_dispose_kern;
1369 setsldescfp(usl, fp);
1379 * Assign a unique message id, issue a syslink message to userland,
1380 * and wait for a reply.
1383 syslink_kdomsg(struct sldesc *ksl, struct slmsg *slmsg)
1385 struct syslink_msg *msg;
1389 * Finish initializing slmsg and post it to the red-black tree for
1390 * reply matching. If the message id is already in use we return
1391 * EEXIST, giving the originator the chance to roll a new msgid.
1394 slmsg->msgsize = msg->sm_bytes;
1395 if ((error = syslink_validate_msg(msg, msg->sm_bytes)) != 0)
1397 msg->sm_msgid = allocsysid();
1400 * Issue the request and wait for a matching reply or failure,
1401 * then remove the message from the matching tree and return.
1403 error = ksl->peer->backend_write(ksl->peer, slmsg);
1404 spin_lock_wr(&ksl->spin);
1406 while (slmsg->rep == NULL) {
1407 error = ssleep(slmsg, &ksl->spin, 0, "kwtmsg", 0);
1408 /* XXX ignore error for now */
1410 if (slmsg->rep == (struct slmsg *)-1) {
1414 error = slmsg->rep->msg->sm_head.se_aux;
1417 spin_unlock_wr(&ksl->spin);
1422 * Similar to syslink_kdomsg but return immediately instead of
1423 * waiting for a reply. The kernel must supply a callback function
1424 * which will be made in the context of the user process replying
1428 syslink_ksendmsg(struct sldesc *ksl, struct slmsg *slmsg,
1429 void (*func)(struct slmsg *, void *, int), void *arg)
1431 struct syslink_msg *msg;
1435 * Finish initializing slmsg and post it to the red-black tree for
1436 * reply matching. If the message id is already in use we return
1437 * EEXIST, giving the originator the chance to roll a new msgid.
1440 slmsg->msgsize = msg->sm_bytes;
1441 slmsg->callback_func = func;
1442 slmsg->callback_data = arg;
1443 if ((error = syslink_validate_msg(msg, msg->sm_bytes)) != 0)
1445 msg->sm_msgid = allocsysid();
1448 * Issue the request. If no error occured the operation will be
1449 * in progress, otherwise the operation is considered to have failed
1450 * and the caller can deallocate the slmsg.
1452 error = ksl->peer->backend_write(ksl->peer, slmsg);
1457 syslink_kwaitmsg(struct sldesc *ksl, struct slmsg *slmsg)
1461 spin_lock_wr(&ksl->spin);
1462 while (slmsg->rep == NULL) {
1463 error = ssleep(slmsg, &ksl->spin, 0, "kwtmsg", 0);
1464 /* XXX ignore error for now */
1466 if (slmsg->rep == (struct slmsg *)-1) {
1470 error = slmsg->rep->msg->sm_head.se_aux;
1472 spin_unlock_wr(&ksl->spin);
1477 syslink_kallocmsg(void)
1479 return(objcache_get(sl_objcache_small, M_WAITOK));
1483 syslink_kfreemsg(struct sldesc *ksl, struct slmsg *slmsg)
1487 if ((rep = slmsg->rep) != NULL) {
1489 ksl->peer->backend_dispose(ksl->peer, rep);
1491 slmsg->callback_func = NULL;
1496 syslink_kshutdown(struct sldesc *ksl, int how)
1498 shutdownsldesc(ksl, how);
1502 syslink_kclose(struct sldesc *ksl)
1504 shutdownsldesc(ksl, SHUT_RDWR);
1509 * Associate a DMA buffer with a kernel syslink message prior to it
1510 * being sent to userland. The DMA buffer is set up from the point
1511 * of view of the target.
1514 syslink_kdmabuf_pages(struct slmsg *slmsg, struct vm_page **mbase, int npages)
1519 xflags = XIOF_VMLINEAR;
1520 if (slmsg->msg->sm_head.se_cmd & SE_CMDF_DMAR)
1521 xflags |= XIOF_READ | XIOF_WRITE;
1522 else if (slmsg->msg->sm_head.se_cmd & SE_CMDF_DMAW)
1523 xflags |= XIOF_READ;
1524 error = xio_init_pages(&slmsg->xio, mbase, npages, xflags);
1525 slmsg->flags |= SLMSGF_HASXIO;
1530 * Associate a DMA buffer with a kernel syslink message prior to it
1531 * being sent to userland. The DMA buffer is set up from the point
1532 * of view of the target.
1535 syslink_kdmabuf_data(struct slmsg *slmsg, char *base, int bytes)
1539 xflags = XIOF_VMLINEAR;
1540 if (slmsg->msg->sm_head.se_cmd & SE_CMDF_DMAR)
1541 xflags |= XIOF_READ | XIOF_WRITE;
1542 else if (slmsg->msg->sm_head.se_cmd & SE_CMDF_DMAW)
1543 xflags |= XIOF_READ;
1544 xio_init_kbuf(&slmsg->xio, base, bytes);
1545 slmsg->xio.xio_flags |= xflags;
1546 slmsg->flags |= SLMSGF_HASXIO;
1550 /************************************************************************
1551 * BACKEND FUNCTIONS FOR KERNEL API *
1552 ************************************************************************
1554 * These are the backend functions for a sldesc associated with a kernel
1559 * Our peer wants to write a syslink message to us and is asking us to
1560 * block if our input queue is full. We don't implement command reception
1561 * so don't block right now.
1565 backend_wblocked_kern(struct sldesc *ksl, int nbio, sl_proto_t proto)
1572 * Our peer is writing a request to the kernel. At the moment we do not
1577 backend_write_kern(struct sldesc *ksl, struct slmsg *slmsg)
1583 * Our peer wants to reply to a syslink message we sent it earlier. The
1584 * original command (that we passed to our peer), and the peer's reply
1585 * is specified. If the peer has failed slrep will be NULL.
1589 backend_reply_kern(struct sldesc *ksl, struct slmsg *slcmd, struct slmsg *slrep)
1593 spin_lock_wr(&ksl->spin);
1594 if (slrep == NULL) {
1595 slcmd->rep = (struct slmsg *)-1;
1599 error = slrep->msg->sm_head.se_aux;
1601 spin_unlock_wr(&ksl->spin);
1604 * Issue callback or wakeup a synchronous waiter.
1606 if (slcmd->callback_func) {
1607 slcmd->callback_func(slcmd, slcmd->callback_data, error);
1614 * Any reply messages we sent to our peer are returned to us for disposal.
1615 * Since we do not currently accept commands from our peer, there will not
1616 * be any replies returned to the peer to dispose of.
1620 backend_dispose_kern(struct sldesc *ksl, struct slmsg *slmsg)
1622 panic("backend_dispose_kern: kernel can't accept commands so it "
1623 "certainly did not reply to one!");