From: Matthew Dillon Date: Mon, 21 Dec 2015 20:17:36 +0000 (-0800) Subject: libc - Add quick version for the context management functions. X-Git-Tag: v4.6.0rc~1140 X-Git-Url: https://gitweb.dragonflybsd.org/dragonfly.git/commitdiff_plain/819d0c163e3cb679c6376a5751ee9d195e1838e2 libc - Add quick version for the context management functions. * Add makecontext_quick(), setcontext_quick(), and swapcontext_quick(). These functions work similarly to the non-quick versions but are designed for fast synchronous switching. These functions do not mess with the signal mask or stack at all and do not save or restore scratch registers. * These functions make no system calls. Switching time can be as low as ~5 nanoseconds. * These functions also provide optimizations for coroutine fall-through linkages. * Note that the coroutine / start-function callback arguments are somewhat different. Start functions are called back as cofunc(ucp, arg). Var-args are not supported and the stack is minimally aligned and initialized. * Remove the old internal set_mcontext() and get_mcontext() routines. --- diff --git a/lib/libc/gen/Makefile.inc b/lib/libc/gen/Makefile.inc index 7148948980..50db75fef2 100644 --- a/lib/libc/gen/Makefile.inc +++ b/lib/libc/gen/Makefile.inc @@ -68,7 +68,7 @@ MAN+= alarm.3 arc4random.3 clock.3 \ getttyent.3 getusershell.3 getvfsbyname.3 getvfsent.3 \ glob.3 initgroups.3 \ isgreater.3 \ - ldexp.3 lockf.3 makecontext.3 modf.3 \ + ldexp.3 lockf.3 makecontext.3 makecontext_quick.3 modf.3 \ nice.3 nlist.3 pause.3 popen.3 posix_spawn.3 \ posix_spawn_file_actions_addopen.3 posix_spawn_file_actions_init.3 \ posix_spawnattr_getflags.3 posix_spawnattr_getpgroup.3 \ @@ -154,6 +154,8 @@ MLINKS+=isgreater.3 isgreaterequal.3 \ MLINKS+=ldexp.3 ldexpf.3 \ ldexp.3 ldexpl.3 MLINKS+=makecontext.3 swapcontext.3 +MLINKS+=makecontext_quick.3 swapcontext_quick.3 +MLINKS+=makecontext_quick.3 setcontext_quick.3 MLINKS+=modf.3 modff.3 \ modf.3 modfl.3 MLINKS+=popen.3 pclose.3 diff --git a/lib/libc/gen/makecontext_quick.3 b/lib/libc/gen/makecontext_quick.3 new file mode 100644 index 0000000000..ed1354ad80 --- /dev/null +++ b/lib/libc/gen/makecontext_quick.3 @@ -0,0 +1,216 @@ +.\" +.\" Copyright (c) 2015 The DragonFly Project. All rights reserved. +.\" +.\" This code is derived from software contributed to The DragonFly Project +.\" by Matthew Dillon +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in +.\" the documentation and/or other materials provided with the +.\" distribution. +.\" 3. Neither the name of The DragonFly Project nor the names of its +.\" contributors may be used to endorse or promote products derived +.\" from this software without specific, prior written permission. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +.\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +.\" FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +.\" COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +.\" INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, +.\" BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +.\" LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +.\" AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +.\" SUCH DAMAGE. +.\" +.Dd December 21, 2015 +.Dt MAKECONTEXT_QUICK 3 +.Os +.Sh NAME +.Nm makecontext_quick , swapcontext_quick , setcontext_quick +.Nd quickly modify and exchange user thread contexts +.Sh LIBRARY +.Lb libc +.Sh SYNOPSIS +.In ucontext.h +.Ft void +.Fn makecontext_quick "ucontext_t *ucp" +.Ft void +.Fn swapcontext_quick "ucontext_t *oucp" "const ucontext_t *ucp" +.Ft void +.Fn setcontext_quick "ucontext_t *ucp" +.Sh DESCRIPTION +The quick context functions work similarly to the non-quick context functions +but are designed for proper coroutine operation and synchronous switching. +The signal mask is not adjusted in any manner by these routines, no system +calls are made, and scratch registers are not required to be retained across +calls. +.Pp +Since no system calls need to be made and the FP state (being scratch across +a procedure call) does not need to be saved or restored, these switching +functions are at least 10 times faster than the non-quick versions. +In addition, callers can setup quick contexts for cofunction chaining +(when one cofunction return-chains to another), and for circular cofunction +chaining loops, avoiding the need to save any register state at all in +those configurations. +.Pp +The +.Fn makecontext_quick +function +initializes all fields of the passed-in context except +.Li "ucp->uc_stack" , +.Li "ucp->uc_cofunc" , +and +.Li "ucp->uc_arg" . +All other structural fields will be zerod. +Note that +.Li "ucp->uc_link" +will also be zerod for safety. +.Pp +The caller must pre-initialize the uc_stack fields. +.Li "ucp->uc_cofunc" , +and +.Li "ucp->uc_arg" +should be initialized prior to making any context switches. +This function will set the context up to call the cofunction as +.Li "ucp->uc_cofunc(ucp, ucp->uc_arg)" . +Note that this calling format is different from the non-quick context calls. +.Pp +If the cofunction returns the wrapper will automatically reinitialize +the context to reissue a cofunction call and then call the next +cofunction via +.Li "ucp->uc_link" . +If the link field is NULL, the wrapper issues an +.Li "exit(0)" . +If the linkages return to the ucontext, the cofunction call is reissued. +The +.Li "ucp->uc_cofunc" , +and +.Li "ucp->uc_arg" +fields may be adjusted at any time to change the cofunction being called. +Using the auto-linkage feature avoids saving register state on cofunction +return and is the absolute quickest context switch possible, almost as +fast as a normal procedure call would be. +.Pp +The +.Fn setcontext_quick +function throws away the current context and switches to the target +context. +Again, the signal mask is not touched and scratch registers are not saved. +If you desire to switch to a signal stack ucontext you must use the +normal +.Fn setcontext +function and not this one. +This function is designed for synchronous switching only. +.Pp +The +.Fn swapcontext_quick +function saves the current register state and switches to the target +context. This function returns when the old context is resumed. +Again, the signal mask is not touched and scratch registers are not saved. +If you desire to switch to a signal stack ucontext you must use the +normal +.Fn swapcontext +function and not this one. +It is acceptable to mix normal context functions with quick functions +as long as you understand the ramifications. +.Pp +There is no quick version for +.Fn getcontext +on purpose. +.Sh RETURN VALUES +These functions have no return value. +.Sh ERRORS +.Bl -tag -width Er +.It Bq Er ENOMEM +There is not enough stack space in +.Fa ucp +to complete the operation. +.El +.Sh EXAMPLE +.Bd -literal +/* + * Example quick context test program. + * + * This program does 300M context switches. 200M using coroutine fall + * through and 100M using swapcontext_quick(). + */ +#include +#include +#include +#include +#include +#include +#include +#include + +static void test1(ucontext_t *ucp, void *arg); +static void test2(ucontext_t *ucp, void *arg); +static void test3(ucontext_t *ucp, void *arg); + +int +main(int ac, char **av) +{ + ucontext_t ucp1; + ucontext_t ucp2; + ucontext_t ucp3; + + ucp1.uc_stack.ss_sp = malloc(32768); + ucp1.uc_stack.ss_size = 32768; + ucp1.uc_cofunc = test1; + makecontext_quick(&ucp1); + + ucp2.uc_stack.ss_sp = malloc(32768); + ucp2.uc_stack.ss_size = 32768; + ucp2.uc_cofunc = test2; + makecontext_quick(&ucp2); + + ucp3.uc_stack.ss_sp = malloc(32768); + ucp3.uc_stack.ss_size = 32768; + ucp3.uc_cofunc = test3; + makecontext_quick(&ucp3); + + ucp1.uc_link = &ucp2; + ucp2.uc_link = &ucp3; + ucp3.uc_link = &ucp1; + setcontext_quick(&ucp1); +} + +int global_counter; + +static void +test1(ucontext_t *ucp, void *arg) +{ +} + +static void +test2(ucontext_t *ucp, void *arg) +{ + ++global_counter; + if (global_counter > 100000000) + exit(1); +} + +static void +test3(ucontext_t *ucp, void *arg) +{ + for (;;) { + swapcontext_quick(ucp, ucp->uc_link); + } +} +.Ed + +.Sh SEE ALSO +.Xr getcontext 3 , +.Xr setcontext 3 , +.Xr makecontext 3 , +.Xr swapcontext 3 , +.Xr ucontext 3 diff --git a/lib/libc/gen/ucontext.3 b/lib/libc/gen/ucontext.3 index a325965d87..532cd2de5c 100644 --- a/lib/libc/gen/ucontext.3 +++ b/lib/libc/gen/ucontext.3 @@ -101,9 +101,21 @@ structures: .It .Ft int .Fn swapcontext "ucontext_t *" "const ucontext_t *" ; +.It +.Ft void +.Fn makecontext_quick "ucontext_t *" ; +.It +.Ft void +.Fn setcontext_quick "ucontext_t *" ; +.It +.Ft void +.Fn swapcontext_quick "ucontext_t *" "ucontex_t *" ; .El .Sh SEE ALSO .Xr sigaltstack 2 , .Xr getcontext 3 , .Xr makecontext 3 , -.Xr setcontext 3 +.Xr setcontext 3 , +.Xr makecontext_quick 3 , +.Xr setcontext_quick 3 , +.Xr swapcontext_quick 3 diff --git a/lib/libc/gen/ucontext.c b/lib/libc/gen/ucontext.c index e060c7c445..291d03c1fb 100644 --- a/lib/libc/gen/ucontext.c +++ b/lib/libc/gen/ucontext.c @@ -40,9 +40,6 @@ __weak_reference(_swapcontext, swapcontext); __weak_reference(_setcontext, setcontext); -int get_mcontext(mcontext_t *); -int set_mcontext(const mcontext_t *); - /* * We need to block most signals during a context switch so we do not * dispatch a signal vector during a context switch. @@ -64,14 +61,6 @@ __sigset_block_all_setup(void) /* * Save the calling context in (oucp) then switch to (ucp). - * - * Block all signals while switching contexts. get_mcontext() returns zero - * when retrieving a context. - * - * When some other thread calls set_mcontext() to resume our thread, - * the resume point causes get_mcontext() to return non-zero to us. - * Signals will be blocked and we must restore the signal mask before - * returning. */ int _swapcontext(ucontext_t *oucp, const ucontext_t *ucp) @@ -101,10 +90,6 @@ _setcontext(ucontext_t *ucp) int ret; ret = sigreturn(ucp); -#if 0 - ret = _sigprocmask(SIG_BLOCK, &sigset_block_all, &ucp->uc_sigmask); - if (ret == 0) - ret = set_mcontext(&ucp->uc_mcontext); -#endif + return(ret); } diff --git a/lib/libc/x86_64/Symbol.map b/lib/libc/x86_64/Symbol.map index f5fccccea9..5f3722cfc4 100644 --- a/lib/libc/x86_64/Symbol.map +++ b/lib/libc/x86_64/Symbol.map @@ -2,7 +2,6 @@ DF404.0 { _setjmp; _longjmp; .mcount; - get_mcontext; getcontext; fabs; fpgetmask; @@ -13,10 +12,12 @@ DF404.0 { fpsetprec; fpsetround; makecontext; + makecontext_quick; + setcontext_quick; + swapcontext_quick; rfork_thread; setjmp; longjmp; - set_mcontext; sigsetjmp; siglongjmp; amd64_get_fsbase; @@ -42,10 +43,11 @@ DFprivate_1.0 { _DYNAMIC; _brk; _end; - _get_mcontext; _getcontext; _makecontext; - _set_mcontext; + _makecontext_quick; + _setcontext_quick; + _swapcontext_quick; .curbrk; .minbrk; .cerror; diff --git a/lib/libc/x86_64/gen/Makefile.inc b/lib/libc/x86_64/gen/Makefile.inc index bb550df7bd..ed89f90f61 100644 --- a/lib/libc/x86_64/gen/Makefile.inc +++ b/lib/libc/x86_64/gen/Makefile.inc @@ -3,6 +3,7 @@ SRCS+= _setjmp.S rfork_thread.S setjmp.S sigsetjmp.S \ mcontext.S makecontext.c nan.c \ + qcontext.S quickcontext.c \ fabs.S flt_rounds.c fpgetmask.c fpgetprec.c fpgetround.c \ fpgetsticky.c fpsetmask.c fpsetprec.c fpsetround.c infinity.c diff --git a/lib/libc/x86_64/gen/mcontext.S b/lib/libc/x86_64/gen/qcontext.S similarity index 53% rename from lib/libc/x86_64/gen/mcontext.S rename to lib/libc/x86_64/gen/qcontext.S index f528797972..7a76b34424 100644 --- a/lib/libc/x86_64/gen/mcontext.S +++ b/lib/libc/x86_64/gen/qcontext.S @@ -1,6 +1,4 @@ /* - * Copyright (c) 2007 Matthew T. Emmerton - * All rights reserved. * Copyright (c) 2007 Matthew Dillon * All rights reserved. * @@ -25,130 +23,110 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ - +/* + * swapcontext_quick() and setcontext_quick(). There is *NO* + * getcontext_quick() routine on purpose. + * + * Quick routines are not required to save or resetore scratch registers, + * FP regs (which are scratch), or flags. + */ #include #include #include /* - * int get_mcontext(mcontext_t *mcp : rdi) + * void setcontext_quick(ucontext_t *ucp) * - * Copy the caller's context into the mcontext, %rax excepted. + * Load the register context, effectively switching to the + * new context. */ - .weak get_mcontext - .set get_mcontext,_get_mcontext -ENTRY(_get_mcontext) - movq $0,MC_ONSTACK(%rdi) /* MC_ONSTACK(%rdi) */ - movq %rdi,MC_RDI(%rdi) - movq %rsi,MC_RSI(%rdi) - movq %rdx,MC_RDX(%rdi) - movq %r8,MC_R8(%rdi) - movq %r9,MC_R9(%rdi) - /* movq %rax,MC_RAX(%rdi) - not needed, replaced below */ - movq %rbx,MC_RBX(%rdi) - /* movq %rcx,MC_RCX(%rdi) - not needed, scratch */ - movq %rbp,MC_RBP(%rdi) - movq %r10,MC_R10(%rdi) - movq %r11,MC_R11(%rdi) - movq %r12,MC_R12(%rdi) - movq %r13,MC_R13(%rdi) - movq %r14,MC_R14(%rdi) - movq %r15,MC_R15(%rdi) + .weak setcontext_quick + .set setcontext_quick,_setcontext_quick +ENTRY(_setcontext_quick) + addq $UC_MCONTEXT,%rdi + + /* MC_ONSTACK(%rdi) */ + /* MC_RDI(%rdi) - see below */ + movq MC_RSI(%rdi),%rsi + movq MC_RDX(%rdi),%rdx + movq MC_R8(%rdi),%r8 + movq MC_R9(%rdi),%r9 + /* MC_RAX(%rdi) - see below */ + movq MC_RBX(%rdi),%rbx + movq MC_RCX(%rdi),%rcx + movq MC_RBP(%rdi),%rbp + movq MC_R10(%rdi),%r10 + movq MC_R11(%rdi),%r11 + movq MC_R12(%rdi),%r12 + movq MC_R13(%rdi),%r13 + movq MC_R14(%rdi),%r14 + movq MC_R15(%rdi),%r15 /* MC_TRAPNO(%rdi) */ /* MC_ADDR(%rdi) */ - /* MC_FLAGS(%rdi) */ + /* MC_FLAGS(%rdi) */ /* MC_ERR(%rdi) */ /* MC_RIP(%rdi) - see below */ /* MC_CS(%rdi) */ - movq $KUCSEL,MC_CS(%rdi) - - pushfq - popq MC_RFLAGS(%rdi) + /* MC_RFLAGS(%rdi) */ /* MC_RSP(%rdi) - see below */ /* MC_SS(%rdi) */ /* - * FP registers are scratch, do not save or restore, but make - * sure the context can be restored by a signal handler (where - * they might not be) by properly initializing the FP control - * fields. - */ - movl $_MC_FPOWNED_NONE,MC_OWNEDFP(%rdi) - movl $_MC_FPFMT_NODEV,MC_FPFORMAT(%rdi) - /* - movl $_MC_FPFMT_YMM,MC_FPFORMAT(%rdi) - */ - movq %rdi,%rsi - movq $CPU_XFEATURE_X87 | CPU_XFEATURE_SSE | CPU_XFEATURE_YMM,%rax - movq $0,%rdx - xsave MC_FPREGS(%rsi) - movq %rsi,%rdi - - /* - * Saved stack pointer as if we had returned from this - * procedure. + * Load the new stack pointer */ - movq %rsp,MC_RSP(%rdi) - addq $8,MC_RSP(%rdi) + movq MC_RSP(%rdi),%rsp /* - * Save rflags - * XXX + * Use %rax to hold the return pc. function returns void. Do + * not mess with the target stack (particularly not the red-zone!). */ + movq MC_RIP(%rdi),%rax /* - * Saved instruction pointer as if we had returned from - * this procedure. + * Finally rdi */ - movq (%rsp),%rdx - movq %rdx,MC_RIP(%rdi) + movq MC_RDI(%rdi),%rdi + jmp *%rax +END(_setcontext_quick) - /* - * On restore as if procedure returned the value 1 - */ - movq $1,MC_RAX(%rdi) /* - * Set MC_LEN + * void swapcontext_quick(ucontex_t *oucp, ucontext_t *nucp) + * + * Save the current state in oucp and switch to nucp. */ - movl $SIZEOF_MCONTEXT_T,MC_LEN(%rdi) - + .weak swapcontext_quick + .set swapcontext_quick,_swapcontext_quick +ENTRY(_swapcontext_quick) /* - * Return 0 + * Save */ - subq %rax,%rax - ret -END(_get_mcontext) + addq $UC_MCONTEXT,%rdi + popq %rax /* return pc */ + movq %rsi,MC_RSI(%rdi) + movq %rbx,MC_RBX(%rdi) + movq %rdx,MC_RDX(%rdi) + movq %rbp,MC_RBP(%rdi) + movq %r10,MC_R10(%rdi) + movq %r11,MC_R11(%rdi) + movq %r12,MC_R12(%rdi) + movq %r13,MC_R13(%rdi) + movq %r14,MC_R14(%rdi) + movq %r15,MC_R15(%rdi) + movq %rsp,MC_RSP(%rdi) + movq %rax,MC_RIP(%rdi) /* - * int set_mcontext(mcontext_t *mcp) - * - * Load the register context, effectively switching to the - * new context. + * Restore (copy of setcontext above) */ - .weak set_mcontext - .set set_mcontext,_set_mcontext -ENTRY(_set_mcontext) - /* - * Restore FP regs (might be needed if the context was - * taken out of a signal context). - */ - cmpl $_MC_FPFMT_NODEV,MC_FPFORMAT(%rdi) - je 1f - movq %rdi,%rsi - movq $CPU_XFEATURE_X87 | CPU_XFEATURE_SSE | CPU_XFEATURE_YMM,%rax - movq $0,%rdx - xrstor MC_FPREGS(%rsi) /* hack assumes cpu supports this */ movq %rsi,%rdi -1: + addq $UC_MCONTEXT,%rdi /* MC_ONSTACK(%rdi) */ /* MC_RDI(%rdi) - see below */ movq MC_RSI(%rdi),%rsi movq MC_RDX(%rdi),%rdx - movq MC_R8(%rdi),%r8 - movq MC_R9(%rdi),%r9 /* MC_RAX(%rdi) - see below */ movq MC_RBX(%rdi),%rbx movq MC_RCX(%rdi),%rcx @@ -165,7 +143,7 @@ ENTRY(_set_mcontext) /* MC_ERR(%rdi) */ /* MC_RIP(%rdi) - see below */ /* MC_CS(%rdi) */ - /* MC_RFLAGS(%rdi) - see below */ + /* MC_RFLAGS(%rdi) */ /* MC_RSP(%rdi) - see below */ /* MC_SS(%rdi) */ @@ -175,25 +153,17 @@ ENTRY(_set_mcontext) movq MC_RSP(%rdi),%rsp /* - * Push the return pc so we can 'ret' to it. - * - * Push the flags so we can restore them just prior to the retq - * (in case setcontext is called - * - * - * XXX this can cream stuff under the stack pointer, which should - * normally be ok but its a side effect I'd rather not have. + * Use %rax to hold the return pc. function returns void. Do + * not mess with the target stack (particularly not the red-zone!). */ - pushq MC_RIP(%rdi) - pushq MC_RFLAGS(%rdi) + movq MC_RIP(%rdi),%rax /* - * Finally rax and rdi + * Finally rdi */ - movq MC_RAX(%rdi),%rax movq MC_RDI(%rdi),%rdi - popfq - retq -END(_set_mcontext) + jmp *%rax + +END(_swapcontext_quick) .section .note.GNU-stack,"",%progbits diff --git a/lib/libc/x86_64/gen/quickcontext.c b/lib/libc/x86_64/gen/quickcontext.c new file mode 100644 index 0000000000..21252b2842 --- /dev/null +++ b/lib/libc/x86_64/gen/quickcontext.c @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2015 Matthew Dillon, All rights reserved. + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +/* Prototypes */ + +static void makectx_quick_wrapper(ucontext_t *ucp, uint64_t *stack_top); + +__weak_reference(_makecontext_quick, makecontext_quick); + +/* + * makecontext_quick() associates a stack with a user thread context + * setup to execute a cofunc sequence. The caller only initializes the + * uc_stack.* fields, uc_cofunc, and uc_arg. This function will zero or + * initialize all other fields. Upon return the caller can optionally + * also initialize uc_link. + * + * These 'quick' calls do not mess with the signal mask and do not require + * kernel intervention. Scratch registers (including FP regs, which are also + * scratch registers) are not saved or restored. Cofunction loops also + * optimize cofunc call loops by not saving the register state when + * switching away to double performance. Of course, swapcontext_quick() + * still saves the register state. There is no getcontext_quick() call + * on purpose. + */ +void +_makecontext_quick(ucontext_t *ucp) +{ + uint64_t *stack_top; + + if (ucp == NULL) + return; + bzero(&ucp->uc_sigmask, sizeof(ucp->uc_sigmask)); + bzero(&ucp->uc_mcontext, sizeof(ucp->uc_mcontext)); + ucp->uc_link = NULL; + ucp->uc_mcontext.mc_len = sizeof(mcontext_t); + + stack_top = (uint64_t *)(ucp->uc_stack.ss_sp + ucp->uc_stack.ss_size); + stack_top = (uint64_t *)((uint64_t)(stack_top) & ~63UL); + stack_top -= 1; + + /* + * Set the machine context to point to the top of the + * stack and the program counter to the context start + * wrapper. Note that setcontext() pushes the return + * address onto the top of the stack, so allow for this + * by adjusting the stack downward 1 slot. Also set + * %rbp to point to the base of the stack where ucp + * is stored. + */ + ucp->uc_mcontext.mc_rdi = (register_t)ucp; + ucp->uc_mcontext.mc_rsi = (register_t)stack_top; + ucp->uc_mcontext.mc_rsp = (register_t)stack_top; + ucp->uc_mcontext.mc_rip = (register_t)makectx_quick_wrapper; + ucp->uc_mcontext.mc_ownedfp = _MC_FPOWNED_NONE; + ucp->uc_mcontext.mc_fpformat = _MC_FPFMT_NODEV; + ucp->uc_mcontext.mc_cs = GSEL(GUCODE_SEL, SEL_UPL); + ucp->uc_mcontext.mc_ss = GSEL(GUDATA_SEL, SEL_UPL); +} + +/* + * If the cofunc call returns set the context up to re-execute the + * wrapper if the linkages eventually link back to this ucp. The + * cofunc can also change uc_cofunc and uc_arg as it desires, allowing + * cofunctions to be optimally linked together. + */ +static void +makectx_quick_wrapper(ucontext_t *ucp, uint64_t *stack_top) +{ + for (;;) { + ucp->uc_cofunc(ucp, ucp->uc_arg); + if (ucp->uc_link == ucp) + continue; + ucp->uc_mcontext.mc_rdi = (register_t)ucp; + ucp->uc_mcontext.mc_rsi = (register_t)stack_top; + ucp->uc_mcontext.mc_rsp = (register_t)stack_top; + ucp->uc_mcontext.mc_rip = (register_t)makectx_quick_wrapper; + if (ucp->uc_link) + setcontext_quick(ucp->uc_link); + exit(0); + } +} diff --git a/sys/sys/ucontext.h b/sys/sys/ucontext.h index 84af083791..077438778d 100644 --- a/sys/sys/ucontext.h +++ b/sys/sys/ucontext.h @@ -53,7 +53,9 @@ typedef struct __ucontext { struct __ucontext *uc_link; stack_t uc_stack; - int __spare__[8]; + void (*uc_cofunc)(struct __ucontext *, void *); + void *uc_arg; + int __spare__[4]; } ucontext_t; #ifndef _KERNEL @@ -65,6 +67,9 @@ int getcontext(ucontext_t *) __returns_twice; int setcontext(const ucontext_t *) __dead2; void makecontext(ucontext_t *, void (*)(void), int, ...); int swapcontext(ucontext_t *, const ucontext_t *); +void setcontext_quick(ucontext_t *); +void makecontext_quick(ucontext_t *); +void swapcontext_quick(ucontext_t *, ucontext_t *); #endif __END_DECLS