4 * Implements inline procedure support for the LWKT subsystem.
6 * Generally speaking these routines only operate on threads associated
7 * with the current cpu. For example, a higher priority thread pending
8 * on a different cpu will not be immediately scheduled by a yield() on
11 * $DragonFly: src/sys/sys/thread2.h,v 1.28 2006/12/23 00:27:03 swildner Exp $
14 #ifndef _SYS_THREAD2_H_
15 #define _SYS_THREAD2_H_
19 #error "This file should not be included by userland programs."
24 * Userland will have its own globaldata which it includes prior to this.
27 #include <sys/systm.h>
29 #ifndef _SYS_GLOBALDATA_H_
30 #include <sys/globaldata.h>
32 #ifndef _MACHINE_CPUFUNC_H_
33 #include <machine/cpufunc.h>
37 * Critical section debugging
39 #ifdef DEBUG_CRIT_SECTIONS
40 #define __DEBUG_CRIT_ARG__ const char *id
41 #define __DEBUG_CRIT_ADD_ARG__ , const char *id
42 #define __DEBUG_CRIT_PASS_ARG__ , id
43 #define __DEBUG_CRIT_ENTER(td) _debug_crit_enter((td), id)
44 #define __DEBUG_CRIT_EXIT(td) _debug_crit_exit((td), id)
45 #define crit_enter() _crit_enter(mycpu, __FUNCTION__)
46 #define crit_enter_id(id) _crit_enter(mycpu, id)
47 #define crit_enter_gd(curgd) _crit_enter((curgd), __FUNCTION__)
48 #define crit_enter_quick(curtd) _crit_enter_quick((curtd), __FUNCTION__)
49 #define crit_enter_hard() _crit_enter_hard(mycpu, __FUNCTION__)
50 #define crit_enter_hard_gd(curgd) _crit_enter_hard((curgd), __FUNCTION__)
51 #define crit_exit() _crit_exit(mycpu, __FUNCTION__)
52 #define crit_exit_id(id) _crit_exit(mycpu, id)
53 #define crit_exit_gd(curgd) _crit_exit((curgd), __FUNCTION__)
54 #define crit_exit_quick(curtd) _crit_exit_quick((curtd), __FUNCTION__)
55 #define crit_exit_hard() _crit_exit_hard(mycpu, __FUNCTION__)
56 #define crit_exit_hard_gd(curgd) _crit_exit_hard((curgd), __FUNCTION__)
57 #define crit_exit_noyield(curtd) _crit_exit_noyield((curtd),__FUNCTION__)
59 #define __DEBUG_CRIT_ARG__ void
60 #define __DEBUG_CRIT_ADD_ARG__
61 #define __DEBUG_CRIT_PASS_ARG__
62 #define __DEBUG_CRIT_ENTER(td)
63 #define __DEBUG_CRIT_EXIT(td)
64 #define crit_enter() _crit_enter(mycpu)
65 #define crit_enter_id(id) _crit_enter(mycpu)
66 #define crit_enter_gd(curgd) _crit_enter((curgd))
67 #define crit_enter_quick(curtd) _crit_enter_quick((curtd))
68 #define crit_enter_hard() _crit_enter_hard(mycpu)
69 #define crit_enter_hard_gd(curgd) _crit_enter_hard((curgd))
70 #define crit_exit() crit_exit_wrapper()
71 #define crit_exit_id(id) _crit_exit(mycpu)
72 #define crit_exit_gd(curgd) _crit_exit((curgd))
73 #define crit_exit_quick(curtd) _crit_exit_quick((curtd))
74 #define crit_exit_hard() _crit_exit_hard(mycpu)
75 #define crit_exit_hard_gd(curgd) _crit_exit_hard((curgd))
76 #define crit_exit_noyield(curtd) _crit_exit_noyield((curtd))
79 extern void crit_exit_wrapper(__DEBUG_CRIT_ARG__);
82 * Track crit_enter()/crit_exit() pairs and warn on mismatches.
84 #ifdef DEBUG_CRIT_SECTIONS
87 _debug_crit_enter(thread_t td, const char *id)
89 int wi = td->td_crit_debug_index;
91 td->td_crit_debug_array[wi & CRIT_DEBUG_ARRAY_MASK] = id;
92 ++td->td_crit_debug_index;
96 _debug_crit_exit(thread_t td, const char *id)
101 wi = td->td_crit_debug_index - 1;
102 if ((gid = td->td_crit_debug_array[wi & CRIT_DEBUG_ARRAY_MASK]) != id) {
103 if (td->td_in_crit_report == 0) {
104 td->td_in_crit_report = 1;
105 kprintf("crit_exit(%s) expected id %s\n", id, gid);
106 td->td_in_crit_report = 0;
109 --td->td_crit_debug_index;
115 * Critical sections prevent preemption, but allowing explicit blocking
116 * and thread switching. Any interrupt occuring while in a critical
117 * section is made pending and returns immediately. Interrupts are not
118 * physically disabled.
120 * Hard critical sections prevent preemption and disallow any blocking
121 * or thread switching, and in addition will assert on any blockable
122 * operation (acquire token not already held, lockmgr, mutex ops, or
123 * splz). Spinlocks can still be used in hard sections.
125 * All critical section routines only operate on the current thread.
126 * Passed gd or td arguments are simply optimizations when mycpu or
127 * curthread is already available to the caller.
134 _crit_enter_quick(thread_t td __DEBUG_CRIT_ADD_ARG__)
137 __DEBUG_CRIT_ENTER(td);
142 _crit_enter(globaldata_t gd __DEBUG_CRIT_ADD_ARG__)
144 _crit_enter_quick(gd->gd_curthread __DEBUG_CRIT_PASS_ARG__);
148 _crit_enter_hard(globaldata_t gd __DEBUG_CRIT_ADD_ARG__)
150 _crit_enter_quick(gd->gd_curthread __DEBUG_CRIT_PASS_ARG__);
151 ++gd->gd_intr_nesting_level;
158 * NOTE: Conditionalizing just gd_reqflags, a case which is virtually
159 * never true regardless of crit_count, should result in 100%
160 * optimal code execution. We don't check crit_count because
161 * it just bloats the inline and does not improve performance.
163 * NOTE: This can produce a considerable amount of code despite the
164 * relatively few lines of code so the non-debug case typically
165 * just wraps it in a real function, crit_exit_wrapper().
168 _crit_exit_noyield(thread_t td __DEBUG_CRIT_ADD_ARG__)
170 __DEBUG_CRIT_EXIT(td);
173 if (__predict_false(td->td_critcount < 0))
176 cpu_ccfence(); /* prevent compiler reordering */
180 _crit_exit_quick(thread_t td __DEBUG_CRIT_ADD_ARG__)
182 _crit_exit_noyield(td __DEBUG_CRIT_PASS_ARG__);
183 if (__predict_false(td->td_gd->gd_reqflags & RQF_IDLECHECK_MASK))
188 _crit_exit(globaldata_t gd __DEBUG_CRIT_ADD_ARG__)
190 _crit_exit_quick(gd->gd_curthread __DEBUG_CRIT_PASS_ARG__);
194 _crit_exit_hard(globaldata_t gd __DEBUG_CRIT_ADD_ARG__)
196 --gd->gd_intr_nesting_level;
197 _crit_exit_quick(gd->gd_curthread __DEBUG_CRIT_PASS_ARG__);
201 crit_test(thread_t td)
203 return(td->td_critcount);
207 * Return whether any threads are runnable, whether they meet mp_lock
208 * requirements or not.
213 return (TAILQ_FIRST(&mycpu->gd_tdrunq) != NULL);
217 lwkt_getpri(thread_t td)
223 lwkt_getpri_self(void)
225 return(lwkt_getpri(curthread));
229 * Reduce our priority in preparation for a return to userland. If
230 * our passive release function was still in place, our priority was
231 * never raised and does not need to be reduced.
233 * See also lwkt_passive_release() and platform/blah/trap.c
236 lwkt_passive_recover(thread_t td)
238 if (td->td_release == NULL)
239 lwkt_setpri_self(TDPRI_USER_NORM);
240 td->td_release = NULL;
246 * IPIQ messaging wrappers. IPIQ remote functions are passed three arguments:
247 * a void * pointer, an integer, and a pointer to the trap frame (or NULL if
248 * the trap frame is not known). However, we wish to provide opaque
249 * interfaces for simpler callbacks... the basic IPI messaging function as
250 * used by the kernel takes a single argument.
253 lwkt_send_ipiq(globaldata_t target, ipifunc1_t func, void *arg)
255 return(lwkt_send_ipiq3(target, (ipifunc3_t)func, arg, 0));
259 lwkt_send_ipiq2(globaldata_t target, ipifunc2_t func, void *arg1, int arg2)
261 return(lwkt_send_ipiq3(target, (ipifunc3_t)func, arg1, arg2));
265 lwkt_send_ipiq_mask(u_int32_t mask, ipifunc1_t func, void *arg)
267 return(lwkt_send_ipiq3_mask(mask, (ipifunc3_t)func, arg, 0));
271 lwkt_send_ipiq2_mask(u_int32_t mask, ipifunc2_t func, void *arg1, int arg2)
273 return(lwkt_send_ipiq3_mask(mask, (ipifunc3_t)func, arg1, arg2));
277 lwkt_send_ipiq_nowait(globaldata_t target, ipifunc1_t func, void *arg)
279 return(lwkt_send_ipiq3_nowait(target, (ipifunc3_t)func, arg, 0));
283 lwkt_send_ipiq2_nowait(globaldata_t target, ipifunc2_t func,
284 void *arg1, int arg2)
286 return(lwkt_send_ipiq3_nowait(target, (ipifunc3_t)func, arg1, arg2));
290 lwkt_send_ipiq_passive(globaldata_t target, ipifunc1_t func, void *arg)
292 return(lwkt_send_ipiq3_passive(target, (ipifunc3_t)func, arg, 0));
296 lwkt_send_ipiq2_passive(globaldata_t target, ipifunc2_t func,
297 void *arg1, int arg2)
299 return(lwkt_send_ipiq3_passive(target, (ipifunc3_t)func, arg1, arg2));
303 lwkt_send_ipiq_bycpu(int dcpu, ipifunc1_t func, void *arg)
305 return(lwkt_send_ipiq3_bycpu(dcpu, (ipifunc3_t)func, arg, 0));
309 lwkt_send_ipiq2_bycpu(int dcpu, ipifunc2_t func, void *arg1, int arg2)
311 return(lwkt_send_ipiq3_bycpu(dcpu, (ipifunc3_t)func, arg1, arg2));
316 #endif /* _SYS_THREAD2_H_ */