4 * Implements inline procedure support for the LWKT subsystem.
6 * Generally speaking these routines only operate on threads associated
7 * with the current cpu. For example, a higher priority thread pending
8 * on a different cpu will not be immediately scheduled by a yield() on
12 #ifndef _SYS_THREAD2_H_
13 #define _SYS_THREAD2_H_
16 #error "This file should not be included by userland programs."
20 * Userland will have its own globaldata which it includes prior to this.
23 #include <sys/systm.h>
25 #ifndef _SYS_GLOBALDATA_H_
26 #include <sys/globaldata.h>
28 #include <machine/cpufunc.h>
29 #include <machine/cpumask.h>
32 * Don't let GCC reorder critical section count adjustments, because it
33 * will BLOW US UP if it does.
36 crit_enter_raw(thread_t td)
44 crit_exit_raw(thread_t td)
52 * Is a token held either by the specified thread or held shared?
54 * We can't inexpensively validate the thread for a shared token
55 * without iterating td->td_toks, so this isn't a perfect test.
58 _lwkt_token_held_any(lwkt_token_t tok, thread_t td)
60 long count = tok->t_count;
63 if (tok->t_ref >= &td->td_toks_base && tok->t_ref < td->td_toks_stop)
65 if ((count & TOK_EXCLUSIVE) == 0 &&
66 (count & ~(TOK_EXCLUSIVE|TOK_EXCLREQ))) {
73 * Is a token held by the specified thread?
76 _lwkt_token_held_excl(lwkt_token_t tok, thread_t td)
78 return ((tok->t_ref >= &td->td_toks_base &&
79 tok->t_ref < td->td_toks_stop));
83 * Critical section debugging
85 #ifdef DEBUG_CRIT_SECTIONS
86 #define __DEBUG_CRIT_ARG__ const char *id
87 #define __DEBUG_CRIT_ADD_ARG__ , const char *id
88 #define __DEBUG_CRIT_PASS_ARG__ , id
89 #define __DEBUG_CRIT_ENTER(td) _debug_crit_enter((td), id)
90 #define __DEBUG_CRIT_EXIT(td) _debug_crit_exit((td), id)
91 #define crit_enter() _crit_enter(mycpu, __func__)
92 #define crit_enter_id(id) _crit_enter(mycpu, id)
93 #define crit_enter_gd(curgd) _crit_enter((curgd), __func__)
94 #define crit_enter_quick(curtd) _crit_enter_quick((curtd), __func__)
95 #define crit_enter_hard() _crit_enter_hard(mycpu, __func__)
96 #define crit_enter_hard_gd(curgd) _crit_enter_hard((curgd), __func__)
97 #define crit_exit() _crit_exit(mycpu, __func__)
98 #define crit_exit_id(id) _crit_exit(mycpu, id)
99 #define crit_exit_gd(curgd) _crit_exit((curgd), __func__)
100 #define crit_exit_quick(curtd) _crit_exit_quick((curtd), __func__)
101 #define crit_exit_hard() _crit_exit_hard(mycpu, __func__)
102 #define crit_exit_hard_gd(curgd) _crit_exit_hard((curgd), __func__)
103 #define crit_exit_noyield(curtd) _crit_exit_noyield((curtd),__func__)
105 #define __DEBUG_CRIT_ARG__ void
106 #define __DEBUG_CRIT_ADD_ARG__
107 #define __DEBUG_CRIT_PASS_ARG__
108 #define __DEBUG_CRIT_ENTER(td)
109 #define __DEBUG_CRIT_EXIT(td)
110 #define crit_enter() _crit_enter(mycpu)
111 #define crit_enter_id(id) _crit_enter(mycpu)
112 #define crit_enter_gd(curgd) _crit_enter((curgd))
113 #define crit_enter_quick(curtd) _crit_enter_quick((curtd))
114 #define crit_enter_hard() _crit_enter_hard(mycpu)
115 #define crit_enter_hard_gd(curgd) _crit_enter_hard((curgd))
116 #define crit_exit() crit_exit_wrapper()
117 #define crit_exit_id(id) _crit_exit(mycpu)
118 #define crit_exit_gd(curgd) _crit_exit((curgd))
119 #define crit_exit_quick(curtd) _crit_exit_quick((curtd))
120 #define crit_exit_hard() _crit_exit_hard(mycpu)
121 #define crit_exit_hard_gd(curgd) _crit_exit_hard((curgd))
122 #define crit_exit_noyield(curtd) _crit_exit_noyield((curtd))
125 extern void crit_exit_wrapper(__DEBUG_CRIT_ARG__);
128 * Track crit_enter()/crit_exit() pairs and warn on mismatches.
130 #ifdef DEBUG_CRIT_SECTIONS
133 _debug_crit_enter(thread_t td, const char *id)
135 int wi = td->td_crit_debug_index;
137 td->td_crit_debug_array[wi & CRIT_DEBUG_ARRAY_MASK] = id;
138 ++td->td_crit_debug_index;
142 _debug_crit_exit(thread_t td, const char *id)
147 wi = td->td_crit_debug_index - 1;
148 if ((gid = td->td_crit_debug_array[wi & CRIT_DEBUG_ARRAY_MASK]) != id) {
149 if (td->td_in_crit_report == 0) {
150 td->td_in_crit_report = 1;
151 kprintf("crit_exit(%s) expected id %s\n", id, gid);
152 td->td_in_crit_report = 0;
155 --td->td_crit_debug_index;
161 * Critical sections prevent preemption, but allowing explicit blocking
162 * and thread switching. Any interrupt occuring while in a critical
163 * section is made pending and returns immediately. Interrupts are not
164 * physically disabled.
166 * Hard critical sections prevent preemption and disallow any blocking
167 * or thread switching, and in addition will assert on any blockable
168 * operation (acquire token not already held, lockmgr, mutex ops, or
169 * splz). Spinlocks can still be used in hard sections.
171 * All critical section routines only operate on the current thread.
172 * Passed gd or td arguments are simply optimizations when mycpu or
173 * curthread is already available to the caller.
180 _crit_enter_quick(thread_t td __DEBUG_CRIT_ADD_ARG__)
183 __DEBUG_CRIT_ENTER(td);
187 _crit_enter(globaldata_t gd __DEBUG_CRIT_ADD_ARG__)
189 _crit_enter_quick(gd->gd_curthread __DEBUG_CRIT_PASS_ARG__);
193 _crit_enter_hard(globaldata_t gd __DEBUG_CRIT_ADD_ARG__)
195 _crit_enter_quick(gd->gd_curthread __DEBUG_CRIT_PASS_ARG__);
196 ++gd->gd_intr_nesting_level;
204 * NOTE: Conditionalizing just gd_reqflags, a case which is virtually
205 * never true regardless of crit_count, should result in 100%
206 * optimal code execution. We don't check crit_count because
207 * it just bloats the inline and does not improve performance.
209 * NOTE: This can produce a considerable amount of code despite the
210 * relatively few lines of code so the non-debug case typically
211 * just wraps it in a real function, crit_exit_wrapper().
214 _crit_exit_noyield(thread_t td __DEBUG_CRIT_ADD_ARG__)
216 __DEBUG_CRIT_EXIT(td);
219 if (__predict_false(td->td_critcount < 0))
225 _crit_exit_quick(thread_t td __DEBUG_CRIT_ADD_ARG__)
227 _crit_exit_noyield(td __DEBUG_CRIT_PASS_ARG__);
228 if (__predict_false(td->td_gd->gd_reqflags & RQF_IDLECHECK_MASK))
233 _crit_exit(globaldata_t gd __DEBUG_CRIT_ADD_ARG__)
235 _crit_exit_quick(gd->gd_curthread __DEBUG_CRIT_PASS_ARG__);
239 _crit_exit_hard(globaldata_t gd __DEBUG_CRIT_ADD_ARG__)
242 --gd->gd_intr_nesting_level;
243 _crit_exit_quick(gd->gd_curthread __DEBUG_CRIT_PASS_ARG__);
247 crit_test(thread_t td)
249 return(td->td_critcount);
253 * Return whether any threads are runnable.
258 return (TAILQ_FIRST(&mycpu->gd_tdrunq) != NULL);
262 lwkt_getpri(thread_t td)
268 lwkt_getpri_self(void)
270 return(lwkt_getpri(curthread));
274 * Reduce our priority in preparation for a return to userland. If
275 * our passive release function was still in place, our priority was
276 * never raised and does not need to be reduced.
278 * See also lwkt_passive_release() and platform/blah/trap.c
281 lwkt_passive_recover(thread_t td)
283 #ifndef NO_LWKT_SPLIT_USERPRI
284 if (td->td_release == NULL)
285 lwkt_setpri_self(TDPRI_USER_NORM);
286 td->td_release = NULL;
294 lwkt_cpusync_init(lwkt_cpusync_t cs, cpumask_t mask,
295 cpusync_func_t func, void *data)
298 /* cs->cs_mack = 0; handled by _interlock */
304 * IPIQ messaging wrappers. IPIQ remote functions are passed three arguments:
305 * a void * pointer, an integer, and a pointer to the trap frame (or NULL if
306 * the trap frame is not known). However, we wish to provide opaque
307 * interfaces for simpler callbacks... the basic IPI messaging function as
308 * used by the kernel takes a single argument.
311 lwkt_send_ipiq(globaldata_t target, ipifunc1_t func, void *arg)
313 return(lwkt_send_ipiq3(target, (ipifunc3_t)func, arg, 0));
317 lwkt_send_ipiq2(globaldata_t target, ipifunc2_t func, void *arg1, int arg2)
319 return(lwkt_send_ipiq3(target, (ipifunc3_t)func, arg1, arg2));
323 lwkt_send_ipiq_mask(cpumask_t mask, ipifunc1_t func, void *arg)
325 return(lwkt_send_ipiq3_mask(mask, (ipifunc3_t)func, arg, 0));
329 lwkt_send_ipiq2_mask(cpumask_t mask, ipifunc2_t func, void *arg1, int arg2)
331 return(lwkt_send_ipiq3_mask(mask, (ipifunc3_t)func, arg1, arg2));
335 lwkt_send_ipiq_passive(globaldata_t target, ipifunc1_t func, void *arg)
337 return(lwkt_send_ipiq3_passive(target, (ipifunc3_t)func, arg, 0));
341 lwkt_send_ipiq2_passive(globaldata_t target, ipifunc2_t func,
342 void *arg1, int arg2)
344 return(lwkt_send_ipiq3_passive(target, (ipifunc3_t)func, arg1, arg2));
348 lwkt_send_ipiq_bycpu(int dcpu, ipifunc1_t func, void *arg)
350 return(lwkt_send_ipiq3_bycpu(dcpu, (ipifunc3_t)func, arg, 0));
354 lwkt_send_ipiq2_bycpu(int dcpu, ipifunc2_t func, void *arg1, int arg2)
356 return(lwkt_send_ipiq3_bycpu(dcpu, (ipifunc3_t)func, arg1, arg2));
360 lwkt_need_ipiq_process(globaldata_t gd)
364 if (CPUMASK_TESTNZERO(gd->gd_ipimask))
367 ipiq = &gd->gd_cpusyncq;
368 return (ipiq->ip_rindex != ipiq->ip_windex);
371 #endif /* _SYS_THREAD2_H_ */