kernel - lwkt_token revamp
[dragonfly.git] / sys / sys / thread2.h
CommitLineData
f1d1c3fa
MD
1/*
2 * SYS/THREAD2.H
3 *
02d8a449 4 * Implements inline procedure support for the LWKT subsystem.
f1d1c3fa 5 *
02d8a449
MD
6 * Generally speaking these routines only operate on threads associated
7 * with the current cpu. For example, a higher priority thread pending
8 * on a different cpu will not be immediately scheduled by a yield() on
9 * this cpu.
f1d1c3fa 10 *
26be20a0 11 * $DragonFly: src/sys/sys/thread2.h,v 1.28 2006/12/23 00:27:03 swildner Exp $
f1d1c3fa
MD
12 */
13
14#ifndef _SYS_THREAD2_H_
15#define _SYS_THREAD2_H_
16
03d6a592
MD
17#ifndef _KERNEL
18
19#error "This file should not be included by userland programs."
20
21#else
22
f1d1c3fa 23/*
05220613
MD
24 * Userland will have its own globaldata which it includes prior to this.
25 */
03d6a592
MD
26#ifndef _SYS_SYSTM_H_
27#include <sys/systm.h>
28#endif
05220613
MD
29#ifndef _SYS_GLOBALDATA_H_
30#include <sys/globaldata.h>
31#endif
853ae338
MD
32#ifndef _MACHINE_CPUFUNC_H_
33#include <machine/cpufunc.h>
34#endif
05220613
MD
35
36/*
02d8a449
MD
37 * Critical section debugging
38 */
39#ifdef DEBUG_CRIT_SECTIONS
40#define __DEBUG_CRIT_ARG__ const char *id
41#define __DEBUG_CRIT_ADD_ARG__ , const char *id
42#define __DEBUG_CRIT_PASS_ARG__ , id
43#define __DEBUG_CRIT_ENTER(td) _debug_crit_enter((td), id)
44#define __DEBUG_CRIT_EXIT(td) _debug_crit_exit((td), id)
45#define crit_enter() _crit_enter(__FUNCTION__)
46#define crit_enter_id(id) _crit_enter(id)
47#define crit_enter_quick(curtd) _crit_enter_quick((curtd), __FUNCTION__)
48#define crit_enter_gd(curgd) _crit_enter_gd(curgd, __FUNCTION__)
49#define crit_exit() _crit_exit(__FUNCTION__)
50#define crit_exit_id(id) _crit_exit(id)
51#define crit_exit_quick(curtd) _crit_exit_quick((curtd), __FUNCTION__)
52#define crit_exit_noyield(curtd) _crit_exit_noyield((curtd),__FUNCTION__)
53#define crit_exit_gd(curgd) _crit_exit_gd((curgd), __FUNCTION__)
54#else
55#define __DEBUG_CRIT_ARG__ void
56#define __DEBUG_CRIT_ADD_ARG__
57#define __DEBUG_CRIT_PASS_ARG__
58#define __DEBUG_CRIT_ENTER(td)
59#define __DEBUG_CRIT_EXIT(td)
60#define crit_enter() _crit_enter()
61#define crit_enter_id(id) _crit_enter()
62#define crit_enter_quick(curtd) _crit_enter_quick(curtd)
63#define crit_enter_gd(curgd) _crit_enter_gd(curgd)
64#define crit_exit() _crit_exit()
65#define crit_exit_id(id) _crit_exit()
66#define crit_exit_quick(curtd) _crit_exit_quick(curtd)
67#define crit_exit_noyield(curtd) _crit_exit_noyield(curtd)
68#define crit_exit_gd(curgd) _crit_exit_gd(curgd)
69#endif
70
71/*
72 * Track crit_enter()/crit_exit() pairs and warn on mismatches.
73 */
74#ifdef DEBUG_CRIT_SECTIONS
75
02d8a449
MD
76static __inline void
77_debug_crit_enter(thread_t td, const char *id)
78{
79 int wi = td->td_crit_debug_index;
80
81 td->td_crit_debug_array[wi & CRIT_DEBUG_ARRAY_MASK] = id;
82 ++td->td_crit_debug_index;
83}
84
85static __inline void
86_debug_crit_exit(thread_t td, const char *id)
87{
88 const char *gid;
89 int wi;
90
91 wi = td->td_crit_debug_index - 1;
92 if ((gid = td->td_crit_debug_array[wi & CRIT_DEBUG_ARRAY_MASK]) != id) {
93 if (td->td_in_crit_report == 0) {
94 td->td_in_crit_report = 1;
26be20a0 95 kprintf("crit_exit(%s) expected id %s\n", id, gid);
02d8a449
MD
96 td->td_in_crit_report = 0;
97 }
98 }
99 --td->td_crit_debug_index;
100}
101
102#endif
103
104/*
f1d1c3fa
MD
105 * Critical sections prevent preemption by raising a thread's priority
106 * above the highest possible interrupting priority. Additionally, the
107 * current cpu will not be able to schedule a new thread but will instead
108 * place it on a pending list (with interrupts physically disabled) and
235957ed 109 * set mycpu->gd_reqflags to indicate that work needs to be done, which
faaeffac 110 * splz_check() takes care of.
f1d1c3fa 111 *
7966cb69
MD
112 * Some of these routines take a struct thread pointer as an argument. This
113 * pointer MUST be curthread and is only passed as an optimization.
114 *
f1d1c3fa
MD
115 * Synchronous switching and blocking is allowed while in a critical section.
116 */
57c254db 117
f1d1c3fa 118static __inline void
02d8a449 119_crit_enter(__DEBUG_CRIT_ARG__)
f1d1c3fa 120{
26a0694b
MD
121 struct thread *td = curthread;
122
7966cb69 123#ifdef INVARIANTS
26a0694b
MD
124 if (td->td_pri < 0)
125 crit_panic();
7966cb69 126#endif
f1cebc1b 127 td->td_pri += TDPRI_CRIT;
02d8a449 128 __DEBUG_CRIT_ENTER(td);
9a35dbc3 129 cpu_ccfence();
f1d1c3fa
MD
130}
131
132static __inline void
02d8a449 133_crit_enter_quick(struct thread *curtd __DEBUG_CRIT_ADD_ARG__)
f1d1c3fa 134{
7966cb69 135 curtd->td_pri += TDPRI_CRIT;
02d8a449 136 __DEBUG_CRIT_ENTER(curtd);
9a35dbc3 137 cpu_ccfence();
7966cb69 138}
f1d1c3fa 139
7966cb69 140static __inline void
02d8a449 141_crit_enter_gd(globaldata_t mygd __DEBUG_CRIT_ADD_ARG__)
37af14fe 142{
02d8a449 143 _crit_enter_quick(mygd->gd_curthread __DEBUG_CRIT_PASS_ARG__);
37af14fe
MD
144}
145
146static __inline void
02d8a449 147_crit_exit_noyield(struct thread *curtd __DEBUG_CRIT_ADD_ARG__)
7966cb69 148{
02d8a449 149 __DEBUG_CRIT_EXIT(curtd);
7966cb69
MD
150 curtd->td_pri -= TDPRI_CRIT;
151#ifdef INVARIANTS
152 if (curtd->td_pri < 0)
26a0694b 153 crit_panic();
7966cb69 154#endif
9a35dbc3 155 cpu_ccfence(); /* prevent compiler reordering */
f1d1c3fa
MD
156}
157
158static __inline void
02d8a449 159_crit_exit(__DEBUG_CRIT_ARG__)
f1d1c3fa
MD
160{
161 thread_t td = curthread;
162
02d8a449 163 __DEBUG_CRIT_EXIT(td);
f1d1c3fa 164 td->td_pri -= TDPRI_CRIT;
7966cb69 165#ifdef INVARIANTS
26a0694b
MD
166 if (td->td_pri < 0)
167 crit_panic();
7966cb69 168#endif
35238fa5 169 cpu_ccfence(); /* prevent compiler reordering */
853ae338 170 if (td->td_gd->gd_reqflags && td->td_pri < TDPRI_CRIT)
faaeffac 171 splz_check();
7966cb69
MD
172}
173
174static __inline void
02d8a449 175_crit_exit_quick(struct thread *curtd __DEBUG_CRIT_ADD_ARG__)
7966cb69 176{
853ae338
MD
177 globaldata_t gd = curtd->td_gd;
178
02d8a449 179 __DEBUG_CRIT_EXIT(curtd);
7966cb69 180 curtd->td_pri -= TDPRI_CRIT;
35238fa5 181 cpu_ccfence(); /* prevent compiler reordering */
853ae338 182 if (gd->gd_reqflags && curtd->td_pri < TDPRI_CRIT)
faaeffac 183 splz_check();
f1d1c3fa
MD
184}
185
37af14fe 186static __inline void
02d8a449 187_crit_exit_gd(globaldata_t mygd __DEBUG_CRIT_ADD_ARG__)
37af14fe 188{
02d8a449 189 _crit_exit_quick(mygd->gd_curthread __DEBUG_CRIT_PASS_ARG__);
37af14fe
MD
190}
191
4b5f931b 192static __inline int
654a39f0
MD
193crit_test(thread_t td)
194{
195 return(td->td_pri >= TDPRI_CRIT);
196}
197
41a01a4d 198/*
96728c05
MD
199 * Return whether any threads are runnable, whether they meet mp_lock
200 * requirements or not.
201 */
f1d1c3fa 202static __inline int
4b5f931b 203lwkt_runnable(void)
f1d1c3fa 204{
4b5f931b 205 return (mycpu->gd_runqmask != 0);
f1d1c3fa
MD
206}
207
234d4a62
MD
208static __inline int
209lwkt_getpri(thread_t td)
210{
211 return(td->td_pri & TDPRI_MASK);
212}
213
214static __inline int
215lwkt_getpri_self(void)
216{
217 return(lwkt_getpri(curthread));
218}
219
3824f392
MD
220/*
221 * Reduce our priority in preparation for a return to userland. If
222 * our passive release function was still in place, our priority was
223 * never raised and does not need to be reduced.
224 *
225 * See also lwkt_passive_release() and platform/blah/trap.c
226 */
227static __inline void
228lwkt_passive_recover(thread_t td)
229{
230 if (td->td_release == NULL)
231 lwkt_setpri_self(TDPRI_USER_NORM);
232 td->td_release = NULL;
233}
234
b8a98473
MD
235#ifdef SMP
236
237/*
238 * IPIQ messaging wrappers. IPIQ remote functions are passed three arguments:
239 * a void * pointer, an integer, and a pointer to the trap frame (or NULL if
240 * the trap frame is not known). However, we wish to provide opaque
241 * interfaces for simpler callbacks... the basic IPI messaging function as
242 * used by the kernel takes a single argument.
243 */
244static __inline int
245lwkt_send_ipiq(globaldata_t target, ipifunc1_t func, void *arg)
246{
247 return(lwkt_send_ipiq3(target, (ipifunc3_t)func, arg, 0));
248}
249
250static __inline int
fc17ad60 251lwkt_send_ipiq2(globaldata_t target, ipifunc2_t func, void *arg1, int arg2)
b8a98473
MD
252{
253 return(lwkt_send_ipiq3(target, (ipifunc3_t)func, arg1, arg2));
254}
255
256static __inline int
257lwkt_send_ipiq_mask(u_int32_t mask, ipifunc1_t func, void *arg)
258{
259 return(lwkt_send_ipiq3_mask(mask, (ipifunc3_t)func, arg, 0));
260}
261
262static __inline int
fc17ad60 263lwkt_send_ipiq2_mask(u_int32_t mask, ipifunc2_t func, void *arg1, int arg2)
b8a98473
MD
264{
265 return(lwkt_send_ipiq3_mask(mask, (ipifunc3_t)func, arg1, arg2));
266}
267
268static __inline int
269lwkt_send_ipiq_nowait(globaldata_t target, ipifunc1_t func, void *arg)
270{
271 return(lwkt_send_ipiq3_nowait(target, (ipifunc3_t)func, arg, 0));
272}
273
274static __inline int
fc17ad60 275lwkt_send_ipiq2_nowait(globaldata_t target, ipifunc2_t func,
b8a98473
MD
276 void *arg1, int arg2)
277{
278 return(lwkt_send_ipiq3_nowait(target, (ipifunc3_t)func, arg1, arg2));
279}
280
281static __inline int
282lwkt_send_ipiq_passive(globaldata_t target, ipifunc1_t func, void *arg)
283{
284 return(lwkt_send_ipiq3_passive(target, (ipifunc3_t)func, arg, 0));
285}
286
287static __inline int
fc17ad60 288lwkt_send_ipiq2_passive(globaldata_t target, ipifunc2_t func,
b8a98473
MD
289 void *arg1, int arg2)
290{
291 return(lwkt_send_ipiq3_passive(target, (ipifunc3_t)func, arg1, arg2));
292}
293
294static __inline int
295lwkt_send_ipiq_bycpu(int dcpu, ipifunc1_t func, void *arg)
296{
297 return(lwkt_send_ipiq3_bycpu(dcpu, (ipifunc3_t)func, arg, 0));
298}
299
300static __inline int
301lwkt_send_ipiq2_bycpu(int dcpu, ipifunc2_t func, void *arg1, int arg2)
302{
303 return(lwkt_send_ipiq3_bycpu(dcpu, (ipifunc3_t)func, arg1, arg2));
304}
305
03d6a592
MD
306#endif /* SMP */
307#endif /* _KERNEL */
308#endif /* _SYS_THREAD2_H_ */
f1d1c3fa 309