Commit | Line | Data |
---|---|---|
c31b1324 | 1 | /* |
c6fbe95a | 2 | * Copyright (c) 2003,2004,2009 The DragonFly Project. All rights reserved. |
8c10bfcf MD |
3 | * |
4 | * This code is derived from software contributed to The DragonFly Project | |
5 | * by Matthew Dillon <dillon@backplane.com> | |
6 | * | |
c31b1324 MD |
7 | * Redistribution and use in source and binary forms, with or without |
8 | * modification, are permitted provided that the following conditions | |
9 | * are met: | |
8c10bfcf | 10 | * |
c31b1324 MD |
11 | * 1. Redistributions of source code must retain the above copyright |
12 | * notice, this list of conditions and the following disclaimer. | |
13 | * 2. Redistributions in binary form must reproduce the above copyright | |
8c10bfcf MD |
14 | * notice, this list of conditions and the following disclaimer in |
15 | * the documentation and/or other materials provided with the | |
16 | * distribution. | |
17 | * 3. Neither the name of The DragonFly Project nor the names of its | |
18 | * contributors may be used to endorse or promote products derived | |
19 | * from this software without specific, prior written permission. | |
20 | * | |
21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
22 | * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
23 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS | |
24 | * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE | |
25 | * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, | |
26 | * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, | |
27 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; | |
28 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED | |
29 | * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, | |
30 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT | |
31 | * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF | |
c31b1324 | 32 | * SUCH DAMAGE. |
c31b1324 MD |
33 | */ |
34 | ||
c6fbe95a MD |
35 | /* |
36 | * lwkt_token - Implement soft token locks. | |
37 | * | |
38 | * Tokens are locks which serialize a thread only while the thread is | |
39 | * running. If the thread blocks all tokens are released, then reacquired | |
40 | * when the thread resumes. | |
41 | * | |
42 | * This implementation requires no critical sections or spin locks, but | |
43 | * does use atomic_cmpset_ptr(). | |
44 | * | |
45 | * Tokens may be recursively acquired by the same thread. However the | |
46 | * caller must be sure to release such tokens in reverse order. | |
47 | */ | |
c31b1324 MD |
48 | #include <sys/param.h> |
49 | #include <sys/systm.h> | |
50 | #include <sys/kernel.h> | |
51 | #include <sys/proc.h> | |
52 | #include <sys/rtprio.h> | |
53 | #include <sys/queue.h> | |
c31b1324 | 54 | #include <sys/sysctl.h> |
4883dbe9 | 55 | #include <sys/ktr.h> |
c31b1324 MD |
56 | #include <sys/kthread.h> |
57 | #include <machine/cpu.h> | |
58 | #include <sys/lock.h> | |
59 | #include <sys/caps.h> | |
9d265729 MD |
60 | #include <sys/spinlock.h> |
61 | ||
62 | #include <sys/thread2.h> | |
63 | #include <sys/spinlock2.h> | |
3b998fa9 | 64 | #include <sys/mplock2.h> |
c31b1324 MD |
65 | |
66 | #include <vm/vm.h> | |
67 | #include <vm/vm_param.h> | |
68 | #include <vm/vm_kern.h> | |
69 | #include <vm/vm_object.h> | |
70 | #include <vm/vm_page.h> | |
71 | #include <vm/vm_map.h> | |
72 | #include <vm/vm_pager.h> | |
73 | #include <vm/vm_extern.h> | |
74 | #include <vm/vm_zone.h> | |
75 | ||
76 | #include <machine/stdarg.h> | |
c31b1324 MD |
77 | #include <machine/smp.h> |
78 | ||
41a01a4d MD |
79 | #ifndef LWKT_NUM_POOL_TOKENS |
80 | #define LWKT_NUM_POOL_TOKENS 1024 /* power of 2 */ | |
81 | #endif | |
82 | #define LWKT_MASK_POOL_TOKENS (LWKT_NUM_POOL_TOKENS - 1) | |
83 | ||
41a01a4d MD |
84 | static lwkt_token pool_tokens[LWKT_NUM_POOL_TOKENS]; |
85 | ||
f917e9bc | 86 | #define TOKEN_STRING "REF=%p TOK=%p TD=%p" |
38717797 HP |
87 | #define CONTENDED_STRING "REF=%p TOK=%p TD=%p (contention started)" |
88 | #define UNCONTENDED_STRING "REF=%p TOK=%p TD=%p (contention stopped)" | |
4883dbe9 MD |
89 | #if !defined(KTR_TOKENS) |
90 | #define KTR_TOKENS KTR_ALL | |
91 | #endif | |
790e4db7 | 92 | |
4883dbe9 | 93 | KTR_INFO_MASTER(tokens); |
c6fbe95a MD |
94 | KTR_INFO(KTR_TOKENS, tokens, fail, 0, TOKEN_STRING, sizeof(void *) * 3); |
95 | KTR_INFO(KTR_TOKENS, tokens, succ, 1, TOKEN_STRING, sizeof(void *) * 3); | |
7cd8d145 | 96 | #if 0 |
c6fbe95a | 97 | KTR_INFO(KTR_TOKENS, tokens, release, 2, TOKEN_STRING, sizeof(void *) * 3); |
f917e9bc MD |
98 | KTR_INFO(KTR_TOKENS, tokens, remote, 3, TOKEN_STRING, sizeof(void *) * 3); |
99 | KTR_INFO(KTR_TOKENS, tokens, reqremote, 4, TOKEN_STRING, sizeof(void *) * 3); | |
100 | KTR_INFO(KTR_TOKENS, tokens, reqfail, 5, TOKEN_STRING, sizeof(void *) * 3); | |
101 | KTR_INFO(KTR_TOKENS, tokens, drain, 6, TOKEN_STRING, sizeof(void *) * 3); | |
38717797 HP |
102 | KTR_INFO(KTR_TOKENS, tokens, contention_start, 7, CONTENDED_STRING, sizeof(void *) * 3); |
103 | KTR_INFO(KTR_TOKENS, tokens, contention_stop, 7, UNCONTENDED_STRING, sizeof(void *) * 3); | |
790e4db7 MD |
104 | #endif |
105 | ||
f917e9bc MD |
106 | #define logtoken(name, ref) \ |
107 | KTR_LOG(tokens_ ## name, ref, ref->tr_tok, curthread) | |
4883dbe9 | 108 | |
c9aa7a82 MD |
109 | /* |
110 | * Global tokens. These replace the MP lock for major subsystem locking. | |
111 | * These tokens are initially used to lockup both global and individual | |
112 | * operations. | |
113 | * | |
114 | * Once individual structures get their own locks these tokens are used | |
115 | * only to protect global lists & other variables and to interlock | |
116 | * allocations and teardowns and such. | |
117 | * | |
118 | * The UP initializer causes token acquisition to also acquire the MP lock | |
119 | * for maximum compatibility. The feature may be enabled and disabled at | |
120 | * any time, the MP state is copied to the tokref when the token is acquired | |
121 | * and will not race against sysctl changes. | |
122 | */ | |
12586b82 MD |
123 | struct lwkt_token pmap_token = LWKT_TOKEN_UP_INITIALIZER(pmap_token); |
124 | struct lwkt_token dev_token = LWKT_TOKEN_UP_INITIALIZER(dev_token); | |
125 | struct lwkt_token vm_token = LWKT_TOKEN_UP_INITIALIZER(vm_token); | |
126 | struct lwkt_token vmspace_token = LWKT_TOKEN_UP_INITIALIZER(vmspace_token); | |
127 | struct lwkt_token kvm_token = LWKT_TOKEN_UP_INITIALIZER(kvm_token); | |
128 | struct lwkt_token proc_token = LWKT_TOKEN_UP_INITIALIZER(proc_token); | |
129 | struct lwkt_token tty_token = LWKT_TOKEN_UP_INITIALIZER(tty_token); | |
130 | struct lwkt_token vnode_token = LWKT_TOKEN_UP_INITIALIZER(vnode_token); | |
2de4f77e | 131 | struct lwkt_token vmobj_token = LWKT_TOKEN_UP_INITIALIZER(vmobj_token); |
c9aa7a82 | 132 | |
0c52fa62 SG |
133 | SYSCTL_INT(_lwkt, OID_AUTO, pmap_mpsafe, CTLFLAG_RW, |
134 | &pmap_token.t_flags, 0, "Require MP lock for pmap_token"); | |
135 | SYSCTL_INT(_lwkt, OID_AUTO, dev_mpsafe, CTLFLAG_RW, | |
136 | &dev_token.t_flags, 0, "Require MP lock for dev_token"); | |
137 | SYSCTL_INT(_lwkt, OID_AUTO, vm_mpsafe, CTLFLAG_RW, | |
138 | &vm_token.t_flags, 0, "Require MP lock for vm_token"); | |
139 | SYSCTL_INT(_lwkt, OID_AUTO, vmspace_mpsafe, CTLFLAG_RW, | |
140 | &vmspace_token.t_flags, 0, "Require MP lock for vmspace_token"); | |
141 | SYSCTL_INT(_lwkt, OID_AUTO, kvm_mpsafe, CTLFLAG_RW, | |
142 | &kvm_token.t_flags, 0, "Require MP lock for kvm_token"); | |
143 | SYSCTL_INT(_lwkt, OID_AUTO, proc_mpsafe, CTLFLAG_RW, | |
144 | &proc_token.t_flags, 0, "Require MP lock for proc_token"); | |
145 | SYSCTL_INT(_lwkt, OID_AUTO, tty_mpsafe, CTLFLAG_RW, | |
146 | &tty_token.t_flags, 0, "Require MP lock for tty_token"); | |
147 | SYSCTL_INT(_lwkt, OID_AUTO, vnode_mpsafe, CTLFLAG_RW, | |
148 | &vnode_token.t_flags, 0, "Require MP lock for vnode_token"); | |
149 | SYSCTL_INT(_lwkt, OID_AUTO, vmobj_mpsafe, CTLFLAG_RW, | |
150 | &vmobj_token.t_flags, 0, "Require MP lock for vmobj_token"); | |
c9aa7a82 MD |
151 | |
152 | /* | |
153 | * The collision count is bumped every time the LWKT scheduler fails | |
154 | * to acquire needed tokens in addition to a normal lwkt_gettoken() | |
155 | * stall. | |
156 | */ | |
0c52fa62 SG |
157 | SYSCTL_LONG(_lwkt, OID_AUTO, pmap_collisions, CTLFLAG_RW, |
158 | &pmap_token.t_collisions, 0, "Collision counter of pmap_token"); | |
159 | SYSCTL_LONG(_lwkt, OID_AUTO, dev_collisions, CTLFLAG_RW, | |
160 | &dev_token.t_collisions, 0, "Collision counter of dev_token"); | |
161 | SYSCTL_LONG(_lwkt, OID_AUTO, vm_collisions, CTLFLAG_RW, | |
162 | &vm_token.t_collisions, 0, "Collision counter of vm_token"); | |
163 | SYSCTL_LONG(_lwkt, OID_AUTO, vmspace_collisions, CTLFLAG_RW, | |
164 | &vmspace_token.t_collisions, 0, "Collision counter of vmspace_token"); | |
165 | SYSCTL_LONG(_lwkt, OID_AUTO, kvm_collisions, CTLFLAG_RW, | |
166 | &kvm_token.t_collisions, 0, "Collision counter of kvm_token"); | |
167 | SYSCTL_LONG(_lwkt, OID_AUTO, proc_collisions, CTLFLAG_RW, | |
168 | &proc_token.t_collisions, 0, "Collision counter of proc_token"); | |
169 | SYSCTL_LONG(_lwkt, OID_AUTO, tty_collisions, CTLFLAG_RW, | |
170 | &tty_token.t_collisions, 0, "Collision counter of tty_token"); | |
171 | SYSCTL_LONG(_lwkt, OID_AUTO, vnode_collisions, CTLFLAG_RW, | |
172 | &vnode_token.t_collisions, 0, "Collision counter of vnode_token"); | |
c9aa7a82 | 173 | |
c6fbe95a MD |
174 | /* |
175 | * Return a pool token given an address | |
176 | */ | |
177 | static __inline | |
178 | lwkt_token_t | |
179 | _lwkt_token_pool_lookup(void *ptr) | |
180 | { | |
181 | int i; | |
182 | ||
183 | i = ((int)(intptr_t)ptr >> 2) ^ ((int)(intptr_t)ptr >> 12); | |
184 | return(&pool_tokens[i & LWKT_MASK_POOL_TOKENS]); | |
185 | } | |
186 | ||
3b998fa9 MD |
187 | /* |
188 | * Initialize a tokref_t prior to making it visible in the thread's | |
189 | * token array. | |
c843cbf7 MD |
190 | * |
191 | * As an optimization we set the MPSAFE flag if the thread is already | |
192 | * holding the MP lock. This bypasses unncessary calls to get_mplock() and | |
193 | * rel_mplock() on tokens which are not normally MPSAFE when the thread | |
194 | * is already holding the MP lock. | |
3933a3ab MD |
195 | * |
196 | * WARNING: The inherited td_xpcount does not count here because a switch | |
197 | * could schedule the preempted thread and blow away the inherited | |
198 | * mplock. | |
3b998fa9 MD |
199 | */ |
200 | static __inline | |
201 | void | |
202 | _lwkt_tokref_init(lwkt_tokref_t ref, lwkt_token_t tok, thread_t td) | |
203 | { | |
204 | ref->tr_tok = tok; | |
205 | ref->tr_owner = td; | |
206 | ref->tr_flags = tok->t_flags; | |
d7ec0cb4 | 207 | #ifdef SMP |
c843cbf7 | 208 | if (td->td_mpcount) |
d7ec0cb4 | 209 | #endif |
c843cbf7 | 210 | ref->tr_flags |= LWKT_TOKEN_MPSAFE; |
3b998fa9 | 211 | } |
c6fbe95a | 212 | |
c31b1324 | 213 | /* |
9d265729 | 214 | * Obtain all the tokens required by the specified thread on the current |
1fe5fad2 MD |
215 | * cpu, return 0 on failure and non-zero on success. If a failure occurs |
216 | * any partially acquired tokens will be released prior to return. | |
dd55d707 | 217 | * |
7eb611ef | 218 | * lwkt_getalltokens is called by the LWKT scheduler to acquire all |
24c80b2b | 219 | * tokens that the thread had acquired prior to going to sleep. |
7eb611ef | 220 | * |
3b998fa9 | 221 | * The scheduler is responsible for maintaining the MP lock count, so |
c9aa7a82 MD |
222 | * we don't need to deal with tr_flags here. We also do not do any |
223 | * logging here. The logging done by lwkt_gettoken() is plenty good | |
224 | * enough to get a feel for it. | |
3b998fa9 | 225 | * |
7eb611ef | 226 | * Called from a critical section. |
c31b1324 | 227 | */ |
41a01a4d | 228 | int |
b37f18d6 | 229 | lwkt_getalltokens(thread_t td, const char **msgp, const void **addrp) |
41a01a4d | 230 | { |
3b998fa9 | 231 | lwkt_tokref_t scan; |
c6fbe95a MD |
232 | lwkt_tokref_t ref; |
233 | lwkt_token_t tok; | |
234 | ||
3b998fa9 MD |
235 | /* |
236 | * Acquire tokens in forward order, assign or validate tok->t_ref. | |
237 | */ | |
238 | for (scan = &td->td_toks_base; scan < td->td_toks_stop; ++scan) { | |
239 | tok = scan->tr_tok; | |
c6fbe95a MD |
240 | for (;;) { |
241 | /* | |
242 | * Try to acquire the token if we do not already have | |
243 | * it. | |
244 | * | |
245 | * NOTE: If atomic_cmpset_ptr() fails we have to | |
246 | * loop and try again. It just means we | |
247 | * lost a cpu race. | |
248 | */ | |
249 | ref = tok->t_ref; | |
c6fbe95a | 250 | if (ref == NULL) { |
3b998fa9 | 251 | if (atomic_cmpset_ptr(&tok->t_ref, NULL, scan)) |
c6fbe95a MD |
252 | break; |
253 | continue; | |
254 | } | |
255 | ||
256 | /* | |
3b998fa9 MD |
257 | * Test if ref is already recursively held by this |
258 | * thread. We cannot safely dereference tok->t_ref | |
259 | * (it might belong to another thread and is thus | |
260 | * unstable), but we don't have to. We can simply | |
261 | * range-check it. | |
c6fbe95a | 262 | */ |
3b998fa9 MD |
263 | if (ref >= &td->td_toks_base && ref < td->td_toks_stop) |
264 | break; | |
265 | ||
266 | /* | |
267 | * Otherwise we failed to acquire all the tokens. | |
268 | * Undo and return. | |
269 | */ | |
b37f18d6 MD |
270 | *msgp = tok->t_desc; |
271 | *addrp = scan->tr_stallpc; | |
c9aa7a82 | 272 | atomic_add_long(&tok->t_collisions, 1); |
3b998fa9 MD |
273 | lwkt_relalltokens(td); |
274 | return(FALSE); | |
38717797 | 275 | } |
41a01a4d | 276 | } |
c6fbe95a | 277 | return (TRUE); |
c31b1324 MD |
278 | } |
279 | ||
41a01a4d | 280 | /* |
9d265729 | 281 | * Release all tokens owned by the specified thread on the current cpu. |
c6fbe95a MD |
282 | * |
283 | * This code is really simple. Even in cases where we own all the tokens | |
284 | * note that t_ref may not match the scan for recursively held tokens, | |
285 | * or for the case where a lwkt_getalltokens() failed. | |
3b998fa9 MD |
286 | * |
287 | * The scheduler is responsible for maintaining the MP lock count, so | |
288 | * we don't need to deal with tr_flags here. | |
7eb611ef MD |
289 | * |
290 | * Called from a critical section. | |
41a01a4d | 291 | */ |
9d265729 MD |
292 | void |
293 | lwkt_relalltokens(thread_t td) | |
41a01a4d | 294 | { |
3b998fa9 | 295 | lwkt_tokref_t scan; |
c6fbe95a MD |
296 | lwkt_token_t tok; |
297 | ||
3b998fa9 MD |
298 | for (scan = &td->td_toks_base; scan < td->td_toks_stop; ++scan) { |
299 | tok = scan->tr_tok; | |
300 | if (tok->t_ref == scan) | |
c6fbe95a | 301 | tok->t_ref = NULL; |
41a01a4d | 302 | } |
41a01a4d MD |
303 | } |
304 | ||
41a01a4d | 305 | /* |
3b998fa9 MD |
306 | * Token acquisition helper function. The caller must have already |
307 | * made nref visible by adjusting td_toks_stop and will be responsible | |
308 | * for the disposition of nref on either success or failure. | |
dd55d707 | 309 | * |
3b998fa9 MD |
310 | * When acquiring tokens recursively we want tok->t_ref to point to |
311 | * the outer (first) acquisition so it gets cleared only on the last | |
312 | * release. | |
41a01a4d | 313 | */ |
41a01a4d | 314 | static __inline |
7eb611ef | 315 | int |
4a28fe22 | 316 | _lwkt_trytokref2(lwkt_tokref_t nref, thread_t td, int blocking) |
41a01a4d | 317 | { |
c6fbe95a | 318 | lwkt_token_t tok; |
3b998fa9 | 319 | lwkt_tokref_t ref; |
c6fbe95a | 320 | |
7eb611ef | 321 | /* |
3b998fa9 MD |
322 | * Make sure the compiler does not reorder prior instructions |
323 | * beyond this demark. | |
7eb611ef | 324 | */ |
c6fbe95a MD |
325 | cpu_ccfence(); |
326 | ||
7eb611ef | 327 | /* |
c6fbe95a | 328 | * Attempt to gain ownership |
7eb611ef | 329 | */ |
c6fbe95a MD |
330 | tok = nref->tr_tok; |
331 | for (;;) { | |
332 | /* | |
333 | * Try to acquire the token if we do not already have | |
4a28fe22 MD |
334 | * it. This is not allowed if we are in a hard code |
335 | * section (because it 'might' have blocked). | |
c6fbe95a MD |
336 | */ |
337 | ref = tok->t_ref; | |
c6fbe95a | 338 | if (ref == NULL) { |
4a28fe22 | 339 | KASSERT((blocking == 0 || |
5fddbda2 MD |
340 | td->td_gd->gd_intr_nesting_level == 0 || |
341 | panic_cpu_gd == mycpu), | |
4a28fe22 MD |
342 | ("Attempt to acquire token %p not already " |
343 | "held in hard code section", tok)); | |
5fddbda2 | 344 | |
c6fbe95a MD |
345 | /* |
346 | * NOTE: If atomic_cmpset_ptr() fails we have to | |
347 | * loop and try again. It just means we | |
348 | * lost a cpu race. | |
349 | */ | |
350 | if (atomic_cmpset_ptr(&tok->t_ref, NULL, nref)) | |
351 | return (TRUE); | |
352 | continue; | |
353 | } | |
354 | ||
355 | /* | |
3b998fa9 MD |
356 | * Test if ref is already recursively held by this |
357 | * thread. We cannot safely dereference tok->t_ref | |
358 | * (it might belong to another thread and is thus | |
359 | * unstable), but we don't have to. We can simply | |
360 | * range-check it. | |
4a28fe22 MD |
361 | * |
362 | * It is ok to acquire a token that is already held | |
363 | * by the current thread when in a hard code section. | |
3b998fa9 MD |
364 | */ |
365 | if (ref >= &td->td_toks_base && ref < td->td_toks_stop) | |
366 | return(TRUE); | |
367 | ||
368 | /* | |
4a28fe22 MD |
369 | * Otherwise we failed, and it is not ok to attempt to |
370 | * acquire a token in a hard code section. | |
c6fbe95a | 371 | */ |
4a28fe22 MD |
372 | KASSERT((blocking == 0 || |
373 | td->td_gd->gd_intr_nesting_level == 0), | |
374 | ("Attempt to acquire token %p not already " | |
375 | "held in hard code section", tok)); | |
376 | ||
c6fbe95a | 377 | return(FALSE); |
dd55d707 | 378 | } |
c31b1324 MD |
379 | } |
380 | ||
c6fbe95a MD |
381 | /* |
382 | * Acquire a serializing token. This routine does not block. | |
383 | */ | |
41a01a4d | 384 | static __inline |
c31b1324 | 385 | int |
c6fbe95a | 386 | _lwkt_trytokref(lwkt_tokref_t ref, thread_t td) |
c31b1324 | 387 | { |
3b998fa9 | 388 | if ((ref->tr_flags & LWKT_TOKEN_MPSAFE) == 0) { |
3933a3ab MD |
389 | if (try_mplock() == 0) { |
390 | --td->td_toks_stop; | |
3b998fa9 | 391 | return (FALSE); |
3933a3ab | 392 | } |
3b998fa9 | 393 | } |
4a28fe22 | 394 | if (_lwkt_trytokref2(ref, td, 0) == FALSE) { |
c6fbe95a | 395 | /* |
3b998fa9 | 396 | * Cleanup, deactivate the failed token. |
c6fbe95a | 397 | */ |
3b998fa9 MD |
398 | if ((ref->tr_flags & LWKT_TOKEN_MPSAFE) == 0) |
399 | rel_mplock(); | |
3933a3ab | 400 | --td->td_toks_stop; |
c6fbe95a MD |
401 | return (FALSE); |
402 | } | |
403 | return (TRUE); | |
c31b1324 MD |
404 | } |
405 | ||
7eb611ef MD |
406 | /* |
407 | * Acquire a serializing token. This routine can block. | |
7eb611ef MD |
408 | */ |
409 | static __inline | |
410 | void | |
b37f18d6 | 411 | _lwkt_gettokref(lwkt_tokref_t ref, thread_t td, const void **stkframe) |
7eb611ef | 412 | { |
3b998fa9 MD |
413 | if ((ref->tr_flags & LWKT_TOKEN_MPSAFE) == 0) |
414 | get_mplock(); | |
4a28fe22 | 415 | if (_lwkt_trytokref2(ref, td, 1) == FALSE) { |
c6fbe95a MD |
416 | /* |
417 | * Give up running if we can't acquire the token right now. | |
c6fbe95a | 418 | * |
3b998fa9 MD |
419 | * Since the tokref is already active the scheduler now |
420 | * takes care of acquisition, so we need only call | |
f9235b6d | 421 | * lwkt_switch(). |
3b998fa9 MD |
422 | * |
423 | * Since we failed this was not a recursive token so upon | |
c6fbe95a MD |
424 | * return tr_tok->t_ref should be assigned to this specific |
425 | * ref. | |
426 | */ | |
b37f18d6 | 427 | ref->tr_stallpc = stkframe[-1]; |
c9aa7a82 | 428 | atomic_add_long(&ref->tr_tok->t_collisions, 1); |
c6fbe95a | 429 | logtoken(fail, ref); |
f9235b6d | 430 | lwkt_switch(); |
c6fbe95a | 431 | logtoken(succ, ref); |
c6fbe95a MD |
432 | KKASSERT(ref->tr_tok->t_ref == ref); |
433 | } | |
7eb611ef MD |
434 | } |
435 | ||
41a01a4d | 436 | void |
3b998fa9 | 437 | lwkt_gettoken(lwkt_token_t tok) |
c31b1324 | 438 | { |
c6fbe95a | 439 | thread_t td = curthread; |
3b998fa9 | 440 | lwkt_tokref_t ref; |
c6fbe95a | 441 | |
3b998fa9 MD |
442 | ref = td->td_toks_stop; |
443 | KKASSERT(ref < &td->td_toks_end); | |
3b998fa9 | 444 | ++td->td_toks_stop; |
b528f10f | 445 | cpu_ccfence(); |
3933a3ab | 446 | _lwkt_tokref_init(ref, tok, td); |
b37f18d6 | 447 | _lwkt_gettokref(ref, td, (const void **)&tok); |
c6fbe95a MD |
448 | } |
449 | ||
4a28fe22 MD |
450 | void |
451 | lwkt_gettoken_hard(lwkt_token_t tok) | |
452 | { | |
453 | thread_t td = curthread; | |
454 | lwkt_tokref_t ref; | |
455 | ||
456 | ref = td->td_toks_stop; | |
457 | KKASSERT(ref < &td->td_toks_end); | |
4a28fe22 | 458 | ++td->td_toks_stop; |
b528f10f | 459 | cpu_ccfence(); |
3933a3ab | 460 | _lwkt_tokref_init(ref, tok, td); |
4a28fe22 MD |
461 | _lwkt_gettokref(ref, td, (const void **)&tok); |
462 | crit_enter_hard_gd(td->td_gd); | |
463 | } | |
464 | ||
3b998fa9 MD |
465 | lwkt_token_t |
466 | lwkt_getpooltoken(void *ptr) | |
c6fbe95a MD |
467 | { |
468 | thread_t td = curthread; | |
3b998fa9 MD |
469 | lwkt_token_t tok; |
470 | lwkt_tokref_t ref; | |
c6fbe95a | 471 | |
3b998fa9 MD |
472 | ref = td->td_toks_stop; |
473 | KKASSERT(ref < &td->td_toks_end); | |
3933a3ab | 474 | ++td->td_toks_stop; |
b528f10f | 475 | cpu_ccfence(); |
3b998fa9 MD |
476 | tok = _lwkt_token_pool_lookup(ptr); |
477 | _lwkt_tokref_init(ref, tok, td); | |
b37f18d6 | 478 | _lwkt_gettokref(ref, td, (const void **)&ptr); |
3b998fa9 | 479 | return(tok); |
c31b1324 MD |
480 | } |
481 | ||
137b3005 MD |
482 | /* |
483 | * Attempt to acquire a token, return TRUE on success, FALSE on failure. | |
484 | */ | |
c31b1324 | 485 | int |
3b998fa9 | 486 | lwkt_trytoken(lwkt_token_t tok) |
c31b1324 | 487 | { |
c6fbe95a | 488 | thread_t td = curthread; |
3b998fa9 | 489 | lwkt_tokref_t ref; |
c6fbe95a | 490 | |
3b998fa9 MD |
491 | ref = td->td_toks_stop; |
492 | KKASSERT(ref < &td->td_toks_end); | |
3b998fa9 | 493 | ++td->td_toks_stop; |
b528f10f | 494 | cpu_ccfence(); |
3933a3ab | 495 | _lwkt_tokref_init(ref, tok, td); |
c6fbe95a | 496 | return(_lwkt_trytokref(ref, td)); |
41a01a4d MD |
497 | } |
498 | ||
c31b1324 | 499 | /* |
c6fbe95a MD |
500 | * Release a serializing token. |
501 | * | |
3b998fa9 MD |
502 | * WARNING! All tokens must be released in reverse order. This will be |
503 | * asserted. | |
c31b1324 | 504 | */ |
41a01a4d | 505 | void |
3b998fa9 | 506 | lwkt_reltoken(lwkt_token_t tok) |
c31b1324 | 507 | { |
3b998fa9 MD |
508 | thread_t td = curthread; |
509 | lwkt_tokref_t ref; | |
c6fbe95a | 510 | |
3b998fa9 MD |
511 | /* |
512 | * Remove ref from thread token list and assert that it matches | |
513 | * the token passed in. Tokens must be released in reverse order. | |
514 | */ | |
515 | ref = td->td_toks_stop - 1; | |
516 | KKASSERT(ref >= &td->td_toks_base && ref->tr_tok == tok); | |
3b998fa9 MD |
517 | |
518 | /* | |
b528f10f MD |
519 | * Only clear the token if it matches ref. If ref was a recursively |
520 | * acquired token it may not match. | |
521 | * | |
3b998fa9 | 522 | * If the token was not MPSAFE release the MP lock. |
b528f10f MD |
523 | * |
524 | * NOTE: We have to do this before adjust td_toks_stop, otherwise | |
525 | * a fast interrupt can come along and reuse our ref while | |
526 | * tok is still attached to it. | |
3b998fa9 | 527 | */ |
b528f10f MD |
528 | if (tok->t_ref == ref) |
529 | tok->t_ref = NULL; | |
530 | cpu_ccfence(); | |
3b998fa9 MD |
531 | if ((ref->tr_flags & LWKT_TOKEN_MPSAFE) == 0) |
532 | rel_mplock(); | |
c6fbe95a MD |
533 | |
534 | /* | |
b528f10f MD |
535 | * Finally adjust td_toks_stop, be very sure that the compiler |
536 | * does not reorder the clearing of tok->t_ref with the | |
537 | * decrementing of td->td_toks_stop. | |
c6fbe95a | 538 | */ |
c6fbe95a | 539 | cpu_ccfence(); |
b528f10f MD |
540 | td->td_toks_stop = ref; |
541 | KKASSERT(tok->t_ref != ref); | |
c31b1324 MD |
542 | } |
543 | ||
4a28fe22 MD |
544 | void |
545 | lwkt_reltoken_hard(lwkt_token_t tok) | |
546 | { | |
547 | lwkt_reltoken(tok); | |
548 | crit_exit_hard(); | |
549 | } | |
550 | ||
177e553a MD |
551 | /* |
552 | * It is faster for users of lwkt_getpooltoken() to use the returned | |
553 | * token and just call lwkt_reltoken(), but for convenience we provide | |
554 | * this function which looks the token up based on the ident. | |
555 | */ | |
556 | void | |
557 | lwkt_relpooltoken(void *ptr) | |
558 | { | |
559 | lwkt_token_t tok = _lwkt_token_pool_lookup(ptr); | |
560 | lwkt_reltoken(tok); | |
561 | } | |
562 | ||
563 | ||
41a01a4d MD |
564 | /* |
565 | * Pool tokens are used to provide a type-stable serializing token | |
566 | * pointer that does not race against disappearing data structures. | |
567 | * | |
568 | * This routine is called in early boot just after we setup the BSP's | |
569 | * globaldata structure. | |
570 | */ | |
571 | void | |
572 | lwkt_token_pool_init(void) | |
573 | { | |
c6fbe95a | 574 | int i; |
41a01a4d | 575 | |
c6fbe95a | 576 | for (i = 0; i < LWKT_NUM_POOL_TOKENS; ++i) |
b37f18d6 | 577 | lwkt_token_init(&pool_tokens[i], 1, "pool"); |
41a01a4d MD |
578 | } |
579 | ||
580 | lwkt_token_t | |
c6fbe95a | 581 | lwkt_token_pool_lookup(void *ptr) |
41a01a4d | 582 | { |
c6fbe95a | 583 | return (_lwkt_token_pool_lookup(ptr)); |
41a01a4d MD |
584 | } |
585 | ||
41a01a4d | 586 | /* |
544c38e8 NT |
587 | * Initialize a token. If mpsafe is 0, the MP lock is acquired before |
588 | * acquiring the token and released after releasing the token. | |
41a01a4d MD |
589 | */ |
590 | void | |
b37f18d6 | 591 | lwkt_token_init(lwkt_token_t tok, int mpsafe, const char *desc) |
41a01a4d | 592 | { |
c6fbe95a | 593 | tok->t_ref = NULL; |
3b998fa9 | 594 | tok->t_flags = mpsafe ? LWKT_TOKEN_MPSAFE : 0; |
73d8e728 | 595 | tok->t_collisions = 0; |
c5724852 | 596 | tok->t_desc = desc; |
c31b1324 MD |
597 | } |
598 | ||
41a01a4d MD |
599 | void |
600 | lwkt_token_uninit(lwkt_token_t tok) | |
601 | { | |
c6fbe95a | 602 | /* empty */ |
41a01a4d | 603 | } |
7eb611ef | 604 | |
c6fbe95a | 605 | #if 0 |
7eb611ef MD |
606 | int |
607 | lwkt_token_is_stale(lwkt_tokref_t ref) | |
608 | { | |
c6fbe95a MD |
609 | lwkt_token_t tok = ref->tr_tok; |
610 | ||
611 | KKASSERT(tok->t_owner == curthread && ref->tr_state == 1 && | |
612 | tok->t_count > 0); | |
613 | ||
614 | /* Token is not stale */ | |
615 | if (tok->t_lastowner == tok->t_owner) | |
616 | return (FALSE); | |
617 | ||
618 | /* | |
619 | * The token is stale. Reset to not stale so that the next call to | |
620 | * lwkt_token_is_stale will return "not stale" unless the token | |
621 | * was acquired in-between by another thread. | |
622 | */ | |
623 | tok->t_lastowner = tok->t_owner; | |
624 | return (TRUE); | |
7eb611ef | 625 | } |
c6fbe95a | 626 | #endif |