2 * Copyright (c) 2003,2004 The DragonFly Project. All rights reserved.
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * $DragonFly: src/sys/kern/lwkt_token.c,v 1.26 2006/05/21 20:23:25 dillon Exp $
39 #include <sys/param.h>
40 #include <sys/systm.h>
41 #include <sys/kernel.h>
43 #include <sys/rtprio.h>
44 #include <sys/queue.h>
45 #include <sys/sysctl.h>
47 #include <sys/kthread.h>
48 #include <machine/cpu.h>
51 #include <sys/spinlock.h>
53 #include <sys/thread2.h>
54 #include <sys/spinlock2.h>
57 #include <vm/vm_param.h>
58 #include <vm/vm_kern.h>
59 #include <vm/vm_object.h>
60 #include <vm/vm_page.h>
61 #include <vm/vm_map.h>
62 #include <vm/vm_pager.h>
63 #include <vm/vm_extern.h>
64 #include <vm/vm_zone.h>
66 #include <machine/stdarg.h>
67 #include <machine/ipl.h>
68 #include <machine/smp.h>
70 #define THREAD_STACK (UPAGES * PAGE_SIZE)
74 #include <sys/stdint.h>
75 #include <libcaps/thread.h>
76 #include <sys/thread.h>
77 #include <sys/msgport.h>
78 #include <sys/errno.h>
79 #include <libcaps/globaldata.h>
80 #include <machine/cpufunc.h>
81 #include <sys/thread2.h>
82 #include <sys/msgport2.h>
86 #include <machine/lock.h>
87 #include <machine/cpu.h>
91 #ifndef LWKT_NUM_POOL_TOKENS
92 #define LWKT_NUM_POOL_TOKENS 1024 /* power of 2 */
94 #define LWKT_MASK_POOL_TOKENS (LWKT_NUM_POOL_TOKENS - 1)
97 static int token_debug = 0;
100 static lwkt_token pool_tokens[LWKT_NUM_POOL_TOKENS];
102 #define TOKEN_STRING "REF=%p TOK=%p TD=%p"
103 #define CONTENDED_STRING "REF=%p TOK=%p TD=%p (contention started)"
104 #define UNCONTENDED_STRING "REF=%p TOK=%p TD=%p (contention stopped)"
105 #if !defined(KTR_TOKENS)
106 #define KTR_TOKENS KTR_ALL
109 KTR_INFO_MASTER(tokens);
110 KTR_INFO(KTR_TOKENS, tokens, try, 0, TOKEN_STRING, sizeof(void *) * 3);
111 KTR_INFO(KTR_TOKENS, tokens, get, 1, TOKEN_STRING, sizeof(void *) * 3);
112 KTR_INFO(KTR_TOKENS, tokens, release, 2, TOKEN_STRING, sizeof(void *) * 3);
114 KTR_INFO(KTR_TOKENS, tokens, remote, 3, TOKEN_STRING, sizeof(void *) * 3);
115 KTR_INFO(KTR_TOKENS, tokens, reqremote, 4, TOKEN_STRING, sizeof(void *) * 3);
116 KTR_INFO(KTR_TOKENS, tokens, reqfail, 5, TOKEN_STRING, sizeof(void *) * 3);
117 KTR_INFO(KTR_TOKENS, tokens, drain, 6, TOKEN_STRING, sizeof(void *) * 3);
118 KTR_INFO(KTR_TOKENS, tokens, contention_start, 7, CONTENDED_STRING, sizeof(void *) * 3);
119 KTR_INFO(KTR_TOKENS, tokens, contention_stop, 7, UNCONTENDED_STRING, sizeof(void *) * 3);
122 #define logtoken(name, ref) \
123 KTR_LOG(tokens_ ## name, ref, ref->tr_tok, curthread)
128 SYSCTL_INT(_lwkt, OID_AUTO, token_debug, CTLFLAG_RW, &token_debug, 0, "");
136 * Obtain all the tokens required by the specified thread on the current
137 * cpu, return 0 on failure and non-zero on success.
139 * NOTE: This code does not work with UP 'degenerate' spinlocks. SMP only.
141 * The preemption code will not allow a target thread holding spinlocks to
142 * preempt the current thread so we do not have to implement this for UP.
145 lwkt_getalltokens(thread_t td)
151 for (refs = td->td_toks; refs; refs = refs->tr_next) {
152 KKASSERT(refs->tr_state == 0);
154 if (tok->t_owner != td) {
155 if (spin_trylock_wr(&tok->t_spinlock) == 0) {
157 * Release the partial list of tokens obtained and return
160 for (undo = td->td_toks; undo != refs; undo = undo->tr_next) {
163 if (--tok->t_count == 0) {
165 spin_unlock_wr(&tok->t_spinlock);
171 KKASSERT(tok->t_count == 0);
180 * Release all tokens owned by the specified thread on the current cpu.
183 lwkt_relalltokens(thread_t td)
188 for (scan = td->td_toks; scan; scan = scan->tr_next) {
189 if (scan->tr_state) {
192 KKASSERT(tok->t_owner == td && tok->t_count > 0);
193 if (--tok->t_count == 0) {
195 spin_unlock_wr(&tok->t_spinlock);
204 * Acquire a serializing token. This routine can block.
206 * On SMP systems we track ownership and a per-owner counter. Tokens are
207 * released when a thread switches out and reacquired when a thread
208 * switches back in. On UP systems we track a global counter for debugging
209 * but otherwise the only issue we have is if a preempting thread wants a
210 * token that is being held by the preempted thread.
214 _lwkt_gettokref(lwkt_tokref_t ref)
222 KKASSERT(mycpu->gd_intr_nesting_level == 0);
227 * Link the tokref to the thread's list
229 ref->tr_next = td->td_toks;
235 * Gain ownership of the token's spinlock, SMP version.
237 if (tok->t_owner != td) {
238 if (spin_trylock_wr(&tok->t_spinlock) == 0) {
242 KKASSERT(tok->t_owner == NULL && tok->t_count == 0);
248 * Gain ownership of the token, UP version. All we have to do
249 * is check the token if we are preempting someone owning the
250 * same token. If we are, we yield the cpu back to the originator
251 * and we will get rescheduled as non-preemptive.
253 while ((td = td->td_preempted) != NULL) {
254 for (scan = td->td_toks; scan; scan = scan->tr_next) {
255 if (scan->tr_tok == tok) {
261 /* NOTE: 'td' invalid after loop */
262 ++tok->t_globalcount;
269 _lwkt_trytokref(lwkt_tokref_t ref)
277 KKASSERT(mycpu->gd_intr_nesting_level == 0);
282 * Link the tokref to the thread's list
284 ref->tr_next = td->td_toks;
290 * Gain ownership of the token's spinlock, SMP version.
292 if (tok->t_owner != td) {
293 if (spin_trylock_wr(&tok->t_spinlock) == 0) {
294 td->td_toks = ref->tr_next;
297 KKASSERT(tok->t_owner == NULL && tok->t_count == 0);
303 * Gain ownership of the token, UP version. All we have to do
304 * is check the token if we are preempting someone owning the
305 * same token. If we are, we yield the cpu back to the originator
306 * and we will get rescheduled as non-preemptive.
308 while ((td = td->td_preempted) != NULL) {
309 for (scan = td->td_toks; scan; scan = scan->tr_next) {
310 if (scan->tr_tok == tok) {
311 td->td_toks = ref->tr_next;
316 /* NOTE: 'td' invalid after loop */
317 ++tok->t_globalcount;
324 lwkt_gettoken(lwkt_tokref_t ref, lwkt_token_t tok)
326 lwkt_tokref_init(ref, tok);
328 _lwkt_gettokref(ref);
332 lwkt_gettokref(lwkt_tokref_t ref)
335 _lwkt_gettokref(ref);
339 lwkt_trytoken(lwkt_tokref_t ref, lwkt_token_t tok)
341 lwkt_tokref_init(ref, tok);
343 return(_lwkt_trytokref(ref));
347 lwkt_trytokref(lwkt_tokref_t ref)
350 return(_lwkt_trytokref(ref));
354 * Release a serializing token
357 lwkt_reltoken(lwkt_tokref *ref)
359 struct lwkt_tokref **scanp;
367 KKASSERT(ref->tr_state == 1 && tok->t_owner == td && tok->t_count > 0);
369 KKASSERT(ref->tr_state == 1 && tok->t_globalcount > 0);
372 for (scanp = &td->td_toks; *scanp != ref; scanp = &((*scanp)->tr_next))
374 *scanp = ref->tr_next;
378 if (--tok->t_count == 0) {
380 spin_unlock_wr(&tok->t_spinlock);
383 --tok->t_globalcount;
385 logtoken(release, ref);
389 * Pool tokens are used to provide a type-stable serializing token
390 * pointer that does not race against disappearing data structures.
392 * This routine is called in early boot just after we setup the BSP's
393 * globaldata structure.
396 lwkt_token_pool_init(void)
400 for (i = 0; i < LWKT_NUM_POOL_TOKENS; ++i)
401 lwkt_token_init(&pool_tokens[i]);
405 lwkt_token_pool_get(void *ptraddr)
409 i = ((int)(intptr_t)ptraddr >> 2) ^ ((int)(intptr_t)ptraddr >> 12);
410 return(&pool_tokens[i & LWKT_MASK_POOL_TOKENS]);
414 * Initialize the owner and release-to cpu to the current cpu
415 * and reset the generation count.
418 lwkt_token_init(lwkt_token_t tok)
421 spin_init(&tok->t_spinlock);
425 tok->t_globalcount = 0;
430 lwkt_token_uninit(lwkt_token_t tok)