2 * Copyright (c) 2003,2004 The DragonFly Project. All rights reserved.
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * $DragonFly: src/sys/kern/lwkt_token.c,v 1.13 2005/04/18 01:03:28 dillon Exp $
39 #include <sys/param.h>
40 #include <sys/systm.h>
41 #include <sys/kernel.h>
43 #include <sys/rtprio.h>
44 #include <sys/queue.h>
45 #include <sys/thread2.h>
46 #include <sys/sysctl.h>
47 #include <sys/kthread.h>
48 #include <machine/cpu.h>
53 #include <vm/vm_param.h>
54 #include <vm/vm_kern.h>
55 #include <vm/vm_object.h>
56 #include <vm/vm_page.h>
57 #include <vm/vm_map.h>
58 #include <vm/vm_pager.h>
59 #include <vm/vm_extern.h>
60 #include <vm/vm_zone.h>
62 #include <machine/stdarg.h>
63 #include <machine/ipl.h>
64 #include <machine/smp.h>
66 #define THREAD_STACK (UPAGES * PAGE_SIZE)
70 #include <sys/stdint.h>
71 #include <libcaps/thread.h>
72 #include <sys/thread.h>
73 #include <sys/msgport.h>
74 #include <sys/errno.h>
75 #include <libcaps/globaldata.h>
76 #include <machine/cpufunc.h>
77 #include <sys/thread2.h>
78 #include <sys/msgport2.h>
82 #include <machine/lock.h>
83 #include <machine/cpu.h>
87 #define MAKE_TOKENS_SPIN
88 /* #define MAKE_TOKENS_YIELD */
90 #ifndef LWKT_NUM_POOL_TOKENS
91 #define LWKT_NUM_POOL_TOKENS 1024 /* power of 2 */
93 #define LWKT_MASK_POOL_TOKENS (LWKT_NUM_POOL_TOKENS - 1)
96 static int token_debug = 0;
99 static void lwkt_reqtoken_remote(void *data);
101 static lwkt_token pool_tokens[LWKT_NUM_POOL_TOKENS];
106 SYSCTL_INT(_lwkt, OID_AUTO, token_debug, CTLFLAG_RW, &token_debug, 0, "");
114 * Determine if we own all the tokens in the token reference list.
115 * Return 1 on success, 0 on failure.
117 * As a side effect, queue requests for tokens we want which are owned
118 * by other cpus. The magic number is used to communicate when the
119 * target cpu has processed the request. Note, however, that the
120 * target cpu may not be able to assign the token to us which is why
121 * the scheduler must spin.
124 lwkt_chktokens(thread_t td)
126 globaldata_t gd = td->td_gd; /* mycpu */
132 for (refs = td->td_toks; refs; refs = refs->tr_next) {
134 if ((dgd = tok->t_cpu) != gd) {
139 * Queue a request to the target cpu, exit the loop early if
140 * we are unable to queue the IPI message. The magic number
141 * flags whether we have a pending ipi request queued or not.
142 * It can be set from MAGIC2 to MAGIC1 by a remote cpu but can
143 * only be set from MAGIC1 to MAGIC2 by our cpu.
145 if (refs->tr_magic == LWKT_TOKREF_MAGIC1) {
146 refs->tr_magic = LWKT_TOKREF_MAGIC2; /* MP synched slowreq*/
148 tok->t_reqcpu = gd; /* MP unsynchronized 'fast' req */
149 if (lwkt_send_ipiq_nowait(dgd, lwkt_reqtoken_remote, refs)) {
151 refs->tr_magic = LWKT_TOKREF_MAGIC1;
163 * Check if we already own the token. Return 1 on success, 0 on failure.
166 lwkt_havetoken(lwkt_token_t tok)
168 globaldata_t gd = mycpu;
169 thread_t td = gd->gd_curthread;
172 for (ref = td->td_toks; ref; ref = ref->tr_next) {
173 if (ref->tr_tok == tok)
180 lwkt_havetokref(lwkt_tokref_t xref)
182 globaldata_t gd = mycpu;
183 thread_t td = gd->gd_curthread;
186 for (ref = td->td_toks; ref; ref = ref->tr_next) {
196 * Returns 1 if it is ok to give a token away, 0 if it is not.
199 lwkt_oktogiveaway_token(lwkt_token_t tok)
201 globaldata_t gd = mycpu;
205 for (td = gd->gd_curthread; td; td = td->td_preempted) {
206 for (ref = td->td_toks; ref; ref = ref->tr_next) {
207 if (ref->tr_tok == tok)
217 * Acquire a serializing token
222 _lwkt_gettokref(lwkt_tokref_t ref)
228 gd = mycpu; /* our cpu */
229 KKASSERT(ref->tr_magic == LWKT_TOKREF_MAGIC1);
230 td = gd->gd_curthread; /* our thread */
233 * Link the request into our thread's list. This interlocks against
234 * remote requests from other cpus and prevents the token from being
235 * given away if our cpu already owns it. This also allows us to
236 * avoid using a critical section.
238 ref->tr_next = td->td_toks;
239 cpu_mb1(); /* order memory / we can be interrupted */
243 * If our cpu does not own the token then let the scheduler deal with
244 * it. We are guarenteed to own the tokens on our thread's token
245 * list when we are switched back in.
247 * Otherwise make sure the token is not held by a thread we are
248 * preempting. If it is, let the scheduler deal with it.
252 if (tok->t_cpu != gd) {
254 * Temporarily operate on tokens synchronously. We have to fix
255 * a number of interlocks and especially the softupdates code to
256 * be able to properly yield. ZZZ
258 #if defined(MAKE_TOKENS_SPIN)
262 while (lwkt_chktokens(td) == 0) {
264 lwkt_drain_token_requests();
267 printf("CHKTOKEN looping on cpu %d\n", gd->gd_cpuid);
270 panic("CHKTOKEN looping on cpu %d", gd->gd_cpuid);
276 #elif defined(MAKE_TOKENS_YIELD)
279 #error MAKE_TOKENS_XXX ?
281 KKASSERT(tok->t_cpu == gd);
282 } else /* NOTE CONDITIONAL */
284 if (td->td_preempted) {
285 while ((td = td->td_preempted) != NULL) {
287 for (scan = td->td_toks; scan; scan = scan->tr_next) {
288 if (scan->tr_tok == tok) {
290 KKASSERT(tok->t_cpu == gd);
297 /* 'td' variable no longer valid due to preempt loop above */
302 * Attempt to acquire a serializing token
306 _lwkt_trytokref(lwkt_tokref_t ref)
312 gd = mycpu; /* our cpu */
313 KKASSERT(ref->tr_magic == LWKT_TOKREF_MAGIC1);
314 td = gd->gd_curthread; /* our thread */
317 * Link the request into our thread's list. This interlocks against
318 * remote requests from other cpus and prevents the token from being
319 * given away if our cpu already owns it. This also allows us to
320 * avoid using a critical section.
322 ref->tr_next = td->td_toks;
323 cpu_mb1(); /* order memory / we can be interrupted */
327 * If our cpu does not own the token then stop now.
329 * Otherwise make sure the token is not held by a thread we are
330 * preempting. If it is, stop.
334 if (tok->t_cpu != gd) {
335 td->td_toks = ref->tr_next; /* remove ref */
337 } else /* NOTE CONDITIONAL */
339 if (td->td_preempted) {
340 while ((td = td->td_preempted) != NULL) {
342 for (scan = td->td_toks; scan; scan = scan->tr_next) {
343 if (scan->tr_tok == tok) {
344 td = gd->gd_curthread; /* our thread */
345 td->td_toks = ref->tr_next; /* remove ref */
351 /* 'td' variable no longer valid */
356 lwkt_gettoken(lwkt_tokref_t ref, lwkt_token_t tok)
358 lwkt_tokref_init(ref, tok);
359 _lwkt_gettokref(ref);
363 lwkt_gettokref(lwkt_tokref_t ref)
365 _lwkt_gettokref(ref);
369 lwkt_trytoken(lwkt_tokref_t ref, lwkt_token_t tok)
371 lwkt_tokref_init(ref, tok);
372 return(_lwkt_trytokref(ref));
376 lwkt_trytokref(lwkt_tokref_t ref)
378 return(_lwkt_trytokref(ref));
382 * Release a serializing token
385 lwkt_reltoken(lwkt_tokref *_ref)
394 * Guard check and stack check (if in the same stack page). We must
395 * also wait for any action pending on remote cpus which we do by
396 * checking the magic number and yielding in a loop.
400 if ((((intptr_t)ref ^ (intptr_t)&_ref) & ~(intptr_t)PAGE_MASK) == 0)
401 KKASSERT((char *)ref > (char *)&_ref);
402 KKASSERT(ref->tr_magic == LWKT_TOKREF_MAGIC1 ||
403 ref->tr_magic == LWKT_TOKREF_MAGIC2);
406 * Locate and unlink the token. Interlock with the token's cpureq
407 * to give the token away before we release it from our thread list,
408 * which allows us to avoid using a critical section.
411 td = gd->gd_curthread;
412 for (pref = &td->td_toks; (ref = *pref) != _ref; pref = &ref->tr_next) {
413 KKASSERT(ref != NULL);
416 KKASSERT(tok->t_cpu == gd);
417 tok->t_cpu = tok->t_reqcpu; /* we do not own 'tok' after this */
418 *pref = ref->tr_next; /* note: also removes giveaway interlock */
421 * If we had gotten the token opportunistically and it still happens to
422 * be queued to a target cpu, we have to wait for the target cpu
423 * to finish processing it. This does not happen very often and does
424 * not need to be optimal.
426 while (ref->tr_magic == LWKT_TOKREF_MAGIC2) {
427 #if defined(MAKE_TOKENS_SPIN)
434 #elif defined(MAKE_TOKENS_YIELD)
437 #error MAKE_TOKENS_XXX ?
443 * Pool tokens are used to provide a type-stable serializing token
444 * pointer that does not race against disappearing data structures.
446 * This routine is called in early boot just after we setup the BSP's
447 * globaldata structure.
450 lwkt_token_pool_init(void)
454 for (i = 0; i < LWKT_NUM_POOL_TOKENS; ++i)
455 lwkt_token_init(&pool_tokens[i]);
459 lwkt_token_pool_get(void *ptraddr)
463 i = ((int)(intptr_t)ptraddr >> 2) ^ ((int)(intptr_t)ptraddr >> 12);
464 return(&pool_tokens[i & LWKT_MASK_POOL_TOKENS]);
470 * This is the receiving side of a remote IPI requesting a token. If we
471 * cannot immediately hand the token off to another cpu we queue it.
473 * NOTE! we 'own' the ref structure, but we only 'own' the token if
477 lwkt_reqtoken_remote(void *data)
479 lwkt_tokref_t ref = data;
480 globaldata_t gd = mycpu;
481 lwkt_token_t tok = ref->tr_tok;
484 * We do not have to queue the token if we can give it away
485 * immediately. Otherwise we queue it to our globaldata structure.
487 KKASSERT(ref->tr_magic == LWKT_TOKREF_MAGIC2);
488 if (lwkt_oktogiveaway_token(tok)) {
489 if (tok->t_cpu == gd)
490 tok->t_cpu = ref->tr_reqgd;
492 ref->tr_magic = LWKT_TOKREF_MAGIC1;
494 ref->tr_gdreqnext = gd->gd_tokreqbase;
495 gd->gd_tokreqbase = ref;
500 * Must be called from a critical section. Satisfy all remote token
501 * requests that are pending on our globaldata structure. The request
502 * does not have to be satisfied with a successful change of ownership
503 * but we do have to acknowledge that we have completed processing the
504 * request by setting the magic number back to MAGIC1.
506 * NOTE! we 'own' the ref structure, but we only 'own' the token if
510 lwkt_drain_token_requests(void)
512 globaldata_t gd = mycpu;
515 while ((ref = gd->gd_tokreqbase) != NULL) {
516 gd->gd_tokreqbase = ref->tr_gdreqnext;
517 KKASSERT(ref->tr_magic == LWKT_TOKREF_MAGIC2);
518 if (ref->tr_tok->t_cpu == gd)
519 ref->tr_tok->t_cpu = ref->tr_reqgd;
521 ref->tr_magic = LWKT_TOKREF_MAGIC1;
528 * Initialize the owner and release-to cpu to the current cpu
529 * and reset the generation count.
532 lwkt_token_init(lwkt_token_t tok)
534 tok->t_cpu = tok->t_reqcpu = mycpu;
538 lwkt_token_uninit(lwkt_token_t tok)