oops, forgot one. Remove another curproc/cred dependancy
[dragonfly.git] / sys / kern / lwkt_thread.c
CommitLineData
8ad65e08
MD
1/*
2 * Copyright (c) 2003 Matthew Dillon <dillon@backplane.com>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
f1d1c3fa
MD
26 * Each cpu in a system has its own self-contained light weight kernel
27 * thread scheduler, which means that generally speaking we only need
28 * to use a critical section to prevent hicups.
29 *
72740893 30 * $DragonFly: src/sys/kern/lwkt_thread.c,v 1.14 2003/07/04 00:32:30 dillon Exp $
8ad65e08
MD
31 */
32
33#include <sys/param.h>
34#include <sys/systm.h>
35#include <sys/kernel.h>
36#include <sys/proc.h>
37#include <sys/rtprio.h>
38#include <sys/queue.h>
f1d1c3fa 39#include <sys/thread2.h>
7d0bac62 40#include <sys/sysctl.h>
99df837e 41#include <sys/kthread.h>
f1d1c3fa 42#include <machine/cpu.h>
99df837e 43#include <sys/lock.h>
f1d1c3fa 44
7d0bac62
MD
45#include <vm/vm.h>
46#include <vm/vm_param.h>
47#include <vm/vm_kern.h>
48#include <vm/vm_object.h>
49#include <vm/vm_page.h>
50#include <vm/vm_map.h>
51#include <vm/vm_pager.h>
52#include <vm/vm_extern.h>
53#include <vm/vm_zone.h>
54
99df837e
MD
55#include <machine/stdarg.h>
56
7d0bac62 57static int untimely_switch = 0;
4b5f931b
MD
58SYSCTL_INT(_lwkt, OID_AUTO, untimely_switch, CTLFLAG_RW, &untimely_switch, 0, "");
59static quad_t switch_count = 0;
60SYSCTL_QUAD(_lwkt, OID_AUTO, switch_count, CTLFLAG_RW, &switch_count, 0, "");
61static quad_t preempt_hit = 0;
62SYSCTL_QUAD(_lwkt, OID_AUTO, preempt_hit, CTLFLAG_RW, &preempt_hit, 0, "");
63static quad_t preempt_miss = 0;
64SYSCTL_QUAD(_lwkt, OID_AUTO, preempt_miss, CTLFLAG_RW, &preempt_miss, 0, "");
26a0694b
MD
65static quad_t preempt_weird = 0;
66SYSCTL_QUAD(_lwkt, OID_AUTO, preempt_weird, CTLFLAG_RW, &preempt_weird, 0, "");
7d0bac62 67
4b5f931b
MD
68/*
69 * These helper procedures handle the runq, they can only be called from
70 * within a critical section.
71 */
f1d1c3fa
MD
72static __inline
73void
74_lwkt_dequeue(thread_t td)
75{
76 if (td->td_flags & TDF_RUNQ) {
4b5f931b
MD
77 int nq = td->td_pri & TDPRI_MASK;
78 struct globaldata *gd = mycpu;
79
f1d1c3fa 80 td->td_flags &= ~TDF_RUNQ;
4b5f931b
MD
81 TAILQ_REMOVE(&gd->gd_tdrunq[nq], td, td_threadq);
82 /* runqmask is passively cleaned up by the switcher */
f1d1c3fa
MD
83 }
84}
85
86static __inline
87void
88_lwkt_enqueue(thread_t td)
89{
90 if ((td->td_flags & TDF_RUNQ) == 0) {
4b5f931b
MD
91 int nq = td->td_pri & TDPRI_MASK;
92 struct globaldata *gd = mycpu;
93
f1d1c3fa 94 td->td_flags |= TDF_RUNQ;
4b5f931b
MD
95 TAILQ_INSERT_TAIL(&gd->gd_tdrunq[nq], td, td_threadq);
96 gd->gd_runqmask |= 1 << nq;
26a0694b
MD
97#if 0
98 /*
99 * YYY needs cli/sti protection? gd_reqpri set by interrupt
100 * when made pending. need better mechanism.
101 */
102 if (gd->gd_reqpri < (td->td_pri & TDPRI_MASK))
103 gd->gd_reqpri = (td->td_pri & TDPRI_MASK);
104#endif
f1d1c3fa
MD
105 }
106}
8ad65e08
MD
107
108/*
109 * LWKTs operate on a per-cpu basis
110 *
73e4f7b9 111 * WARNING! Called from early boot, 'mycpu' may not work yet.
8ad65e08
MD
112 */
113void
114lwkt_gdinit(struct globaldata *gd)
115{
4b5f931b
MD
116 int i;
117
118 for (i = 0; i < sizeof(gd->gd_tdrunq)/sizeof(gd->gd_tdrunq[0]); ++i)
119 TAILQ_INIT(&gd->gd_tdrunq[i]);
120 gd->gd_runqmask = 0;
73e4f7b9 121 TAILQ_INIT(&gd->gd_tdallq);
8ad65e08
MD
122}
123
7d0bac62
MD
124/*
125 * Initialize a thread wait structure prior to first use.
126 *
127 * NOTE! called from low level boot code, we cannot do anything fancy!
128 */
129void
130lwkt_init_wait(lwkt_wait_t w)
131{
132 TAILQ_INIT(&w->wa_waitq);
133}
134
135/*
136 * Create a new thread. The thread must be associated with a process context
137 * or LWKT start address before it can be scheduled.
0cfcada1
MD
138 *
139 * If you intend to create a thread without a process context this function
140 * does everything except load the startup and switcher function.
7d0bac62
MD
141 */
142thread_t
ef0fdad1 143lwkt_alloc_thread(struct thread *td)
7d0bac62 144{
99df837e 145 void *stack;
ef0fdad1 146 int flags = 0;
7d0bac62 147
ef0fdad1 148 if (td == NULL) {
26a0694b 149 crit_enter();
ef0fdad1
MD
150 if (mycpu->gd_tdfreecount > 0) {
151 --mycpu->gd_tdfreecount;
152 td = TAILQ_FIRST(&mycpu->gd_tdfreeq);
153 KASSERT(td != NULL && (td->td_flags & TDF_EXITED),
154 ("lwkt_alloc_thread: unexpected NULL or corrupted td"));
155 TAILQ_REMOVE(&mycpu->gd_tdfreeq, td, td_threadq);
156 crit_exit();
157 stack = td->td_kstack;
158 flags = td->td_flags & (TDF_ALLOCATED_STACK|TDF_ALLOCATED_THREAD);
159 } else {
160 crit_exit();
161 td = zalloc(thread_zone);
162 td->td_kstack = NULL;
163 flags |= TDF_ALLOCATED_THREAD;
164 }
165 }
166 if ((stack = td->td_kstack) == NULL) {
99df837e 167 stack = (void *)kmem_alloc(kernel_map, UPAGES * PAGE_SIZE);
ef0fdad1 168 flags |= TDF_ALLOCATED_STACK;
99df837e 169 }
26a0694b 170 lwkt_init_thread(td, stack, flags, mycpu);
99df837e 171 return(td);
7d0bac62
MD
172}
173
174/*
175 * Initialize a preexisting thread structure. This function is used by
176 * lwkt_alloc_thread() and also used to initialize the per-cpu idlethread.
177 *
178 * NOTE! called from low level boot code, we cannot do anything fancy!
179 */
180void
26a0694b 181lwkt_init_thread(thread_t td, void *stack, int flags, struct globaldata *gd)
7d0bac62 182{
99df837e
MD
183 bzero(td, sizeof(struct thread));
184 td->td_kstack = stack;
185 td->td_flags |= flags;
26a0694b
MD
186 td->td_gd = gd;
187 td->td_pri = TDPRI_CRIT;
99df837e 188 pmap_init_thread(td);
73e4f7b9
MD
189 crit_enter();
190 TAILQ_INSERT_TAIL(&mycpu->gd_tdallq, td, td_allq);
191 crit_exit();
192}
193
194void
195lwkt_set_comm(thread_t td, const char *ctl, ...)
196{
197 va_list va;
198
199 va_start(va, ctl);
200 vsnprintf(td->td_comm, sizeof(td->td_comm), ctl, va);
201 va_end(va);
7d0bac62
MD
202}
203
99df837e 204void
73e4f7b9 205lwkt_hold(thread_t td)
99df837e 206{
73e4f7b9
MD
207 ++td->td_refs;
208}
209
210void
211lwkt_rele(thread_t td)
212{
213 KKASSERT(td->td_refs > 0);
214 --td->td_refs;
215}
216
217void
218lwkt_wait_free(thread_t td)
219{
220 while (td->td_refs)
221 tsleep(td, PWAIT, "tdreap", hz);
222}
223
224void
225lwkt_free_thread(thread_t td)
226{
227 struct globaldata *gd = mycpu;
228
99df837e
MD
229 KASSERT(td->td_flags & TDF_EXITED,
230 ("lwkt_free_thread: did not exit! %p", td));
231
232 crit_enter();
73e4f7b9
MD
233 TAILQ_REMOVE(&gd->gd_tdallq, td, td_allq);
234 if (gd->gd_tdfreecount < CACHE_NTHREADS &&
99df837e
MD
235 (td->td_flags & TDF_ALLOCATED_THREAD)
236 ) {
73e4f7b9
MD
237 ++gd->gd_tdfreecount;
238 TAILQ_INSERT_HEAD(&gd->gd_tdfreeq, td, td_threadq);
99df837e
MD
239 crit_exit();
240 } else {
241 crit_exit();
242 if (td->td_kstack && (td->td_flags & TDF_ALLOCATED_STACK)) {
243 kmem_free(kernel_map,
244 (vm_offset_t)td->td_kstack, UPAGES * PAGE_SIZE);
73e4f7b9 245 /* gd invalid */
99df837e
MD
246 td->td_kstack = NULL;
247 }
248 if (td->td_flags & TDF_ALLOCATED_THREAD)
249 zfree(thread_zone, td);
250 }
251}
252
253
8ad65e08
MD
254/*
255 * Switch to the next runnable lwkt. If no LWKTs are runnable then
f1d1c3fa
MD
256 * switch to the idlethread. Switching must occur within a critical
257 * section to avoid races with the scheduling queue.
258 *
259 * We always have full control over our cpu's run queue. Other cpus
260 * that wish to manipulate our queue must use the cpu_*msg() calls to
261 * talk to our cpu, so a critical section is all that is needed and
262 * the result is very, very fast thread switching.
263 *
264 * We always 'own' our own thread and the threads on our run queue,l
265 * due to TDF_RUNNING or TDF_RUNQ being set. We can safely clear
266 * TDF_RUNNING while in a critical section.
267 *
268 * The td_switch() function must be called while in the critical section.
269 * This function saves as much state as is appropriate for the type of
270 * thread.
271 *
272 * (self contained on a per cpu basis)
8ad65e08
MD
273 */
274void
275lwkt_switch(void)
276{
4b5f931b 277 struct globaldata *gd;
f1d1c3fa 278 thread_t td = curthread;
8ad65e08
MD
279 thread_t ntd;
280
b68b7282 281 if (mycpu->gd_intr_nesting_level && td->td_preempted == NULL)
26a0694b 282 panic("lwkt_switch: cannot switch from within an interrupt, yet\n");
ef0fdad1 283
f1d1c3fa 284 crit_enter();
4b5f931b 285 ++switch_count;
99df837e
MD
286 if ((ntd = td->td_preempted) != NULL) {
287 /*
288 * We had preempted another thread on this cpu, resume the preempted
26a0694b
MD
289 * thread. This occurs transparently, whether the preempted thread
290 * was scheduled or not (it may have been preempted after descheduling
291 * itself).
99df837e 292 */
26a0694b
MD
293 KKASSERT(ntd->td_flags & TDF_PREEMPT_LOCK);
294 ntd->td_flags |= TDF_PREEMPT_DONE;
8ad65e08 295 } else {
4b5f931b
MD
296 /*
297 * Priority queue / round-robin at each priority. Note that user
298 * processes run at a fixed, low priority and the user process
299 * scheduler deals with interactions between user processes
300 * by scheduling and descheduling them from the LWKT queue as
301 * necessary.
302 */
303 gd = mycpu;
304
305again:
306 if (gd->gd_runqmask) {
307 int nq = bsrl(gd->gd_runqmask);
308 if ((ntd = TAILQ_FIRST(&gd->gd_tdrunq[nq])) == NULL) {
309 gd->gd_runqmask &= ~(1 << nq);
310 goto again;
311 }
312 TAILQ_REMOVE(&gd->gd_tdrunq[nq], ntd, td_threadq);
313 TAILQ_INSERT_TAIL(&gd->gd_tdrunq[nq], ntd, td_threadq);
314 } else {
315 ntd = gd->gd_idletd;
316 }
f1d1c3fa 317 }
26a0694b
MD
318 KASSERT(ntd->td_pri >= TDPRI_CRIT,
319 ("priority problem in lwkt_switch %d %d", td->td_pri, ntd->td_pri));
99df837e 320 if (td != ntd)
f1d1c3fa 321 td->td_switch(ntd);
f1d1c3fa 322 crit_exit();
8ad65e08
MD
323}
324
b68b7282 325/*
26a0694b
MD
326 * Request that the target thread preempt the current thread. This only
327 * works if:
b68b7282 328 *
26a0694b
MD
329 * + We aren't trying to preempt ourselves (it can happen!)
330 * + We are not currently being preempted
331 * + the target is not currently being preempted
b68b7282 332 *
26a0694b
MD
333 * XXX at the moment we run the target thread in a critical section during
334 * the preemption in order to prevent the target from taking interrupts
335 * that *WE* can't. Preemption is strictly limited to interrupt threads
336 * and interrupt-like threads, outside of a critical section, and the
337 * preempted source thread will be resumed the instant the target blocks
338 * whether or not the source is scheduled (i.e. preemption is supposed to
339 * be as transparent as possible).
4b5f931b
MD
340 *
341 * This call is typically made from an interrupt handler like sched_ithd()
342 * which will only run if the current thread is not in a critical section,
343 * so we optimize the priority check a bit.
26a0694b
MD
344 *
345 * CAREFUL! either we or the target thread may get interrupted during the
346 * switch.
b68b7282
MD
347 */
348void
73e4f7b9 349lwkt_preempt(thread_t ntd, int id)
b68b7282 350{
73e4f7b9 351 thread_t td = curthread;
b68b7282 352
26a0694b
MD
353 /*
354 * The caller has put us in a critical section, and in order to have
355 * gotten here in the first place the thread the caller interrupted
356 * cannot have been in a critical section before.
357 */
358 KASSERT(ntd->td_pri >= TDPRI_CRIT, ("BADCRIT0 %d", ntd->td_pri));
359 KASSERT((td->td_pri & ~TDPRI_MASK) == TDPRI_CRIT, ("BADPRI %d", td->td_pri));
360
361 if (td == ntd || ((td->td_flags | ntd->td_flags) & TDF_PREEMPT_LOCK)) {
362 ++preempt_weird;
363 return;
364 }
365 if (ntd->td_preempted) {
4b5f931b 366 ++preempt_hit;
26a0694b
MD
367 return;
368 }
369 if ((ntd->td_pri & TDPRI_MASK) <= (td->td_pri & TDPRI_MASK)) {
4b5f931b 370 ++preempt_miss;
26a0694b 371 return;
b68b7282 372 }
26a0694b
MD
373
374 ++preempt_hit;
375 ntd->td_preempted = td;
376 td->td_flags |= TDF_PREEMPT_LOCK;
377 td->td_switch(ntd);
378 KKASSERT(ntd->td_preempted && (td->td_flags & TDF_PREEMPT_DONE));
379 ntd->td_preempted = NULL;
380 td->td_flags &= ~(TDF_PREEMPT_LOCK|TDF_PREEMPT_DONE);
b68b7282
MD
381}
382
f1d1c3fa
MD
383/*
384 * Yield our thread while higher priority threads are pending. This is
385 * typically called when we leave a critical section but it can be safely
386 * called while we are in a critical section.
387 *
388 * This function will not generally yield to equal priority threads but it
389 * can occur as a side effect. Note that lwkt_switch() is called from
390 * inside the critical section to pervent its own crit_exit() from reentering
391 * lwkt_yield_quick().
392 *
ef0fdad1
MD
393 * gd_reqpri indicates that *something* changed, e.g. an interrupt or softint
394 * came along but was blocked and made pending.
395 *
f1d1c3fa
MD
396 * (self contained on a per cpu basis)
397 */
398void
399lwkt_yield_quick(void)
400{
401 thread_t td = curthread;
ef0fdad1
MD
402
403 if ((td->td_pri & TDPRI_MASK) < mycpu->gd_reqpri) {
404 mycpu->gd_reqpri = 0;
f1d1c3fa
MD
405 splz();
406 }
407
408 /*
409 * YYY enabling will cause wakeup() to task-switch, which really
410 * confused the old 4.x code. This is a good way to simulate
7d0bac62
MD
411 * preemption and MP without actually doing preemption or MP, because a
412 * lot of code assumes that wakeup() does not block.
f1d1c3fa 413 */
ef0fdad1 414 if (untimely_switch && mycpu->gd_intr_nesting_level == 0) {
f1d1c3fa
MD
415 crit_enter();
416 /*
417 * YYY temporary hacks until we disassociate the userland scheduler
418 * from the LWKT scheduler.
419 */
420 if (td->td_flags & TDF_RUNQ) {
421 lwkt_switch(); /* will not reenter yield function */
422 } else {
423 lwkt_schedule_self(); /* make sure we are scheduled */
424 lwkt_switch(); /* will not reenter yield function */
425 lwkt_deschedule_self(); /* make sure we are descheduled */
426 }
427 crit_exit_noyield();
428 }
f1d1c3fa
MD
429}
430
8ad65e08 431/*
f1d1c3fa
MD
432 * This implements a normal yield which, unlike _quick, will yield to equal
433 * priority threads as well. Note that gd_reqpri tests will be handled by
434 * the crit_exit() call in lwkt_switch().
435 *
436 * (self contained on a per cpu basis)
8ad65e08
MD
437 */
438void
f1d1c3fa 439lwkt_yield(void)
8ad65e08 440{
f1d1c3fa
MD
441 lwkt_schedule_self();
442 lwkt_switch();
443}
444
445/*
446 * Schedule a thread to run. As the current thread we can always safely
447 * schedule ourselves, and a shortcut procedure is provided for that
448 * function.
449 *
450 * (non-blocking, self contained on a per cpu basis)
451 */
452void
453lwkt_schedule_self(void)
454{
455 thread_t td = curthread;
456
457 crit_enter();
458 KASSERT(td->td_wait == NULL, ("lwkt_schedule_self(): td_wait not NULL!"));
f1d1c3fa 459 _lwkt_enqueue(td);
26a0694b
MD
460 if (td->td_proc && td->td_proc->p_stat == SSLEEP)
461 panic("SCHED SELF PANIC");
f1d1c3fa 462 crit_exit();
8ad65e08 463}
8ad65e08
MD
464
465/*
f1d1c3fa
MD
466 * Generic schedule. Possibly schedule threads belonging to other cpus and
467 * deal with threads that might be blocked on a wait queue.
468 *
469 * This function will queue requests asynchronously when possible, but may
470 * block if no request structures are available. Upon return the caller
471 * should note that the scheduling request may not yet have been processed
472 * by the target cpu.
473 *
474 * YYY this is one of the best places to implement any load balancing code.
475 * Load balancing can be accomplished by requesting other sorts of actions
476 * for the thread in question.
8ad65e08
MD
477 */
478void
479lwkt_schedule(thread_t td)
480{
26a0694b
MD
481 if ((td->td_flags & TDF_PREEMPT_LOCK) == 0 && td->td_proc
482 && td->td_proc->p_stat == SSLEEP
483 ) {
484 printf("PANIC schedule curtd = %p (%d %d) target %p (%d %d)\n",
485 curthread,
486 curthread->td_proc ? curthread->td_proc->p_pid : -1,
487 curthread->td_proc ? curthread->td_proc->p_stat : -1,
488 td,
489 td->td_proc ? curthread->td_proc->p_pid : -1,
490 td->td_proc ? curthread->td_proc->p_stat : -1
491 );
492 panic("SCHED PANIC");
493 }
f1d1c3fa
MD
494 crit_enter();
495 if (td == curthread) {
496 _lwkt_enqueue(td);
497 } else {
498 lwkt_wait_t w;
499
500 /*
501 * If the thread is on a wait list we have to send our scheduling
502 * request to the owner of the wait structure. Otherwise we send
503 * the scheduling request to the cpu owning the thread. Races
504 * are ok, the target will forward the message as necessary (the
505 * message may chase the thread around before it finally gets
506 * acted upon).
507 *
508 * (remember, wait structures use stable storage)
509 */
510 if ((w = td->td_wait) != NULL) {
511 if (lwkt_havetoken(&w->wa_token)) {
512 TAILQ_REMOVE(&w->wa_waitq, td, td_threadq);
513 --w->wa_count;
514 td->td_wait = NULL;
d0e06f83 515 if (td->td_cpu == mycpu->gd_cpuid) {
f1d1c3fa
MD
516 _lwkt_enqueue(td);
517 } else {
518 panic("lwkt_schedule: cpu mismatch1");
8ad65e08 519#if 0
f1d1c3fa
MD
520 lwkt_cpu_msg_union_t msg = lwkt_getcpumsg();
521 initScheduleReqMsg_Wait(&msg.mu_SchedReq, td, w);
522 cpu_sendnormsg(&msg.mu_Msg);
8ad65e08 523#endif
f1d1c3fa
MD
524 }
525 } else {
526 panic("lwkt_schedule: cpu mismatch2");
527#if 0
528 lwkt_cpu_msg_union_t msg = lwkt_getcpumsg();
529 initScheduleReqMsg_Wait(&msg.mu_SchedReq, td, w);
530 cpu_sendnormsg(&msg.mu_Msg);
531#endif
532 }
533 } else {
534 /*
535 * If the wait structure is NULL and we own the thread, there
536 * is no race (since we are in a critical section). If we
537 * do not own the thread there might be a race but the
538 * target cpu will deal with it.
539 */
d0e06f83 540 if (td->td_cpu == mycpu->gd_cpuid) {
f1d1c3fa
MD
541 _lwkt_enqueue(td);
542 } else {
543 panic("lwkt_schedule: cpu mismatch3");
544#if 0
545 lwkt_cpu_msg_union_t msg = lwkt_getcpumsg();
546 initScheduleReqMsg_Thread(&msg.mu_SchedReq, td);
547 cpu_sendnormsg(&msg.mu_Msg);
548#endif
549 }
550 }
8ad65e08 551 }
f1d1c3fa 552 crit_exit();
8ad65e08
MD
553}
554
555/*
f1d1c3fa
MD
556 * Deschedule a thread.
557 *
558 * (non-blocking, self contained on a per cpu basis)
559 */
560void
561lwkt_deschedule_self(void)
562{
563 thread_t td = curthread;
564
565 crit_enter();
566 KASSERT(td->td_wait == NULL, ("lwkt_schedule_self(): td_wait not NULL!"));
f1d1c3fa
MD
567 _lwkt_dequeue(td);
568 crit_exit();
569}
570
571/*
572 * Generic deschedule. Descheduling threads other then your own should be
573 * done only in carefully controlled circumstances. Descheduling is
574 * asynchronous.
575 *
576 * This function may block if the cpu has run out of messages.
8ad65e08
MD
577 */
578void
579lwkt_deschedule(thread_t td)
580{
f1d1c3fa
MD
581 crit_enter();
582 if (td == curthread) {
583 _lwkt_dequeue(td);
584 } else {
d0e06f83 585 if (td->td_cpu == mycpu->gd_cpuid) {
f1d1c3fa
MD
586 _lwkt_dequeue(td);
587 } else {
588 panic("lwkt_deschedule: cpu mismatch");
589#if 0
590 lwkt_cpu_msg_union_t msg = lwkt_getcpumsg();
591 initDescheduleReqMsg_Thread(&msg.mu_DeschedReq, td);
592 cpu_sendnormsg(&msg.mu_Msg);
593#endif
594 }
595 }
596 crit_exit();
597}
598
4b5f931b
MD
599/*
600 * Set the target thread's priority. This routine does not automatically
601 * switch to a higher priority thread, LWKT threads are not designed for
602 * continuous priority changes. Yield if you want to switch.
603 *
604 * We have to retain the critical section count which uses the high bits
26a0694b
MD
605 * of the td_pri field. The specified priority may also indicate zero or
606 * more critical sections by adding TDPRI_CRIT*N.
4b5f931b
MD
607 */
608void
609lwkt_setpri(thread_t td, int pri)
610{
26a0694b
MD
611 KKASSERT(pri >= 0);
612 crit_enter();
613 if (td->td_flags & TDF_RUNQ) {
614 _lwkt_dequeue(td);
615 td->td_pri = (td->td_pri & ~TDPRI_MASK) + pri;
616 _lwkt_enqueue(td);
617 } else {
618 td->td_pri = (td->td_pri & ~TDPRI_MASK) + pri;
619 }
620 crit_exit();
621}
622
623void
624lwkt_setpri_self(int pri)
625{
626 thread_t td = curthread;
627
4b5f931b
MD
628 KKASSERT(pri >= 0 && pri <= TDPRI_MAX);
629 crit_enter();
630 if (td->td_flags & TDF_RUNQ) {
631 _lwkt_dequeue(td);
632 td->td_pri = (td->td_pri & ~TDPRI_MASK) + pri;
633 _lwkt_enqueue(td);
634 } else {
635 td->td_pri = (td->td_pri & ~TDPRI_MASK) + pri;
636 }
637 crit_exit();
638}
639
640struct proc *
641lwkt_preempted_proc(void)
642{
73e4f7b9 643 thread_t td = curthread;
4b5f931b
MD
644 while (td->td_preempted)
645 td = td->td_preempted;
646 return(td->td_proc);
647}
648
649
f1d1c3fa
MD
650/*
651 * This function deschedules the current thread and blocks on the specified
652 * wait queue. We obtain ownership of the wait queue in order to block
653 * on it. A generation number is used to interlock the wait queue in case
654 * it gets signalled while we are blocked waiting on the token.
655 *
656 * Note: alternatively we could dequeue our thread and then message the
657 * target cpu owning the wait queue. YYY implement as sysctl.
658 *
659 * Note: wait queue signals normally ping-pong the cpu as an optimization.
660 */
661void
ae8050a4 662lwkt_block(lwkt_wait_t w, const char *wmesg, int *gen)
f1d1c3fa
MD
663{
664 thread_t td = curthread;
f1d1c3fa 665
f1d1c3fa 666 lwkt_gettoken(&w->wa_token);
ae8050a4 667 if (w->wa_gen == *gen) {
f1d1c3fa
MD
668 _lwkt_dequeue(td);
669 TAILQ_INSERT_TAIL(&w->wa_waitq, td, td_threadq);
670 ++w->wa_count;
671 td->td_wait = w;
ae8050a4 672 td->td_wmesg = wmesg;
f1d1c3fa 673 lwkt_switch();
8ad65e08 674 }
ae8050a4
MD
675 /* token might be lost, doesn't matter for gen update */
676 *gen = w->wa_gen;
f1d1c3fa
MD
677 lwkt_reltoken(&w->wa_token);
678}
679
680/*
681 * Signal a wait queue. We gain ownership of the wait queue in order to
682 * signal it. Once a thread is removed from the wait queue we have to
683 * deal with the cpu owning the thread.
684 *
685 * Note: alternatively we could message the target cpu owning the wait
686 * queue. YYY implement as sysctl.
687 */
688void
689lwkt_signal(lwkt_wait_t w)
690{
691 thread_t td;
692 int count;
693
694 lwkt_gettoken(&w->wa_token);
695 ++w->wa_gen;
696 count = w->wa_count;
697 while ((td = TAILQ_FIRST(&w->wa_waitq)) != NULL && count) {
698 --count;
699 --w->wa_count;
700 TAILQ_REMOVE(&w->wa_waitq, td, td_threadq);
701 td->td_wait = NULL;
ae8050a4 702 td->td_wmesg = NULL;
d0e06f83 703 if (td->td_cpu == mycpu->gd_cpuid) {
f1d1c3fa
MD
704 _lwkt_enqueue(td);
705 } else {
706#if 0
707 lwkt_cpu_msg_union_t msg = lwkt_getcpumsg();
708 initScheduleReqMsg_Thread(&msg.mu_SchedReq, td);
709 cpu_sendnormsg(&msg.mu_Msg);
710#endif
711 panic("lwkt_signal: cpu mismatch");
712 }
713 lwkt_regettoken(&w->wa_token);
714 }
715 lwkt_reltoken(&w->wa_token);
716}
717
718/*
719 * Aquire ownership of a token
720 *
721 * Aquire ownership of a token. The token may have spl and/or critical
722 * section side effects, depending on its purpose. These side effects
723 * guarentee that you will maintain ownership of the token as long as you
724 * do not block. If you block you may lose access to the token (but you
725 * must still release it even if you lose your access to it).
726 *
727 * Note that the spl and critical section characteristics of a token
728 * may not be changed once the token has been initialized.
729 */
730void
731lwkt_gettoken(lwkt_token_t tok)
732{
733 /*
734 * Prevent preemption so the token can't be taken away from us once
735 * we gain ownership of it. Use a synchronous request which might
736 * block. The request will be forwarded as necessary playing catchup
737 * to the token.
738 */
739 crit_enter();
740#if 0
d0e06f83 741 while (tok->t_cpu != mycpu->gd_cpuid) {
f1d1c3fa
MD
742 lwkt_cpu_msg_union msg;
743 initTokenReqMsg(&msg.mu_TokenReq);
744 cpu_domsg(&msg);
745 }
746#endif
747 /*
748 * leave us in a critical section on return. This will be undone
749 * by lwkt_reltoken()
750 */
751}
752
753/*
754 * Release your ownership of a token. Releases must occur in reverse
755 * order to aquisitions, eventually so priorities can be unwound properly
756 * like SPLs. At the moment the actual implemention doesn't care.
757 *
758 * We can safely hand a token that we own to another cpu without notifying
759 * it, but once we do we can't get it back without requesting it (unless
760 * the other cpu hands it back to us before we check).
761 *
762 * We might have lost the token, so check that.
763 */
764void
765lwkt_reltoken(lwkt_token_t tok)
766{
d0e06f83 767 if (tok->t_cpu == mycpu->gd_cpuid) {
f1d1c3fa
MD
768 tok->t_cpu = tok->t_reqcpu;
769 }
770 crit_exit();
771}
772
773/*
774 * Reaquire a token that might have been lost. Returns 1 if we blocked
775 * while reaquiring the token (meaning that you might have lost other
776 * tokens you held when you made this call), return 0 if we did not block.
777 */
778int
779lwkt_regettoken(lwkt_token_t tok)
780{
781#if 0
d0e06f83
MD
782 if (tok->t_cpu != mycpu->gd_cpuid) {
783 while (tok->t_cpu != mycpu->gd_cpuid) {
f1d1c3fa
MD
784 lwkt_cpu_msg_union msg;
785 initTokenReqMsg(&msg.mu_TokenReq);
786 cpu_domsg(&msg);
787 }
788 return(1);
789 }
790#endif
791 return(0);
8ad65e08
MD
792}
793
72740893
MD
794void
795lwkt_inittoken(lwkt_token_t tok)
796{
797 /*
798 * Zero structure and set cpu owner and reqcpu to cpu 0.
799 */
800 bzero(tok, sizeof(*tok));
801}
802
99df837e
MD
803/*
804 * Create a kernel process/thread/whatever. It shares it's address space
805 * with proc0 - ie: kernel only.
806 *
807 * XXX should be renamed to lwkt_create()
808 */
809int
810lwkt_create(void (*func)(void *), void *arg,
73e4f7b9 811 struct thread **tdp, thread_t template, int tdflags,
ef0fdad1 812 const char *fmt, ...)
99df837e 813{
73e4f7b9 814 thread_t td;
99df837e
MD
815 va_list ap;
816
ef0fdad1 817 td = *tdp = lwkt_alloc_thread(template);
99df837e 818 cpu_set_thread_handler(td, kthread_exit, func, arg);
ef0fdad1 819 td->td_flags |= TDF_VERBOSE | tdflags;
99df837e
MD
820
821 /*
822 * Set up arg0 for 'ps' etc
823 */
824 va_start(ap, fmt);
825 vsnprintf(td->td_comm, sizeof(td->td_comm), fmt, ap);
826 va_end(ap);
827
828 /*
829 * Schedule the thread to run
830 */
ef0fdad1
MD
831 if ((td->td_flags & TDF_STOPREQ) == 0)
832 lwkt_schedule(td);
833 else
834 td->td_flags &= ~TDF_STOPREQ;
99df837e
MD
835 return 0;
836}
837
838/*
839 * Destroy an LWKT thread. Warning! This function is not called when
840 * a process exits, cpu_proc_exit() directly calls cpu_thread_exit() and
841 * uses a different reaping mechanism.
842 */
843void
844lwkt_exit(void)
845{
846 thread_t td = curthread;
847
848 if (td->td_flags & TDF_VERBOSE)
849 printf("kthread %p %s has exited\n", td, td->td_comm);
850 crit_enter();
851 lwkt_deschedule_self();
852 ++mycpu->gd_tdfreecount;
853 TAILQ_INSERT_TAIL(&mycpu->gd_tdfreeq, td, td_threadq);
854 cpu_thread_exit();
855}
856
857/*
858 * Create a kernel process/thread/whatever. It shares it's address space
ef0fdad1 859 * with proc0 - ie: kernel only. 5.x compatible.
99df837e
MD
860 */
861int
862kthread_create(void (*func)(void *), void *arg,
863 struct thread **tdp, const char *fmt, ...)
864{
73e4f7b9 865 thread_t td;
99df837e
MD
866 va_list ap;
867
ef0fdad1 868 td = *tdp = lwkt_alloc_thread(NULL);
99df837e
MD
869 cpu_set_thread_handler(td, kthread_exit, func, arg);
870 td->td_flags |= TDF_VERBOSE;
871
872 /*
873 * Set up arg0 for 'ps' etc
874 */
875 va_start(ap, fmt);
876 vsnprintf(td->td_comm, sizeof(td->td_comm), fmt, ap);
877 va_end(ap);
878
879 /*
880 * Schedule the thread to run
881 */
882 lwkt_schedule(td);
883 return 0;
884}
885
26a0694b
MD
886void
887crit_panic(void)
888{
73e4f7b9 889 thread_t td = curthread;
26a0694b
MD
890 int lpri = td->td_pri;
891
892 td->td_pri = 0;
893 panic("td_pri is/would-go negative! %p %d", td, lpri);
894}
895
99df837e
MD
896/*
897 * Destroy an LWKT thread. Warning! This function is not called when
898 * a process exits, cpu_proc_exit() directly calls cpu_thread_exit() and
899 * uses a different reaping mechanism.
900 *
901 * XXX duplicates lwkt_exit()
902 */
903void
904kthread_exit(void)
905{
906 lwkt_exit();
907}
908