2 * Copyright (c) 2003 Matthew Dillon <dillon@backplane.com>
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * Each cpu in a system has its own self-contained light weight kernel
27 * thread scheduler, which means that generally speaking we only need
28 * to use a critical section to prevent hicups.
30 * $DragonFly: src/sys/kern/lwkt_thread.c,v 1.2 2003/06/21 07:54:57 dillon Exp $
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/kernel.h>
37 #include <sys/rtprio.h>
38 #include <sys/queue.h>
39 #include <sys/thread2.h>
40 #include <machine/cpu.h>
44 _lwkt_dequeue(thread_t td)
46 if (td->td_flags & TDF_RUNQ) {
47 td->td_flags &= ~TDF_RUNQ;
48 TAILQ_REMOVE(&mycpu->gd_tdrunq, td, td_threadq);
54 _lwkt_enqueue(thread_t td)
56 if ((td->td_flags & TDF_RUNQ) == 0) {
57 td->td_flags |= TDF_RUNQ;
58 TAILQ_INSERT_TAIL(&mycpu->gd_tdrunq, td, td_threadq);
63 * LWKTs operate on a per-cpu basis
65 * YYY implement strict priorities & round-robin at the same priority
68 lwkt_gdinit(struct globaldata *gd)
70 TAILQ_INIT(&gd->gd_tdrunq);
74 * Switch to the next runnable lwkt. If no LWKTs are runnable then
75 * switch to the idlethread. Switching must occur within a critical
76 * section to avoid races with the scheduling queue.
78 * We always have full control over our cpu's run queue. Other cpus
79 * that wish to manipulate our queue must use the cpu_*msg() calls to
80 * talk to our cpu, so a critical section is all that is needed and
81 * the result is very, very fast thread switching.
83 * We always 'own' our own thread and the threads on our run queue,l
84 * due to TDF_RUNNING or TDF_RUNQ being set. We can safely clear
85 * TDF_RUNNING while in a critical section.
87 * The td_switch() function must be called while in the critical section.
88 * This function saves as much state as is appropriate for the type of
91 * (self contained on a per cpu basis)
96 thread_t td = curthread;
100 if ((ntd = TAILQ_FIRST(&mycpu->gd_tdrunq)) != NULL) {
101 TAILQ_REMOVE(&mycpu->gd_tdrunq, ntd, td_threadq);
102 TAILQ_INSERT_TAIL(&mycpu->gd_tdrunq, ntd, td_threadq);
104 ntd = &mycpu->gd_idlethread;
107 td->td_flags &= ~TDF_RUNNING;
108 ntd->td_flags |= TDF_RUNNING;
115 * Yield our thread while higher priority threads are pending. This is
116 * typically called when we leave a critical section but it can be safely
117 * called while we are in a critical section.
119 * This function will not generally yield to equal priority threads but it
120 * can occur as a side effect. Note that lwkt_switch() is called from
121 * inside the critical section to pervent its own crit_exit() from reentering
122 * lwkt_yield_quick().
124 * (self contained on a per cpu basis)
127 lwkt_yield_quick(void)
129 thread_t td = curthread;
130 while ((td->td_pri & TDPRI_MASK) < mycpu->gd_reqpri) {
132 cpu_schedule_reqs(); /* resets gd_reqpri */
138 * YYY enabling will cause wakeup() to task-switch, which really
139 * confused the old 4.x code. This is a good way to simulate
140 * preemption without actually doing preemption, because a lot
141 * of code (including schedule, deschedule) uses critical sections
142 * which devolve to here if an interrupt occured.
145 if (intr_nesting_level == 0) {
148 * YYY temporary hacks until we disassociate the userland scheduler
149 * from the LWKT scheduler.
151 if (td->td_flags & TDF_RUNQ) {
152 lwkt_switch(); /* will not reenter yield function */
154 lwkt_schedule_self(); /* make sure we are scheduled */
155 lwkt_switch(); /* will not reenter yield function */
156 lwkt_deschedule_self(); /* make sure we are descheduled */
164 * This implements a normal yield which, unlike _quick, will yield to equal
165 * priority threads as well. Note that gd_reqpri tests will be handled by
166 * the crit_exit() call in lwkt_switch().
168 * (self contained on a per cpu basis)
173 lwkt_schedule_self();
178 * Schedule a thread to run. As the current thread we can always safely
179 * schedule ourselves, and a shortcut procedure is provided for that
182 * (non-blocking, self contained on a per cpu basis)
185 lwkt_schedule_self(void)
187 thread_t td = curthread;
190 KASSERT(td->td_wait == NULL, ("lwkt_schedule_self(): td_wait not NULL!"));
191 KASSERT(td->td_flags & TDF_RUNNING, ("lwkt_schedule_self(): TDF_RUNNING not set!"));
197 * Generic schedule. Possibly schedule threads belonging to other cpus and
198 * deal with threads that might be blocked on a wait queue.
200 * This function will queue requests asynchronously when possible, but may
201 * block if no request structures are available. Upon return the caller
202 * should note that the scheduling request may not yet have been processed
205 * YYY this is one of the best places to implement any load balancing code.
206 * Load balancing can be accomplished by requesting other sorts of actions
207 * for the thread in question.
210 lwkt_schedule(thread_t td)
213 if (td == curthread) {
219 * If the thread is on a wait list we have to send our scheduling
220 * request to the owner of the wait structure. Otherwise we send
221 * the scheduling request to the cpu owning the thread. Races
222 * are ok, the target will forward the message as necessary (the
223 * message may chase the thread around before it finally gets
226 * (remember, wait structures use stable storage)
228 if ((w = td->td_wait) != NULL) {
229 if (lwkt_havetoken(&w->wa_token)) {
230 TAILQ_REMOVE(&w->wa_waitq, td, td_threadq);
233 if (td->td_cpu == mycpu->gd_cpu) {
236 panic("lwkt_schedule: cpu mismatch1");
238 lwkt_cpu_msg_union_t msg = lwkt_getcpumsg();
239 initScheduleReqMsg_Wait(&msg.mu_SchedReq, td, w);
240 cpu_sendnormsg(&msg.mu_Msg);
244 panic("lwkt_schedule: cpu mismatch2");
246 lwkt_cpu_msg_union_t msg = lwkt_getcpumsg();
247 initScheduleReqMsg_Wait(&msg.mu_SchedReq, td, w);
248 cpu_sendnormsg(&msg.mu_Msg);
253 * If the wait structure is NULL and we own the thread, there
254 * is no race (since we are in a critical section). If we
255 * do not own the thread there might be a race but the
256 * target cpu will deal with it.
258 if (td->td_cpu == mycpu->gd_cpu) {
261 panic("lwkt_schedule: cpu mismatch3");
263 lwkt_cpu_msg_union_t msg = lwkt_getcpumsg();
264 initScheduleReqMsg_Thread(&msg.mu_SchedReq, td);
265 cpu_sendnormsg(&msg.mu_Msg);
274 * Deschedule a thread.
276 * (non-blocking, self contained on a per cpu basis)
279 lwkt_deschedule_self(void)
281 thread_t td = curthread;
284 KASSERT(td->td_wait == NULL, ("lwkt_schedule_self(): td_wait not NULL!"));
285 KASSERT(td->td_flags & TDF_RUNNING, ("lwkt_schedule_self(): TDF_RUNNING not set!"));
291 * Generic deschedule. Descheduling threads other then your own should be
292 * done only in carefully controlled circumstances. Descheduling is
295 * This function may block if the cpu has run out of messages.
298 lwkt_deschedule(thread_t td)
301 if (td == curthread) {
304 if (td->td_cpu == mycpu->gd_cpu) {
307 panic("lwkt_deschedule: cpu mismatch");
309 lwkt_cpu_msg_union_t msg = lwkt_getcpumsg();
310 initDescheduleReqMsg_Thread(&msg.mu_DeschedReq, td);
311 cpu_sendnormsg(&msg.mu_Msg);
319 * This function deschedules the current thread and blocks on the specified
320 * wait queue. We obtain ownership of the wait queue in order to block
321 * on it. A generation number is used to interlock the wait queue in case
322 * it gets signalled while we are blocked waiting on the token.
324 * Note: alternatively we could dequeue our thread and then message the
325 * target cpu owning the wait queue. YYY implement as sysctl.
327 * Note: wait queue signals normally ping-pong the cpu as an optimization.
330 lwkt_block(lwkt_wait_t w)
332 thread_t td = curthread;
336 lwkt_gettoken(&w->wa_token);
337 if (w->wa_gen == gen) {
339 TAILQ_INSERT_TAIL(&w->wa_waitq, td, td_threadq);
344 lwkt_reltoken(&w->wa_token);
348 * Signal a wait queue. We gain ownership of the wait queue in order to
349 * signal it. Once a thread is removed from the wait queue we have to
350 * deal with the cpu owning the thread.
352 * Note: alternatively we could message the target cpu owning the wait
353 * queue. YYY implement as sysctl.
356 lwkt_signal(lwkt_wait_t w)
361 lwkt_gettoken(&w->wa_token);
364 while ((td = TAILQ_FIRST(&w->wa_waitq)) != NULL && count) {
367 TAILQ_REMOVE(&w->wa_waitq, td, td_threadq);
369 if (td->td_cpu == mycpu->gd_cpu) {
373 lwkt_cpu_msg_union_t msg = lwkt_getcpumsg();
374 initScheduleReqMsg_Thread(&msg.mu_SchedReq, td);
375 cpu_sendnormsg(&msg.mu_Msg);
377 panic("lwkt_signal: cpu mismatch");
379 lwkt_regettoken(&w->wa_token);
381 lwkt_reltoken(&w->wa_token);
385 * Aquire ownership of a token
387 * Aquire ownership of a token. The token may have spl and/or critical
388 * section side effects, depending on its purpose. These side effects
389 * guarentee that you will maintain ownership of the token as long as you
390 * do not block. If you block you may lose access to the token (but you
391 * must still release it even if you lose your access to it).
393 * Note that the spl and critical section characteristics of a token
394 * may not be changed once the token has been initialized.
397 lwkt_gettoken(lwkt_token_t tok)
400 * Prevent preemption so the token can't be taken away from us once
401 * we gain ownership of it. Use a synchronous request which might
402 * block. The request will be forwarded as necessary playing catchup
407 while (tok->t_cpu != mycpu->gd_cpu) {
408 lwkt_cpu_msg_union msg;
409 initTokenReqMsg(&msg.mu_TokenReq);
414 * leave us in a critical section on return. This will be undone
420 * Release your ownership of a token. Releases must occur in reverse
421 * order to aquisitions, eventually so priorities can be unwound properly
422 * like SPLs. At the moment the actual implemention doesn't care.
424 * We can safely hand a token that we own to another cpu without notifying
425 * it, but once we do we can't get it back without requesting it (unless
426 * the other cpu hands it back to us before we check).
428 * We might have lost the token, so check that.
431 lwkt_reltoken(lwkt_token_t tok)
433 if (tok->t_cpu == mycpu->gd_cpu) {
434 tok->t_cpu = tok->t_reqcpu;
440 * Reaquire a token that might have been lost. Returns 1 if we blocked
441 * while reaquiring the token (meaning that you might have lost other
442 * tokens you held when you made this call), return 0 if we did not block.
445 lwkt_regettoken(lwkt_token_t tok)
448 if (tok->t_cpu != mycpu->gd_cpu) {
449 while (tok->t_cpu != mycpu->gd_cpu) {
450 lwkt_cpu_msg_union msg;
451 initTokenReqMsg(&msg.mu_TokenReq);