oops, forgot one. Remove another curproc/cred dependancy
[dragonfly.git] / sys / i386 / i386 / swtch.s
CommitLineData
984263bc
MD
1/*-
2 * Copyright (c) 1990 The Regents of the University of California.
3 * All rights reserved.
f1d1c3fa 4 * LWKT threads Copyright (c) 2003 Matthew Dillon
984263bc
MD
5 *
6 * This code is derived from software contributed to Berkeley by
7 * William Jolitz.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 * 3. All advertising materials mentioning features or use of this software
18 * must display the following acknowledgement:
19 * This product includes software developed by the University of
20 * California, Berkeley and its contributors.
21 * 4. Neither the name of the University nor the names of its contributors
22 * may be used to endorse or promote products derived from this software
23 * without specific prior written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
28 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
29 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
30 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
31 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
32 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
33 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
34 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
35 * SUCH DAMAGE.
36 *
37 * $FreeBSD: src/sys/i386/i386/swtch.s,v 1.89.2.10 2003/01/23 03:36:24 ps Exp $
5fb1f500 38 * $DragonFly: src/sys/i386/i386/Attic/swtch.s,v 1.20 2003/07/05 05:54:00 dillon Exp $
984263bc
MD
39 */
40
41#include "npx.h"
42#include "opt_user_ldt.h"
43
44#include <sys/rtprio.h>
45
46#include <machine/asmacros.h>
47#include <machine/ipl.h>
48
49#ifdef SMP
50#include <machine/pmap.h>
51#include <machine/smptests.h> /** GRAB_LOPRIO */
52#include <machine/apic.h>
53#include <machine/lock.h>
54#endif /* SMP */
55
56#include "assym.s"
57
984263bc
MD
58 .data
59
2954c92f 60 .globl panic
984263bc
MD
61
62#if defined(SWTCH_OPTIM_STATS)
2954c92f
MD
63 .globl swtch_optim_stats, tlb_flush_count
64swtch_optim_stats: .long 0 /* number of _swtch_optims */
65tlb_flush_count: .long 0
984263bc
MD
66#endif
67
68 .text
69
984263bc
MD
70
71/*
8ad65e08
MD
72 * cpu_heavy_switch(next_thread)
73 *
74 * Switch from the current thread to a new thread. This entry
75 * is normally called via the thread->td_switch function, and will
76 * only be called when the current thread is a heavy weight process.
77 *
78 * YYY disable interrupts once giant is removed.
984263bc 79 */
8ad65e08 80ENTRY(cpu_heavy_switch)
2954c92f 81 movl PCPU(curthread),%ecx
84b592ba 82 movl TD_PROC(%ecx),%ecx
984263bc 83
8ad65e08 84 cli
984263bc 85 movl P_VMSPACE(%ecx), %edx
72740893 86 movl PCPU(cpuid), %eax
984263bc
MD
87 btrl %eax, VM_PMAP+PM_ACTIVE(%edx)
88
8ad65e08
MD
89 /*
90 * Save general regs
91 */
92 movl P_THREAD(%ecx),%edx
b7c628e4 93 movl TD_PCB(%edx),%edx
984263bc
MD
94 movl (%esp),%eax /* Hardware registers */
95 movl %eax,PCB_EIP(%edx)
96 movl %ebx,PCB_EBX(%edx)
97 movl %esp,PCB_ESP(%edx)
98 movl %ebp,PCB_EBP(%edx)
99 movl %esi,PCB_ESI(%edx)
100 movl %edi,PCB_EDI(%edx)
101 movl %gs,PCB_GS(%edx)
102
8ad65e08
MD
103 /*
104 * Push the LWKT switch restore function, which resumes a heavy
105 * weight process. Note that the LWKT switcher is based on
106 * TD_SP, while the heavy weight process switcher is based on
107 * PCB_ESP. TD_SP is usually one pointer pushed relative to
108 * PCB_ESP.
109 */
110 movl P_THREAD(%ecx),%eax
111 pushl $cpu_heavy_restore
112 movl %esp,TD_SP(%eax)
113
114 /*
115 * Save debug regs if necessary
116 */
984263bc
MD
117 movb PCB_FLAGS(%edx),%al
118 andb $PCB_DBREGS,%al
119 jz 1f /* no, skip over */
120 movl %dr7,%eax /* yes, do the save */
121 movl %eax,PCB_DR7(%edx)
122 andl $0x0000fc00, %eax /* disable all watchpoints */
123 movl %eax,%dr7
124 movl %dr6,%eax
125 movl %eax,PCB_DR6(%edx)
126 movl %dr3,%eax
127 movl %eax,PCB_DR3(%edx)
128 movl %dr2,%eax
129 movl %eax,PCB_DR2(%edx)
130 movl %dr1,%eax
131 movl %eax,PCB_DR1(%edx)
132 movl %dr0,%eax
133 movl %eax,PCB_DR0(%edx)
1341:
135
8ad65e08
MD
136 /*
137 * Save the FP state if we have used the FP.
138 */
984263bc 139#if NNPX > 0
263e4574 140 movl P_THREAD(%ecx),%ecx
2954c92f 141 cmpl %ecx,PCPU(npxthread)
984263bc
MD
142 jne 1f
143 addl $PCB_SAVEFPU,%edx /* h/w bugs make saving complicated */
144 pushl %edx
2954c92f 145 call npxsave /* do it in a big C function */
984263bc
MD
146 popl %eax
1471:
af0bff84 148 /* %ecx,%edx trashed */
984263bc
MD
149#endif /* NNPX > 0 */
150
84b592ba 151 /*
8ad65e08
MD
152 * Switch to the next thread, which was passed as an argument
153 * to cpu_heavy_switch(). Due to the switch-restore function we pushed,
154 * the argument is at 8(%esp). Set the current thread, load the
155 * stack pointer, and 'ret' into the switch-restore function.
84b592ba 156 */
8ad65e08 157 movl 8(%esp),%eax
2954c92f 158 movl %eax,PCPU(curthread)
8ad65e08
MD
159 movl TD_SP(%eax),%esp
160 ret
984263bc 161
8ad65e08
MD
162/*
163 * cpu_exit_switch()
164 *
165 * The switch function is changed to this when a thread is going away
166 * for good. We have to ensure that the MMU state is not cached, and
167 * we don't bother saving the existing thread state before switching.
ae8050a4
MD
168 *
169 * At this point we are in a critical section and this cpu owns the
170 * thread's token, which serves as an interlock until the switchout is
171 * complete.
8ad65e08
MD
172 */
173ENTRY(cpu_exit_switch)
ae8050a4
MD
174 /*
175 * Get us out of the vmspace
176 */
2954c92f 177 movl IdlePTD,%ecx
8ad65e08
MD
178 movl %cr3,%eax
179 cmpl %ecx,%eax
180 je 1f
181 movl %ecx,%cr3
984263bc 1821:
5fb1f500 183 movl PCPU(curthread),%ecx
ae8050a4
MD
184 /*
185 * Switch to the next thread.
186 */
8ad65e08
MD
187 cli
188 movl 4(%esp),%eax
2954c92f 189 movl %eax,PCPU(curthread)
8ad65e08 190 movl TD_SP(%eax),%esp
ae8050a4
MD
191
192 /*
99df837e
MD
193 * We are now the next thread, set the exited flag and wakeup
194 * any waiters.
ae8050a4 195 */
99df837e 196 orl $TDF_EXITED,TD_FLAGS(%ecx)
ae8050a4 197 pushl %eax
99df837e
MD
198 pushl %ecx /* wakeup(oldthread) */
199 call wakeup
ae8050a4 200 addl $4,%esp
99df837e 201 popl %eax /* note: next thread expects curthread in %eax */
ae8050a4
MD
202
203 /*
204 * Restore the next thread's state and resume it. Note: the
205 * restore function assumes that the next thread's address is
206 * in %eax.
207 */
8ad65e08 208 ret
984263bc 209
8ad65e08
MD
210/*
211 * cpu_heavy_restore() (current thread in %eax on entry)
212 *
213 * Restore the thread after an LWKT switch. This entry is normally
214 * called via the LWKT switch restore function, which was pulled
215 * off the thread stack and jumped to.
216 *
217 * This entry is only called if the thread was previously saved
218 * using cpu_heavy_switch() (the heavy weight process thread switcher).
219 *
220 * YYY theoretically we do not have to restore everything here, a lot
221 * of this junk can wait until we return to usermode. But for now
222 * we restore everything.
223 *
224 * YYY STI/CLI sequencing.
7d0bac62
MD
225 *
226 * YYY note: spl check is done in mi_switch when it splx()'s.
8ad65e08 227 */
26a0694b 228
8ad65e08
MD
229ENTRY(cpu_heavy_restore)
230 /* interrupts are disabled */
8f41e33b 231 movl TD_PCB(%eax),%edx
8ad65e08 232 movl TD_PROC(%eax),%ecx
984263bc 233#ifdef DIAGNOSTIC
984263bc
MD
234 cmpb $SRUN,P_STAT(%ecx)
235 jne badsw2
236#endif
984263bc
MD
237
238#if defined(SWTCH_OPTIM_STATS)
239 incl _swtch_optim_stats
240#endif
8ad65e08
MD
241 /*
242 * Restore the MMU address space
243 */
984263bc
MD
244 movl %cr3,%ebx
245 cmpl PCB_CR3(%edx),%ebx
246 je 4f
247#if defined(SWTCH_OPTIM_STATS)
248 decl _swtch_optim_stats
249 incl _tlb_flush_count
250#endif
251 movl PCB_CR3(%edx),%ebx
252 movl %ebx,%cr3
2534:
254
8ad65e08
MD
255 /*
256 * Deal with the PCB extension, restore the private tss
257 */
72740893 258 movl PCPU(cpuid), %esi
984263bc
MD
259 cmpl $0, PCB_EXT(%edx) /* has pcb extension? */
260 je 1f
2954c92f 261 btsl %esi, private_tss /* mark use of private tss */
984263bc
MD
262 movl PCB_EXT(%edx), %edi /* new tss descriptor */
263 jmp 2f
2641:
265
b7c628e4
MD
266 /*
267 * update common_tss.tss_esp0 pointer. This is the supervisor
268 * stack pointer on entry from user mode. Since the pcb is
269 * at the top of the supervisor stack esp0 starts just below it.
270 * We leave enough space for vm86 (16 bytes).
8ad65e08
MD
271 *
272 * common_tss.tss_esp0 is needed when user mode traps into the
273 * kernel.
b7c628e4
MD
274 */
275 leal -16(%edx),%ebx
2954c92f 276 movl %ebx, PCPU(common_tss) + TSS_ESP0
984263bc 277
2954c92f 278 btrl %esi, private_tss
984263bc 279 jae 3f
17a9f566
MD
280
281 /*
282 * There is no way to get the address of a segment-accessed variable
283 * so we store a self-referential pointer at the base of the per-cpu
284 * data area and add the appropriate offset.
285 */
984263bc
MD
286 movl $gd_common_tssd, %edi
287 addl %fs:0, %edi
17a9f566 288
8ad65e08
MD
289 /*
290 * Move the correct TSS descriptor into the GDT slot, then reload
291 * tr. YYY not sure what is going on here
292 */
984263bc 2932:
2954c92f 294 movl PCPU(tss_gdt), %ebx /* entry in GDT */
984263bc
MD
295 movl 0(%edi), %eax
296 movl %eax, 0(%ebx)
297 movl 4(%edi), %eax
298 movl %eax, 4(%ebx)
299 movl $GPROC0_SEL*8, %esi /* GSEL(entry, SEL_KPL) */
300 ltr %si
8ad65e08
MD
301
302 /*
303 * Tell the pmap that our cpu is using the VMSPACE now.
304 */
984263bc
MD
3053:
306 movl P_VMSPACE(%ecx), %ebx
2954c92f 307 movl PCPU(cpuid), %eax
984263bc
MD
308 btsl %eax, VM_PMAP+PM_ACTIVE(%ebx)
309
8ad65e08
MD
310 /*
311 * Restore general registers.
312 */
984263bc
MD
313 movl PCB_EBX(%edx),%ebx
314 movl PCB_ESP(%edx),%esp
315 movl PCB_EBP(%edx),%ebp
316 movl PCB_ESI(%edx),%esi
317 movl PCB_EDI(%edx),%edi
318 movl PCB_EIP(%edx),%eax
319 movl %eax,(%esp)
320
8ad65e08
MD
321 /*
322 * SMP ickyness to direct interrupts.
323 */
324
984263bc
MD
325#ifdef SMP
326#ifdef GRAB_LOPRIO /* hold LOPRIO for INTs */
327#ifdef CHEAP_TPR
328 movl $0, lapic_tpr
329#else
330 andl $~APIC_TPR_PRIO, lapic_tpr
331#endif /** CHEAP_TPR */
332#endif /** GRAB_LOPRIO */
984263bc
MD
333#endif /* SMP */
334
8ad65e08
MD
335 /*
336 * Restore the user LDT if we have one
337 */
984263bc
MD
338#ifdef USER_LDT
339 cmpl $0, PCB_USERLDT(%edx)
340 jnz 1f
2954c92f
MD
341 movl _default_ldt,%eax
342 cmpl PCPU(currentldt),%eax
984263bc 343 je 2f
2954c92f
MD
344 lldt _default_ldt
345 movl %eax,PCPU(currentldt)
984263bc
MD
346 jmp 2f
3471: pushl %edx
2954c92f 348 call set_user_ldt
984263bc
MD
349 popl %edx
3502:
351#endif
8ad65e08
MD
352 /*
353 * Restore the %gs segment register, which must be done after
354 * loading the user LDT. Since user processes can modify the
355 * register via procfs, this may result in a fault which is
356 * detected by checking the fault address against cpu_switch_load_gs
357 * in i386/i386/trap.c
358 */
984263bc
MD
359 .globl cpu_switch_load_gs
360cpu_switch_load_gs:
361 movl PCB_GS(%edx),%gs
362
8ad65e08
MD
363 /*
364 * Restore the DEBUG register state if necessary.
365 */
984263bc
MD
366 movb PCB_FLAGS(%edx),%al
367 andb $PCB_DBREGS,%al
368 jz 1f /* no, skip over */
369 movl PCB_DR6(%edx),%eax /* yes, do the restore */
370 movl %eax,%dr6
371 movl PCB_DR3(%edx),%eax
372 movl %eax,%dr3
373 movl PCB_DR2(%edx),%eax
374 movl %eax,%dr2
375 movl PCB_DR1(%edx),%eax
376 movl %eax,%dr1
377 movl PCB_DR0(%edx),%eax
378 movl %eax,%dr0
379 movl %dr7,%eax /* load dr7 so as not to disturb */
380 andl $0x0000fc00,%eax /* reserved bits */
381 pushl %ebx
382 movl PCB_DR7(%edx),%ebx
383 andl $~0x0000fc00,%ebx
384 orl %ebx,%eax
385 popl %ebx
386 movl %eax,%dr7
3871:
388
8ad65e08 389 sti /* XXX */
984263bc
MD
390 ret
391
392CROSSJUMPTARGET(sw1a)
393
ef0fdad1
MD
394badsw0:
395 pushl %eax
396 pushl $sw0_1
2954c92f 397 call panic
ef0fdad1
MD
398
399sw0_1: .asciz "cpu_switch: panic: %p"
400
984263bc
MD
401#ifdef DIAGNOSTIC
402badsw1:
403 pushl $sw0_1
2954c92f 404 call panic
984263bc
MD
405
406sw0_1: .asciz "cpu_switch: has wchan"
407
408badsw2:
409 pushl $sw0_2
2954c92f 410 call panic
984263bc
MD
411
412sw0_2: .asciz "cpu_switch: not SRUN"
413#endif
414
415#if defined(SMP) && defined(DIAGNOSTIC)
416badsw4:
417 pushl $sw0_4
2954c92f 418 call panic
984263bc
MD
419
420sw0_4: .asciz "cpu_switch: do not have lock"
421#endif /* SMP && DIAGNOSTIC */
422
0cfcada1
MD
423string: .asciz "SWITCHING\n"
424
984263bc
MD
425/*
426 * savectx(pcb)
427 * Update pcb, saving current processor state.
428 */
429ENTRY(savectx)
430 /* fetch PCB */
431 movl 4(%esp),%ecx
432
433 /* caller's return address - child won't execute this routine */
434 movl (%esp),%eax
435 movl %eax,PCB_EIP(%ecx)
436
437 movl %cr3,%eax
438 movl %eax,PCB_CR3(%ecx)
439
440 movl %ebx,PCB_EBX(%ecx)
441 movl %esp,PCB_ESP(%ecx)
442 movl %ebp,PCB_EBP(%ecx)
443 movl %esi,PCB_ESI(%ecx)
444 movl %edi,PCB_EDI(%ecx)
445 movl %gs,PCB_GS(%ecx)
446
447#if NNPX > 0
448 /*
af0bff84 449 * If npxthread == NULL, then the npx h/w state is irrelevant and the
984263bc
MD
450 * state had better already be in the pcb. This is true for forks
451 * but not for dumps (the old book-keeping with FP flags in the pcb
452 * always lost for dumps because the dump pcb has 0 flags).
453 *
af0bff84
MD
454 * If npxthread != NULL, then we have to save the npx h/w state to
455 * npxthread's pcb and copy it to the requested pcb, or save to the
984263bc
MD
456 * requested pcb and reload. Copying is easier because we would
457 * have to handle h/w bugs for reloading. We used to lose the
458 * parent's npx state for forks by forgetting to reload.
459 */
2954c92f 460 movl PCPU(npxthread),%eax
984263bc
MD
461 testl %eax,%eax
462 je 1f
463
464 pushl %ecx
b7c628e4 465 movl TD_PCB(%eax),%eax
984263bc
MD
466 leal PCB_SAVEFPU(%eax),%eax
467 pushl %eax
468 pushl %eax
2954c92f 469 call npxsave
984263bc
MD
470 addl $4,%esp
471 popl %eax
472 popl %ecx
473
474 pushl $PCB_SAVEFPU_SIZE
475 leal PCB_SAVEFPU(%ecx),%ecx
476 pushl %ecx
477 pushl %eax
2954c92f 478 call bcopy
984263bc
MD
479 addl $12,%esp
480#endif /* NNPX > 0 */
481
4821:
483 ret
8ad65e08
MD
484
485/*
486 * cpu_idle_restore() (current thread in %eax on entry)
487 *
488 * Don't bother setting up any regs other then %ebp so backtraces
489 * don't die. This restore function is used to bootstrap into the
490 * cpu_idle() LWKT only, after that cpu_lwkt_*() will be used for
491 * switching.
72740893
MD
492 *
493 * If we are an AP we have to call ap_init() before jumping to
494 * cpu_idle(). ap_init() will synchronize with the BP and finish
495 * setting up various ncpu-dependant globaldata fields. This may
496 * happen on UP as well as SMP if we happen to be simulating multiple
497 * cpus.
8ad65e08
MD
498 */
499ENTRY(cpu_idle_restore)
500 movl $0,%ebp
501 pushl $0
72740893
MD
502#ifdef SMP
503 cmpl $0,PCPU(cpuid)
504 je 1f
505 call ap_init
5061:
507#endif
ef0fdad1 508 sti
8ad65e08
MD
509 jmp cpu_idle
510
0cfcada1
MD
511/*
512 * cpu_kthread_restore() (current thread is %eax on entry)
513 *
514 * Don't bother setting up any regs other then %ebp so backtraces
515 * don't die. This restore function is used to bootstrap into an
516 * LWKT based kernel thread only. cpu_lwkt_switch() will be used
517 * after this.
26a0694b
MD
518 *
519 * Since all of our context is on the stack we are reentrant and
520 * we can release our critical section and enable interrupts early.
0cfcada1
MD
521 */
522ENTRY(cpu_kthread_restore)
523 movl TD_PCB(%eax),%ebx
524 movl $0,%ebp
26a0694b 525 subl $TDPRI_CRIT,TD_PRI(%eax)
ef0fdad1 526 sti
0cfcada1
MD
527 popl %edx /* kthread exit function */
528 pushl PCB_EBX(%ebx) /* argument to ESI function */
529 pushl %edx /* set exit func as return address */
530 movl PCB_ESI(%ebx),%eax
531 jmp *%eax
532
8ad65e08
MD
533/*
534 * cpu_lwkt_switch()
535 *
536 * Standard LWKT switching function. Only non-scratch registers are
537 * saved and we don't bother with the MMU state or anything else.
26a0694b
MD
538 *
539 * This function is always called while in a critical section.
540 *
8ad65e08
MD
541 * YYY BGL, SPL
542 */
543ENTRY(cpu_lwkt_switch)
544 movl 4(%esp),%eax
545 pushl %ebp
546 pushl %ebx
547 pushl %esi
548 pushl %edi
549 pushfl
2954c92f 550 movl PCPU(curthread),%ecx
8ad65e08
MD
551 pushl $cpu_lwkt_restore
552 cli
553 movl %esp,TD_SP(%ecx)
2954c92f 554 movl %eax,PCPU(curthread)
8ad65e08
MD
555 movl TD_SP(%eax),%esp
556 ret
557
558/*
26a0694b 559 * cpu_lwkt_restore() (current thread in %eax on entry)
8ad65e08 560 *
26a0694b
MD
561 * Standard LWKT restore function. This function is always called
562 * while in a critical section.
563 *
564 * Warning: due to preemption the restore function can be used to
565 * 'return' to the original thread. Interrupt disablement must be
566 * protected through the switch so we cannot run splz here.
8ad65e08
MD
567 */
568ENTRY(cpu_lwkt_restore)
569 popfl
570 popl %edi
571 popl %esi
572 popl %ebx
573 popl %ebp
574 ret
575