Initial import from FreeBSD RELENG_4:
[dragonfly.git] / sys / i386 / i386 / swtch.s
CommitLineData
984263bc
MD
1/*-
2 * Copyright (c) 1990 The Regents of the University of California.
3 * All rights reserved.
4 *
5 * This code is derived from software contributed to Berkeley by
6 * William Jolitz.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. All advertising materials mentioning features or use of this software
17 * must display the following acknowledgement:
18 * This product includes software developed by the University of
19 * California, Berkeley and its contributors.
20 * 4. Neither the name of the University nor the names of its contributors
21 * may be used to endorse or promote products derived from this software
22 * without specific prior written permission.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * SUCH DAMAGE.
35 *
36 * $FreeBSD: src/sys/i386/i386/swtch.s,v 1.89.2.10 2003/01/23 03:36:24 ps Exp $
37 */
38
39#include "npx.h"
40#include "opt_user_ldt.h"
41
42#include <sys/rtprio.h>
43
44#include <machine/asmacros.h>
45#include <machine/ipl.h>
46
47#ifdef SMP
48#include <machine/pmap.h>
49#include <machine/smptests.h> /** GRAB_LOPRIO */
50#include <machine/apic.h>
51#include <machine/lock.h>
52#endif /* SMP */
53
54#include "assym.s"
55
56
57/*****************************************************************************/
58/* Scheduling */
59/*****************************************************************************/
60
61 .data
62
63 .globl _hlt_vector
64_hlt_vector: .long _cpu_idle /* pointer to halt routine */
65
66 .globl _panic
67
68#if defined(SWTCH_OPTIM_STATS)
69 .globl _swtch_optim_stats, _tlb_flush_count
70_swtch_optim_stats: .long 0 /* number of _swtch_optims */
71_tlb_flush_count: .long 0
72#endif
73
74 .text
75
76/*
77 * When no processes are on the runq, cpu_switch() branches to _idle
78 * to wait for something to come ready.
79 */
80 ALIGN_TEXT
81 .type _idle,@function
82_idle:
83 xorl %ebp,%ebp
84 movl %ebp,_switchtime
85
86#ifdef SMP
87
88 /* when called, we have the mplock, intr disabled */
89 /* use our idleproc's "context" */
90 movl _IdlePTD, %ecx
91 movl %cr3, %eax
92 cmpl %ecx, %eax
93 je 2f
94#if defined(SWTCH_OPTIM_STATS)
95 decl _swtch_optim_stats
96 incl _tlb_flush_count
97#endif
98 movl %ecx, %cr3
992:
100 /* Keep space for nonexisting return addr, or profiling bombs */
101 movl $gd_idlestack_top-4, %ecx
102 addl %fs:0, %ecx
103 movl %ecx, %esp
104
105 /* update common_tss.tss_esp0 pointer */
106 movl %ecx, _common_tss + TSS_ESP0
107
108 movl _cpuid, %esi
109 btrl %esi, _private_tss
110 jae 1f
111
112 movl $gd_common_tssd, %edi
113 addl %fs:0, %edi
114
115 /* move correct tss descriptor into GDT slot, then reload tr */
116 movl _tss_gdt, %ebx /* entry in GDT */
117 movl 0(%edi), %eax
118 movl %eax, 0(%ebx)
119 movl 4(%edi), %eax
120 movl %eax, 4(%ebx)
121 movl $GPROC0_SEL*8, %esi /* GSEL(entry, SEL_KPL) */
122 ltr %si
1231:
124
125 sti
126
127 /*
128 * XXX callers of cpu_switch() do a bogus splclock(). Locking should
129 * be left to cpu_switch().
130 *
131 * NOTE: spl*() may only be called while we hold the MP lock (which
132 * we do).
133 */
134 call _spl0
135
136 cli
137
138 /*
139 * _REALLY_ free the lock, no matter how deep the prior nesting.
140 * We will recover the nesting on the way out when we have a new
141 * proc to load.
142 *
143 * XXX: we had damn well better be sure we had it before doing this!
144 */
145 movl $FREE_LOCK, %eax
146 movl %eax, _mp_lock
147
148 /* do NOT have lock, intrs disabled */
149 .globl idle_loop
150idle_loop:
151
152 cmpl $0,_smp_active
153 jne 1f
154 cmpl $0,_cpuid
155 je 1f
156 jmp 2f
157
1581:
159 call _procrunnable
160 testl %eax,%eax
161 jnz 3f
162
163 /*
164 * Handle page-zeroing in the idle loop. Called with interrupts
165 * disabled and the MP lock released. Inside vm_page_zero_idle
166 * we enable interrupts and grab the mplock as required.
167 */
168 cmpl $0,_do_page_zero_idle
169 je 2f
170
171 call _vm_page_zero_idle /* internal locking */
172 testl %eax, %eax
173 jnz idle_loop
1742:
175
176 /* enable intrs for a halt */
177 movl $0, lapic_tpr /* 1st candidate for an INT */
178 call *_hlt_vector /* wait for interrupt */
179 cli
180 jmp idle_loop
181
182 /*
183 * Note that interrupts must be enabled while obtaining the MP lock
184 * in order to be able to take IPI's while blocked.
185 */
1863:
187#ifdef GRAB_LOPRIO
188 movl $LOPRIO_LEVEL, lapic_tpr /* arbitrate for INTs */
189#endif
190 sti
191 call _get_mplock
192 cli
193 call _procrunnable
194 testl %eax,%eax
195 CROSSJUMP(jnz, sw1a, jz)
196 call _rel_mplock
197 jmp idle_loop
198
199#else /* !SMP */
200
201 movl $HIDENAME(tmpstk),%esp
202#if defined(OVERLY_CONSERVATIVE_PTD_MGMT)
203#if defined(SWTCH_OPTIM_STATS)
204 incl _swtch_optim_stats
205#endif
206 movl _IdlePTD, %ecx
207 movl %cr3, %eax
208 cmpl %ecx, %eax
209 je 2f
210#if defined(SWTCH_OPTIM_STATS)
211 decl _swtch_optim_stats
212 incl _tlb_flush_count
213#endif
214 movl %ecx, %cr3
2152:
216#endif
217
218 /* update common_tss.tss_esp0 pointer */
219 movl %esp, _common_tss + TSS_ESP0
220
221 movl $0, %esi
222 btrl %esi, _private_tss
223 jae 1f
224
225 movl $_common_tssd, %edi
226
227 /* move correct tss descriptor into GDT slot, then reload tr */
228 movl _tss_gdt, %ebx /* entry in GDT */
229 movl 0(%edi), %eax
230 movl %eax, 0(%ebx)
231 movl 4(%edi), %eax
232 movl %eax, 4(%ebx)
233 movl $GPROC0_SEL*8, %esi /* GSEL(entry, SEL_KPL) */
234 ltr %si
2351:
236
237 sti
238
239 /*
240 * XXX callers of cpu_switch() do a bogus splclock(). Locking should
241 * be left to cpu_switch().
242 */
243 call _spl0
244
245 ALIGN_TEXT
246idle_loop:
247 cli
248 call _procrunnable
249 testl %eax,%eax
250 CROSSJUMP(jnz, sw1a, jz)
251#ifdef DEVICE_POLLING
252 call _idle_poll
253#else /* standard code */
254 call _vm_page_zero_idle
255#endif
256 testl %eax, %eax
257 jnz idle_loop
258 call *_hlt_vector /* wait for interrupt */
259 jmp idle_loop
260
261#endif /* SMP */
262
263CROSSJUMPTARGET(_idle)
264
265#if 0
266
267ENTRY(default_halt)
268 sti
269#ifndef SMP
270 hlt /* XXX: until a wakeup IPI */
271#endif
272 ret
273
274#endif
275
276/*
277 * cpu_switch()
278 */
279ENTRY(cpu_switch)
280
281 /* switch to new process. first, save context as needed */
282 movl _curproc,%ecx
283
284 /* if no process to save, don't bother */
285 testl %ecx,%ecx
286 je sw1
287
288#ifdef SMP
289 movb P_ONCPU(%ecx), %al /* save "last" cpu */
290 movb %al, P_LASTCPU(%ecx)
291 movb $0xff, P_ONCPU(%ecx) /* "leave" the cpu */
292#endif /* SMP */
293 movl P_VMSPACE(%ecx), %edx
294#ifdef SMP
295 movl _cpuid, %eax
296#else
297 xorl %eax, %eax
298#endif /* SMP */
299 btrl %eax, VM_PMAP+PM_ACTIVE(%edx)
300
301 movl P_ADDR(%ecx),%edx
302
303 movl (%esp),%eax /* Hardware registers */
304 movl %eax,PCB_EIP(%edx)
305 movl %ebx,PCB_EBX(%edx)
306 movl %esp,PCB_ESP(%edx)
307 movl %ebp,PCB_EBP(%edx)
308 movl %esi,PCB_ESI(%edx)
309 movl %edi,PCB_EDI(%edx)
310 movl %gs,PCB_GS(%edx)
311
312 /* test if debug regisers should be saved */
313 movb PCB_FLAGS(%edx),%al
314 andb $PCB_DBREGS,%al
315 jz 1f /* no, skip over */
316 movl %dr7,%eax /* yes, do the save */
317 movl %eax,PCB_DR7(%edx)
318 andl $0x0000fc00, %eax /* disable all watchpoints */
319 movl %eax,%dr7
320 movl %dr6,%eax
321 movl %eax,PCB_DR6(%edx)
322 movl %dr3,%eax
323 movl %eax,PCB_DR3(%edx)
324 movl %dr2,%eax
325 movl %eax,PCB_DR2(%edx)
326 movl %dr1,%eax
327 movl %eax,PCB_DR1(%edx)
328 movl %dr0,%eax
329 movl %eax,PCB_DR0(%edx)
3301:
331
332#ifdef SMP
333 movl _mp_lock, %eax
334 /* XXX FIXME: we should be saving the local APIC TPR */
335#ifdef DIAGNOSTIC
336 cmpl $FREE_LOCK, %eax /* is it free? */
337 je badsw4 /* yes, bad medicine! */
338#endif /* DIAGNOSTIC */
339 andl $COUNT_FIELD, %eax /* clear CPU portion */
340 movl %eax, PCB_MPNEST(%edx) /* store it */
341#endif /* SMP */
342
343#if NNPX > 0
344 /* have we used fp, and need a save? */
345 cmpl %ecx,_npxproc
346 jne 1f
347 addl $PCB_SAVEFPU,%edx /* h/w bugs make saving complicated */
348 pushl %edx
349 call _npxsave /* do it in a big C function */
350 popl %eax
3511:
352#endif /* NNPX > 0 */
353
354 movl $0,_curproc /* out of process */
355
356 /* save is done, now choose a new process or idle */
357sw1:
358 cli
359
360#ifdef SMP
361 /* Stop scheduling if smp_active goes zero and we are not BSP */
362 cmpl $0,_smp_active
363 jne 1f
364 cmpl $0,_cpuid
365 CROSSJUMP(je, _idle, jne) /* wind down */
3661:
367#endif
368
369sw1a:
370 call _chooseproc /* trash ecx, edx, ret eax*/
371 testl %eax,%eax
372 CROSSJUMP(je, _idle, jne) /* if no proc, idle */
373 movl %eax,%ecx
374
375 xorl %eax,%eax
376 andl $~AST_RESCHED,_astpending
377
378#ifdef DIAGNOSTIC
379 cmpl %eax,P_WCHAN(%ecx)
380 jne badsw1
381 cmpb $SRUN,P_STAT(%ecx)
382 jne badsw2
383#endif
384
385 movl P_ADDR(%ecx),%edx
386
387#if defined(SWTCH_OPTIM_STATS)
388 incl _swtch_optim_stats
389#endif
390 /* switch address space */
391 movl %cr3,%ebx
392 cmpl PCB_CR3(%edx),%ebx
393 je 4f
394#if defined(SWTCH_OPTIM_STATS)
395 decl _swtch_optim_stats
396 incl _tlb_flush_count
397#endif
398 movl PCB_CR3(%edx),%ebx
399 movl %ebx,%cr3
4004:
401
402#ifdef SMP
403 movl _cpuid, %esi
404#else
405 xorl %esi, %esi
406#endif
407 cmpl $0, PCB_EXT(%edx) /* has pcb extension? */
408 je 1f
409 btsl %esi, _private_tss /* mark use of private tss */
410 movl PCB_EXT(%edx), %edi /* new tss descriptor */
411 jmp 2f
4121:
413
414 /* update common_tss.tss_esp0 pointer */
415 movl %edx, %ebx /* pcb */
416 addl $(UPAGES * PAGE_SIZE - 16), %ebx
417 movl %ebx, _common_tss + TSS_ESP0
418
419 btrl %esi, _private_tss
420 jae 3f
421#ifdef SMP
422 movl $gd_common_tssd, %edi
423 addl %fs:0, %edi
424#else
425 movl $_common_tssd, %edi
426#endif
4272:
428 /* move correct tss descriptor into GDT slot, then reload tr */
429 movl _tss_gdt, %ebx /* entry in GDT */
430 movl 0(%edi), %eax
431 movl %eax, 0(%ebx)
432 movl 4(%edi), %eax
433 movl %eax, 4(%ebx)
434 movl $GPROC0_SEL*8, %esi /* GSEL(entry, SEL_KPL) */
435 ltr %si
4363:
437 movl P_VMSPACE(%ecx), %ebx
438#ifdef SMP
439 movl _cpuid, %eax
440#else
441 xorl %eax, %eax
442#endif
443 btsl %eax, VM_PMAP+PM_ACTIVE(%ebx)
444
445 /* restore context */
446 movl PCB_EBX(%edx),%ebx
447 movl PCB_ESP(%edx),%esp
448 movl PCB_EBP(%edx),%ebp
449 movl PCB_ESI(%edx),%esi
450 movl PCB_EDI(%edx),%edi
451 movl PCB_EIP(%edx),%eax
452 movl %eax,(%esp)
453
454#ifdef SMP
455#ifdef GRAB_LOPRIO /* hold LOPRIO for INTs */
456#ifdef CHEAP_TPR
457 movl $0, lapic_tpr
458#else
459 andl $~APIC_TPR_PRIO, lapic_tpr
460#endif /** CHEAP_TPR */
461#endif /** GRAB_LOPRIO */
462 movl _cpuid,%eax
463 movb %al, P_ONCPU(%ecx)
464#endif /* SMP */
465 movl %edx, _curpcb
466 movl %ecx, _curproc /* into next process */
467
468#ifdef SMP
469 movl _cpu_lockid, %eax
470 orl PCB_MPNEST(%edx), %eax /* add next count from PROC */
471 movl %eax, _mp_lock /* load the mp_lock */
472 /* XXX FIXME: we should be restoring the local APIC TPR */
473#endif /* SMP */
474
475#ifdef USER_LDT
476 cmpl $0, PCB_USERLDT(%edx)
477 jnz 1f
478 movl __default_ldt,%eax
479 cmpl _currentldt,%eax
480 je 2f
481 lldt __default_ldt
482 movl %eax,_currentldt
483 jmp 2f
4841: pushl %edx
485 call _set_user_ldt
486 popl %edx
4872:
488#endif
489
490 /* This must be done after loading the user LDT. */
491 .globl cpu_switch_load_gs
492cpu_switch_load_gs:
493 movl PCB_GS(%edx),%gs
494
495 /* test if debug regisers should be restored */
496 movb PCB_FLAGS(%edx),%al
497 andb $PCB_DBREGS,%al
498 jz 1f /* no, skip over */
499 movl PCB_DR6(%edx),%eax /* yes, do the restore */
500 movl %eax,%dr6
501 movl PCB_DR3(%edx),%eax
502 movl %eax,%dr3
503 movl PCB_DR2(%edx),%eax
504 movl %eax,%dr2
505 movl PCB_DR1(%edx),%eax
506 movl %eax,%dr1
507 movl PCB_DR0(%edx),%eax
508 movl %eax,%dr0
509 movl %dr7,%eax /* load dr7 so as not to disturb */
510 andl $0x0000fc00,%eax /* reserved bits */
511 pushl %ebx
512 movl PCB_DR7(%edx),%ebx
513 andl $~0x0000fc00,%ebx
514 orl %ebx,%eax
515 popl %ebx
516 movl %eax,%dr7
5171:
518
519 sti
520 ret
521
522CROSSJUMPTARGET(sw1a)
523
524#ifdef DIAGNOSTIC
525badsw1:
526 pushl $sw0_1
527 call _panic
528
529sw0_1: .asciz "cpu_switch: has wchan"
530
531badsw2:
532 pushl $sw0_2
533 call _panic
534
535sw0_2: .asciz "cpu_switch: not SRUN"
536#endif
537
538#if defined(SMP) && defined(DIAGNOSTIC)
539badsw4:
540 pushl $sw0_4
541 call _panic
542
543sw0_4: .asciz "cpu_switch: do not have lock"
544#endif /* SMP && DIAGNOSTIC */
545
546/*
547 * savectx(pcb)
548 * Update pcb, saving current processor state.
549 */
550ENTRY(savectx)
551 /* fetch PCB */
552 movl 4(%esp),%ecx
553
554 /* caller's return address - child won't execute this routine */
555 movl (%esp),%eax
556 movl %eax,PCB_EIP(%ecx)
557
558 movl %cr3,%eax
559 movl %eax,PCB_CR3(%ecx)
560
561 movl %ebx,PCB_EBX(%ecx)
562 movl %esp,PCB_ESP(%ecx)
563 movl %ebp,PCB_EBP(%ecx)
564 movl %esi,PCB_ESI(%ecx)
565 movl %edi,PCB_EDI(%ecx)
566 movl %gs,PCB_GS(%ecx)
567
568#if NNPX > 0
569 /*
570 * If npxproc == NULL, then the npx h/w state is irrelevant and the
571 * state had better already be in the pcb. This is true for forks
572 * but not for dumps (the old book-keeping with FP flags in the pcb
573 * always lost for dumps because the dump pcb has 0 flags).
574 *
575 * If npxproc != NULL, then we have to save the npx h/w state to
576 * npxproc's pcb and copy it to the requested pcb, or save to the
577 * requested pcb and reload. Copying is easier because we would
578 * have to handle h/w bugs for reloading. We used to lose the
579 * parent's npx state for forks by forgetting to reload.
580 */
581 movl _npxproc,%eax
582 testl %eax,%eax
583 je 1f
584
585 pushl %ecx
586 movl P_ADDR(%eax),%eax
587 leal PCB_SAVEFPU(%eax),%eax
588 pushl %eax
589 pushl %eax
590 call _npxsave
591 addl $4,%esp
592 popl %eax
593 popl %ecx
594
595 pushl $PCB_SAVEFPU_SIZE
596 leal PCB_SAVEFPU(%ecx),%ecx
597 pushl %ecx
598 pushl %eax
599 call _bcopy
600 addl $12,%esp
601#endif /* NNPX > 0 */
602
6031:
604 ret