kernel - Fix serious issue w/ smp_invltlb(), plus other issues (3)
[dragonfly.git] / sys / platform / pc32 / isa / ipl.s
CommitLineData
984263bc
MD
1/*-
2 * Copyright (c) 1989, 1990 William F. Jolitz.
3 * Copyright (c) 1990 The Regents of the University of California.
4 * All rights reserved.
5 *
6 * This code is derived from software contributed to Berkeley by
7 * William Jolitz.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 * 3. All advertising materials mentioning features or use of this software
18 * must display the following acknowledgement:
19 * This product includes software developed by the University of
20 * California, Berkeley and its contributors.
21 * 4. Neither the name of the University nor the names of its contributors
22 * may be used to endorse or promote products derived from this software
23 * without specific prior written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
28 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
29 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
30 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
31 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
32 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
33 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
34 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
35 * SUCH DAMAGE.
36 *
37 * @(#)ipl.s
38 *
39 * $FreeBSD: src/sys/i386/isa/ipl.s,v 1.32.2.3 2002/05/16 16:03:56 bde Exp $
c7eb0589 40 * $DragonFly: src/sys/platform/pc32/isa/ipl.s,v 1.28 2007/01/22 19:37:04 corecode Exp $
984263bc
MD
41 */
42
bdc560a1
MD
43#include "use_npx.h"
44
45#include <machine/asmacros.h>
46#include <machine/segments.h>
47#include <machine/ipl.h>
48#include <machine/lock.h>
49#include <machine/psl.h>
50#include <machine/trap.h>
bdc560a1
MD
51
52#include "assym.s"
984263bc
MD
53
54/*
55 * AT/386
56 * Vector interrupt control section
57 *
729e15a8 58 * fpending - Pending interrupts (set when a masked interrupt occurs)
5f456c40 59 * spending - Pending software interrupts
984263bc 60 */
bdc560a1
MD
61 .data
62 ALIGN_DATA
984263bc 63
46b26c5e
MD
64 .globl fastunpend_count
65fastunpend_count: .long 0
66
984263bc 67 .text
bdc560a1
MD
68 SUPERALIGN_TEXT
69
46a3f46d
MD
70 /*
71 * GENERAL NOTES
72 *
73 * - fast interrupts are always called with a critical section
74 * held
75 *
76 * - we release our critical section when scheduling interrupt
77 * or softinterrupt threads in order so they can preempt
78 * (unless we are called manually from a critical section, in
79 * which case there will still be a critical section and
80 * they won't preempt anyway).
81 *
82 * - TD_NEST_COUNT prevents splz from nesting too deeply within
83 * itself. It is *not* actually an interrupt nesting count.
84 * PCPU(intr_nesting_level) is an interrupt nesting count.
85 *
86 * - We have to be careful in regards to local interrupts
87 * occuring simultaniously with our doreti and splz
88 * processing.
89 */
984263bc 90
ef0fdad1
MD
91 /*
92 * DORETI
93 *
94 * Handle return from interrupts, traps and syscalls. This function
95 * checks the cpl for unmasked pending interrupts (fast, normal, or
96 * soft) and schedules them if appropriate, then irets.
71ef2f5c
MD
97 *
98 * If we are in a critical section we cannot run any pending ints
99 * nor can be play with mp_lock.
46a3f46d 100 *
38787eef
MD
101 * NOTE: Since SPLs no longer exist, all callers of this function
102 * push $0 for the CPL. HOWEVER, we *STILL* use the cpl mask within
103 * this function to mark fast interrupts which could not be dispatched
104 * do to the unavailability of the BGL.
ef0fdad1 105 */
984263bc 106 SUPERALIGN_TEXT
bdc560a1 107 .globl doreti
2954c92f
MD
108 .type doreti,@function
109doreti:
110 FAKE_MCOUNT(bintr) /* init "from" bintr -> doreti */
38787eef
MD
111 popl %eax /* cpl to restore XXX */
112 movl $0,%eax /* irq mask unavailable due to BGL */
2954c92f 113 movl PCPU(curthread),%ebx
f9235b6d 114 cli /* interlock with td_critcount */
235957ed
MD
115 cmpl $0,PCPU(reqflags) /* short cut if nothing to do */
116 je 5f
f9235b6d
MD
117 testl $-1,TD_CRITCOUNT(%ebx) /* can't unpend if in critical sec */
118 jne 5f
119 incl TD_CRITCOUNT(%ebx) /* force all ints to pending */
984263bc 120doreti_next:
ecf9dec6 121 cli /* re-assert cli on loop */
38787eef 122 movl %eax,%ecx /* irq mask unavailable due to BGL */
ef0fdad1 123 notl %ecx
96728c05 124#ifdef SMP
235957ed 125 testl $RQF_IPIQ,PCPU(reqflags)
96728c05 126 jnz doreti_ipiq
78ea5a2a
SZ
127 testl $RQF_TIMER,PCPU(reqflags)
128 jnz doreti_timer
96728c05 129#endif
a2a5ad0d
MD
130 testl PCPU(fpending),%ecx /* check for an unmasked fast int */
131 jnz doreti_fast
132
5f456c40
MD
133 movl PCPU(spending),%ecx /* check for a pending software int */
134 cmpl $0,%ecx
135 jnz doreti_soft
136
235957ed 137 testl $RQF_AST_MASK,PCPU(reqflags) /* any pending ASTs? */
26a0694b 138 jz 2f
984263bc 139 testl $PSL_VM,TF_EFLAGS(%esp)
ef0fdad1 140 jz 1f
235957ed
MD
141 cmpl $1,in_vm86call /* YYY make per 'cpu'? */
142 jnz doreti_ast
ef0fdad1 1431:
0a3f9b47 144 /* ASTs are only applicable when returning to userland */
984263bc 145 testb $SEL_RPL_MASK,TF_CS(%esp)
235957ed 146 jnz doreti_ast
96728c05 1472:
984263bc 148 /*
ef0fdad1 149 * Nothing left to do, finish up. Interrupts are still disabled.
38787eef
MD
150 * %eax contains the mask of IRQ's that are not available due to
151 * BGL requirements. We can only clear RQF_INTPEND if *ALL* pending
152 * interrupts have been processed.
984263bc 153 */
f9235b6d 154 decl TD_CRITCOUNT(%ebx) /* interlocked with cli */
a2a5ad0d
MD
155 testl %eax,%eax
156 jnz 5f
5c323556 157 andl $~RQF_INTPEND,PCPU(reqflags)
ef0fdad1 1585:
984263bc 159 MEXITCOUNT
4e7c41c5
MD
160
161 /*
162 * Restore the segment registers. Since segment register values
163 * can be set from user mode, this can result in a kernel mode
164 * exception. The trap code will revector to the *_fault code
165 * which then sets up a T_PROTFLT signal. If the signal is
166 * sent to userland, sendsig() will automatically clean up all
167 * the segment registers to avoid a loop.
168 */
169 .globl doreti_popl_gs
984263bc 170 .globl doreti_popl_fs
ef0fdad1
MD
171 .globl doreti_popl_es
172 .globl doreti_popl_ds
173 .globl doreti_iret
984263bc
MD
174 .globl doreti_syscall_ret
175doreti_syscall_ret:
4e7c41c5
MD
176doreti_popl_gs:
177 popl %gs
984263bc
MD
178doreti_popl_fs:
179 popl %fs
984263bc
MD
180doreti_popl_es:
181 popl %es
984263bc
MD
182doreti_popl_ds:
183 popl %ds
184 popal
4e7c41c5 185 addl $3*4,%esp /* xflags, trap, err */
984263bc
MD
186doreti_iret:
187 iret
188
ecf9dec6
MD
189 /*
190 * Interrupts are likely disabled due to the above interlock
191 * between cli/iretq. We must enable them before calling any
192 * high level function.
193 */
984263bc
MD
194 ALIGN_TEXT
195 .globl doreti_iret_fault
196doreti_iret_fault:
4e7c41c5 197 subl $3*4,%esp /* xflags, trap, err */
984263bc
MD
198 pushal
199 pushl %ds
200 .globl doreti_popl_ds_fault
201doreti_popl_ds_fault:
202 pushl %es
203 .globl doreti_popl_es_fault
204doreti_popl_es_fault:
205 pushl %fs
206 .globl doreti_popl_fs_fault
207doreti_popl_fs_fault:
4e7c41c5
MD
208 pushl %gs
209 .globl doreti_popl_gs_fault
210doreti_popl_gs_fault:
ecf9dec6 211 sti
984263bc
MD
212 movl $0,TF_ERR(%esp) /* XXX should be the error code */
213 movl $T_PROTFLT,TF_TRAPNO(%esp)
214 jmp alltraps_with_regs_pushed
215
984263bc 216 /*
38787eef
MD
217 * FAST interrupt pending. NOTE: stack context holds frame structure
218 * for fast interrupt procedure, do not do random pushes or pops!
984263bc 219 */
ef0fdad1
MD
220 ALIGN_TEXT
221doreti_fast:
2954c92f 222 andl PCPU(fpending),%ecx /* only check fast ints */
ecf9dec6 223 sti
984263bc 224 bsfl %ecx, %ecx /* locate the next dispatchable int */
2954c92f 225 btrl %ecx, PCPU(fpending) /* is it really still pending? */
ef0fdad1 226 jnc doreti_next
38787eef
MD
227 pushl %eax /* save IRQ mask unavailable for BGL */
228 /* NOTE: is also CPL in frame */
46b26c5e 229 call dofastunpend /* unpend fast intr %ecx */
984263bc 230 popl %eax
ef0fdad1 231 jmp doreti_next
984263bc 232
984263bc 233 /*
5f456c40
MD
234 * SOFT interrupt pending
235 *
236 * Temporarily back-out our critical section to allow an interrupt
237 * preempt us when we schedule it. Bump intr_nesting_level to
238 * prevent the switch code from recursing via splz too deeply.
239 */
240 ALIGN_TEXT
241doreti_soft:
ecf9dec6 242 sti
5f456c40
MD
243 bsfl %ecx,%ecx /* locate the next pending softint */
244 btrl %ecx,PCPU(spending) /* make sure its still pending */
245 jnc doreti_next
246 addl $FIRST_SOFTINT,%ecx /* actual intr number */
247 pushl %eax
248 pushl %ecx
249 incl TD_NEST_COUNT(%ebx) /* prevent doreti/splz nesting */
f9235b6d 250 decl TD_CRITCOUNT(%ebx) /* so we can preempt */
5f456c40 251 call sched_ithd /* YYY must pull in imasks */
f9235b6d 252 incl TD_CRITCOUNT(%ebx)
5f456c40
MD
253 decl TD_NEST_COUNT(%ebx)
254 addl $4,%esp
255 popl %eax
256 jmp doreti_next
257
258 /*
235957ed
MD
259 * AST pending. We clear RQF_AST_SIGNAL automatically, the others
260 * are cleared by the trap as they are processed.
a2a5ad0d
MD
261 *
262 * Temporarily back-out our critical section because trap() can be
263 * a long-winded call, and we want to be more syscall-like.
264 *
235957ed
MD
265 * YYY theoretically we can call lwkt_switch directly if all we need
266 * to do is a reschedule.
984263bc 267 */
235957ed 268doreti_ast:
a722be49 269 andl $~(RQF_AST_SIGNAL|RQF_AST_UPCALL),PCPU(reqflags)
a2a5ad0d
MD
270 sti
271 movl %eax,%esi /* save cpl (can't use stack) */
272 movl $T_ASTFLT,TF_TRAPNO(%esp)
c7eb0589 273 pushl %esp /* pass frame by reference */
f9235b6d 274 decl TD_CRITCOUNT(%ebx)
d080fbe8 275 call trap
f9235b6d 276 incl TD_CRITCOUNT(%ebx)
c7eb0589 277 addl $4,%esp
96728c05
MD
278 movl %esi,%eax /* restore cpl for loop */
279 jmp doreti_next
280
281#ifdef SMP
282 /*
235957ed 283 * IPIQ message pending. We clear RQF_IPIQ automatically.
96728c05
MD
284 */
285doreti_ipiq:
38787eef 286 movl %eax,%esi /* save cpl (can't use stack) */
03aa8d99 287 incl PCPU(intr_nesting_level)
235957ed 288 andl $~RQF_IPIQ,PCPU(reqflags)
ecf9dec6 289 sti
88c4d2f6 290 subl $8,%esp /* add dummy vec and ppl */
c7eb0589 291 pushl %esp /* pass frame by reference */
88c4d2f6 292 call lwkt_process_ipiq_frame
c7eb0589 293 addl $12,%esp
03aa8d99 294 decl PCPU(intr_nesting_level)
38787eef 295 movl %esi,%eax /* restore cpl for loop */
984263bc
MD
296 jmp doreti_next
297
78ea5a2a
SZ
298doreti_timer:
299 movl %eax,%esi /* save cpl (can't use stack) */
300 incl PCPU(intr_nesting_level)
301 andl $~RQF_TIMER,PCPU(reqflags)
ecf9dec6 302 sti
78ea5a2a
SZ
303 subl $8,%esp /* add dummy vec and ppl */
304 pushl %esp /* pass frame by reference */
305 call lapic_timer_process_frame
306 addl $12,%esp
307 decl PCPU(intr_nesting_level)
308 movl %esi,%eax /* restore cpl for loop */
309 jmp doreti_next
310
96728c05 311#endif
ef0fdad1
MD
312
313 /*
314 * SPLZ() a C callable procedure to dispatch any unmasked pending
315 * interrupts regardless of critical section nesting. ASTs
316 * are not dispatched.
26a0694b 317 *
38787eef
MD
318 * Use %eax to track those IRQs that could not be processed
319 * due to BGL requirements.
ef0fdad1
MD
320 */
321 SUPERALIGN_TEXT
322
323ENTRY(splz)
26a0694b 324 pushfl
ef0fdad1 325 pushl %ebx
2954c92f 326 movl PCPU(curthread),%ebx
f9235b6d 327 incl TD_CRITCOUNT(%ebx)
38787eef 328 movl $0,%eax
ef0fdad1
MD
329
330splz_next:
26a0694b 331 cli
ef0fdad1
MD
332 movl %eax,%ecx /* ecx = ~CPL */
333 notl %ecx
96728c05 334#ifdef SMP
235957ed 335 testl $RQF_IPIQ,PCPU(reqflags)
96728c05 336 jnz splz_ipiq
78ea5a2a
SZ
337 testl $RQF_TIMER,PCPU(reqflags)
338 jnz splz_timer
96728c05 339#endif
a2a5ad0d
MD
340 testl PCPU(fpending),%ecx /* check for an unmasked fast int */
341 jnz splz_fast
342
5f456c40
MD
343 movl PCPU(spending),%ecx
344 cmpl $0,%ecx
345 jnz splz_soft
346
f9235b6d 347 decl TD_CRITCOUNT(%ebx)
235957ed 348
3c23a41a
MD
349 /*
350 * Nothing left to do, finish up. Interrupts are still disabled.
38787eef
MD
351 * If our mask of IRQs we couldn't process due to BGL requirements
352 * is 0 then there are no pending interrupt sources left and we
353 * can clear RQF_INTPEND.
3c23a41a 354 */
a2a5ad0d
MD
355 testl %eax,%eax
356 jnz 5f
235957ed 357 andl $~RQF_INTPEND,PCPU(reqflags)
a2a5ad0d 3585:
ef0fdad1 359 popl %ebx
26a0694b 360 popfl
984263bc
MD
361 ret
362
ef0fdad1
MD
363 /*
364 * FAST interrupt pending
365 */
984263bc 366 ALIGN_TEXT
ef0fdad1 367splz_fast:
2954c92f 368 andl PCPU(fpending),%ecx /* only check fast ints */
ecf9dec6 369 sti
ef0fdad1 370 bsfl %ecx, %ecx /* locate the next dispatchable int */
2954c92f 371 btrl %ecx, PCPU(fpending) /* is it really still pending? */
ef0fdad1
MD
372 jnc splz_next
373 pushl %eax
46b26c5e 374 call dofastunpend /* unpend fast intr %ecx */
ef0fdad1
MD
375 popl %eax
376 jmp splz_next
984263bc 377
5f456c40
MD
378 /*
379 * SOFT interrupt pending
380 *
381 * Temporarily back-out our critical section to allow the interrupt
382 * preempt us.
383 */
384 ALIGN_TEXT
385splz_soft:
ecf9dec6 386 sti
5f456c40
MD
387 bsfl %ecx,%ecx /* locate the next pending softint */
388 btrl %ecx,PCPU(spending) /* make sure its still pending */
389 jnc splz_next
390 addl $FIRST_SOFTINT,%ecx /* actual intr number */
5f456c40
MD
391 pushl %eax
392 pushl %ecx
f9235b6d 393 decl TD_CRITCOUNT(%ebx)
5f456c40
MD
394 incl TD_NEST_COUNT(%ebx) /* prevent doreti/splz nesting */
395 call sched_ithd /* YYY must pull in imasks */
f9235b6d 396 incl TD_CRITCOUNT(%ebx)
5f456c40
MD
397 decl TD_NEST_COUNT(%ebx) /* prevent doreti/splz nesting */
398 addl $4,%esp
399 popl %eax
400 jmp splz_next
401
96728c05
MD
402#ifdef SMP
403splz_ipiq:
235957ed 404 andl $~RQF_IPIQ,PCPU(reqflags)
ecf9dec6 405 sti
96728c05
MD
406 pushl %eax
407 call lwkt_process_ipiq
408 popl %eax
409 jmp splz_next
78ea5a2a
SZ
410
411splz_timer:
412 andl $~RQF_TIMER,PCPU(reqflags)
ecf9dec6 413 sti
78ea5a2a
SZ
414 pushl %eax
415 call lapic_timer_process
416 popl %eax
417 jmp splz_next
96728c05
MD
418#endif
419
46b26c5e
MD
420 /*
421 * dofastunpend(%ecx:intr)
422 *
423 * A FAST interrupt previously made pending can now be run,
424 * execute it by pushing a dummy interrupt frame and
425 * calling ithread_fast_handler to execute or schedule it.
426 *
427 * ithread_fast_handler() returns 0 if it wants us to unmask
428 * further interrupts.
429 */
430#define PUSH_DUMMY \
431 pushfl ; /* phys int frame / flags */ \
432 pushl %cs ; /* phys int frame / cs */ \
433 pushl 12(%esp) ; /* original caller eip */ \
434 pushl $0 ; /* dummy error code */ \
435 pushl $0 ; /* dummy trap type */ \
4e7c41c5
MD
436 pushl $0 ; /* dummy xflags */ \
437 subl $13*4,%esp ; /* pushal + 4 seg regs (dummy) + CPL */ \
46b26c5e
MD
438
439#define POP_DUMMY \
4e7c41c5 440 addl $19*4,%esp ; \
46b26c5e
MD
441
442dofastunpend:
443 pushl %ebp /* frame for backtrace */
444 movl %esp,%ebp
445 PUSH_DUMMY
446 pushl %ecx /* last part of intrframe = intr */
447 incl fastunpend_count
c7eb0589 448 pushl %esp /* pass frame by reference */
46b26c5e 449 call ithread_fast_handler /* returns 0 to unmask */
c7eb0589 450 addl $4,%esp /* remove pointer, now intr on top */
46b26c5e
MD
451 cmpl $0,%eax
452 jnz 1f
453 movl MachIntrABI + MACHINTR_INTREN, %eax
454 call *%eax /* MachIntrABI.intren(intr) */
4551:
456 addl $4,%esp
457 POP_DUMMY
458 popl %ebp
459 ret
460