2 * Copyright (c) 1989, 1990 William F. Jolitz.
3 * Copyright (c) 1990 The Regents of the University of California.
6 * This code is derived from software contributed to Berkeley by
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 * 3. All advertising materials mentioning features or use of this software
18 * must display the following acknowledgement:
19 * This product includes software developed by the University of
20 * California, Berkeley and its contributors.
21 * 4. Neither the name of the University nor the names of its contributors
22 * may be used to endorse or promote products derived from this software
23 * without specific prior written permission.
25 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
28 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
29 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
30 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
31 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
32 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
33 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
34 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
39 * $FreeBSD: src/sys/i386/isa/ipl.s,v 1.32.2.3 2002/05/16 16:03:56 bde Exp $
40 * $DragonFly: src/sys/platform/pc32/isa/ipl.s,v 1.28 2007/01/22 19:37:04 corecode Exp $
45 #include <machine/asmacros.h>
46 #include <machine/segments.h>
47 #include <machine/ipl.h>
48 #include <machine/lock.h>
49 #include <machine/psl.h>
50 #include <machine/trap.h>
56 * Vector interrupt control section
58 * ipending - Pending interrupts (set when a masked interrupt occurs)
59 * spending - Pending software interrupts
64 .globl fastunpend_count
65 fastunpend_count: .long 0
73 * - fast interrupts are always called with a critical section
76 * - we release our critical section when scheduling interrupt
77 * or softinterrupt threads in order so they can preempt
78 * (unless we are called manually from a critical section, in
79 * which case there will still be a critical section and
80 * they won't preempt anyway).
82 * - TD_NEST_COUNT prevents splz from nesting too deeply within
83 * itself. It is *not* actually an interrupt nesting count.
84 * PCPU(intr_nesting_level) is an interrupt nesting count.
86 * - We have to be careful in regards to local interrupts
87 * occuring simultaniously with our doreti and splz
94 * Handle return from interrupts, traps and syscalls. This function
95 * checks the cpl for unmasked pending interrupts (fast, normal, or
96 * soft) and schedules them if appropriate, then irets.
98 * If we are in a critical section we cannot run any pending ints
99 * nor can be play with mp_lock.
101 * NOTE: Since SPLs no longer exist, all callers of this function
102 * push $0 for the CPL. HOWEVER, we *STILL* use the cpl mask within
103 * this function to mark fast interrupts which could not be dispatched
104 * do to the unavailability of the BGL.
108 .type doreti,@function
110 FAKE_MCOUNT(bintr) /* init "from" bintr -> doreti */
111 popl %eax /* cpl to restore XXX */
112 movl $0,%eax /* irq mask unavailable due to BGL */
113 movl PCPU(curthread),%ebx
114 cli /* interlock with TDPRI_CRIT */
115 cmpl $0,PCPU(reqflags) /* short cut if nothing to do */
117 cmpl $TDPRI_CRIT,TD_PRI(%ebx) /* can't unpend if in critical sec */
119 addl $TDPRI_CRIT,TD_PRI(%ebx) /* force all ints to pending */
121 sti /* allow new interrupts */
122 movl %eax,%ecx /* irq mask unavailable due to BGL */
124 cli /* disallow YYY remove */
126 testl $RQF_IPIQ,PCPU(reqflags)
128 testl $RQF_TIMER,PCPU(reqflags)
131 testl PCPU(fpending),%ecx /* check for an unmasked fast int */
134 testl PCPU(ipending),%ecx /* check for an unmasked slow int */
137 movl PCPU(spending),%ecx /* check for a pending software int */
141 testl $RQF_AST_MASK,PCPU(reqflags) /* any pending ASTs? */
143 testl $PSL_VM,TF_EFLAGS(%esp)
145 cmpl $1,in_vm86call /* YYY make per 'cpu'? */
148 /* ASTs are only applicable when returning to userland */
149 testb $SEL_RPL_MASK,TF_CS(%esp)
153 * Nothing left to do, finish up. Interrupts are still disabled.
154 * %eax contains the mask of IRQ's that are not available due to
155 * BGL requirements. We can only clear RQF_INTPEND if *ALL* pending
156 * interrupts have been processed.
158 subl $TDPRI_CRIT,TD_PRI(%ebx) /* interlocked with cli */
161 andl $~RQF_INTPEND,PCPU(reqflags)
166 * Restore the segment registers. Since segment register values
167 * can be set from user mode, this can result in a kernel mode
168 * exception. The trap code will revector to the *_fault code
169 * which then sets up a T_PROTFLT signal. If the signal is
170 * sent to userland, sendsig() will automatically clean up all
171 * the segment registers to avoid a loop.
173 .globl doreti_popl_gs
174 .globl doreti_popl_fs
175 .globl doreti_popl_es
176 .globl doreti_popl_ds
178 .globl doreti_syscall_ret
189 addl $3*4,%esp /* xflags, trap, err */
194 .globl doreti_iret_fault
196 subl $3*4,%esp /* xflags, trap, err */
199 .globl doreti_popl_ds_fault
200 doreti_popl_ds_fault:
202 .globl doreti_popl_es_fault
203 doreti_popl_es_fault:
205 .globl doreti_popl_fs_fault
206 doreti_popl_fs_fault:
208 .globl doreti_popl_gs_fault
209 doreti_popl_gs_fault:
210 movl $0,TF_ERR(%esp) /* XXX should be the error code */
211 movl $T_PROTFLT,TF_TRAPNO(%esp)
212 jmp alltraps_with_regs_pushed
215 * FAST interrupt pending. NOTE: stack context holds frame structure
216 * for fast interrupt procedure, do not do random pushes or pops!
220 andl PCPU(fpending),%ecx /* only check fast ints */
221 bsfl %ecx, %ecx /* locate the next dispatchable int */
222 btrl %ecx, PCPU(fpending) /* is it really still pending? */
224 pushl %eax /* save IRQ mask unavailable for BGL */
225 /* NOTE: is also CPL in frame */
228 pushl %ecx /* save ecx */
233 /* MP lock successful */
236 incl PCPU(intr_nesting_level)
237 call dofastunpend /* unpend fast intr %ecx */
238 decl PCPU(intr_nesting_level)
247 btsl %ecx, PCPU(fpending) /* oops, couldn't get the MP lock */
248 popl %eax /* add to temp. cpl mask to ignore */
249 orl PCPU(fpending),%eax
253 * INTR interrupt pending
255 * Temporarily back-out our critical section to allow an interrupt
256 * preempt us when we schedule it. Bump intr_nesting_level to
257 * prevent the switch code from recursing via splz too deeply.
261 andl PCPU(ipending),%ecx /* only check normal ints */
262 bsfl %ecx, %ecx /* locate the next dispatchable int */
263 btrl %ecx, PCPU(ipending) /* is it really still pending? */
267 incl TD_NEST_COUNT(%ebx) /* prevent doreti/splz nesting */
268 subl $TDPRI_CRIT,TD_PRI(%ebx) /* so we can preempt */
269 call sched_ithd /* YYY must pull in imasks */
270 addl $TDPRI_CRIT,TD_PRI(%ebx)
271 decl TD_NEST_COUNT(%ebx)
277 * SOFT interrupt pending
279 * Temporarily back-out our critical section to allow an interrupt
280 * preempt us when we schedule it. Bump intr_nesting_level to
281 * prevent the switch code from recursing via splz too deeply.
285 bsfl %ecx,%ecx /* locate the next pending softint */
286 btrl %ecx,PCPU(spending) /* make sure its still pending */
288 addl $FIRST_SOFTINT,%ecx /* actual intr number */
291 incl TD_NEST_COUNT(%ebx) /* prevent doreti/splz nesting */
292 subl $TDPRI_CRIT,TD_PRI(%ebx) /* so we can preempt */
293 call sched_ithd /* YYY must pull in imasks */
294 addl $TDPRI_CRIT,TD_PRI(%ebx)
295 decl TD_NEST_COUNT(%ebx)
301 * AST pending. We clear RQF_AST_SIGNAL automatically, the others
302 * are cleared by the trap as they are processed.
304 * Temporarily back-out our critical section because trap() can be
305 * a long-winded call, and we want to be more syscall-like.
307 * YYY theoretically we can call lwkt_switch directly if all we need
308 * to do is a reschedule.
311 andl $~(RQF_AST_SIGNAL|RQF_AST_UPCALL),PCPU(reqflags)
313 movl %eax,%esi /* save cpl (can't use stack) */
314 movl $T_ASTFLT,TF_TRAPNO(%esp)
315 pushl %esp /* pass frame by reference */
316 subl $TDPRI_CRIT,TD_PRI(%ebx)
318 addl $TDPRI_CRIT,TD_PRI(%ebx)
320 movl %esi,%eax /* restore cpl for loop */
325 * IPIQ message pending. We clear RQF_IPIQ automatically.
328 movl %eax,%esi /* save cpl (can't use stack) */
329 incl PCPU(intr_nesting_level)
330 andl $~RQF_IPIQ,PCPU(reqflags)
331 subl $8,%esp /* add dummy vec and ppl */
332 pushl %esp /* pass frame by reference */
333 call lwkt_process_ipiq_frame
335 decl PCPU(intr_nesting_level)
336 movl %esi,%eax /* restore cpl for loop */
340 movl %eax,%esi /* save cpl (can't use stack) */
341 incl PCPU(intr_nesting_level)
342 andl $~RQF_TIMER,PCPU(reqflags)
343 subl $8,%esp /* add dummy vec and ppl */
344 pushl %esp /* pass frame by reference */
345 call lapic_timer_process_frame
347 decl PCPU(intr_nesting_level)
348 movl %esi,%eax /* restore cpl for loop */
354 * SPLZ() a C callable procedure to dispatch any unmasked pending
355 * interrupts regardless of critical section nesting. ASTs
356 * are not dispatched.
358 * Use %eax to track those IRQs that could not be processed
359 * due to BGL requirements.
366 movl PCPU(curthread),%ebx
367 addl $TDPRI_CRIT,TD_PRI(%ebx)
372 movl %eax,%ecx /* ecx = ~CPL */
375 testl $RQF_IPIQ,PCPU(reqflags)
377 testl $RQF_TIMER,PCPU(reqflags)
380 testl PCPU(fpending),%ecx /* check for an unmasked fast int */
383 testl PCPU(ipending),%ecx
386 movl PCPU(spending),%ecx
390 subl $TDPRI_CRIT,TD_PRI(%ebx)
393 * Nothing left to do, finish up. Interrupts are still disabled.
394 * If our mask of IRQs we couldn't process due to BGL requirements
395 * is 0 then there are no pending interrupt sources left and we
396 * can clear RQF_INTPEND.
400 andl $~RQF_INTPEND,PCPU(reqflags)
407 * FAST interrupt pending
411 andl PCPU(fpending),%ecx /* only check fast ints */
412 bsfl %ecx, %ecx /* locate the next dispatchable int */
413 btrl %ecx, PCPU(fpending) /* is it really still pending? */
425 incl PCPU(intr_nesting_level)
426 call dofastunpend /* unpend fast intr %ecx */
427 decl PCPU(intr_nesting_level)
436 btsl %ecx, PCPU(fpending) /* oops, couldn't get the MP lock */
438 orl PCPU(fpending),%eax
442 * INTR interrupt pending
444 * Temporarily back-out our critical section to allow the interrupt
449 andl PCPU(ipending),%ecx /* only check normal ints */
450 bsfl %ecx, %ecx /* locate the next dispatchable int */
451 btrl %ecx, PCPU(ipending) /* is it really still pending? */
456 subl $TDPRI_CRIT,TD_PRI(%ebx)
457 incl TD_NEST_COUNT(%ebx) /* prevent doreti/splz nesting */
458 call sched_ithd /* YYY must pull in imasks */
459 addl $TDPRI_CRIT,TD_PRI(%ebx)
460 decl TD_NEST_COUNT(%ebx) /* prevent doreti/splz nesting */
466 * SOFT interrupt pending
468 * Temporarily back-out our critical section to allow the interrupt
473 bsfl %ecx,%ecx /* locate the next pending softint */
474 btrl %ecx,PCPU(spending) /* make sure its still pending */
476 addl $FIRST_SOFTINT,%ecx /* actual intr number */
480 subl $TDPRI_CRIT,TD_PRI(%ebx)
481 incl TD_NEST_COUNT(%ebx) /* prevent doreti/splz nesting */
482 call sched_ithd /* YYY must pull in imasks */
483 addl $TDPRI_CRIT,TD_PRI(%ebx)
484 decl TD_NEST_COUNT(%ebx) /* prevent doreti/splz nesting */
491 andl $~RQF_IPIQ,PCPU(reqflags)
493 call lwkt_process_ipiq
498 andl $~RQF_TIMER,PCPU(reqflags)
500 call lapic_timer_process
506 * dofastunpend(%ecx:intr)
508 * A FAST interrupt previously made pending can now be run,
509 * execute it by pushing a dummy interrupt frame and
510 * calling ithread_fast_handler to execute or schedule it.
512 * ithread_fast_handler() returns 0 if it wants us to unmask
513 * further interrupts.
516 pushfl ; /* phys int frame / flags */ \
517 pushl %cs ; /* phys int frame / cs */ \
518 pushl 12(%esp) ; /* original caller eip */ \
519 pushl $0 ; /* dummy error code */ \
520 pushl $0 ; /* dummy trap type */ \
521 pushl $0 ; /* dummy xflags */ \
522 subl $13*4,%esp ; /* pushal + 4 seg regs (dummy) + CPL */ \
528 pushl %ebp /* frame for backtrace */
531 pushl %ecx /* last part of intrframe = intr */
532 incl fastunpend_count
533 pushl %esp /* pass frame by reference */
534 call ithread_fast_handler /* returns 0 to unmask */
535 addl $4,%esp /* remove pointer, now intr on top */
538 movl MachIntrABI + MACHINTR_INTREN, %eax
539 call *%eax /* MachIntrABI.intren(intr) */