2 * Copyright (c) 1989, 1990 William F. Jolitz.
3 * Copyright (c) 1990 The Regents of the University of California.
6 * This code is derived from software contributed to Berkeley by
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 * 3. All advertising materials mentioning features or use of this software
18 * must display the following acknowledgement:
19 * This product includes software developed by the University of
20 * California, Berkeley and its contributors.
21 * 4. Neither the name of the University nor the names of its contributors
22 * may be used to endorse or promote products derived from this software
23 * without specific prior written permission.
25 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
28 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
29 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
30 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
31 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
32 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
33 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
34 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
39 * $FreeBSD: src/sys/i386/isa/ipl.s,v 1.32.2.3 2002/05/16 16:03:56 bde Exp $
42 #include <machine/asmacros.h>
43 #include <machine/segments.h>
44 #include <machine/ipl.h>
45 #include <machine/lock.h>
46 #include <machine/psl.h>
47 #include <machine/trap.h>
53 * Vector interrupt control section
55 * ipending - Pending interrupts (set when a masked interrupt occurs)
56 * spending - Pending software interrupts
61 .globl fastunpend_count
62 fastunpend_count: .long 0
70 * - interrupts are always called with a critical section held
72 * - we release our critical section when scheduling interrupt
73 * or softinterrupt threads in order so they can preempt
74 * (unless we are called manually from a critical section, in
75 * which case there will still be a critical section and
76 * they won't preempt anyway).
78 * - TD_NEST_COUNT prevents splz from nesting too deeply within
79 * itself. It is *not* actually an interrupt nesting count.
80 * PCPU(intr_nesting_level) is an interrupt nesting count.
82 * - We have to be careful in regards to local interrupts
83 * occuring simultaniously with our doreti and splz
90 * Handle return from interrupts, traps and syscalls. This function
91 * checks the cpl for unmasked pending interrupts (hardware or soft)
92 * and schedules them if appropriate, then irets.
94 * If we are in a critical section we cannot run any pending ints.
96 * NOTE: Since SPLs no longer exist, all callers of this function
97 * push $0 for the CPL. HOWEVER, we *STILL* use the cpl mask within
98 * this function to mark interrupts which could not be dispatched
99 * do to the unavailability of the BGL.
103 .type doreti,@function
105 FAKE_MCOUNT(bintr) /* init "from" bintr -> doreti */
106 popl %eax /* cpl to restore XXX */
107 movl $0,%eax /* irq mask unavailable due to BGL */
108 movl PCPU(curthread),%ebx
109 cli /* interlock with td_critcount */
110 cmpl $0,PCPU(reqflags) /* short cut if nothing to do */
112 testl $-1,TD_CRITCOUNT(%ebx) /* can't unpend if in critical sec */
114 incl TD_CRITCOUNT(%ebx) /* force all ints to pending */
116 cli /* re-assert cli on loop */
117 movl %eax,%ecx /* irq mask unavailable due to BGL */
120 testl $RQF_IPIQ,PCPU(reqflags)
122 testl $RQF_TIMER,PCPU(reqflags)
126 * check for an unmasked int (6 groups)
129 testl PCPU_E4(ipending,%edx),%ecx
133 testl PCPU_E4(ipending,%edx),%ecx
137 testl PCPU_E4(ipending,%edx),%ecx
141 testl PCPU_E4(ipending,%edx),%ecx
145 testl PCPU_E4(ipending,%edx),%ecx
149 testl PCPU_E4(ipending,%edx),%ecx
152 movl PCPU(spending),%ecx /* check for a pending software int */
156 testl $RQF_AST_MASK,PCPU(reqflags) /* any pending ASTs? */
158 testl $PSL_VM,TF_EFLAGS(%esp)
160 cmpl $1,in_vm86call /* YYY make per 'cpu'? */
163 /* ASTs are only applicable when returning to userland */
164 testb $SEL_RPL_MASK,TF_CS(%esp)
168 * Nothing left to do, finish up. Interrupts are still disabled.
169 * %eax contains the mask of IRQ's that are not available due to
170 * BGL requirements. We can only clear RQF_INTPEND if *ALL* pending
171 * interrupts have been processed.
173 decl TD_CRITCOUNT(%ebx) /* interlocked with cli */
176 andl $~RQF_INTPEND,PCPU(reqflags)
181 * Restore the segment registers. Since segment register values
182 * can be set from user mode, this can result in a kernel mode
183 * exception. The trap code will revector to the *_fault code
184 * which then sets up a T_PROTFLT signal. If the signal is
185 * sent to userland, sendsig() will automatically clean up all
186 * the segment registers to avoid a loop.
188 .globl doreti_popl_gs
189 .globl doreti_popl_fs
190 .globl doreti_popl_es
191 .globl doreti_popl_ds
193 .globl doreti_syscall_ret
204 addl $3*4,%esp /* xflags, trap, err */
209 * Interrupts are likely disabled due to the above interlock
210 * between cli/iretq. We must enable them before calling any
211 * high level function.
214 .globl doreti_iret_fault
216 subl $3*4,%esp /* xflags, trap, err */
219 .globl doreti_popl_ds_fault
220 doreti_popl_ds_fault:
222 .globl doreti_popl_es_fault
223 doreti_popl_es_fault:
225 .globl doreti_popl_fs_fault
226 doreti_popl_fs_fault:
228 .globl doreti_popl_gs_fault
229 doreti_popl_gs_fault:
231 movl $0,TF_ERR(%esp) /* XXX should be the error code */
232 movl $T_PROTFLT,TF_TRAPNO(%esp)
233 jmp alltraps_with_regs_pushed
236 * Interrupt pending. NOTE: stack context holds frame structure
237 * for interrupt procedure, do not do random pushes or pops!
241 andl PCPU_E4(ipending,%edx),%ecx
243 bsfl %ecx, %ecx /* locate the next dispatchable int */
244 btrl %ecx, PCPU_E4(ipending,%edx)
245 /* is it really still pending? */
249 orl %edx, %ecx /* form intr number */
251 pushl %eax /* save IRQ mask unavailable for BGL */
252 /* NOTE: is also CPL in frame */
253 call dofastunpend /* unpend intr %ecx */
258 * SOFT interrupt pending
260 * Temporarily back-out our critical section to allow an interrupt
261 * preempt us when we schedule it. Bump intr_nesting_level to
262 * prevent the switch code from recursing via splz too deeply.
267 bsfl %ecx,%ecx /* locate the next pending softint */
268 btrl %ecx,PCPU(spending) /* make sure its still pending */
270 addl $FIRST_SOFTINT,%ecx /* actual intr number */
273 incl TD_NEST_COUNT(%ebx) /* prevent doreti/splz nesting */
274 decl TD_CRITCOUNT(%ebx) /* so we can preempt */
275 call sched_ithd /* YYY must pull in imasks */
276 incl TD_CRITCOUNT(%ebx)
277 decl TD_NEST_COUNT(%ebx)
283 * AST pending. We clear RQF_AST_SIGNAL automatically, the others
284 * are cleared by the trap as they are processed.
286 * Temporarily back-out our critical section because trap() can be
287 * a long-winded call, and we want to be more syscall-like.
289 * YYY theoretically we can call lwkt_switch directly if all we need
290 * to do is a reschedule.
293 andl $~(RQF_AST_SIGNAL|RQF_AST_UPCALL),PCPU(reqflags)
295 movl %eax,%esi /* save cpl (can't use stack) */
296 movl $T_ASTFLT,TF_TRAPNO(%esp)
297 pushl %esp /* pass frame by reference */
298 decl TD_CRITCOUNT(%ebx)
300 incl TD_CRITCOUNT(%ebx)
302 movl %esi,%eax /* restore cpl for loop */
307 * IPIQ message pending. We clear RQF_IPIQ automatically.
310 movl %eax,%esi /* save cpl (can't use stack) */
311 incl PCPU(intr_nesting_level)
312 andl $~RQF_IPIQ,PCPU(reqflags)
314 subl $8,%esp /* add dummy vec and ppl */
315 pushl %esp /* pass frame by reference */
316 call lwkt_process_ipiq_frame
318 decl PCPU(intr_nesting_level)
319 movl %esi,%eax /* restore cpl for loop */
323 movl %eax,%esi /* save cpl (can't use stack) */
324 incl PCPU(intr_nesting_level)
325 andl $~RQF_TIMER,PCPU(reqflags)
327 subl $8,%esp /* add dummy vec and ppl */
328 pushl %esp /* pass frame by reference */
329 call lapic_timer_process_frame
331 decl PCPU(intr_nesting_level)
332 movl %esi,%eax /* restore cpl for loop */
338 * SPLZ() a C callable procedure to dispatch any unmasked pending
339 * interrupts regardless of critical section nesting. ASTs
340 * are not dispatched.
342 * Use %eax to track those IRQs that could not be processed
343 * due to BGL requirements.
350 movl PCPU(curthread),%ebx
351 incl TD_CRITCOUNT(%ebx)
356 movl %eax,%ecx /* ecx = ~CPL */
359 testl $RQF_IPIQ,PCPU(reqflags)
361 testl $RQF_TIMER,PCPU(reqflags)
365 * check for an unmasked int (6 groups)
368 testl PCPU_E4(ipending,%edx),%ecx
372 testl PCPU_E4(ipending,%edx),%ecx
376 testl PCPU_E4(ipending,%edx),%ecx
380 testl PCPU_E4(ipending,%edx),%ecx
384 testl PCPU_E4(ipending,%edx),%ecx
388 testl PCPU_E4(ipending,%edx),%ecx
391 movl PCPU(spending),%ecx
395 decl TD_CRITCOUNT(%ebx)
398 * Nothing left to do, finish up. Interrupts are still disabled.
399 * If our mask of IRQs we couldn't process due to BGL requirements
400 * is 0 then there are no pending interrupt sources left and we
401 * can clear RQF_INTPEND.
405 andl $~RQF_INTPEND,PCPU(reqflags)
416 andl PCPU_E4(ipending,%edx),%ecx
418 bsfl %ecx, %ecx /* locate the next dispatchable int */
419 btrl %ecx, PCPU_E4(ipending,%edx)
420 /* is it really still pending? */
424 orl %edx, %ecx /* form intr number */
427 call dofastunpend /* unpend intr %ecx */
432 * SOFT interrupt pending
434 * Temporarily back-out our critical section to allow the interrupt
440 bsfl %ecx,%ecx /* locate the next pending softint */
441 btrl %ecx,PCPU(spending) /* make sure its still pending */
443 addl $FIRST_SOFTINT,%ecx /* actual intr number */
446 decl TD_CRITCOUNT(%ebx)
447 incl TD_NEST_COUNT(%ebx) /* prevent doreti/splz nesting */
448 call sched_ithd /* YYY must pull in imasks */
449 incl TD_CRITCOUNT(%ebx)
450 decl TD_NEST_COUNT(%ebx) /* prevent doreti/splz nesting */
457 andl $~RQF_IPIQ,PCPU(reqflags)
460 call lwkt_process_ipiq
465 andl $~RQF_TIMER,PCPU(reqflags)
468 call lapic_timer_process
474 * dofastunpend(%ecx:intr)
476 * An interrupt previously made pending can now be run,
477 * execute it by pushing a dummy interrupt frame and
478 * calling ithread_fast_handler to execute or schedule it.
480 * ithread_fast_handler() returns 0 if it wants us to unmask
481 * further interrupts.
484 pushfl ; /* phys int frame / flags */ \
485 pushl %cs ; /* phys int frame / cs */ \
486 pushl 12(%esp) ; /* original caller eip */ \
487 pushl $0 ; /* dummy error code */ \
488 pushl $0 ; /* dummy trap type */ \
489 pushl $0 ; /* dummy xflags */ \
490 subl $13*4,%esp ; /* pushal + 4 seg regs (dummy) + CPL */ \
496 pushl %ebp /* frame for backtrace */
499 pushl %ecx /* last part of intrframe = intr */
500 incl fastunpend_count
501 pushl %esp /* pass frame by reference */
502 call ithread_fast_handler /* returns 0 to unmask */
503 addl $4,%esp /* remove pointer, now intr on top */
506 movl MachIntrABI + MACHINTR_INTREN, %eax
507 call *%eax /* MachIntrABI.intren(intr) */