2 * Copyright (c) 2008 The DragonFly Project. All rights reserved.
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 * Copyright (c) 1989, 1990 William F. Jolitz.
37 * Copyright (c) 1990 The Regents of the University of California.
38 * All rights reserved.
40 * This code is derived from software contributed to Berkeley by
43 * Redistribution and use in source and binary forms, with or without
44 * modification, are permitted provided that the following conditions
46 * 1. Redistributions of source code must retain the above copyright
47 * notice, this list of conditions and the following disclaimer.
48 * 2. Redistributions in binary form must reproduce the above copyright
49 * notice, this list of conditions and the following disclaimer in the
50 * documentation and/or other materials provided with the distribution.
51 * 3. All advertising materials mentioning features or use of this software
52 * must display the following acknowledgement:
53 * This product includes software developed by the University of
54 * California, Berkeley and its contributors.
55 * 4. Neither the name of the University nor the names of its contributors
56 * may be used to endorse or promote products derived from this software
57 * without specific prior written permission.
59 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
60 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
61 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
62 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
63 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
64 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
65 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
66 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
67 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
68 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
73 * $FreeBSD: src/sys/i386/isa/ipl.s,v 1.32.2.3 2002/05/16 16:03:56 bde Exp $
76 #include <machine/asmacros.h>
77 #include <machine/segments.h>
78 #include <machine/ipl.h>
79 #include <machine/lock.h>
80 #include <machine/psl.h>
81 #include <machine/trap.h>
87 * Vector interrupt control section
89 * fpending - Pending interrupts (set when a masked interrupt occurs)
90 * spending - Pending software interrupts
95 .globl fastunpend_count
96 fastunpend_count: .long 0
104 * - fast interrupts are always called with a critical section
107 * - we release our critical section when scheduling interrupt
108 * or softinterrupt threads in order so they can preempt
109 * (unless we are called manually from a critical section, in
110 * which case there will still be a critical section and
111 * they won't preempt anyway).
113 * - TD_NEST_COUNT prevents splz from nesting too deeply within
114 * itself. It is *not* actually an interrupt nesting count.
115 * PCPU(intr_nesting_level) is an interrupt nesting count.
117 * - We have to be careful in regards to local interrupts
118 * occuring simultaniously with our doreti and splz
125 * Handle return from interrupts, traps and syscalls. This function
126 * checks the cpl for unmasked pending interrupts (fast, normal, or
127 * soft) and schedules them if appropriate, then irets.
129 * If we are in a critical section we cannot run any pending ints
130 * nor can be play with mp_lock.
132 * The stack contains a trapframe at the start of doreti.
136 .type doreti,@function
138 FAKE_MCOUNT(bintr) /* init "from" bintr -> doreti */
139 movq $0,%rax /* irq mask unavailable due to BGL */
140 movq PCPU(curthread),%rbx
141 cli /* interlock with critical section */
142 cmpl $0,PCPU(reqflags) /* short cut if nothing to do */
144 testl $-1,TD_CRITCOUNT(%rbx) /* can't unpend if in critical sec */
146 incl TD_CRITCOUNT(%rbx) /* force all ints to pending */
148 sti /* allow new interrupts */
149 movl %eax,%ecx /* irq mask unavailable due to BGL */
151 cli /* disallow YYY remove */
153 testl $RQF_IPIQ,PCPU(reqflags)
155 testl $RQF_TIMER,PCPU(reqflags)
158 testl PCPU(fpending),%ecx /* check for an unmasked fast int */
161 movl PCPU(spending),%ecx /* check for a pending software int */
165 testl $RQF_AST_MASK,PCPU(reqflags) /* any pending ASTs? */
168 /* ASTs are only applicable when returning to userland */
169 testb $SEL_RPL_MASK,TF_CS(%rsp)
173 * Nothing left to do, finish up. Interrupts are still disabled.
174 * %eax contains the mask of IRQ's that are not available due to
175 * BGL requirements. We can only clear RQF_INTPEND if *ALL* pending
176 * interrupts have been processed.
178 decl TD_CRITCOUNT(%rbx) /* interlocked with cli */
181 andl $~RQF_INTPEND,PCPU(reqflags)
186 * Restore register and iret. iret can fault on %rip (which is
187 * really stupid). If this occurs we re-fault and vector to
188 * doreti_iret_fault().
191 * can be set from user mode, this can result in a kernel mode
192 * exception. The trap code will revector to the *_fault code
193 * which then sets up a T_PROTFLT signal. If the signal is
194 * sent to userland, sendsig() will automatically clean up all
195 * the segment registers to avoid a loop.
198 .globl doreti_syscall_ret
200 POP_FRAME /* registers and %gs (+cli) */
201 /* special global also used by exception.S */
206 * doreti_iret_fault. Alternative return code for
207 * the case where we get a fault in the doreti_exit code
208 * above. trap() (sys/platform/pc64/x86_64/trap.c) catches this specific
209 * case, sends the process a signal and continues in the
210 * corresponding place in the code below.
213 .globl doreti_iret_fault
216 testq $PSL_I,TF_RFLAGS(%rsp)
220 movq $T_PROTFLT,TF_TRAPNO(%rsp)
221 movq $0,TF_ERR(%rsp) /* XXX should be the error code */
222 movq $0,TF_ADDR(%rsp)
223 FAKE_MCOUNT(TF_RIP(%rsp))
227 * FAST interrupt pending. NOTE: stack context holds frame structure
228 * for fast interrupt procedure, do not do random pushes or pops!
232 andl PCPU(fpending),%ecx /* only check fast ints */
233 bsfl %ecx, %ecx /* locate the next dispatchable int */
234 btrl %ecx, PCPU(fpending) /* is it really still pending? */
236 pushq %rax /* save IRQ mask unavailable for BGL */
237 /* NOTE: is also CPL in frame */
240 pushq %rcx /* save ecx */
245 /* MP lock successful */
248 call dofastunpend /* unpend fast intr %ecx */
257 btsl %ecx, PCPU(fpending) /* oops, couldn't get the MP lock */
258 popq %rax /* add to temp. cpl mask to ignore */
259 orl PCPU(fpending),%eax
263 * SOFT interrupt pending
265 * Temporarily back-out our critical section to allow an interrupt
266 * preempt us when we schedule it. Bump intr_nesting_level to
267 * prevent the switch code from recursing via splz too deeply.
271 bsfl %ecx,%ecx /* locate the next pending softint */
272 btrl %ecx,PCPU(spending) /* make sure its still pending */
274 addl $FIRST_SOFTINT,%ecx /* actual intr number */
276 movl %ecx,%edi /* argument to C call */
277 incl TD_NEST_COUNT(%rbx) /* prevent doreti/splz nesting */
278 decl TD_CRITCOUNT(%rbx) /* so we can preempt */
279 call sched_ithd /* YYY must pull in imasks */
280 incl TD_CRITCOUNT(%rbx)
281 decl TD_NEST_COUNT(%rbx)
286 * AST pending. We clear RQF_AST_SIGNAL automatically, the others
287 * are cleared by the trap as they are processed.
289 * Temporarily back-out our critical section because trap() can be
290 * a long-winded call, and we want to be more syscall-like.
292 * YYY theoretically we can call lwkt_switch directly if all we need
293 * to do is a reschedule.
296 andl $~(RQF_AST_SIGNAL|RQF_AST_UPCALL),PCPU(reqflags)
298 movl %eax,%r12d /* save cpl (can't use stack) */
299 movl $T_ASTFLT,TF_TRAPNO(%rsp)
300 movq %rsp,%rdi /* pass frame by ref (%edi = C arg) */
301 decl TD_CRITCOUNT(%rbx)
303 incl TD_CRITCOUNT(%rbx)
304 movl %r12d,%eax /* restore cpl for loop */
309 * IPIQ message pending. We clear RQF_IPIQ automatically.
312 movl %eax,%r12d /* save cpl (can't use stack) */
313 incl PCPU(intr_nesting_level)
314 andl $~RQF_IPIQ,PCPU(reqflags)
315 subq $8,%rsp /* trapframe->intrframe */
316 movq %rsp,%rdi /* pass frame by ref (C arg) */
317 call lwkt_process_ipiq_frame
318 addq $8,%rsp /* intrframe->trapframe */
319 decl PCPU(intr_nesting_level)
320 movl %r12d,%eax /* restore cpl for loop */
324 movl %eax,%r12d /* save cpl (can't use stack) */
325 incl PCPU(intr_nesting_level)
326 andl $~RQF_TIMER,PCPU(reqflags)
327 subq $8,%rsp /* trapframe->intrframe */
328 movq %rsp,%rdi /* pass frame by ref (C arg) */
329 call lapic_timer_process_frame
330 addq $8,%rsp /* intrframe->trapframe */
331 decl PCPU(intr_nesting_level)
332 movl %r12d,%eax /* restore cpl for loop */
338 * SPLZ() a C callable procedure to dispatch any unmasked pending
339 * interrupts regardless of critical section nesting. ASTs
340 * are not dispatched.
342 * Use %eax to track those IRQs that could not be processed
343 * due to BGL requirements.
350 movq PCPU(curthread),%rbx
351 incl TD_CRITCOUNT(%rbx)
356 movl %eax,%ecx /* ecx = ~CPL */
359 testl $RQF_IPIQ,PCPU(reqflags)
361 testl $RQF_TIMER,PCPU(reqflags)
364 testl PCPU(fpending),%ecx /* check for an unmasked fast int */
367 movl PCPU(spending),%ecx
371 decl TD_CRITCOUNT(%rbx)
374 * Nothing left to do, finish up. Interrupts are still disabled.
375 * If our mask of IRQs we couldn't process due to BGL requirements
376 * is 0 then there are no pending interrupt sources left and we
377 * can clear RQF_INTPEND.
381 andl $~RQF_INTPEND,PCPU(reqflags)
388 * FAST interrupt pending
392 andl PCPU(fpending),%ecx /* only check fast ints */
393 bsfl %ecx, %ecx /* locate the next dispatchable int */
394 btrl %ecx, PCPU(fpending) /* is it really still pending? */
399 movl %ecx,%edi /* argument to try_mplock */
405 call dofastunpend /* unpend fast intr %ecx */
414 btsl %ecx, PCPU(fpending) /* oops, couldn't get the MP lock */
416 orl PCPU(fpending),%eax
420 * SOFT interrupt pending
422 * Temporarily back-out our critical section to allow the interrupt
427 bsfl %ecx,%ecx /* locate the next pending softint */
428 btrl %ecx,PCPU(spending) /* make sure its still pending */
430 addl $FIRST_SOFTINT,%ecx /* actual intr number */
433 movl %ecx,%edi /* C argument */
434 decl TD_CRITCOUNT(%rbx)
435 incl TD_NEST_COUNT(%rbx) /* prevent doreti/splz nesting */
436 call sched_ithd /* YYY must pull in imasks */
437 incl TD_CRITCOUNT(%rbx)
438 decl TD_NEST_COUNT(%rbx) /* prevent doreti/splz nesting */
444 andl $~RQF_IPIQ,PCPU(reqflags)
446 call lwkt_process_ipiq
451 andl $~RQF_TIMER,PCPU(reqflags)
453 call lapic_timer_process
459 * dofastunpend(%ecx:intr)
461 * A FAST interrupt previously made pending can now be run,
462 * execute it by pushing a dummy interrupt frame and
463 * calling ithread_fast_handler to execute or schedule it.
465 * ithread_fast_handler() returns 0 if it wants us to unmask
466 * further interrupts.
469 pushfq ; /* phys int frame / flags */ \
471 pushq %rax ; /* phys int frame / cs */ \
472 pushq 3*8(%rsp) ; /* original caller eip */ \
473 subq $TF_RIP,%rsp ; /* trap frame */ \
474 movq $0,TF_XFLAGS(%rsp) ; /* extras */ \
475 movq $0,TF_TRAPNO(%rsp) ; /* extras */ \
476 movq $0,TF_ADDR(%rsp) ; /* extras */ \
477 movq $0,TF_FLAGS(%rsp) ; /* extras */ \
478 movq $0,TF_ERR(%rsp) ; /* extras */ \
481 addq $TF_RIP+(3*8),%rsp ; \
484 pushq %rbp /* frame for backtrace */
487 pushq %rcx /* last part of intrframe = intr */
488 incl fastunpend_count
489 movq %rsp,%rdi /* pass frame by reference C arg */
490 call ithread_fast_handler /* returns 0 to unmask */
491 popq %rdi /* intrframe->trapframe */
492 /* + also rdi C arg to next call */
495 movq MachIntrABI + MACHINTR_INTREN, %rax
496 callq *%rax /* MachIntrABI.intren(intr) */