2 * from: vector.s, 386BSD 0.1 unknown origin
3 * $FreeBSD: src/sys/i386/isa/apic_vector.s,v 1.47.2.5 2001/09/01 22:33:38 tegge Exp $
6 #include "opt_auto_eoi.h"
8 #include <machine/asmacros.h>
9 #include <machine/lock.h>
10 #include <machine/psl.h>
11 #include <machine/trap.h>
13 #include <machine_base/icu/icu.h>
14 #include <bus/isa/isa.h>
19 #include <machine_base/apic/ioapic_ipl.h>
20 #include <machine/intr_machdep.h>
22 /* convert an absolute IRQ# into bitmask */
23 #define IRQ_LBIT(irq_num) (1 << ((irq_num) & 0x1f))
25 /* convert an absolute IRQ# into ipending index */
26 #define IRQ_LIDX(irq_num) ((irq_num) >> 5)
28 #define MPLOCKED lock ;
31 * Push an interrupt frame in a format acceptable to doreti, reload
32 * the segment registers for the kernel.
35 pushl $0 ; /* dummy error code */ \
36 pushl $0 ; /* dummy trap type */ \
37 pushl $0 ; /* dummy xflags type */ \
39 pushl %ds ; /* save data and extra segments ... */ \
52 * Warning: POP_FRAME can only be used if there is no chance of a
53 * segment register being changed (e.g. by procfs), which is why syscalls
62 addl $3*4,%esp ; /* dummy xflags, trap & error codes */ \
64 #define IOAPICADDR(irq_num) \
65 CNAME(ioapic_irqs) + IOAPIC_IRQI_SIZE * (irq_num) + IOAPIC_IRQI_ADDR
66 #define REDIRIDX(irq_num) \
67 CNAME(ioapic_irqs) + IOAPIC_IRQI_SIZE * (irq_num) + IOAPIC_IRQI_IDX
68 #define IOAPICFLAGS(irq_num) \
69 CNAME(ioapic_irqs) + IOAPIC_IRQI_SIZE * (irq_num) + IOAPIC_IRQI_FLAGS
71 #define MASK_IRQ(irq_num) \
72 IOAPIC_IMASK_LOCK ; /* into critical reg */ \
73 testl $IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
74 jne 7f ; /* masked, don't mask */ \
75 orl $IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
76 /* set the mask bit */ \
77 movl IOAPICADDR(irq_num), %ecx ; /* ioapic addr */ \
78 movl REDIRIDX(irq_num), %eax ; /* get the index */ \
79 movl %eax, (%ecx) ; /* write the index */ \
80 orl $IOART_INTMASK,IOAPIC_WINDOW(%ecx) ;/* set the mask */ \
81 7: ; /* already masked */ \
82 IOAPIC_IMASK_UNLOCK ; \
85 * Test to see whether we are handling an edge or level triggered INT.
86 * Level-triggered INTs must still be masked as we don't clear the source,
87 * and the EOI cycle would cause redundant INTs to occur.
89 #define MASK_LEVEL_IRQ(irq_num) \
90 testl $IOAPIC_IRQI_FLAG_LEVEL, IOAPICFLAGS(irq_num) ; \
91 jz 9f ; /* edge, don't mask */ \
96 * Test to see if the source is currntly masked, clear if so.
98 #define UNMASK_IRQ(irq_num) \
101 IOAPIC_IMASK_LOCK ; /* into critical reg */ \
102 testl $IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
103 je 7f ; /* bit clear, not masked */ \
104 andl $~IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
105 /* clear mask bit */ \
106 movl IOAPICADDR(irq_num),%ecx ; /* ioapic addr */ \
107 movl REDIRIDX(irq_num), %eax ; /* get the index */ \
108 movl %eax,(%ecx) ; /* write the index */ \
109 andl $~IOART_INTMASK,IOAPIC_WINDOW(%ecx) ;/* clear the mask */ \
111 IOAPIC_IMASK_UNLOCK ; \
115 * Interrupt call handlers run in the following sequence:
117 * - Push the trap frame required by doreti
118 * - Mask the interrupt and reenable its source
119 * - If we cannot take the interrupt set its ipending bit and
121 * - If we can take the interrupt clear its ipending bit,
122 * call the handler, then unmask and doreti.
124 * YYY can cache gd base opitner instead of using hidden %fs prefixes.
127 #define INTR_HANDLER(irq_num) \
130 IDTVEC(ioapic_intr##irq_num) ; \
132 FAKE_MCOUNT(15*4(%esp)) ; \
133 MASK_LEVEL_IRQ(irq_num) ; \
135 movl $0,LA_EOI(%eax) ; \
136 movl PCPU(curthread),%ebx ; \
137 movl $0,%eax ; /* CURRENT CPL IN FRAME (REMOVED) */ \
139 testl $-1,TD_NEST_COUNT(%ebx) ; \
141 testl $-1,TD_CRITCOUNT(%ebx) ; \
144 /* in critical section, make interrupt pending */ \
145 /* set the pending bit and return, leave interrupt masked */ \
146 movl $IRQ_LIDX(irq_num),%edx ; \
147 orl $IRQ_LBIT(irq_num),PCPU_E4(ipending,%edx) ; \
148 orl $RQF_INTPEND,PCPU(reqflags) ; \
151 /* clear pending bit, run handler */ \
152 movl $IRQ_LIDX(irq_num),%edx ; \
153 andl $~IRQ_LBIT(irq_num),PCPU_E4(ipending,%edx) ; \
155 pushl %esp ; /* pass frame by reference */ \
156 incl TD_CRITCOUNT(%ebx) ; \
158 call ithread_fast_handler ; /* returns 0 to unmask */ \
159 decl TD_CRITCOUNT(%ebx) ; \
161 UNMASK_IRQ(irq_num) ; \
167 * Handle "spurious INTerrupts".
169 * This is different than the "spurious INTerrupt" generated by an
170 * 8259 PIC for missing INTs. See the APIC documentation for details.
171 * This routine should NOT do an 'EOI' cycle.
178 /* No EOI cycle used here */
183 * Handle TLB shootdowns.
185 * NOTE: Interrupts remain disabled.
193 movl $0,LA_EOI(%eax) /* End Of Interrupt to APIC */
194 FAKE_MCOUNT(15*4(%esp))
196 subl $8,%esp /* make same as interrupt frame */
197 pushl %esp /* pass frame by reference */
198 call smp_invltlb_intr
202 jmp doreti_syscall_ret
205 * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
207 * - Signals its receipt.
208 * - Waits for permission to restart.
209 * - Processing pending IPIQ events while waiting.
210 * - Signals its restart.
222 pushl %ds /* save current data segment */
226 mov %ax, %ds /* use KERNEL data segment */
231 movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */
233 movl PCPU(cpuid), %eax
234 imull $PCB_SIZE, %eax
235 leal CNAME(stoppcbs)(%eax), %eax
237 call CNAME(savectx) /* Save process context */
241 movl PCPU(cpuid), %eax
244 * Indicate that we have stopped and loop waiting for permission
245 * to start again. We must still process IPI events while in a
248 * Interrupts must remain enabled for non-IPI'd per-cpu interrupts
249 * (e.g. Xtimer, Xinvltlb).
252 btsl %eax, stopped_cpus /* stopped_cpus |= (1<<id) */
255 andl $~RQF_IPIQ,PCPU(reqflags)
257 call lwkt_smp_stopped
259 btl %eax, started_cpus /* while (!(started_cpus & (1<<id))) */
263 btrl %eax, started_cpus /* started_cpus &= ~(1<<id) */
265 btrl %eax, stopped_cpus /* stopped_cpus &= ~(1<<id) */
270 movl CNAME(cpustop_restartfunc), %eax
273 movl $0, CNAME(cpustop_restartfunc) /* One-shot */
278 popl %ds /* restore previous data segment */
287 * For now just have one ipiq IPI, but what we really want is
288 * to have one for each source cpu to the APICs don't get stalled
289 * backlogging the requests.
297 movl $0,LA_EOI(%eax) /* End Of Interrupt to APIC */
298 FAKE_MCOUNT(15*4(%esp))
300 incl PCPU(cnt) + V_IPI
301 movl PCPU(curthread),%ebx
302 testl $-1,TD_CRITCOUNT(%ebx)
304 subl $8,%esp /* make same as interrupt frame */
305 pushl %esp /* pass frame by reference */
306 incl PCPU(intr_nesting_level)
307 incl TD_CRITCOUNT(%ebx)
309 call lwkt_process_ipiq_frame
310 decl TD_CRITCOUNT(%ebx)
311 decl PCPU(intr_nesting_level)
313 pushl $0 /* CPL for frame (REMOVED) */
317 orl $RQF_IPIQ,PCPU(reqflags)
319 jmp doreti_syscall_ret
327 movl $0,LA_EOI(%eax) /* End Of Interrupt to APIC */
328 FAKE_MCOUNT(15*4(%esp))
330 incl PCPU(cnt) + V_TIMER
331 movl PCPU(curthread),%ebx
332 testl $-1,TD_CRITCOUNT(%ebx)
334 testl $-1,TD_NEST_COUNT(%ebx)
336 subl $8,%esp /* make same as interrupt frame */
337 pushl %esp /* pass frame by reference */
338 incl PCPU(intr_nesting_level)
339 incl TD_CRITCOUNT(%ebx)
341 call lapic_timer_process_frame
342 decl TD_CRITCOUNT(%ebx)
343 decl PCPU(intr_nesting_level)
345 pushl $0 /* CPL for frame (REMOVED) */
349 orl $RQF_TIMER,PCPU(reqflags)
351 jmp doreti_syscall_ret
550 /* variables used by stop_cpus()/restart_cpus()/Xcpustop */
551 .globl stopped_cpus, started_cpus
557 .globl CNAME(cpustop_restartfunc)
558 CNAME(cpustop_restartfunc):