2 * from: vector.s, 386BSD 0.1 unknown origin
3 * $FreeBSD: src/sys/i386/isa/apic_vector.s,v 1.47.2.5 2001/09/01 22:33:38 tegge Exp $
6 #include "opt_auto_eoi.h"
8 #include <machine/asmacros.h>
9 #include <machine/lock.h>
10 #include <machine/psl.h>
11 #include <machine/trap.h>
13 #include <machine_base/icu/icu.h>
14 #include <bus/isa/isa.h>
19 #include <machine_base/apic/ioapic_ipl.h>
20 #include <machine/intr_machdep.h>
22 /* convert an absolute IRQ# into bitmask */
23 #define IRQ_LBIT(irq_num) (1 << ((irq_num) & 0x1f))
25 /* convert an absolute IRQ# into ipending index */
26 #define IRQ_LIDX(irq_num) ((irq_num) >> 5)
29 #define MPLOCKED lock ;
35 * Push an interrupt frame in a format acceptable to doreti, reload
36 * the segment registers for the kernel.
39 pushl $0 ; /* dummy error code */ \
40 pushl $0 ; /* dummy trap type */ \
41 pushl $0 ; /* dummy xflags type */ \
43 pushl %ds ; /* save data and extra segments ... */ \
56 * Warning: POP_FRAME can only be used if there is no chance of a
57 * segment register being changed (e.g. by procfs), which is why syscalls
66 addl $3*4,%esp ; /* dummy xflags, trap & error codes */ \
68 #define IOAPICADDR(irq_num) \
69 CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_ADDR
70 #define REDIRIDX(irq_num) \
71 CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_ENTIDX
72 #define IOAPICFLAGS(irq_num) \
73 CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_FLAGS
75 #define MASK_IRQ(irq_num) \
76 IOAPIC_IMASK_LOCK ; /* into critical reg */ \
77 testl $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
78 jne 7f ; /* masked, don't mask */ \
79 orl $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
80 /* set the mask bit */ \
81 movl IOAPICADDR(irq_num), %ecx ; /* ioapic addr */ \
82 movl REDIRIDX(irq_num), %eax ; /* get the index */ \
83 movl %eax, (%ecx) ; /* write the index */ \
84 orl $IOART_INTMASK,IOAPIC_WINDOW(%ecx) ;/* set the mask */ \
85 7: ; /* already masked */ \
86 IOAPIC_IMASK_UNLOCK ; \
89 * Test to see whether we are handling an edge or level triggered INT.
90 * Level-triggered INTs must still be masked as we don't clear the source,
91 * and the EOI cycle would cause redundant INTs to occur.
93 #define MASK_LEVEL_IRQ(irq_num) \
94 testl $IOAPIC_IM_FLAG_LEVEL, IOAPICFLAGS(irq_num) ; \
95 jz 9f ; /* edge, don't mask */ \
100 * Test to see if the source is currntly masked, clear if so.
102 #define UNMASK_IRQ(irq_num) \
105 IOAPIC_IMASK_LOCK ; /* into critical reg */ \
106 testl $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
107 je 7f ; /* bit clear, not masked */ \
108 andl $~IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
109 /* clear mask bit */ \
110 movl IOAPICADDR(irq_num),%ecx ; /* ioapic addr */ \
111 movl REDIRIDX(irq_num), %eax ; /* get the index */ \
112 movl %eax,(%ecx) ; /* write the index */ \
113 andl $~IOART_INTMASK,IOAPIC_WINDOW(%ecx) ;/* clear the mask */ \
115 IOAPIC_IMASK_UNLOCK ; \
118 #ifdef SMP /* APIC-IO */
121 * Interrupt call handlers run in the following sequence:
123 * - Push the trap frame required by doreti
124 * - Mask the interrupt and reenable its source
125 * - If we cannot take the interrupt set its ipending bit and
127 * - If we can take the interrupt clear its ipending bit,
128 * call the handler, then unmask and doreti.
130 * YYY can cache gd base opitner instead of using hidden %fs prefixes.
133 #define INTR_HANDLER(irq_num) \
136 IDTVEC(ioapic_intr##irq_num) ; \
138 FAKE_MCOUNT(15*4(%esp)) ; \
139 MASK_LEVEL_IRQ(irq_num) ; \
141 movl $0,LA_EOI(%eax) ; \
142 movl PCPU(curthread),%ebx ; \
143 movl $0,%eax ; /* CURRENT CPL IN FRAME (REMOVED) */ \
145 testl $-1,TD_NEST_COUNT(%ebx) ; \
147 testl $-1,TD_CRITCOUNT(%ebx) ; \
150 /* in critical section, make interrupt pending */ \
151 /* set the pending bit and return, leave interrupt masked */ \
152 movl $IRQ_LIDX(irq_num),%edx ; \
153 orl $IRQ_LBIT(irq_num),PCPU_E4(ipending,%edx) ; \
154 orl $RQF_INTPEND,PCPU(reqflags) ; \
157 /* clear pending bit, run handler */ \
158 movl $IRQ_LIDX(irq_num),%edx ; \
159 andl $~IRQ_LBIT(irq_num),PCPU_E4(ipending,%edx) ; \
161 pushl %esp ; /* pass frame by reference */ \
162 incl TD_CRITCOUNT(%ebx) ; \
164 call ithread_fast_handler ; /* returns 0 to unmask */ \
165 decl TD_CRITCOUNT(%ebx) ; \
167 UNMASK_IRQ(irq_num) ; \
175 * Handle "spurious INTerrupts".
177 * This is different than the "spurious INTerrupt" generated by an
178 * 8259 PIC for missing INTs. See the APIC documentation for details.
179 * This routine should NOT do an 'EOI' cycle.
186 /* No EOI cycle used here */
192 * Handle TLB shootdowns.
194 * NOTE: Interrupts remain disabled.
202 movl $0,LA_EOI(%eax) /* End Of Interrupt to APIC */
203 FAKE_MCOUNT(15*4(%esp))
205 subl $8,%esp /* make same as interrupt frame */
206 pushl %esp /* pass frame by reference */
207 call smp_invltlb_intr
211 jmp doreti_syscall_ret
214 * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
216 * - Signals its receipt.
217 * - Waits for permission to restart.
218 * - Processing pending IPIQ events while waiting.
219 * - Signals its restart.
231 pushl %ds /* save current data segment */
235 mov %ax, %ds /* use KERNEL data segment */
240 movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */
242 movl PCPU(cpuid), %eax
243 imull $PCB_SIZE, %eax
244 leal CNAME(stoppcbs)(%eax), %eax
246 call CNAME(savectx) /* Save process context */
250 movl PCPU(cpuid), %eax
253 * Indicate that we have stopped and loop waiting for permission
254 * to start again. We must still process IPI events while in a
257 * Interrupts must remain enabled for non-IPI'd per-cpu interrupts
258 * (e.g. Xtimer, Xinvltlb).
261 btsl %eax, stopped_cpus /* stopped_cpus |= (1<<id) */
264 andl $~RQF_IPIQ,PCPU(reqflags)
266 call lwkt_smp_stopped
268 btl %eax, started_cpus /* while (!(started_cpus & (1<<id))) */
272 btrl %eax, started_cpus /* started_cpus &= ~(1<<id) */
274 btrl %eax, stopped_cpus /* stopped_cpus &= ~(1<<id) */
279 movl CNAME(cpustop_restartfunc), %eax
282 movl $0, CNAME(cpustop_restartfunc) /* One-shot */
287 popl %ds /* restore previous data segment */
296 * For now just have one ipiq IPI, but what we really want is
297 * to have one for each source cpu to the APICs don't get stalled
298 * backlogging the requests.
306 movl $0,LA_EOI(%eax) /* End Of Interrupt to APIC */
307 FAKE_MCOUNT(15*4(%esp))
309 incl PCPU(cnt) + V_IPI
310 movl PCPU(curthread),%ebx
311 testl $-1,TD_CRITCOUNT(%ebx)
313 subl $8,%esp /* make same as interrupt frame */
314 pushl %esp /* pass frame by reference */
315 incl PCPU(intr_nesting_level)
316 incl TD_CRITCOUNT(%ebx)
318 call lwkt_process_ipiq_frame
319 decl TD_CRITCOUNT(%ebx)
320 decl PCPU(intr_nesting_level)
322 pushl $0 /* CPL for frame (REMOVED) */
326 orl $RQF_IPIQ,PCPU(reqflags)
328 jmp doreti_syscall_ret
336 movl $0,LA_EOI(%eax) /* End Of Interrupt to APIC */
337 FAKE_MCOUNT(15*4(%esp))
339 incl PCPU(cnt) + V_TIMER
340 movl PCPU(curthread),%ebx
341 testl $-1,TD_CRITCOUNT(%ebx)
343 testl $-1,TD_NEST_COUNT(%ebx)
345 subl $8,%esp /* make same as interrupt frame */
346 pushl %esp /* pass frame by reference */
347 incl PCPU(intr_nesting_level)
348 incl TD_CRITCOUNT(%ebx)
350 call lapic_timer_process_frame
351 decl TD_CRITCOUNT(%ebx)
352 decl PCPU(intr_nesting_level)
354 pushl $0 /* CPL for frame (REMOVED) */
358 orl $RQF_TIMER,PCPU(reqflags)
360 jmp doreti_syscall_ret
362 #ifdef SMP /* APIC-IO */
563 /* variables used by stop_cpus()/restart_cpus()/Xcpustop */
564 .globl stopped_cpus, started_cpus
570 .globl CNAME(cpustop_restartfunc)
571 CNAME(cpustop_restartfunc):