2 * from: vector.s, 386BSD 0.1 unknown origin
3 * $FreeBSD: src/sys/i386/isa/apic_vector.s,v 1.47.2.5 2001/09/01 22:33:38 tegge Exp $
4 * $DragonFly: src/sys/platform/pc32/apic/apic_vector.s,v 1.39 2008/08/02 01:14:43 dillon Exp $
9 #include "opt_auto_eoi.h"
12 #include <machine/asmacros.h>
13 #include <machine/lock.h>
14 #include <machine/psl.h>
15 #include <machine/trap.h>
16 #include <machine/segments.h>
18 #include <machine_base/icu/icu.h>
19 #include <bus/isa/isa.h>
25 #include <machine/smp.h>
26 #include <machine_base/isa/intr_machdep.h>
28 /* convert an absolute IRQ# into a bitmask */
29 #define IRQ_LBIT(irq_num) (1 << (irq_num))
31 /* make an index into the IO APIC from the IRQ# */
32 #define REDTBL_IDX(irq_num) (0x10 + ((irq_num) * 2))
35 #define MPLOCKED lock ;
40 #define APIC_PUSH_FRAME \
41 PUSH_FRAME ; /* 15 regs + space for 5 extras */ \
42 movq $0,TF_XFLAGS(%rsp) ; \
43 movq $0,TF_TRAPNO(%rsp) ; \
44 movq $0,TF_ADDR(%rsp) ; \
45 movq $0,TF_FLAGS(%rsp) ; \
46 movq $0,TF_ERR(%rsp) ; \
50 * JG stale? Warning: POP_FRAME can only be used if there is no chance of a
51 * segment register being changed (e.g. by procfs), which is why syscalls
54 #define APIC_POP_FRAME POP_FRAME
56 #define IOAPICADDR(irq_num) \
57 CNAME(int_to_apicintpin) + AIMI_SIZE * (irq_num) + AIMI_APIC_ADDRESS
58 #define REDIRIDX(irq_num) \
59 CNAME(int_to_apicintpin) + AIMI_SIZE * (irq_num) + AIMI_REDIRINDEX
60 #define IOAPICFLAGS(irq_num) \
61 CNAME(int_to_apicintpin) + AIMI_SIZE * (irq_num) + AIMI_FLAGS
63 #define MASK_IRQ(irq_num) \
64 APIC_IMASK_LOCK ; /* into critical reg */ \
65 testl $AIMI_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
66 jne 7f ; /* masked, don't mask */ \
67 orl $AIMI_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
68 /* set the mask bit */ \
69 movq IOAPICADDR(irq_num), %rcx ; /* ioapic addr */ \
70 movl REDIRIDX(irq_num), %eax ; /* get the index */ \
71 movl %eax, (%rcx) ; /* write the index */ \
72 orl $IOART_INTMASK,IOAPIC_WINDOW(%rcx) ;/* set the mask */ \
73 7: ; /* already masked */ \
77 * Test to see whether we are handling an edge or level triggered INT.
78 * Level-triggered INTs must still be masked as we don't clear the source,
79 * and the EOI cycle would cause redundant INTs to occur.
81 #define MASK_LEVEL_IRQ(irq_num) \
82 testl $AIMI_FLAG_LEVEL, IOAPICFLAGS(irq_num) ; \
83 jz 9f ; /* edge, don't mask */ \
88 * Test to see if the source is currntly masked, clear if so.
90 #define UNMASK_IRQ(irq_num) \
93 APIC_IMASK_LOCK ; /* into critical reg */ \
94 testl $AIMI_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
95 je 7f ; /* bit clear, not masked */ \
96 andl $~AIMI_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
97 /* clear mask bit */ \
98 movq IOAPICADDR(irq_num),%rcx ; /* ioapic addr */ \
99 movl REDIRIDX(irq_num), %eax ; /* get the index */ \
100 movl %eax,(%rcx) ; /* write the index */ \
101 andl $~IOART_INTMASK,IOAPIC_WINDOW(%rcx) ;/* clear the mask */ \
103 APIC_IMASK_UNLOCK ; \
106 #ifdef SMP /* APIC-IO */
109 * Fast interrupt call handlers run in the following sequence:
111 * - Push the trap frame required by doreti
112 * - Mask the interrupt and reenable its source
113 * - If we cannot take the interrupt set its fpending bit and
114 * doreti. Note that we cannot mess with mp_lock at all
115 * if we entered from a critical section!
116 * - If we can take the interrupt clear its fpending bit,
117 * call the handler, then unmask and doreti.
119 * YYY can cache gd base opitner instead of using hidden %fs prefixes.
122 #define FAST_INTR(irq_num, vec_name) \
127 FAKE_MCOUNT(15*4(%esp)) ; \
128 MASK_LEVEL_IRQ(irq_num) ; \
130 movl $0, LA_EOI(%rax) ; \
131 movq PCPU(curthread),%rbx ; \
132 testl $-1,TD_NEST_COUNT(%rbx) ; \
134 testl $-1,TD_CRITCOUNT(%rbx) ; \
137 /* in critical section, make interrupt pending */ \
138 /* set the pending bit and return, leave interrupt masked */ \
139 orl $IRQ_LBIT(irq_num),PCPU(fpending) ; \
140 orl $RQF_INTPEND,PCPU(reqflags) ; \
143 /* clear pending bit, run handler */ \
144 andl $~IRQ_LBIT(irq_num),PCPU(fpending) ; \
145 pushq $irq_num ; /* trapframe -> intrframe */ \
146 movq %rsp, %rdi ; /* pass frame by reference */ \
147 incl TD_CRITCOUNT(%rbx) ; \
148 call ithread_fast_handler ; /* returns 0 to unmask */ \
149 decl TD_CRITCOUNT(%rbx) ; \
150 addq $8, %rsp ; /* intrframe -> trapframe */ \
151 UNMASK_IRQ(irq_num) ; \
159 * Handle "spurious INTerrupts".
161 * This is different than the "spurious INTerrupt" generated by an
162 * 8259 PIC for missing INTs. See the APIC documentation for details.
163 * This routine should NOT do an 'EOI' cycle.
170 /* No EOI cycle used here */
176 * Handle TLB shootdowns.
184 movq %cr3, %rax /* invalidate the TLB */
188 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
195 * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
197 * - Signals its receipt.
198 * - Waits for permission to restart.
199 * - Processing pending IPIQ events while waiting.
200 * - Signals its restart.
209 /* We save registers that are not preserved across function calls. */
210 /* JG can be re-written with mov's */
222 /* JGXXX switch to kernel %gs? */
223 pushl %ds /* save current data segment */
227 mov %ax, %ds /* use KERNEL data segment */
233 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
236 movl PCPU(cpuid), %eax
237 imull $PCB_SIZE, %eax
238 leaq CNAME(stoppcbs), %rdi
240 call CNAME(savectx) /* Save process context */
243 movl PCPU(cpuid), %eax
246 * Indicate that we have stopped and loop waiting for permission
247 * to start again. We must still process IPI events while in a
251 btsl %eax, stopped_cpus /* stopped_cpus |= (1<<id) */
253 andl $~RQF_IPIQ,PCPU(reqflags)
255 call lwkt_smp_stopped
257 btl %eax, started_cpus /* while (!(started_cpus & (1<<id))) */
261 btrl %eax, started_cpus /* started_cpus &= ~(1<<id) */
263 btrl %eax, stopped_cpus /* stopped_cpus &= ~(1<<id) */
268 movq CNAME(cpustop_restartfunc), %rax
271 movq $0, CNAME(cpustop_restartfunc) /* One-shot */
287 popl %ds /* restore previous data segment */
294 * For now just have one ipiq IPI, but what we really want is
295 * to have one for each source cpu to the APICs don't get stalled
296 * backlogging the requests.
304 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
305 FAKE_MCOUNT(15*4(%esp))
307 incl PCPU(cnt) + V_IPI
308 movq PCPU(curthread),%rbx
309 testl $-1,TD_CRITCOUNT(%rbx)
311 subq $8,%rsp /* make same as interrupt frame */
312 movq %rsp,%rdi /* pass frame by reference */
313 incl PCPU(intr_nesting_level)
314 incl TD_CRITCOUNT(%rbx)
315 call lwkt_process_ipiq_frame
316 decl TD_CRITCOUNT(%rbx)
317 decl PCPU(intr_nesting_level)
318 addq $8,%rsp /* turn into trapframe */
322 orl $RQF_IPIQ,PCPU(reqflags)
333 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
334 FAKE_MCOUNT(15*4(%esp))
336 incl PCPU(cnt) + V_TIMER
337 movq PCPU(curthread),%rbx
338 testl $-1,TD_CRITCOUNT(%rbx)
340 testl $-1,TD_NEST_COUNT(%rbx)
342 subq $8,%rsp /* make same as interrupt frame */
343 movq %rsp,%rdi /* pass frame by reference */
344 incl PCPU(intr_nesting_level)
345 incl TD_CRITCOUNT(%rbx)
346 call lapic_timer_process_frame
347 decl TD_CRITCOUNT(%rbx)
348 decl PCPU(intr_nesting_level)
349 addq $8,%rsp /* turn into trapframe */
353 orl $RQF_TIMER,PCPU(reqflags)
358 #ifdef SMP /* APIC-IO */
361 FAST_INTR(0,apic_fastintr0)
362 FAST_INTR(1,apic_fastintr1)
363 FAST_INTR(2,apic_fastintr2)
364 FAST_INTR(3,apic_fastintr3)
365 FAST_INTR(4,apic_fastintr4)
366 FAST_INTR(5,apic_fastintr5)
367 FAST_INTR(6,apic_fastintr6)
368 FAST_INTR(7,apic_fastintr7)
369 FAST_INTR(8,apic_fastintr8)
370 FAST_INTR(9,apic_fastintr9)
371 FAST_INTR(10,apic_fastintr10)
372 FAST_INTR(11,apic_fastintr11)
373 FAST_INTR(12,apic_fastintr12)
374 FAST_INTR(13,apic_fastintr13)
375 FAST_INTR(14,apic_fastintr14)
376 FAST_INTR(15,apic_fastintr15)
377 FAST_INTR(16,apic_fastintr16)
378 FAST_INTR(17,apic_fastintr17)
379 FAST_INTR(18,apic_fastintr18)
380 FAST_INTR(19,apic_fastintr19)
381 FAST_INTR(20,apic_fastintr20)
382 FAST_INTR(21,apic_fastintr21)
383 FAST_INTR(22,apic_fastintr22)
384 FAST_INTR(23,apic_fastintr23)
391 /* variables used by stop_cpus()/restart_cpus()/Xcpustop */
392 .globl stopped_cpus, started_cpus
398 .globl CNAME(cpustop_restartfunc)
399 CNAME(cpustop_restartfunc):