2 * from: vector.s, 386BSD 0.1 unknown origin
3 * $FreeBSD: src/sys/i386/isa/apic_vector.s,v 1.47.2.5 2001/09/01 22:33:38 tegge Exp $
7 #include "opt_auto_eoi.h"
10 #include <machine/asmacros.h>
11 #include <machine/lock.h>
12 #include <machine/psl.h>
13 #include <machine/trap.h>
14 #include <machine/segments.h>
16 #include <machine_base/icu/icu.h>
17 #include <bus/isa/isa.h>
23 #include <machine/smp.h>
24 #include <machine/intr_machdep.h>
27 /* convert an absolute IRQ# into bitmask */
28 #define IRQ_LBIT(irq_num) (1UL << (irq_num & 0x3f))
31 #define IRQ_SBITS(irq_num) ((irq_num) & 0x3f)
33 /* convert an absolute IRQ# into gd_ipending index */
34 #define IRQ_LIDX(irq_num) ((irq_num) >> 6)
37 #define MPLOCKED lock ;
42 #define APIC_PUSH_FRAME \
43 PUSH_FRAME ; /* 15 regs + space for 5 extras */ \
44 movq $0,TF_XFLAGS(%rsp) ; \
45 movq $0,TF_TRAPNO(%rsp) ; \
46 movq $0,TF_ADDR(%rsp) ; \
47 movq $0,TF_FLAGS(%rsp) ; \
48 movq $0,TF_ERR(%rsp) ; \
52 * JG stale? Warning: POP_FRAME can only be used if there is no chance of a
53 * segment register being changed (e.g. by procfs), which is why syscalls
56 #define APIC_POP_FRAME POP_FRAME
58 #define IOAPICADDR(irq_num) \
59 CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_ADDR
60 #define REDIRIDX(irq_num) \
61 CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_ENTIDX
62 #define IOAPICFLAGS(irq_num) \
63 CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_FLAGS
65 #define MASK_IRQ(irq_num) \
66 APIC_IMASK_LOCK ; /* into critical reg */ \
67 testl $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
68 jne 7f ; /* masked, don't mask */ \
69 orl $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
70 /* set the mask bit */ \
71 movq IOAPICADDR(irq_num), %rcx ; /* ioapic addr */ \
72 movl REDIRIDX(irq_num), %eax ; /* get the index */ \
73 movl %eax, (%rcx) ; /* write the index */ \
74 orl $IOART_INTMASK,IOAPIC_WINDOW(%rcx) ;/* set the mask */ \
75 7: ; /* already masked */ \
79 * Test to see whether we are handling an edge or level triggered INT.
80 * Level-triggered INTs must still be masked as we don't clear the source,
81 * and the EOI cycle would cause redundant INTs to occur.
83 #define MASK_LEVEL_IRQ(irq_num) \
84 testl $IOAPIC_IM_FLAG_LEVEL, IOAPICFLAGS(irq_num) ; \
85 jz 9f ; /* edge, don't mask */ \
90 * Test to see if the source is currntly masked, clear if so.
92 #define UNMASK_IRQ(irq_num) \
95 APIC_IMASK_LOCK ; /* into critical reg */ \
96 testl $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
97 je 7f ; /* bit clear, not masked */ \
98 andl $~IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
99 /* clear mask bit */ \
100 movq IOAPICADDR(irq_num),%rcx ; /* ioapic addr */ \
101 movl REDIRIDX(irq_num), %eax ; /* get the index */ \
102 movl %eax,(%rcx) ; /* write the index */ \
103 andl $~IOART_INTMASK,IOAPIC_WINDOW(%rcx) ;/* clear the mask */ \
105 APIC_IMASK_UNLOCK ; \
108 #ifdef SMP /* APIC-IO */
111 * Interrupt call handlers run in the following sequence:
113 * - Push the trap frame required by doreti
114 * - Mask the interrupt and reenable its source
115 * - If we cannot take the interrupt set its ipending bit and
117 * - If we can take the interrupt clear its ipending bit,
118 * call the handler, then unmask and doreti.
120 * YYY can cache gd base opitner instead of using hidden %fs prefixes.
123 #define INTR_HANDLER(irq_num) \
126 IDTVEC(apic_intr##irq_num) ; \
128 FAKE_MCOUNT(TF_RIP(%rsp)) ; \
129 MASK_LEVEL_IRQ(irq_num) ; \
131 movl $0, LA_EOI(%rax) ; \
132 movq PCPU(curthread),%rbx ; \
133 testl $-1,TD_NEST_COUNT(%rbx) ; \
135 testl $-1,TD_CRITCOUNT(%rbx) ; \
138 /* in critical section, make interrupt pending */ \
139 /* set the pending bit and return, leave interrupt masked */ \
141 shlq $IRQ_SBITS(irq_num),%rcx ; \
142 movl $IRQ_LIDX(irq_num),%edx ; \
143 orq %rcx,PCPU_E8(ipending,%edx) ; \
144 orl $RQF_INTPEND,PCPU(reqflags) ; \
147 /* clear pending bit, run handler */ \
149 shlq $IRQ_SBITS(irq_num),%rcx ; \
151 movl $IRQ_LIDX(irq_num),%edx ; \
152 andq %rcx,PCPU_E8(ipending,%edx) ; \
153 pushq $irq_num ; /* trapframe -> intrframe */ \
154 movq %rsp, %rdi ; /* pass frame by reference */ \
155 incl TD_CRITCOUNT(%rbx) ; \
157 call ithread_fast_handler ; /* returns 0 to unmask */ \
158 decl TD_CRITCOUNT(%rbx) ; \
159 addq $8, %rsp ; /* intrframe -> trapframe */ \
160 UNMASK_IRQ(irq_num) ; \
168 * Handle "spurious INTerrupts".
170 * This is different than the "spurious INTerrupt" generated by an
171 * 8259 PIC for missing INTs. See the APIC documentation for details.
172 * This routine should NOT do an 'EOI' cycle.
179 /* No EOI cycle used here */
185 * Handle TLB shootdowns.
187 * NOTE: interrupts are left disabled.
195 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
196 FAKE_MCOUNT(TF_RIP(%rsp))
197 subq $8,%rsp /* make same as interrupt frame */
198 movq %rsp,%rdi /* pass frame by reference */
199 call smp_invltlb_intr
200 addq $8,%rsp /* turn into trapframe */
206 * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
208 * - We cannot call doreti
209 * - Signals its receipt.
210 * - Waits for permission to restart.
211 * - Processing pending IPIQ events while waiting.
212 * - Signals its restart.
221 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
223 movl PCPU(cpuid), %eax
224 imull $PCB_SIZE, %eax
225 leaq CNAME(stoppcbs), %rdi
227 call CNAME(savectx) /* Save process context */
229 movslq PCPU(cpuid), %rax
232 * Indicate that we have stopped and loop waiting for permission
233 * to start again. We must still process IPI events while in a
236 * Interrupts must remain enabled for non-IPI'd per-cpu interrupts
237 * (e.g. Xtimer, Xinvltlb).
240 btsq %rax, stopped_cpus /* stopped_cpus |= (1<<id) */
243 andl $~RQF_IPIQ,PCPU(reqflags)
245 call lwkt_smp_stopped
248 btq %rax, started_cpus /* while (!(started_cpus & (1<<id))) */
252 btrq %rax, started_cpus /* started_cpus &= ~(1<<id) */
254 btrq %rax, stopped_cpus /* stopped_cpus &= ~(1<<id) */
259 movq CNAME(cpustop_restartfunc), %rax
262 movq $0, CNAME(cpustop_restartfunc) /* One-shot */
271 * For now just have one ipiq IPI, but what we really want is
272 * to have one for each source cpu to the APICs don't get stalled
273 * backlogging the requests.
281 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
282 FAKE_MCOUNT(TF_RIP(%rsp))
284 incl PCPU(cnt) + V_IPI
285 movq PCPU(curthread),%rbx
286 testl $-1,TD_CRITCOUNT(%rbx)
288 subq $8,%rsp /* make same as interrupt frame */
289 movq %rsp,%rdi /* pass frame by reference */
290 incl PCPU(intr_nesting_level)
291 incl TD_CRITCOUNT(%rbx)
293 call lwkt_process_ipiq_frame
294 decl TD_CRITCOUNT(%rbx)
295 decl PCPU(intr_nesting_level)
296 addq $8,%rsp /* turn into trapframe */
300 orl $RQF_IPIQ,PCPU(reqflags)
311 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
312 FAKE_MCOUNT(TF_RIP(%rsp))
314 subq $8,%rsp /* make same as interrupt frame */
315 movq %rsp,%rdi /* pass frame by reference */
316 call lapic_timer_always
317 addq $8,%rsp /* turn into trapframe */
319 incl PCPU(cnt) + V_TIMER
320 movq PCPU(curthread),%rbx
321 testl $-1,TD_CRITCOUNT(%rbx)
323 testl $-1,TD_NEST_COUNT(%rbx)
325 subq $8,%rsp /* make same as interrupt frame */
326 movq %rsp,%rdi /* pass frame by reference */
327 incl PCPU(intr_nesting_level)
328 incl TD_CRITCOUNT(%rbx)
330 call lapic_timer_process_frame
331 decl TD_CRITCOUNT(%rbx)
332 decl PCPU(intr_nesting_level)
333 addq $8,%rsp /* turn into trapframe */
337 orl $RQF_TIMER,PCPU(reqflags)
342 #ifdef SMP /* APIC-IO */
543 /* variables used by stop_cpus()/restart_cpus()/Xcpustop */
544 .globl stopped_cpus, started_cpus
550 .globl CNAME(cpustop_restartfunc)
551 CNAME(cpustop_restartfunc):