2 * from: vector.s, 386BSD 0.1 unknown origin
3 * $FreeBSD: src/sys/i386/isa/apic_vector.s,v 1.47.2.5 2001/09/01 22:33:38 tegge Exp $
4 * $DragonFly: src/sys/platform/pc32/apic/apic_vector.s,v 1.34 2006/11/07 18:50:06 dillon Exp $
8 #include "opt_auto_eoi.h"
10 #include <machine/asmacros.h>
11 #include <machine/lock.h>
12 #include <machine/psl.h>
13 #include <machine/trap.h>
15 #include <machine_base/icu/icu.h>
16 #include <bus/isa/i386/isa.h>
22 #include <machine/smp.h>
23 #include <machine_base/isa/intr_machdep.h>
25 /* convert an absolute IRQ# into a bitmask */
26 #define IRQ_LBIT(irq_num) (1 << (irq_num))
28 /* make an index into the IO APIC from the IRQ# */
29 #define REDTBL_IDX(irq_num) (0x10 + ((irq_num) * 2))
32 #define MPLOCKED lock ;
38 * Push an interrupt frame in a format acceptable to doreti, reload
39 * the segment registers for the kernel.
42 pushl $0 ; /* dummy error code */ \
43 pushl $0 ; /* dummy trap type */ \
45 pushl %ds ; /* save data and extra segments ... */ \
55 pushfl ; /* phys int frame / flags */ \
56 pushl %cs ; /* phys int frame / cs */ \
57 pushl 12(%esp) ; /* original caller eip */ \
58 pushl $0 ; /* dummy error code */ \
59 pushl $0 ; /* dummy trap type */ \
60 subl $12*4,%esp ; /* pushal + 3 seg regs (dummy) + CPL */ \
63 * Warning: POP_FRAME can only be used if there is no chance of a
64 * segment register being changed (e.g. by procfs), which is why syscalls
72 addl $2*4,%esp ; /* dummy trap & error codes */ \
77 #define IOAPICADDR(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 8
78 #define REDIRIDX(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 12
80 #define MASK_IRQ(irq_num) \
81 APIC_IMASK_LOCK ; /* into critical reg */ \
82 testl $IRQ_LBIT(irq_num), apic_imen ; \
83 jne 7f ; /* masked, don't mask */ \
84 orl $IRQ_LBIT(irq_num), apic_imen ; /* set the mask bit */ \
85 movl IOAPICADDR(irq_num), %ecx ; /* ioapic addr */ \
86 movl REDIRIDX(irq_num), %eax ; /* get the index */ \
87 movl %eax, (%ecx) ; /* write the index */ \
88 movl IOAPIC_WINDOW(%ecx), %eax ; /* current value */ \
89 orl $IOART_INTMASK, %eax ; /* set the mask */ \
90 movl %eax, IOAPIC_WINDOW(%ecx) ; /* new value */ \
91 7: ; /* already masked */ \
95 * Test to see whether we are handling an edge or level triggered INT.
96 * Level-triggered INTs must still be masked as we don't clear the source,
97 * and the EOI cycle would cause redundant INTs to occur.
99 #define MASK_LEVEL_IRQ(irq_num) \
100 testl $IRQ_LBIT(irq_num), apic_pin_trigger ; \
101 jz 9f ; /* edge, don't mask */ \
102 MASK_IRQ(irq_num) ; \
106 * Test to see if the source is currntly masked, clear if so.
108 #define UNMASK_IRQ(irq_num) \
111 APIC_IMASK_LOCK ; /* into critical reg */ \
112 testl $IRQ_LBIT(irq_num), apic_imen ; \
113 je 7f ; /* bit clear, not masked */ \
114 andl $~IRQ_LBIT(irq_num), apic_imen ;/* clear mask bit */ \
115 movl IOAPICADDR(irq_num),%ecx ; /* ioapic addr */ \
116 movl REDIRIDX(irq_num), %eax ; /* get the index */ \
117 movl %eax,(%ecx) ; /* write the index */ \
118 movl IOAPIC_WINDOW(%ecx),%eax ; /* current value */ \
119 andl $~IOART_INTMASK,%eax ; /* clear the mask */ \
120 movl %eax,IOAPIC_WINDOW(%ecx) ; /* new value */ \
122 APIC_IMASK_UNLOCK ; \
128 * Fast interrupt call handlers run in the following sequence:
130 * - Push the trap frame required by doreti
131 * - Mask the interrupt and reenable its source
132 * - If we cannot take the interrupt set its fpending bit and
133 * doreti. Note that we cannot mess with mp_lock at all
134 * if we entered from a critical section!
135 * - If we can take the interrupt clear its fpending bit,
136 * call the handler, then unmask and doreti.
138 * YYY can cache gd base opitner instead of using hidden %fs prefixes.
141 #define FAST_INTR(irq_num, vec_name) \
146 FAKE_MCOUNT(13*4(%esp)) ; \
147 MASK_LEVEL_IRQ(irq_num) ; \
148 movl $0, lapic_eoi ; \
149 movl PCPU(curthread),%ebx ; \
150 movl $0,%eax ; /* CURRENT CPL IN FRAME (REMOVED) */ \
152 cmpl $TDPRI_CRIT,TD_PRI(%ebx) ; \
155 /* in critical section, make interrupt pending */ \
156 /* set the pending bit and return, leave interrupt masked */ \
157 orl $IRQ_LBIT(irq_num),PCPU(fpending) ; \
158 orl $RQF_INTPEND,PCPU(reqflags) ; \
161 /* clear pending bit, run handler */ \
162 andl $~IRQ_LBIT(irq_num),PCPU(fpending) ; \
164 call ithread_fast_handler ; /* returns 0 to unmask */ \
166 UNMASK_IRQ(irq_num) ; \
172 * Slow interrupt call handlers run in the following sequence:
174 * - Push the trap frame required by doreti.
175 * - Mask the interrupt and reenable its source.
176 * - If we cannot take the interrupt set its ipending bit and
177 * doreti. In addition to checking for a critical section
178 * and cpl mask we also check to see if the thread is still
179 * running. Note that we cannot mess with mp_lock at all
180 * if we entered from a critical section!
181 * - If we can take the interrupt clear its ipending bit
182 * and schedule the thread. Leave interrupts masked and doreti.
184 * Note that calls to sched_ithd() are made with interrupts enabled
185 * and outside a critical section. YYY sched_ithd may preempt us
186 * synchronously (fix interrupt stacking).
188 * YYY can cache gd base pointer instead of using hidden %fs
192 #define SLOW_INTR(irq_num, vec_name, maybe_extra_ipending) \
197 maybe_extra_ipending ; \
199 MASK_LEVEL_IRQ(irq_num) ; \
200 incl PCPU(cnt) + V_INTR ; \
201 movl $0, lapic_eoi ; \
202 movl PCPU(curthread),%ebx ; \
203 movl $0,%eax ; /* CURRENT CPL IN FRAME (REMOVED) */ \
204 pushl %eax ; /* cpl do restore */ \
205 cmpl $TDPRI_CRIT,TD_PRI(%ebx) ; \
208 /* set the pending bit and return, leave the interrupt masked */ \
209 orl $IRQ_LBIT(irq_num), PCPU(ipending) ; \
210 orl $RQF_INTPEND,PCPU(reqflags) ; \
213 /* set running bit, clear pending bit, run handler */ \
214 andl $~IRQ_LBIT(irq_num), PCPU(ipending) ; \
224 * Wrong interrupt call handlers. We program these into APIC vectors
225 * that should otherwise never occur. For example, we program the SLOW
226 * vector for irq N with this when we program the FAST vector with the
229 * XXX for now all we can do is EOI it. We can't call do_wrongintr
230 * (yet) because we could be in a critical section.
232 #define WRONGINTR(irq_num,vec_name) \
237 movl $0, lapic_eoi ; /* End Of Interrupt to APIC */ \
238 /*pushl $irq_num ;*/ \
239 /*call do_wrongintr ;*/ \
247 * Handle "spurious INTerrupts".
249 * This is different than the "spurious INTerrupt" generated by an
250 * 8259 PIC for missing INTs. See the APIC documentation for details.
251 * This routine should NOT do an 'EOI' cycle.
258 /* No EOI cycle used here */
264 * Handle TLB shootdowns.
272 movl %cr3, %eax /* invalidate the TLB */
275 ss /* stack segment, avoid %ds load */
276 movl $0, lapic_eoi /* End Of Interrupt to APIC */
283 * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
285 * - Signals its receipt.
286 * - Waits for permission to restart.
287 * - Processing pending IPIQ events while waiting.
288 * - Signals its restart.
300 pushl %ds /* save current data segment */
304 mov %ax, %ds /* use KERNEL data segment */
308 movl $0, lapic_eoi /* End Of Interrupt to APIC */
310 movl PCPU(cpuid), %eax
311 imull $PCB_SIZE, %eax
312 leal CNAME(stoppcbs)(%eax), %eax
314 call CNAME(savectx) /* Save process context */
318 movl PCPU(cpuid), %eax
321 * Indicate that we have stopped and loop waiting for permission
322 * to start again. We must still process IPI events while in a
326 btsl %eax, stopped_cpus /* stopped_cpus |= (1<<id) */
328 andl $~RQF_IPIQ,PCPU(reqflags)
330 call lwkt_smp_stopped
332 btl %eax, started_cpus /* while (!(started_cpus & (1<<id))) */
336 btrl %eax, started_cpus /* started_cpus &= ~(1<<id) */
338 btrl %eax, stopped_cpus /* stopped_cpus &= ~(1<<id) */
343 movl CNAME(cpustop_restartfunc), %eax
346 movl $0, CNAME(cpustop_restartfunc) /* One-shot */
351 popl %ds /* restore previous data segment */
360 * For now just have one ipiq IPI, but what we really want is
361 * to have one for each source cpu to the APICs don't get stalled
362 * backlogging the requests.
369 movl $0, lapic_eoi /* End Of Interrupt to APIC */
370 FAKE_MCOUNT(13*4(%esp))
372 movl PCPU(curthread),%ebx
373 cmpl $TDPRI_CRIT,TD_PRI(%ebx)
375 subl $8,%esp /* make same as interrupt frame */
376 incl PCPU(intr_nesting_level)
377 addl $TDPRI_CRIT,TD_PRI(%ebx)
378 call lwkt_process_ipiq_frame
379 subl $TDPRI_CRIT,TD_PRI(%ebx)
380 decl PCPU(intr_nesting_level)
382 pushl $0 /* CPL for frame (REMOVED) */
386 orl $RQF_IPIQ,PCPU(reqflags)
394 FAST_INTR(0,apic_fastintr0)
395 FAST_INTR(1,apic_fastintr1)
396 FAST_INTR(2,apic_fastintr2)
397 FAST_INTR(3,apic_fastintr3)
398 FAST_INTR(4,apic_fastintr4)
399 FAST_INTR(5,apic_fastintr5)
400 FAST_INTR(6,apic_fastintr6)
401 FAST_INTR(7,apic_fastintr7)
402 FAST_INTR(8,apic_fastintr8)
403 FAST_INTR(9,apic_fastintr9)
404 FAST_INTR(10,apic_fastintr10)
405 FAST_INTR(11,apic_fastintr11)
406 FAST_INTR(12,apic_fastintr12)
407 FAST_INTR(13,apic_fastintr13)
408 FAST_INTR(14,apic_fastintr14)
409 FAST_INTR(15,apic_fastintr15)
410 FAST_INTR(16,apic_fastintr16)
411 FAST_INTR(17,apic_fastintr17)
412 FAST_INTR(18,apic_fastintr18)
413 FAST_INTR(19,apic_fastintr19)
414 FAST_INTR(20,apic_fastintr20)
415 FAST_INTR(21,apic_fastintr21)
416 FAST_INTR(22,apic_fastintr22)
417 FAST_INTR(23,apic_fastintr23)
419 /* YYY what is this garbage? */
421 SLOW_INTR(0,apic_slowintr0,)
422 SLOW_INTR(1,apic_slowintr1,)
423 SLOW_INTR(2,apic_slowintr2,)
424 SLOW_INTR(3,apic_slowintr3,)
425 SLOW_INTR(4,apic_slowintr4,)
426 SLOW_INTR(5,apic_slowintr5,)
427 SLOW_INTR(6,apic_slowintr6,)
428 SLOW_INTR(7,apic_slowintr7,)
429 SLOW_INTR(8,apic_slowintr8,)
430 SLOW_INTR(9,apic_slowintr9,)
431 SLOW_INTR(10,apic_slowintr10,)
432 SLOW_INTR(11,apic_slowintr11,)
433 SLOW_INTR(12,apic_slowintr12,)
434 SLOW_INTR(13,apic_slowintr13,)
435 SLOW_INTR(14,apic_slowintr14,)
436 SLOW_INTR(15,apic_slowintr15,)
437 SLOW_INTR(16,apic_slowintr16,)
438 SLOW_INTR(17,apic_slowintr17,)
439 SLOW_INTR(18,apic_slowintr18,)
440 SLOW_INTR(19,apic_slowintr19,)
441 SLOW_INTR(20,apic_slowintr20,)
442 SLOW_INTR(21,apic_slowintr21,)
443 SLOW_INTR(22,apic_slowintr22,)
444 SLOW_INTR(23,apic_slowintr23,)
446 WRONGINTR(0,apic_wrongintr0)
447 WRONGINTR(1,apic_wrongintr1)
448 WRONGINTR(2,apic_wrongintr2)
449 WRONGINTR(3,apic_wrongintr3)
450 WRONGINTR(4,apic_wrongintr4)
451 WRONGINTR(5,apic_wrongintr5)
452 WRONGINTR(6,apic_wrongintr6)
453 WRONGINTR(7,apic_wrongintr7)
454 WRONGINTR(8,apic_wrongintr8)
455 WRONGINTR(9,apic_wrongintr9)
456 WRONGINTR(10,apic_wrongintr10)
457 WRONGINTR(11,apic_wrongintr11)
458 WRONGINTR(12,apic_wrongintr12)
459 WRONGINTR(13,apic_wrongintr13)
460 WRONGINTR(14,apic_wrongintr14)
461 WRONGINTR(15,apic_wrongintr15)
462 WRONGINTR(16,apic_wrongintr16)
463 WRONGINTR(17,apic_wrongintr17)
464 WRONGINTR(18,apic_wrongintr18)
465 WRONGINTR(19,apic_wrongintr19)
466 WRONGINTR(20,apic_wrongintr20)
467 WRONGINTR(21,apic_wrongintr21)
468 WRONGINTR(22,apic_wrongintr22)
469 WRONGINTR(23,apic_wrongintr23)
476 /* variables used by stop_cpus()/restart_cpus()/Xcpustop */
477 .globl stopped_cpus, started_cpus
483 .globl CNAME(cpustop_restartfunc)
484 CNAME(cpustop_restartfunc):
487 .globl apic_pin_trigger