i386 - Get completely rid of APIC_IO
[dragonfly.git] / sys / platform / pc32 / apic / apic_vector.s
CommitLineData
984263bc
MD
1/*
2 * from: vector.s, 386BSD 0.1 unknown origin
3 * $FreeBSD: src/sys/i386/isa/apic_vector.s,v 1.47.2.5 2001/09/01 22:33:38 tegge Exp $
21ce0dfa 4 * $DragonFly: src/sys/platform/pc32/apic/apic_vector.s,v 1.39 2008/08/02 01:14:43 dillon Exp $
984263bc
MD
5 */
6
06f5be02
MD
7#include "use_npx.h"
8#include "opt_auto_eoi.h"
9
10#include <machine/asmacros.h>
06f5be02
MD
11#include <machine/lock.h>
12#include <machine/psl.h>
13#include <machine/trap.h>
06f5be02 14
a9295349 15#include <machine_base/icu/icu.h>
21ce0dfa 16#include <bus/isa/isa.h>
06f5be02
MD
17
18#include "assym.s"
984263bc 19
bdc560a1
MD
20#include "apicreg.h"
21#include "apic_ipl.h"
984263bc 22#include <machine/smp.h>
a9295349 23#include <machine_base/isa/intr_machdep.h>
984263bc
MD
24
25/* convert an absolute IRQ# into a bitmask */
8a8d5d85 26#define IRQ_LBIT(irq_num) (1 << (irq_num))
984263bc
MD
27
28/* make an index into the IO APIC from the IRQ# */
29#define REDTBL_IDX(irq_num) (0x10 + ((irq_num) * 2))
30
97359a5b
MD
31#ifdef SMP
32#define MPLOCKED lock ;
33#else
34#define MPLOCKED
35#endif
36
984263bc 37/*
8a8d5d85
MD
38 * Push an interrupt frame in a format acceptable to doreti, reload
39 * the segment registers for the kernel.
984263bc
MD
40 */
41#define PUSH_FRAME \
42 pushl $0 ; /* dummy error code */ \
43 pushl $0 ; /* dummy trap type */ \
4e7c41c5 44 pushl $0 ; /* dummy xflags type */ \
984263bc
MD
45 pushal ; \
46 pushl %ds ; /* save data and extra segments ... */ \
47 pushl %es ; \
8a8d5d85 48 pushl %fs ; \
4e7c41c5 49 pushl %gs ; \
c885c20e 50 cld ; \
8a8d5d85
MD
51 mov $KDSEL,%ax ; \
52 mov %ax,%ds ; \
53 mov %ax,%es ; \
4e7c41c5 54 mov %ax,%gs ; \
8a8d5d85
MD
55 mov $KPSEL,%ax ; \
56 mov %ax,%fs ; \
984263bc 57
8a8d5d85
MD
58/*
59 * Warning: POP_FRAME can only be used if there is no chance of a
60 * segment register being changed (e.g. by procfs), which is why syscalls
61 * have to use doreti.
62 */
984263bc 63#define POP_FRAME \
4e7c41c5 64 popl %gs ; \
984263bc
MD
65 popl %fs ; \
66 popl %es ; \
67 popl %ds ; \
68 popal ; \
4e7c41c5 69 addl $3*4,%esp ; /* dummy xflags, trap & error codes */ \
8a8d5d85 70
3d911e0a
SZ
71#define IOAPICADDR(irq_num) \
72 CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_ADDR
73#define REDIRIDX(irq_num) \
74 CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_ENTIDX
0f546930
SZ
75#define IOAPICFLAGS(irq_num) \
76 CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_FLAGS
8a8d5d85 77
984263bc 78#define MASK_IRQ(irq_num) \
97359a5b 79 APIC_IMASK_LOCK ; /* into critical reg */ \
ea689d1c 80 testl $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
984263bc 81 jne 7f ; /* masked, don't mask */ \
ea689d1c
SZ
82 orl $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
83 /* set the mask bit */ \
984263bc
MD
84 movl IOAPICADDR(irq_num), %ecx ; /* ioapic addr */ \
85 movl REDIRIDX(irq_num), %eax ; /* get the index */ \
86 movl %eax, (%ecx) ; /* write the index */ \
bda48b43 87 orl $IOART_INTMASK,IOAPIC_WINDOW(%ecx) ;/* set the mask */ \
984263bc 887: ; /* already masked */ \
97359a5b 89 APIC_IMASK_UNLOCK ; \
8a8d5d85 90
984263bc
MD
91/*
92 * Test to see whether we are handling an edge or level triggered INT.
93 * Level-triggered INTs must still be masked as we don't clear the source,
94 * and the EOI cycle would cause redundant INTs to occur.
95 */
96#define MASK_LEVEL_IRQ(irq_num) \
0f546930 97 testl $IOAPIC_IM_FLAG_LEVEL, IOAPICFLAGS(irq_num) ; \
984263bc
MD
98 jz 9f ; /* edge, don't mask */ \
99 MASK_IRQ(irq_num) ; \
8a8d5d85 1009: ; \
984263bc 101
984263bc
MD
102/*
103 * Test to see if the source is currntly masked, clear if so.
104 */
ea689d1c 105#define UNMASK_IRQ(irq_num) \
477d3c1c
MD
106 cmpl $0,%eax ; \
107 jnz 8f ; \
97359a5b 108 APIC_IMASK_LOCK ; /* into critical reg */ \
ea689d1c 109 testl $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
984263bc 110 je 7f ; /* bit clear, not masked */ \
ea689d1c
SZ
111 andl $~IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
112 /* clear mask bit */ \
984263bc
MD
113 movl IOAPICADDR(irq_num),%ecx ; /* ioapic addr */ \
114 movl REDIRIDX(irq_num), %eax ; /* get the index */ \
115 movl %eax,(%ecx) ; /* write the index */ \
bda48b43 116 andl $~IOART_INTMASK,IOAPIC_WINDOW(%ecx) ;/* clear the mask */ \
984263bc 1177: ; \
97359a5b 118 APIC_IMASK_UNLOCK ; \
477d3c1c 1198: ; \
984263bc 120
79b62055 121#ifdef SMP /* APIC-IO */
97359a5b 122
8a8d5d85
MD
123/*
124 * Fast interrupt call handlers run in the following sequence:
125 *
126 * - Push the trap frame required by doreti
127 * - Mask the interrupt and reenable its source
128 * - If we cannot take the interrupt set its fpending bit and
71ef2f5c
MD
129 * doreti. Note that we cannot mess with mp_lock at all
130 * if we entered from a critical section!
8a8d5d85
MD
131 * - If we can take the interrupt clear its fpending bit,
132 * call the handler, then unmask and doreti.
133 *
134 * YYY can cache gd base opitner instead of using hidden %fs prefixes.
135 */
136
137#define FAST_INTR(irq_num, vec_name) \
138 .text ; \
139 SUPERALIGN_TEXT ; \
140IDTVEC(vec_name) ; \
141 PUSH_FRAME ; \
4e7c41c5 142 FAKE_MCOUNT(15*4(%esp)) ; \
8a8d5d85 143 MASK_LEVEL_IRQ(irq_num) ; \
35408d22 144 movl $0, lapic_eoi ; \
8a8d5d85 145 movl PCPU(curthread),%ebx ; \
38787eef 146 movl $0,%eax ; /* CURRENT CPL IN FRAME (REMOVED) */ \
984263bc 147 pushl %eax ; \
1be5027b
MD
148 testl $-1,TD_NEST_COUNT(%ebx) ; \
149 jne 1f ; \
f9235b6d
MD
150 testl $-1,TD_CRITCOUNT(%ebx) ; \
151 je 2f ; \
8a8d5d85 1521: ; \
545a1cd3 153 /* in critical section, make interrupt pending */ \
8a8d5d85
MD
154 /* set the pending bit and return, leave interrupt masked */ \
155 orl $IRQ_LBIT(irq_num),PCPU(fpending) ; \
235957ed 156 orl $RQF_INTPEND,PCPU(reqflags) ; \
8a8d5d85
MD
157 jmp 5f ; \
1582: ; \
159 /* clear pending bit, run handler */ \
8a8d5d85 160 andl $~IRQ_LBIT(irq_num),PCPU(fpending) ; \
477d3c1c 161 pushl $irq_num ; \
c7eb0589 162 pushl %esp ; /* pass frame by reference */ \
f9235b6d 163 incl TD_CRITCOUNT(%ebx) ; \
477d3c1c 164 call ithread_fast_handler ; /* returns 0 to unmask */ \
f9235b6d 165 decl TD_CRITCOUNT(%ebx) ; \
c7eb0589 166 addl $8, %esp ; \
8a8d5d85
MD
167 UNMASK_IRQ(irq_num) ; \
1685: ; \
169 MEXITCOUNT ; \
170 jmp doreti ; \
984263bc 171
97359a5b
MD
172#endif
173
984263bc
MD
174/*
175 * Handle "spurious INTerrupts".
176 * Notes:
177 * This is different than the "spurious INTerrupt" generated by an
178 * 8259 PIC for missing INTs. See the APIC documentation for details.
179 * This routine should NOT do an 'EOI' cycle.
180 */
181 .text
182 SUPERALIGN_TEXT
2954c92f
MD
183 .globl Xspuriousint
184Xspuriousint:
984263bc
MD
185
186 /* No EOI cycle used here */
187
188 iret
189
190
191/*
192 * Handle TLB shootdowns.
193 */
194 .text
195 SUPERALIGN_TEXT
8a8d5d85
MD
196 .globl Xinvltlb
197Xinvltlb:
984263bc
MD
198 pushl %eax
199
984263bc
MD
200 movl %cr3, %eax /* invalidate the TLB */
201 movl %eax, %cr3
202
203 ss /* stack segment, avoid %ds load */
204 movl $0, lapic_eoi /* End Of Interrupt to APIC */
205
206 popl %eax
207 iret
208
209
984263bc 210/*
984263bc
MD
211 * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
212 *
213 * - Signals its receipt.
214 * - Waits for permission to restart.
bd8015ca 215 * - Processing pending IPIQ events while waiting.
984263bc
MD
216 * - Signals its restart.
217 */
218
219 .text
220 SUPERALIGN_TEXT
2954c92f
MD
221 .globl Xcpustop
222Xcpustop:
984263bc
MD
223 pushl %ebp
224 movl %esp, %ebp
225 pushl %eax
226 pushl %ecx
227 pushl %edx
228 pushl %ds /* save current data segment */
229 pushl %fs
230
231 movl $KDSEL, %eax
232 mov %ax, %ds /* use KERNEL data segment */
233 movl $KPSEL, %eax
234 mov %ax, %fs
235
236 movl $0, lapic_eoi /* End Of Interrupt to APIC */
237
2954c92f 238 movl PCPU(cpuid), %eax
984263bc
MD
239 imull $PCB_SIZE, %eax
240 leal CNAME(stoppcbs)(%eax), %eax
241 pushl %eax
242 call CNAME(savectx) /* Save process context */
243 addl $4, %esp
244
245
2954c92f 246 movl PCPU(cpuid), %eax
984263bc 247
bd8015ca
MD
248 /*
249 * Indicate that we have stopped and loop waiting for permission
250 * to start again. We must still process IPI events while in a
251 * stopped state.
252 */
97359a5b 253 MPLOCKED
2954c92f 254 btsl %eax, stopped_cpus /* stopped_cpus |= (1<<id) */
984263bc 2551:
bd8015ca
MD
256 andl $~RQF_IPIQ,PCPU(reqflags)
257 pushl %eax
258 call lwkt_smp_stopped
259 popl %eax
2954c92f 260 btl %eax, started_cpus /* while (!(started_cpus & (1<<id))) */
984263bc
MD
261 jnc 1b
262
97359a5b 263 MPLOCKED
2954c92f 264 btrl %eax, started_cpus /* started_cpus &= ~(1<<id) */
97359a5b 265 MPLOCKED
2954c92f 266 btrl %eax, stopped_cpus /* stopped_cpus &= ~(1<<id) */
984263bc
MD
267
268 test %eax, %eax
269 jnz 2f
270
271 movl CNAME(cpustop_restartfunc), %eax
272 test %eax, %eax
273 jz 2f
274 movl $0, CNAME(cpustop_restartfunc) /* One-shot */
275
276 call *%eax
2772:
278 popl %fs
279 popl %ds /* restore previous data segment */
280 popl %edx
281 popl %ecx
282 popl %eax
283 movl %ebp, %esp
284 popl %ebp
285 iret
286
96728c05
MD
287 /*
288 * For now just have one ipiq IPI, but what we really want is
289 * to have one for each source cpu to the APICs don't get stalled
290 * backlogging the requests.
291 */
292 .text
293 SUPERALIGN_TEXT
294 .globl Xipiq
295Xipiq:
296 PUSH_FRAME
297 movl $0, lapic_eoi /* End Of Interrupt to APIC */
4e7c41c5 298 FAKE_MCOUNT(15*4(%esp))
96728c05 299
b785701b 300 incl PCPU(cnt) + V_IPI
96728c05 301 movl PCPU(curthread),%ebx
f9235b6d
MD
302 testl $-1,TD_CRITCOUNT(%ebx)
303 jne 1f
88c4d2f6 304 subl $8,%esp /* make same as interrupt frame */
c7eb0589 305 pushl %esp /* pass frame by reference */
03aa8d99 306 incl PCPU(intr_nesting_level)
f9235b6d 307 incl TD_CRITCOUNT(%ebx)
88c4d2f6 308 call lwkt_process_ipiq_frame
f9235b6d 309 decl TD_CRITCOUNT(%ebx)
03aa8d99 310 decl PCPU(intr_nesting_level)
c7eb0589 311 addl $12,%esp
38787eef 312 pushl $0 /* CPL for frame (REMOVED) */
96728c05
MD
313 MEXITCOUNT
314 jmp doreti
3151:
235957ed 316 orl $RQF_IPIQ,PCPU(reqflags)
96728c05
MD
317 MEXITCOUNT
318 POP_FRAME
319 iret
984263bc 320
78ea5a2a
SZ
321 .text
322 SUPERALIGN_TEXT
323 .globl Xtimer
324Xtimer:
325 PUSH_FRAME
326 movl $0, lapic_eoi /* End Of Interrupt to APIC */
327 FAKE_MCOUNT(15*4(%esp))
328
b785701b 329 incl PCPU(cnt) + V_TIMER
78ea5a2a 330 movl PCPU(curthread),%ebx
f9235b6d
MD
331 testl $-1,TD_CRITCOUNT(%ebx)
332 jne 1f
5119f32b
SZ
333 testl $-1,TD_NEST_COUNT(%ebx)
334 jne 1f
78ea5a2a
SZ
335 subl $8,%esp /* make same as interrupt frame */
336 pushl %esp /* pass frame by reference */
337 incl PCPU(intr_nesting_level)
f9235b6d 338 incl TD_CRITCOUNT(%ebx)
78ea5a2a 339 call lapic_timer_process_frame
f9235b6d 340 decl TD_CRITCOUNT(%ebx)
78ea5a2a
SZ
341 decl PCPU(intr_nesting_level)
342 addl $12,%esp
343 pushl $0 /* CPL for frame (REMOVED) */
344 MEXITCOUNT
345 jmp doreti
3461:
347 orl $RQF_TIMER,PCPU(reqflags)
348 MEXITCOUNT
349 POP_FRAME
350 iret
351
79b62055 352#ifdef SMP /* APIC-IO */
97359a5b 353
984263bc 354MCOUNT_LABEL(bintr)
10ff1029
MD
355 FAST_INTR(0,apic_fastintr0)
356 FAST_INTR(1,apic_fastintr1)
357 FAST_INTR(2,apic_fastintr2)
358 FAST_INTR(3,apic_fastintr3)
359 FAST_INTR(4,apic_fastintr4)
360 FAST_INTR(5,apic_fastintr5)
361 FAST_INTR(6,apic_fastintr6)
362 FAST_INTR(7,apic_fastintr7)
363 FAST_INTR(8,apic_fastintr8)
364 FAST_INTR(9,apic_fastintr9)
365 FAST_INTR(10,apic_fastintr10)
366 FAST_INTR(11,apic_fastintr11)
367 FAST_INTR(12,apic_fastintr12)
368 FAST_INTR(13,apic_fastintr13)
369 FAST_INTR(14,apic_fastintr14)
370 FAST_INTR(15,apic_fastintr15)
371 FAST_INTR(16,apic_fastintr16)
372 FAST_INTR(17,apic_fastintr17)
373 FAST_INTR(18,apic_fastintr18)
374 FAST_INTR(19,apic_fastintr19)
375 FAST_INTR(20,apic_fastintr20)
376 FAST_INTR(21,apic_fastintr21)
377 FAST_INTR(22,apic_fastintr22)
378 FAST_INTR(23,apic_fastintr23)
fe5f755a
SZ
379 FAST_INTR(24,apic_fastintr24)
380 FAST_INTR(25,apic_fastintr25)
381 FAST_INTR(26,apic_fastintr26)
382 FAST_INTR(27,apic_fastintr27)
383 FAST_INTR(28,apic_fastintr28)
384 FAST_INTR(29,apic_fastintr29)
385 FAST_INTR(30,apic_fastintr30)
386 FAST_INTR(31,apic_fastintr31)
984263bc
MD
387MCOUNT_LABEL(eintr)
388
97359a5b
MD
389#endif
390
984263bc 391 .data
ef0fdad1 392
984263bc 393/* variables used by stop_cpus()/restart_cpus()/Xcpustop */
2954c92f
MD
394 .globl stopped_cpus, started_cpus
395stopped_cpus:
984263bc 396 .long 0
2954c92f 397started_cpus:
984263bc
MD
398 .long 0
399
984263bc 400 .globl CNAME(cpustop_restartfunc)
984263bc
MD
401CNAME(cpustop_restartfunc):
402 .long 0
403
984263bc 404 .text
06f5be02 405