lapic timer: Add necessary bits for lapic timer interrupt delivery
[dragonfly.git] / sys / platform / pc32 / apic / apic_vector.s
CommitLineData
984263bc
MD
1/*
2 * from: vector.s, 386BSD 0.1 unknown origin
3 * $FreeBSD: src/sys/i386/isa/apic_vector.s,v 1.47.2.5 2001/09/01 22:33:38 tegge Exp $
21ce0dfa 4 * $DragonFly: src/sys/platform/pc32/apic/apic_vector.s,v 1.39 2008/08/02 01:14:43 dillon Exp $
984263bc
MD
5 */
6
06f5be02
MD
7#include "use_npx.h"
8#include "opt_auto_eoi.h"
9
10#include <machine/asmacros.h>
06f5be02
MD
11#include <machine/lock.h>
12#include <machine/psl.h>
13#include <machine/trap.h>
06f5be02 14
a9295349 15#include <machine_base/icu/icu.h>
21ce0dfa 16#include <bus/isa/isa.h>
06f5be02
MD
17
18#include "assym.s"
984263bc 19
bdc560a1
MD
20#include "apicreg.h"
21#include "apic_ipl.h"
984263bc 22#include <machine/smp.h>
a9295349 23#include <machine_base/isa/intr_machdep.h>
984263bc
MD
24
25/* convert an absolute IRQ# into a bitmask */
8a8d5d85 26#define IRQ_LBIT(irq_num) (1 << (irq_num))
984263bc
MD
27
28/* make an index into the IO APIC from the IRQ# */
29#define REDTBL_IDX(irq_num) (0x10 + ((irq_num) * 2))
30
97359a5b
MD
31#ifdef SMP
32#define MPLOCKED lock ;
33#else
34#define MPLOCKED
35#endif
36
984263bc 37/*
8a8d5d85
MD
38 * Push an interrupt frame in a format acceptable to doreti, reload
39 * the segment registers for the kernel.
984263bc
MD
40 */
41#define PUSH_FRAME \
42 pushl $0 ; /* dummy error code */ \
43 pushl $0 ; /* dummy trap type */ \
4e7c41c5 44 pushl $0 ; /* dummy xflags type */ \
984263bc
MD
45 pushal ; \
46 pushl %ds ; /* save data and extra segments ... */ \
47 pushl %es ; \
8a8d5d85 48 pushl %fs ; \
4e7c41c5 49 pushl %gs ; \
c885c20e 50 cld ; \
8a8d5d85
MD
51 mov $KDSEL,%ax ; \
52 mov %ax,%ds ; \
53 mov %ax,%es ; \
4e7c41c5 54 mov %ax,%gs ; \
8a8d5d85
MD
55 mov $KPSEL,%ax ; \
56 mov %ax,%fs ; \
984263bc 57
8a8d5d85
MD
58#define PUSH_DUMMY \
59 pushfl ; /* phys int frame / flags */ \
60 pushl %cs ; /* phys int frame / cs */ \
61 pushl 12(%esp) ; /* original caller eip */ \
62 pushl $0 ; /* dummy error code */ \
63 pushl $0 ; /* dummy trap type */ \
4e7c41c5
MD
64 pushl $0 ; /* dummy xflags type */ \
65 subl $13*4,%esp ; /* pushal + 4 seg regs (dummy) + CPL */ \
8a8d5d85
MD
66
67/*
68 * Warning: POP_FRAME can only be used if there is no chance of a
69 * segment register being changed (e.g. by procfs), which is why syscalls
70 * have to use doreti.
71 */
984263bc 72#define POP_FRAME \
4e7c41c5 73 popl %gs ; \
984263bc
MD
74 popl %fs ; \
75 popl %es ; \
76 popl %ds ; \
77 popal ; \
4e7c41c5 78 addl $3*4,%esp ; /* dummy xflags, trap & error codes */ \
8a8d5d85
MD
79
80#define POP_DUMMY \
4e7c41c5 81 addl $19*4,%esp ; \
984263bc
MD
82
83#define IOAPICADDR(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 8
84#define REDIRIDX(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 12
8a8d5d85 85
984263bc 86#define MASK_IRQ(irq_num) \
97359a5b 87 APIC_IMASK_LOCK ; /* into critical reg */ \
8a8d5d85 88 testl $IRQ_LBIT(irq_num), apic_imen ; \
984263bc 89 jne 7f ; /* masked, don't mask */ \
8a8d5d85 90 orl $IRQ_LBIT(irq_num), apic_imen ; /* set the mask bit */ \
984263bc
MD
91 movl IOAPICADDR(irq_num), %ecx ; /* ioapic addr */ \
92 movl REDIRIDX(irq_num), %eax ; /* get the index */ \
93 movl %eax, (%ecx) ; /* write the index */ \
94 movl IOAPIC_WINDOW(%ecx), %eax ; /* current value */ \
95 orl $IOART_INTMASK, %eax ; /* set the mask */ \
96 movl %eax, IOAPIC_WINDOW(%ecx) ; /* new value */ \
977: ; /* already masked */ \
97359a5b 98 APIC_IMASK_UNLOCK ; \
8a8d5d85 99
984263bc
MD
100/*
101 * Test to see whether we are handling an edge or level triggered INT.
102 * Level-triggered INTs must still be masked as we don't clear the source,
103 * and the EOI cycle would cause redundant INTs to occur.
104 */
105#define MASK_LEVEL_IRQ(irq_num) \
8a8d5d85 106 testl $IRQ_LBIT(irq_num), apic_pin_trigger ; \
984263bc
MD
107 jz 9f ; /* edge, don't mask */ \
108 MASK_IRQ(irq_num) ; \
8a8d5d85 1099: ; \
984263bc 110
984263bc
MD
111/*
112 * Test to see if the source is currntly masked, clear if so.
113 */
114#define UNMASK_IRQ(irq_num) \
477d3c1c
MD
115 cmpl $0,%eax ; \
116 jnz 8f ; \
97359a5b 117 APIC_IMASK_LOCK ; /* into critical reg */ \
8a8d5d85 118 testl $IRQ_LBIT(irq_num), apic_imen ; \
984263bc 119 je 7f ; /* bit clear, not masked */ \
8a8d5d85 120 andl $~IRQ_LBIT(irq_num), apic_imen ;/* clear mask bit */ \
984263bc
MD
121 movl IOAPICADDR(irq_num),%ecx ; /* ioapic addr */ \
122 movl REDIRIDX(irq_num), %eax ; /* get the index */ \
123 movl %eax,(%ecx) ; /* write the index */ \
124 movl IOAPIC_WINDOW(%ecx),%eax ; /* current value */ \
125 andl $~IOART_INTMASK,%eax ; /* clear the mask */ \
126 movl %eax,IOAPIC_WINDOW(%ecx) ; /* new value */ \
1277: ; \
97359a5b 128 APIC_IMASK_UNLOCK ; \
477d3c1c 1298: ; \
984263bc 130
97359a5b
MD
131#ifdef APIC_IO
132
8a8d5d85
MD
133/*
134 * Fast interrupt call handlers run in the following sequence:
135 *
136 * - Push the trap frame required by doreti
137 * - Mask the interrupt and reenable its source
138 * - If we cannot take the interrupt set its fpending bit and
71ef2f5c
MD
139 * doreti. Note that we cannot mess with mp_lock at all
140 * if we entered from a critical section!
8a8d5d85
MD
141 * - If we can take the interrupt clear its fpending bit,
142 * call the handler, then unmask and doreti.
143 *
144 * YYY can cache gd base opitner instead of using hidden %fs prefixes.
145 */
146
147#define FAST_INTR(irq_num, vec_name) \
148 .text ; \
149 SUPERALIGN_TEXT ; \
150IDTVEC(vec_name) ; \
151 PUSH_FRAME ; \
4e7c41c5 152 FAKE_MCOUNT(15*4(%esp)) ; \
8a8d5d85 153 MASK_LEVEL_IRQ(irq_num) ; \
35408d22 154 movl $0, lapic_eoi ; \
8a8d5d85 155 movl PCPU(curthread),%ebx ; \
38787eef 156 movl $0,%eax ; /* CURRENT CPL IN FRAME (REMOVED) */ \
984263bc 157 pushl %eax ; \
1be5027b
MD
158 testl $-1,TD_NEST_COUNT(%ebx) ; \
159 jne 1f ; \
8a8d5d85 160 cmpl $TDPRI_CRIT,TD_PRI(%ebx) ; \
38787eef 161 jl 2f ; \
8a8d5d85 1621: ; \
545a1cd3 163 /* in critical section, make interrupt pending */ \
8a8d5d85
MD
164 /* set the pending bit and return, leave interrupt masked */ \
165 orl $IRQ_LBIT(irq_num),PCPU(fpending) ; \
235957ed 166 orl $RQF_INTPEND,PCPU(reqflags) ; \
8a8d5d85
MD
167 jmp 5f ; \
1682: ; \
169 /* clear pending bit, run handler */ \
8a8d5d85 170 andl $~IRQ_LBIT(irq_num),PCPU(fpending) ; \
477d3c1c 171 pushl $irq_num ; \
c7eb0589 172 pushl %esp ; /* pass frame by reference */ \
477d3c1c 173 call ithread_fast_handler ; /* returns 0 to unmask */ \
c7eb0589 174 addl $8, %esp ; \
8a8d5d85
MD
175 UNMASK_IRQ(irq_num) ; \
1765: ; \
177 MEXITCOUNT ; \
178 jmp doreti ; \
984263bc 179
8a8d5d85 180/*
8a8d5d85
MD
181 * Slow interrupt call handlers run in the following sequence:
182 *
183 * - Push the trap frame required by doreti.
184 * - Mask the interrupt and reenable its source.
185 * - If we cannot take the interrupt set its ipending bit and
186 * doreti. In addition to checking for a critical section
187 * and cpl mask we also check to see if the thread is still
71ef2f5c
MD
188 * running. Note that we cannot mess with mp_lock at all
189 * if we entered from a critical section!
96728c05
MD
190 * - If we can take the interrupt clear its ipending bit
191 * and schedule the thread. Leave interrupts masked and doreti.
8a8d5d85 192 *
96728c05
MD
193 * Note that calls to sched_ithd() are made with interrupts enabled
194 * and outside a critical section. YYY sched_ithd may preempt us
03aa8d99 195 * synchronously (fix interrupt stacking).
8a8d5d85
MD
196 *
197 * YYY can cache gd base pointer instead of using hidden %fs
198 * prefixes.
199 */
200
10ff1029 201#define SLOW_INTR(irq_num, vec_name, maybe_extra_ipending) \
984263bc
MD
202 .text ; \
203 SUPERALIGN_TEXT ; \
984263bc
MD
204IDTVEC(vec_name) ; \
205 PUSH_FRAME ; \
984263bc
MD
206 maybe_extra_ipending ; \
207; \
984263bc 208 MASK_LEVEL_IRQ(irq_num) ; \
1e9de339 209 incl PCPU(cnt) + V_INTR ; \
35408d22 210 movl $0, lapic_eoi ; \
2954c92f 211 movl PCPU(curthread),%ebx ; \
38787eef 212 movl $0,%eax ; /* CURRENT CPL IN FRAME (REMOVED) */ \
8a8d5d85 213 pushl %eax ; /* cpl do restore */ \
1be5027b
MD
214 testl $-1,TD_NEST_COUNT(%ebx) ; \
215 jne 1f ; \
8f41e33b 216 cmpl $TDPRI_CRIT,TD_PRI(%ebx) ; \
38787eef 217 jl 2f ; \
8a8d5d85
MD
2181: ; \
219 /* set the pending bit and return, leave the interrupt masked */ \
220 orl $IRQ_LBIT(irq_num), PCPU(ipending) ; \
235957ed 221 orl $RQF_INTPEND,PCPU(reqflags) ; \
8a8d5d85
MD
222 jmp 5f ; \
2232: ; \
8a8d5d85 224 /* set running bit, clear pending bit, run handler */ \
8a8d5d85 225 andl $~IRQ_LBIT(irq_num), PCPU(ipending) ; \
1be5027b 226 incl TD_NEST_COUNT(%ebx) ; \
984263bc 227 sti ; \
8a8d5d85
MD
228 pushl $irq_num ; \
229 call sched_ithd ; \
ef0fdad1 230 addl $4,%esp ; \
1be5027b
MD
231 cli ; \
232 decl TD_NEST_COUNT(%ebx) ; \
8a8d5d85 2335: ; \
984263bc 234 MEXITCOUNT ; \
2954c92f 235 jmp doreti ; \
8a8d5d85 236
385b2dc2
MD
237/*
238 * Wrong interrupt call handlers. We program these into APIC vectors
239 * that should otherwise never occur. For example, we program the SLOW
240 * vector for irq N with this when we program the FAST vector with the
241 * real interrupt.
242 *
243 * XXX for now all we can do is EOI it. We can't call do_wrongintr
244 * (yet) because we could be in a critical section.
245 */
246#define WRONGINTR(irq_num,vec_name) \
247 .text ; \
248 SUPERALIGN_TEXT ; \
249IDTVEC(vec_name) ; \
250 PUSH_FRAME ; \
251 movl $0, lapic_eoi ; /* End Of Interrupt to APIC */ \
252 /*pushl $irq_num ;*/ \
253 /*call do_wrongintr ;*/ \
254 /*addl $4,%esp ;*/ \
255 POP_FRAME ; \
256 iret ; \
984263bc 257
97359a5b
MD
258#endif
259
984263bc
MD
260/*
261 * Handle "spurious INTerrupts".
262 * Notes:
263 * This is different than the "spurious INTerrupt" generated by an
264 * 8259 PIC for missing INTs. See the APIC documentation for details.
265 * This routine should NOT do an 'EOI' cycle.
266 */
267 .text
268 SUPERALIGN_TEXT
2954c92f
MD
269 .globl Xspuriousint
270Xspuriousint:
984263bc
MD
271
272 /* No EOI cycle used here */
273
274 iret
275
276
277/*
278 * Handle TLB shootdowns.
279 */
280 .text
281 SUPERALIGN_TEXT
8a8d5d85
MD
282 .globl Xinvltlb
283Xinvltlb:
984263bc
MD
284 pushl %eax
285
984263bc
MD
286 movl %cr3, %eax /* invalidate the TLB */
287 movl %eax, %cr3
288
289 ss /* stack segment, avoid %ds load */
290 movl $0, lapic_eoi /* End Of Interrupt to APIC */
291
292 popl %eax
293 iret
294
295
984263bc 296/*
984263bc
MD
297 * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
298 *
299 * - Signals its receipt.
300 * - Waits for permission to restart.
bd8015ca 301 * - Processing pending IPIQ events while waiting.
984263bc
MD
302 * - Signals its restart.
303 */
304
305 .text
306 SUPERALIGN_TEXT
2954c92f
MD
307 .globl Xcpustop
308Xcpustop:
984263bc
MD
309 pushl %ebp
310 movl %esp, %ebp
311 pushl %eax
312 pushl %ecx
313 pushl %edx
314 pushl %ds /* save current data segment */
315 pushl %fs
316
317 movl $KDSEL, %eax
318 mov %ax, %ds /* use KERNEL data segment */
319 movl $KPSEL, %eax
320 mov %ax, %fs
321
322 movl $0, lapic_eoi /* End Of Interrupt to APIC */
323
2954c92f 324 movl PCPU(cpuid), %eax
984263bc
MD
325 imull $PCB_SIZE, %eax
326 leal CNAME(stoppcbs)(%eax), %eax
327 pushl %eax
328 call CNAME(savectx) /* Save process context */
329 addl $4, %esp
330
331
2954c92f 332 movl PCPU(cpuid), %eax
984263bc 333
bd8015ca
MD
334 /*
335 * Indicate that we have stopped and loop waiting for permission
336 * to start again. We must still process IPI events while in a
337 * stopped state.
338 */
97359a5b 339 MPLOCKED
2954c92f 340 btsl %eax, stopped_cpus /* stopped_cpus |= (1<<id) */
984263bc 3411:
bd8015ca
MD
342 andl $~RQF_IPIQ,PCPU(reqflags)
343 pushl %eax
344 call lwkt_smp_stopped
345 popl %eax
2954c92f 346 btl %eax, started_cpus /* while (!(started_cpus & (1<<id))) */
984263bc
MD
347 jnc 1b
348
97359a5b 349 MPLOCKED
2954c92f 350 btrl %eax, started_cpus /* started_cpus &= ~(1<<id) */
97359a5b 351 MPLOCKED
2954c92f 352 btrl %eax, stopped_cpus /* stopped_cpus &= ~(1<<id) */
984263bc
MD
353
354 test %eax, %eax
355 jnz 2f
356
357 movl CNAME(cpustop_restartfunc), %eax
358 test %eax, %eax
359 jz 2f
360 movl $0, CNAME(cpustop_restartfunc) /* One-shot */
361
362 call *%eax
3632:
364 popl %fs
365 popl %ds /* restore previous data segment */
366 popl %edx
367 popl %ecx
368 popl %eax
369 movl %ebp, %esp
370 popl %ebp
371 iret
372
96728c05
MD
373 /*
374 * For now just have one ipiq IPI, but what we really want is
375 * to have one for each source cpu to the APICs don't get stalled
376 * backlogging the requests.
377 */
378 .text
379 SUPERALIGN_TEXT
380 .globl Xipiq
381Xipiq:
382 PUSH_FRAME
383 movl $0, lapic_eoi /* End Of Interrupt to APIC */
4e7c41c5 384 FAKE_MCOUNT(15*4(%esp))
96728c05
MD
385
386 movl PCPU(curthread),%ebx
387 cmpl $TDPRI_CRIT,TD_PRI(%ebx)
388 jge 1f
88c4d2f6 389 subl $8,%esp /* make same as interrupt frame */
c7eb0589 390 pushl %esp /* pass frame by reference */
03aa8d99 391 incl PCPU(intr_nesting_level)
96728c05 392 addl $TDPRI_CRIT,TD_PRI(%ebx)
88c4d2f6 393 call lwkt_process_ipiq_frame
96728c05 394 subl $TDPRI_CRIT,TD_PRI(%ebx)
03aa8d99 395 decl PCPU(intr_nesting_level)
c7eb0589 396 addl $12,%esp
38787eef 397 pushl $0 /* CPL for frame (REMOVED) */
96728c05
MD
398 MEXITCOUNT
399 jmp doreti
4001:
235957ed 401 orl $RQF_IPIQ,PCPU(reqflags)
96728c05
MD
402 MEXITCOUNT
403 POP_FRAME
404 iret
984263bc 405
78ea5a2a
SZ
406 .text
407 SUPERALIGN_TEXT
408 .globl Xtimer
409Xtimer:
410 PUSH_FRAME
411 movl $0, lapic_eoi /* End Of Interrupt to APIC */
412 FAKE_MCOUNT(15*4(%esp))
413
414 movl PCPU(curthread),%ebx
415 cmpl $TDPRI_CRIT,TD_PRI(%ebx)
416 jge 1f
417 subl $8,%esp /* make same as interrupt frame */
418 pushl %esp /* pass frame by reference */
419 incl PCPU(intr_nesting_level)
420 addl $TDPRI_CRIT,TD_PRI(%ebx)
421 call lapic_timer_process_frame
422 subl $TDPRI_CRIT,TD_PRI(%ebx)
423 decl PCPU(intr_nesting_level)
424 addl $12,%esp
425 pushl $0 /* CPL for frame (REMOVED) */
426 MEXITCOUNT
427 jmp doreti
4281:
429 orl $RQF_TIMER,PCPU(reqflags)
430 MEXITCOUNT
431 POP_FRAME
432 iret
433
97359a5b
MD
434#ifdef APIC_IO
435
984263bc 436MCOUNT_LABEL(bintr)
10ff1029
MD
437 FAST_INTR(0,apic_fastintr0)
438 FAST_INTR(1,apic_fastintr1)
439 FAST_INTR(2,apic_fastintr2)
440 FAST_INTR(3,apic_fastintr3)
441 FAST_INTR(4,apic_fastintr4)
442 FAST_INTR(5,apic_fastintr5)
443 FAST_INTR(6,apic_fastintr6)
444 FAST_INTR(7,apic_fastintr7)
445 FAST_INTR(8,apic_fastintr8)
446 FAST_INTR(9,apic_fastintr9)
447 FAST_INTR(10,apic_fastintr10)
448 FAST_INTR(11,apic_fastintr11)
449 FAST_INTR(12,apic_fastintr12)
450 FAST_INTR(13,apic_fastintr13)
451 FAST_INTR(14,apic_fastintr14)
452 FAST_INTR(15,apic_fastintr15)
453 FAST_INTR(16,apic_fastintr16)
454 FAST_INTR(17,apic_fastintr17)
455 FAST_INTR(18,apic_fastintr18)
456 FAST_INTR(19,apic_fastintr19)
457 FAST_INTR(20,apic_fastintr20)
458 FAST_INTR(21,apic_fastintr21)
459 FAST_INTR(22,apic_fastintr22)
460 FAST_INTR(23,apic_fastintr23)
984263bc 461
8a8d5d85 462 /* YYY what is this garbage? */
984263bc 463
10ff1029
MD
464 SLOW_INTR(0,apic_slowintr0,)
465 SLOW_INTR(1,apic_slowintr1,)
466 SLOW_INTR(2,apic_slowintr2,)
467 SLOW_INTR(3,apic_slowintr3,)
468 SLOW_INTR(4,apic_slowintr4,)
469 SLOW_INTR(5,apic_slowintr5,)
470 SLOW_INTR(6,apic_slowintr6,)
471 SLOW_INTR(7,apic_slowintr7,)
472 SLOW_INTR(8,apic_slowintr8,)
473 SLOW_INTR(9,apic_slowintr9,)
474 SLOW_INTR(10,apic_slowintr10,)
475 SLOW_INTR(11,apic_slowintr11,)
476 SLOW_INTR(12,apic_slowintr12,)
477 SLOW_INTR(13,apic_slowintr13,)
478 SLOW_INTR(14,apic_slowintr14,)
479 SLOW_INTR(15,apic_slowintr15,)
480 SLOW_INTR(16,apic_slowintr16,)
481 SLOW_INTR(17,apic_slowintr17,)
482 SLOW_INTR(18,apic_slowintr18,)
483 SLOW_INTR(19,apic_slowintr19,)
484 SLOW_INTR(20,apic_slowintr20,)
485 SLOW_INTR(21,apic_slowintr21,)
486 SLOW_INTR(22,apic_slowintr22,)
487 SLOW_INTR(23,apic_slowintr23,)
8a8d5d85 488
10ff1029
MD
489 WRONGINTR(0,apic_wrongintr0)
490 WRONGINTR(1,apic_wrongintr1)
491 WRONGINTR(2,apic_wrongintr2)
492 WRONGINTR(3,apic_wrongintr3)
493 WRONGINTR(4,apic_wrongintr4)
494 WRONGINTR(5,apic_wrongintr5)
495 WRONGINTR(6,apic_wrongintr6)
496 WRONGINTR(7,apic_wrongintr7)
497 WRONGINTR(8,apic_wrongintr8)
498 WRONGINTR(9,apic_wrongintr9)
499 WRONGINTR(10,apic_wrongintr10)
500 WRONGINTR(11,apic_wrongintr11)
501 WRONGINTR(12,apic_wrongintr12)
502 WRONGINTR(13,apic_wrongintr13)
503 WRONGINTR(14,apic_wrongintr14)
504 WRONGINTR(15,apic_wrongintr15)
505 WRONGINTR(16,apic_wrongintr16)
506 WRONGINTR(17,apic_wrongintr17)
507 WRONGINTR(18,apic_wrongintr18)
508 WRONGINTR(19,apic_wrongintr19)
509 WRONGINTR(20,apic_wrongintr20)
510 WRONGINTR(21,apic_wrongintr21)
511 WRONGINTR(22,apic_wrongintr22)
512 WRONGINTR(23,apic_wrongintr23)
984263bc
MD
513MCOUNT_LABEL(eintr)
514
97359a5b
MD
515#endif
516
984263bc 517 .data
ef0fdad1 518
984263bc 519/* variables used by stop_cpus()/restart_cpus()/Xcpustop */
2954c92f
MD
520 .globl stopped_cpus, started_cpus
521stopped_cpus:
984263bc 522 .long 0
2954c92f 523started_cpus:
984263bc
MD
524 .long 0
525
984263bc 526 .globl CNAME(cpustop_restartfunc)
984263bc
MD
527CNAME(cpustop_restartfunc):
528 .long 0
529
2954c92f
MD
530 .globl apic_pin_trigger
531apic_pin_trigger:
984263bc
MD
532 .long 0
533
534 .text
06f5be02 535