kernel: Make SMP support default (and non-optional).
[dragonfly.git] / sys / platform / pc32 / apic / apic_vector.s
CommitLineData
984263bc
MD
1/*
2 * from: vector.s, 386BSD 0.1 unknown origin
3 * $FreeBSD: src/sys/i386/isa/apic_vector.s,v 1.47.2.5 2001/09/01 22:33:38 tegge Exp $
4 */
5
06f5be02
MD
6#include "opt_auto_eoi.h"
7
8#include <machine/asmacros.h>
06f5be02
MD
9#include <machine/lock.h>
10#include <machine/psl.h>
11#include <machine/trap.h>
06f5be02 12
a9295349 13#include <machine_base/icu/icu.h>
21ce0dfa 14#include <bus/isa/isa.h>
06f5be02
MD
15
16#include "assym.s"
984263bc 17
bdc560a1 18#include "apicreg.h"
77f86d14 19#include <machine_base/apic/ioapic_ipl.h>
87cf6827 20#include <machine/intr_machdep.h>
984263bc 21
c263294b
SZ
22/* convert an absolute IRQ# into bitmask */
23#define IRQ_LBIT(irq_num) (1 << ((irq_num) & 0x1f))
984263bc 24
c263294b
SZ
25/* convert an absolute IRQ# into ipending index */
26#define IRQ_LIDX(irq_num) ((irq_num) >> 5)
984263bc 27
97359a5b 28#define MPLOCKED lock ;
97359a5b 29
984263bc 30/*
8a8d5d85
MD
31 * Push an interrupt frame in a format acceptable to doreti, reload
32 * the segment registers for the kernel.
984263bc
MD
33 */
34#define PUSH_FRAME \
35 pushl $0 ; /* dummy error code */ \
36 pushl $0 ; /* dummy trap type */ \
4e7c41c5 37 pushl $0 ; /* dummy xflags type */ \
984263bc
MD
38 pushal ; \
39 pushl %ds ; /* save data and extra segments ... */ \
40 pushl %es ; \
8a8d5d85 41 pushl %fs ; \
4e7c41c5 42 pushl %gs ; \
c885c20e 43 cld ; \
8a8d5d85
MD
44 mov $KDSEL,%ax ; \
45 mov %ax,%ds ; \
46 mov %ax,%es ; \
4e7c41c5 47 mov %ax,%gs ; \
8a8d5d85
MD
48 mov $KPSEL,%ax ; \
49 mov %ax,%fs ; \
984263bc 50
8a8d5d85
MD
51/*
52 * Warning: POP_FRAME can only be used if there is no chance of a
53 * segment register being changed (e.g. by procfs), which is why syscalls
54 * have to use doreti.
55 */
984263bc 56#define POP_FRAME \
4e7c41c5 57 popl %gs ; \
984263bc
MD
58 popl %fs ; \
59 popl %es ; \
60 popl %ds ; \
61 popal ; \
4e7c41c5 62 addl $3*4,%esp ; /* dummy xflags, trap & error codes */ \
8a8d5d85 63
3d911e0a 64#define IOAPICADDR(irq_num) \
7a54dec9 65 CNAME(ioapic_irqs) + IOAPIC_IRQI_SIZE * (irq_num) + IOAPIC_IRQI_ADDR
3d911e0a 66#define REDIRIDX(irq_num) \
7a54dec9 67 CNAME(ioapic_irqs) + IOAPIC_IRQI_SIZE * (irq_num) + IOAPIC_IRQI_IDX
0f546930 68#define IOAPICFLAGS(irq_num) \
7a54dec9 69 CNAME(ioapic_irqs) + IOAPIC_IRQI_SIZE * (irq_num) + IOAPIC_IRQI_FLAGS
8a8d5d85 70
984263bc 71#define MASK_IRQ(irq_num) \
9e0e3f85 72 IOAPIC_IMASK_LOCK ; /* into critical reg */ \
7a54dec9 73 testl $IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
984263bc 74 jne 7f ; /* masked, don't mask */ \
7a54dec9 75 orl $IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
ea689d1c 76 /* set the mask bit */ \
984263bc
MD
77 movl IOAPICADDR(irq_num), %ecx ; /* ioapic addr */ \
78 movl REDIRIDX(irq_num), %eax ; /* get the index */ \
79 movl %eax, (%ecx) ; /* write the index */ \
bda48b43 80 orl $IOART_INTMASK,IOAPIC_WINDOW(%ecx) ;/* set the mask */ \
984263bc 817: ; /* already masked */ \
9e0e3f85 82 IOAPIC_IMASK_UNLOCK ; \
8a8d5d85 83
984263bc
MD
84/*
85 * Test to see whether we are handling an edge or level triggered INT.
86 * Level-triggered INTs must still be masked as we don't clear the source,
87 * and the EOI cycle would cause redundant INTs to occur.
88 */
89#define MASK_LEVEL_IRQ(irq_num) \
7a54dec9 90 testl $IOAPIC_IRQI_FLAG_LEVEL, IOAPICFLAGS(irq_num) ; \
984263bc
MD
91 jz 9f ; /* edge, don't mask */ \
92 MASK_IRQ(irq_num) ; \
8a8d5d85 939: ; \
984263bc 94
984263bc
MD
95/*
96 * Test to see if the source is currntly masked, clear if so.
97 */
ea689d1c 98#define UNMASK_IRQ(irq_num) \
477d3c1c
MD
99 cmpl $0,%eax ; \
100 jnz 8f ; \
9e0e3f85 101 IOAPIC_IMASK_LOCK ; /* into critical reg */ \
7a54dec9 102 testl $IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
984263bc 103 je 7f ; /* bit clear, not masked */ \
7a54dec9 104 andl $~IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ; \
ea689d1c 105 /* clear mask bit */ \
984263bc
MD
106 movl IOAPICADDR(irq_num),%ecx ; /* ioapic addr */ \
107 movl REDIRIDX(irq_num), %eax ; /* get the index */ \
108 movl %eax,(%ecx) ; /* write the index */ \
bda48b43 109 andl $~IOART_INTMASK,IOAPIC_WINDOW(%ecx) ;/* clear the mask */ \
984263bc 1107: ; \
9e0e3f85 111 IOAPIC_IMASK_UNLOCK ; \
477d3c1c 1128: ; \
984263bc 113
8a8d5d85 114/*
081be8a5 115 * Interrupt call handlers run in the following sequence:
8a8d5d85
MD
116 *
117 * - Push the trap frame required by doreti
118 * - Mask the interrupt and reenable its source
c263294b 119 * - If we cannot take the interrupt set its ipending bit and
b5d16701 120 * doreti.
c263294b 121 * - If we can take the interrupt clear its ipending bit,
8a8d5d85
MD
122 * call the handler, then unmask and doreti.
123 *
124 * YYY can cache gd base opitner instead of using hidden %fs prefixes.
125 */
126
20a6b6c7 127#define INTR_HANDLER(irq_num) \
8a8d5d85
MD
128 .text ; \
129 SUPERALIGN_TEXT ; \
9e0e3f85 130IDTVEC(ioapic_intr##irq_num) ; \
8a8d5d85 131 PUSH_FRAME ; \
4e7c41c5 132 FAKE_MCOUNT(15*4(%esp)) ; \
8a8d5d85 133 MASK_LEVEL_IRQ(irq_num) ; \
cb7d6921
SZ
134 movl lapic,%eax ; \
135 movl $0,LA_EOI(%eax) ; \
8a8d5d85 136 movl PCPU(curthread),%ebx ; \
38787eef 137 movl $0,%eax ; /* CURRENT CPL IN FRAME (REMOVED) */ \
984263bc 138 pushl %eax ; \
1be5027b
MD
139 testl $-1,TD_NEST_COUNT(%ebx) ; \
140 jne 1f ; \
f9235b6d
MD
141 testl $-1,TD_CRITCOUNT(%ebx) ; \
142 je 2f ; \
8a8d5d85 1431: ; \
545a1cd3 144 /* in critical section, make interrupt pending */ \
8a8d5d85 145 /* set the pending bit and return, leave interrupt masked */ \
c263294b
SZ
146 movl $IRQ_LIDX(irq_num),%edx ; \
147 orl $IRQ_LBIT(irq_num),PCPU_E4(ipending,%edx) ; \
235957ed 148 orl $RQF_INTPEND,PCPU(reqflags) ; \
8a8d5d85
MD
149 jmp 5f ; \
1502: ; \
151 /* clear pending bit, run handler */ \
c263294b
SZ
152 movl $IRQ_LIDX(irq_num),%edx ; \
153 andl $~IRQ_LBIT(irq_num),PCPU_E4(ipending,%edx) ; \
477d3c1c 154 pushl $irq_num ; \
c7eb0589 155 pushl %esp ; /* pass frame by reference */ \
f9235b6d 156 incl TD_CRITCOUNT(%ebx) ; \
2d910aaf 157 sti ; \
477d3c1c 158 call ithread_fast_handler ; /* returns 0 to unmask */ \
f9235b6d 159 decl TD_CRITCOUNT(%ebx) ; \
c7eb0589 160 addl $8, %esp ; \
8a8d5d85
MD
161 UNMASK_IRQ(irq_num) ; \
1625: ; \
163 MEXITCOUNT ; \
164 jmp doreti ; \
984263bc 165
984263bc
MD
166/*
167 * Handle "spurious INTerrupts".
168 * Notes:
169 * This is different than the "spurious INTerrupt" generated by an
170 * 8259 PIC for missing INTs. See the APIC documentation for details.
171 * This routine should NOT do an 'EOI' cycle.
172 */
173 .text
174 SUPERALIGN_TEXT
2954c92f
MD
175 .globl Xspuriousint
176Xspuriousint:
984263bc
MD
177
178 /* No EOI cycle used here */
179
180 iret
181
984263bc
MD
182/*
183 * Handle TLB shootdowns.
2d910aaf
MD
184 *
185 * NOTE: Interrupts remain disabled.
984263bc
MD
186 */
187 .text
188 SUPERALIGN_TEXT
8a8d5d85
MD
189 .globl Xinvltlb
190Xinvltlb:
7d4d6fdb 191 PUSH_FRAME
cb7d6921
SZ
192 movl lapic,%eax
193 movl $0,LA_EOI(%eax) /* End Of Interrupt to APIC */
7d4d6fdb 194 FAKE_MCOUNT(15*4(%esp))
984263bc 195
7d4d6fdb
MD
196 subl $8,%esp /* make same as interrupt frame */
197 pushl %esp /* pass frame by reference */
198 call smp_invltlb_intr
199 addl $12,%esp
984263bc 200
7d4d6fdb
MD
201 MEXITCOUNT
202 jmp doreti_syscall_ret
984263bc 203
984263bc
MD
204/*
205 * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
206 *
207 * - Signals its receipt.
208 * - Waits for permission to restart.
bd8015ca 209 * - Processing pending IPIQ events while waiting.
984263bc
MD
210 * - Signals its restart.
211 */
212
213 .text
214 SUPERALIGN_TEXT
2954c92f
MD
215 .globl Xcpustop
216Xcpustop:
984263bc
MD
217 pushl %ebp
218 movl %esp, %ebp
219 pushl %eax
220 pushl %ecx
221 pushl %edx
222 pushl %ds /* save current data segment */
223 pushl %fs
224
225 movl $KDSEL, %eax
226 mov %ax, %ds /* use KERNEL data segment */
227 movl $KPSEL, %eax
228 mov %ax, %fs
229
cb7d6921
SZ
230 movl lapic, %eax
231 movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */
984263bc 232
2954c92f 233 movl PCPU(cpuid), %eax
984263bc
MD
234 imull $PCB_SIZE, %eax
235 leal CNAME(stoppcbs)(%eax), %eax
236 pushl %eax
237 call CNAME(savectx) /* Save process context */
238 addl $4, %esp
239
240
2954c92f 241 movl PCPU(cpuid), %eax
984263bc 242
bd8015ca
MD
243 /*
244 * Indicate that we have stopped and loop waiting for permission
245 * to start again. We must still process IPI events while in a
246 * stopped state.
2d910aaf
MD
247 *
248 * Interrupts must remain enabled for non-IPI'd per-cpu interrupts
249 * (e.g. Xtimer, Xinvltlb).
bd8015ca 250 */
97359a5b 251 MPLOCKED
2954c92f 252 btsl %eax, stopped_cpus /* stopped_cpus |= (1<<id) */
2d910aaf 253 sti
984263bc 2541:
bd8015ca
MD
255 andl $~RQF_IPIQ,PCPU(reqflags)
256 pushl %eax
257 call lwkt_smp_stopped
258 popl %eax
2954c92f 259 btl %eax, started_cpus /* while (!(started_cpus & (1<<id))) */
984263bc
MD
260 jnc 1b
261
97359a5b 262 MPLOCKED
2954c92f 263 btrl %eax, started_cpus /* started_cpus &= ~(1<<id) */
97359a5b 264 MPLOCKED
2954c92f 265 btrl %eax, stopped_cpus /* stopped_cpus &= ~(1<<id) */
984263bc
MD
266
267 test %eax, %eax
268 jnz 2f
269
270 movl CNAME(cpustop_restartfunc), %eax
271 test %eax, %eax
272 jz 2f
273 movl $0, CNAME(cpustop_restartfunc) /* One-shot */
274
275 call *%eax
2762:
277 popl %fs
278 popl %ds /* restore previous data segment */
279 popl %edx
280 popl %ecx
281 popl %eax
282 movl %ebp, %esp
283 popl %ebp
284 iret
285
96728c05
MD
286 /*
287 * For now just have one ipiq IPI, but what we really want is
288 * to have one for each source cpu to the APICs don't get stalled
289 * backlogging the requests.
290 */
291 .text
292 SUPERALIGN_TEXT
293 .globl Xipiq
294Xipiq:
295 PUSH_FRAME
cb7d6921
SZ
296 movl lapic,%eax
297 movl $0,LA_EOI(%eax) /* End Of Interrupt to APIC */
4e7c41c5 298 FAKE_MCOUNT(15*4(%esp))
96728c05 299
b785701b 300 incl PCPU(cnt) + V_IPI
96728c05 301 movl PCPU(curthread),%ebx
f9235b6d
MD
302 testl $-1,TD_CRITCOUNT(%ebx)
303 jne 1f
88c4d2f6 304 subl $8,%esp /* make same as interrupt frame */
c7eb0589 305 pushl %esp /* pass frame by reference */
03aa8d99 306 incl PCPU(intr_nesting_level)
f9235b6d 307 incl TD_CRITCOUNT(%ebx)
2d910aaf 308 sti
88c4d2f6 309 call lwkt_process_ipiq_frame
f9235b6d 310 decl TD_CRITCOUNT(%ebx)
03aa8d99 311 decl PCPU(intr_nesting_level)
c7eb0589 312 addl $12,%esp
38787eef 313 pushl $0 /* CPL for frame (REMOVED) */
96728c05
MD
314 MEXITCOUNT
315 jmp doreti
3161:
235957ed 317 orl $RQF_IPIQ,PCPU(reqflags)
96728c05 318 MEXITCOUNT
7d4d6fdb 319 jmp doreti_syscall_ret
984263bc 320
78ea5a2a
SZ
321 .text
322 SUPERALIGN_TEXT
323 .globl Xtimer
324Xtimer:
325 PUSH_FRAME
cb7d6921
SZ
326 movl lapic,%eax
327 movl $0,LA_EOI(%eax) /* End Of Interrupt to APIC */
78ea5a2a
SZ
328 FAKE_MCOUNT(15*4(%esp))
329
b785701b 330 incl PCPU(cnt) + V_TIMER
78ea5a2a 331 movl PCPU(curthread),%ebx
f9235b6d
MD
332 testl $-1,TD_CRITCOUNT(%ebx)
333 jne 1f
5119f32b
SZ
334 testl $-1,TD_NEST_COUNT(%ebx)
335 jne 1f
78ea5a2a
SZ
336 subl $8,%esp /* make same as interrupt frame */
337 pushl %esp /* pass frame by reference */
338 incl PCPU(intr_nesting_level)
f9235b6d 339 incl TD_CRITCOUNT(%ebx)
2d910aaf 340 sti
78ea5a2a 341 call lapic_timer_process_frame
f9235b6d 342 decl TD_CRITCOUNT(%ebx)
78ea5a2a
SZ
343 decl PCPU(intr_nesting_level)
344 addl $12,%esp
345 pushl $0 /* CPL for frame (REMOVED) */
346 MEXITCOUNT
347 jmp doreti
3481:
349 orl $RQF_TIMER,PCPU(reqflags)
350 MEXITCOUNT
7d4d6fdb 351 jmp doreti_syscall_ret
78ea5a2a 352
984263bc 353MCOUNT_LABEL(bintr)
20a6b6c7
SZ
354 INTR_HANDLER(0)
355 INTR_HANDLER(1)
356 INTR_HANDLER(2)
357 INTR_HANDLER(3)
358 INTR_HANDLER(4)
359 INTR_HANDLER(5)
360 INTR_HANDLER(6)
361 INTR_HANDLER(7)
362 INTR_HANDLER(8)
363 INTR_HANDLER(9)
364 INTR_HANDLER(10)
365 INTR_HANDLER(11)
366 INTR_HANDLER(12)
367 INTR_HANDLER(13)
368 INTR_HANDLER(14)
369 INTR_HANDLER(15)
370 INTR_HANDLER(16)
371 INTR_HANDLER(17)
372 INTR_HANDLER(18)
373 INTR_HANDLER(19)
374 INTR_HANDLER(20)
375 INTR_HANDLER(21)
376 INTR_HANDLER(22)
377 INTR_HANDLER(23)
378 INTR_HANDLER(24)
379 INTR_HANDLER(25)
380 INTR_HANDLER(26)
381 INTR_HANDLER(27)
382 INTR_HANDLER(28)
383 INTR_HANDLER(29)
384 INTR_HANDLER(30)
385 INTR_HANDLER(31)
c571da4a
SZ
386 INTR_HANDLER(32)
387 INTR_HANDLER(33)
388 INTR_HANDLER(34)
389 INTR_HANDLER(35)
390 INTR_HANDLER(36)
391 INTR_HANDLER(37)
392 INTR_HANDLER(38)
393 INTR_HANDLER(39)
394 INTR_HANDLER(40)
395 INTR_HANDLER(41)
396 INTR_HANDLER(42)
397 INTR_HANDLER(43)
398 INTR_HANDLER(44)
399 INTR_HANDLER(45)
400 INTR_HANDLER(46)
401 INTR_HANDLER(47)
402 INTR_HANDLER(48)
403 INTR_HANDLER(49)
404 INTR_HANDLER(50)
405 INTR_HANDLER(51)
406 INTR_HANDLER(52)
407 INTR_HANDLER(53)
408 INTR_HANDLER(54)
409 INTR_HANDLER(55)
410 INTR_HANDLER(56)
411 INTR_HANDLER(57)
412 INTR_HANDLER(58)
413 INTR_HANDLER(59)
414 INTR_HANDLER(60)
415 INTR_HANDLER(61)
416 INTR_HANDLER(62)
417 INTR_HANDLER(63)
418 INTR_HANDLER(64)
419 INTR_HANDLER(65)
420 INTR_HANDLER(66)
421 INTR_HANDLER(67)
422 INTR_HANDLER(68)
423 INTR_HANDLER(69)
424 INTR_HANDLER(70)
425 INTR_HANDLER(71)
426 INTR_HANDLER(72)
427 INTR_HANDLER(73)
428 INTR_HANDLER(74)
429 INTR_HANDLER(75)
430 INTR_HANDLER(76)
431 INTR_HANDLER(77)
432 INTR_HANDLER(78)
433 INTR_HANDLER(79)
434 INTR_HANDLER(80)
435 INTR_HANDLER(81)
436 INTR_HANDLER(82)
437 INTR_HANDLER(83)
438 INTR_HANDLER(84)
439 INTR_HANDLER(85)
440 INTR_HANDLER(86)
441 INTR_HANDLER(87)
442 INTR_HANDLER(88)
443 INTR_HANDLER(89)
444 INTR_HANDLER(90)
445 INTR_HANDLER(91)
446 INTR_HANDLER(92)
447 INTR_HANDLER(93)
448 INTR_HANDLER(94)
449 INTR_HANDLER(95)
450 INTR_HANDLER(96)
451 INTR_HANDLER(97)
452 INTR_HANDLER(98)
453 INTR_HANDLER(99)
454 INTR_HANDLER(100)
455 INTR_HANDLER(101)
456 INTR_HANDLER(102)
457 INTR_HANDLER(103)
458 INTR_HANDLER(104)
459 INTR_HANDLER(105)
460 INTR_HANDLER(106)
461 INTR_HANDLER(107)
462 INTR_HANDLER(108)
463 INTR_HANDLER(109)
464 INTR_HANDLER(110)
465 INTR_HANDLER(111)
466 INTR_HANDLER(112)
467 INTR_HANDLER(113)
468 INTR_HANDLER(114)
469 INTR_HANDLER(115)
470 INTR_HANDLER(116)
471 INTR_HANDLER(117)
472 INTR_HANDLER(118)
473 INTR_HANDLER(119)
474 INTR_HANDLER(120)
475 INTR_HANDLER(121)
476 INTR_HANDLER(122)
477 INTR_HANDLER(123)
478 INTR_HANDLER(124)
479 INTR_HANDLER(125)
480 INTR_HANDLER(126)
481 INTR_HANDLER(127)
482 INTR_HANDLER(128)
483 INTR_HANDLER(129)
484 INTR_HANDLER(130)
485 INTR_HANDLER(131)
486 INTR_HANDLER(132)
487 INTR_HANDLER(133)
488 INTR_HANDLER(134)
489 INTR_HANDLER(135)
490 INTR_HANDLER(136)
491 INTR_HANDLER(137)
492 INTR_HANDLER(138)
493 INTR_HANDLER(139)
494 INTR_HANDLER(140)
495 INTR_HANDLER(141)
496 INTR_HANDLER(142)
497 INTR_HANDLER(143)
498 INTR_HANDLER(144)
499 INTR_HANDLER(145)
500 INTR_HANDLER(146)
501 INTR_HANDLER(147)
502 INTR_HANDLER(148)
503 INTR_HANDLER(149)
504 INTR_HANDLER(150)
505 INTR_HANDLER(151)
506 INTR_HANDLER(152)
507 INTR_HANDLER(153)
508 INTR_HANDLER(154)
509 INTR_HANDLER(155)
510 INTR_HANDLER(156)
511 INTR_HANDLER(157)
512 INTR_HANDLER(158)
513 INTR_HANDLER(159)
514 INTR_HANDLER(160)
515 INTR_HANDLER(161)
516 INTR_HANDLER(162)
517 INTR_HANDLER(163)
518 INTR_HANDLER(164)
519 INTR_HANDLER(165)
520 INTR_HANDLER(166)
521 INTR_HANDLER(167)
522 INTR_HANDLER(168)
523 INTR_HANDLER(169)
524 INTR_HANDLER(170)
525 INTR_HANDLER(171)
526 INTR_HANDLER(172)
527 INTR_HANDLER(173)
528 INTR_HANDLER(174)
529 INTR_HANDLER(175)
530 INTR_HANDLER(176)
531 INTR_HANDLER(177)
532 INTR_HANDLER(178)
533 INTR_HANDLER(179)
534 INTR_HANDLER(180)
535 INTR_HANDLER(181)
536 INTR_HANDLER(182)
537 INTR_HANDLER(183)
538 INTR_HANDLER(184)
539 INTR_HANDLER(185)
540 INTR_HANDLER(186)
541 INTR_HANDLER(187)
542 INTR_HANDLER(188)
543 INTR_HANDLER(189)
544 INTR_HANDLER(190)
545 INTR_HANDLER(191)
984263bc
MD
546MCOUNT_LABEL(eintr)
547
984263bc 548 .data
ef0fdad1 549
984263bc 550/* variables used by stop_cpus()/restart_cpus()/Xcpustop */
2954c92f
MD
551 .globl stopped_cpus, started_cpus
552stopped_cpus:
984263bc 553 .long 0
2954c92f 554started_cpus:
984263bc
MD
555 .long 0
556
984263bc 557 .globl CNAME(cpustop_restartfunc)
984263bc
MD
558CNAME(cpustop_restartfunc):
559 .long 0
560
984263bc 561 .text
06f5be02 562