kernel - Misc fixes and debugging
[dragonfly.git] / sys / platform / pc64 / apic / apic_vector.s
1 /*
2  *      from: vector.s, 386BSD 0.1 unknown origin
3  * $FreeBSD: src/sys/i386/isa/apic_vector.s,v 1.47.2.5 2001/09/01 22:33:38 tegge Exp $
4  */
5
6 #if 0
7 #include "opt_auto_eoi.h"
8 #endif
9
10 #include <machine/asmacros.h>
11 #include <machine/lock.h>
12 #include <machine/psl.h>
13 #include <machine/trap.h>
14 #include <machine/segments.h>
15
16 #include <machine_base/icu/icu.h>
17 #include <bus/isa/isa.h>
18
19 #include "assym.s"
20
21 #include "apicreg.h"
22 #include <machine_base/apic/ioapic_ipl.h>
23 #include <machine/intr_machdep.h>
24
25 #ifdef foo
26 /* convert an absolute IRQ# into bitmask */
27 #define IRQ_LBIT(irq_num)       (1UL << (irq_num & 0x3f))
28 #endif
29
30 #define IRQ_SBITS(irq_num)      ((irq_num) & 0x3f)
31
32 /* convert an absolute IRQ# into gd_ipending index */
33 #define IRQ_LIDX(irq_num)       ((irq_num) >> 6)
34
35 #ifdef SMP
36 #define MPLOCKED     lock ;
37 #else
38 #define MPLOCKED
39 #endif
40
41 #define APIC_PUSH_FRAME                                                 \
42         PUSH_FRAME ;            /* 15 regs + space for 5 extras */      \
43         movq $0,TF_XFLAGS(%rsp) ;                                       \
44         movq $0,TF_TRAPNO(%rsp) ;                                       \
45         movq $0,TF_ADDR(%rsp) ;                                         \
46         movq $0,TF_FLAGS(%rsp) ;                                        \
47         movq $0,TF_ERR(%rsp) ;                                          \
48         cld ;                                                           \
49
50 /*
51  * JG stale? Warning: POP_FRAME can only be used if there is no chance of a
52  * segment register being changed (e.g. by procfs), which is why syscalls
53  * have to use doreti.
54  */
55 #define APIC_POP_FRAME                                                  \
56         POP_FRAME ;                                                     \
57
58 #define IOAPICADDR(irq_num) \
59         CNAME(ioapic_irqs) + IOAPIC_IRQI_SIZE * (irq_num) + IOAPIC_IRQI_ADDR
60 #define REDIRIDX(irq_num) \
61         CNAME(ioapic_irqs) + IOAPIC_IRQI_SIZE * (irq_num) + IOAPIC_IRQI_IDX
62 #define IOAPICFLAGS(irq_num) \
63         CNAME(ioapic_irqs) + IOAPIC_IRQI_SIZE * (irq_num) + IOAPIC_IRQI_FLAGS
64  
65 #define MASK_IRQ(irq_num)                                               \
66         IOAPIC_IMASK_LOCK ;                     /* into critical reg */ \
67         testl   $IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ;        \
68         jne     7f ;                    /* masked, don't mask */        \
69         orl     $IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ;        \
70                                                 /* set the mask bit */  \
71         movq    IOAPICADDR(irq_num), %rcx ;     /* ioapic addr */       \
72         movl    REDIRIDX(irq_num), %eax ;       /* get the index */     \
73         movl    %eax, (%rcx) ;                  /* write the index */   \
74         orl     $IOART_INTMASK,IOAPIC_WINDOW(%rcx) ;/* set the mask */  \
75 7: ;                                            /* already masked */    \
76         IOAPIC_IMASK_UNLOCK ;                                           \
77
78 /*
79  * Test to see whether we are handling an edge or level triggered INT.
80  *  Level-triggered INTs must still be masked as we don't clear the source,
81  *  and the EOI cycle would cause redundant INTs to occur.
82  */
83 #define MASK_LEVEL_IRQ(irq_num)                                         \
84         testl   $IOAPIC_IRQI_FLAG_LEVEL, IOAPICFLAGS(irq_num) ;         \
85         jz      9f ;                            /* edge, don't mask */  \
86         MASK_IRQ(irq_num) ;                                             \
87 9: ;                                                                    \
88
89 /*
90  * Test to see if the source is currntly masked, clear if so.
91  */
92 #define UNMASK_IRQ(irq_num)                                     \
93         cmpl    $0,%eax ;                                               \
94         jnz     8f ;                                                    \
95         IOAPIC_IMASK_LOCK ;                     /* into critical reg */ \
96         testl   $IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ;        \
97         je      7f ;                    /* bit clear, not masked */     \
98         andl    $~IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ;       \
99                                                 /* clear mask bit */    \
100         movq    IOAPICADDR(irq_num),%rcx ;      /* ioapic addr */       \
101         movl    REDIRIDX(irq_num), %eax ;       /* get the index */     \
102         movl    %eax,(%rcx) ;                   /* write the index */   \
103         andl    $~IOART_INTMASK,IOAPIC_WINDOW(%rcx) ;/* clear the mask */ \
104 7: ;                                                                    \
105         IOAPIC_IMASK_UNLOCK ;                                           \
106 8: ;                                                                    \
107
108 /*
109  * Interrupt call handlers run in the following sequence:
110  *
111  *      - Push the trap frame required by doreti
112  *      - Mask the interrupt and reenable its source
113  *      - If we cannot take the interrupt set its ipending bit and
114  *        doreti.
115  *      - If we can take the interrupt clear its ipending bit,
116  *        call the handler, then unmask and doreti.
117  *
118  * YYY can cache gd base opitner instead of using hidden %fs prefixes.
119  */
120
121 #define INTR_HANDLER(irq_num)                                           \
122         .text ;                                                         \
123         SUPERALIGN_TEXT ;                                               \
124 IDTVEC(ioapic_intr##irq_num) ;                                          \
125         APIC_PUSH_FRAME ;                                               \
126         FAKE_MCOUNT(TF_RIP(%rsp)) ;                                     \
127         MASK_LEVEL_IRQ(irq_num) ;                                       \
128         movq    lapic, %rax ;                                           \
129         movl    $0, LA_EOI(%rax) ;                                      \
130         movq    PCPU(curthread),%rbx ;                                  \
131         testl   $-1,TD_NEST_COUNT(%rbx) ;                               \
132         jne     1f ;                                                    \
133         testl   $-1,TD_CRITCOUNT(%rbx) ;                                \
134         je      2f ;                                                    \
135 1: ;                                                                    \
136         /* in critical section, make interrupt pending */               \
137         /* set the pending bit and return, leave interrupt masked */    \
138         movq    $1,%rcx ;                                               \
139         shlq    $IRQ_SBITS(irq_num),%rcx ;                              \
140         movq    $IRQ_LIDX(irq_num),%rdx ;                               \
141         orq     %rcx,PCPU_E8(ipending,%rdx) ;                           \
142         orl     $RQF_INTPEND,PCPU(reqflags) ;                           \
143         jmp     5f ;                                                    \
144 2: ;                                                                    \
145         /* clear pending bit, run handler */                            \
146         movq    $1,%rcx ;                                               \
147         shlq    $IRQ_SBITS(irq_num),%rcx ;                              \
148         notq    %rcx ;                                                  \
149         movq    $IRQ_LIDX(irq_num),%rdx ;                               \
150         andq    %rcx,PCPU_E8(ipending,%rdx) ;                           \
151         pushq   $irq_num ;              /* trapframe -> intrframe */    \
152         movq    %rsp, %rdi ;            /* pass frame by reference */   \
153         incl    TD_CRITCOUNT(%rbx) ;                                    \
154         sti ;                                                           \
155         call    ithread_fast_handler ;  /* returns 0 to unmask */       \
156         decl    TD_CRITCOUNT(%rbx) ;                                    \
157         addq    $8, %rsp ;              /* intrframe -> trapframe */    \
158         UNMASK_IRQ(irq_num) ;                                           \
159 5: ;                                                                    \
160         MEXITCOUNT ;                                                    \
161         jmp     doreti ;                                                \
162
163 /*
164  * Handle "spurious INTerrupts".
165  *
166  * NOTE: This is different than the "spurious INTerrupt" generated by an
167  *       8259 PIC for missing INTs.  See the APIC documentation for details.
168  *       This routine should NOT do an 'EOI' cycle.
169  *
170  * NOTE: Even though we don't do anything here we must still swapgs if
171  *       coming from a user frame in case the iretq faults... just use
172  *       the nominal APIC_PUSH_FRAME sequence to get it done.
173  */
174         .text
175         SUPERALIGN_TEXT
176         .globl Xspuriousint
177 Xspuriousint:
178         APIC_PUSH_FRAME
179         /* No EOI cycle used here */
180         FAKE_MCOUNT(TF_RIP(%rsp))
181         MEXITCOUNT
182         APIC_POP_FRAME
183         jmp     doreti_iret
184
185 #ifdef SMP
186
187 /*
188  * Handle TLB shootdowns.
189  *
190  * NOTE: interrupts are left disabled.
191  */
192         .text
193         SUPERALIGN_TEXT
194         .globl  Xinvltlb
195 Xinvltlb:
196         APIC_PUSH_FRAME
197         movq    lapic, %rax
198         movl    $0, LA_EOI(%rax)        /* End Of Interrupt to APIC */
199         FAKE_MCOUNT(TF_RIP(%rsp))
200         subq    $8,%rsp                 /* make same as interrupt frame */
201         movq    %rsp,%rdi               /* pass frame by reference */
202         call    smp_invltlb_intr
203         addq    $8,%rsp                 /* turn into trapframe */
204         MEXITCOUNT
205         APIC_POP_FRAME
206         jmp     doreti_iret
207
208 /*
209  * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
210  *
211  *  - We cannot call doreti
212  *  - Signals its receipt.
213  *  - Waits for permission to restart.
214  *  - Processing pending IPIQ events while waiting.
215  *  - Signals its restart.
216  */
217
218         .text
219         SUPERALIGN_TEXT
220         .globl Xcpustop
221 Xcpustop:
222         APIC_PUSH_FRAME
223         movq    lapic, %rax
224         movl    $0, LA_EOI(%rax)        /* End Of Interrupt to APIC */
225
226         movl    PCPU(cpuid), %eax
227         imull   $PCB_SIZE, %eax
228         leaq    CNAME(stoppcbs), %rdi
229         addq    %rax, %rdi
230         call    CNAME(savectx)          /* Save process context */
231
232         movslq  PCPU(cpuid), %rax
233
234         /*
235          * Indicate that we have stopped and loop waiting for permission
236          * to start again.  We must still process IPI events while in a
237          * stopped state.
238          *
239          * Interrupts must remain enabled for non-IPI'd per-cpu interrupts
240          * (e.g. Xtimer, Xinvltlb).
241          */
242         MPLOCKED
243         btsq    %rax, stopped_cpus      /* stopped_cpus |= (1<<id) */
244         sti
245 1:
246         andl    $~RQF_IPIQ,PCPU(reqflags)
247         pushq   %rax
248         call    lwkt_smp_stopped
249         popq    %rax
250         pause
251         btq     %rax, started_cpus      /* while (!(started_cpus & (1<<id))) */
252         jnc     1b
253
254         MPLOCKED
255         btrq    %rax, started_cpus      /* started_cpus &= ~(1<<id) */
256         MPLOCKED
257         btrq    %rax, stopped_cpus      /* stopped_cpus &= ~(1<<id) */
258
259         test    %eax, %eax
260         jnz     2f
261
262         movq    CNAME(cpustop_restartfunc), %rax
263         test    %rax, %rax
264         jz      2f
265         movq    $0, CNAME(cpustop_restartfunc)  /* One-shot */
266
267         call    *%rax
268 2:
269         MEXITCOUNT
270         APIC_POP_FRAME
271         jmp     doreti_iret
272
273         /*
274          * For now just have one ipiq IPI, but what we really want is
275          * to have one for each source cpu to the APICs don't get stalled
276          * backlogging the requests.
277          */
278         .text
279         SUPERALIGN_TEXT
280         .globl Xipiq
281 Xipiq:
282         APIC_PUSH_FRAME
283         movq    lapic, %rax
284         movl    $0, LA_EOI(%rax)        /* End Of Interrupt to APIC */
285         FAKE_MCOUNT(TF_RIP(%rsp))
286
287         incl    PCPU(cnt) + V_IPI
288         movq    PCPU(curthread),%rbx
289         testl   $-1,TD_CRITCOUNT(%rbx)
290         jne     1f
291         subq    $8,%rsp                 /* make same as interrupt frame */
292         movq    %rsp,%rdi               /* pass frame by reference */
293         incl    PCPU(intr_nesting_level)
294         incl    TD_CRITCOUNT(%rbx)
295         sti
296         call    lwkt_process_ipiq_frame
297         decl    TD_CRITCOUNT(%rbx)
298         decl    PCPU(intr_nesting_level)
299         addq    $8,%rsp                 /* turn into trapframe */
300         MEXITCOUNT
301         jmp     doreti
302 1:
303         orl     $RQF_IPIQ,PCPU(reqflags)
304         MEXITCOUNT
305         APIC_POP_FRAME
306         jmp     doreti_iret
307
308 #endif  /* SMP */
309
310         .text
311         SUPERALIGN_TEXT
312         .globl Xtimer
313 Xtimer:
314         APIC_PUSH_FRAME
315         movq    lapic, %rax
316         movl    $0, LA_EOI(%rax)        /* End Of Interrupt to APIC */
317         FAKE_MCOUNT(TF_RIP(%rsp))
318
319         subq    $8,%rsp                 /* make same as interrupt frame */
320         movq    %rsp,%rdi               /* pass frame by reference */
321         call    lapic_timer_always
322         addq    $8,%rsp                 /* turn into trapframe */
323
324         incl    PCPU(cnt) + V_TIMER
325         movq    PCPU(curthread),%rbx
326         testl   $-1,TD_CRITCOUNT(%rbx)
327         jne     1f
328         testl   $-1,TD_NEST_COUNT(%rbx)
329         jne     1f
330         subq    $8,%rsp                 /* make same as interrupt frame */
331         movq    %rsp,%rdi               /* pass frame by reference */
332         incl    PCPU(intr_nesting_level)
333         incl    TD_CRITCOUNT(%rbx)
334         sti
335         call    lapic_timer_process_frame
336         decl    TD_CRITCOUNT(%rbx)
337         decl    PCPU(intr_nesting_level)
338         addq    $8,%rsp                 /* turn into trapframe */
339         MEXITCOUNT
340         jmp     doreti
341 1:
342         orl     $RQF_TIMER,PCPU(reqflags)
343         MEXITCOUNT
344         APIC_POP_FRAME
345         jmp     doreti_iret
346
347 MCOUNT_LABEL(bintr)
348         INTR_HANDLER(0)
349         INTR_HANDLER(1)
350         INTR_HANDLER(2)
351         INTR_HANDLER(3)
352         INTR_HANDLER(4)
353         INTR_HANDLER(5)
354         INTR_HANDLER(6)
355         INTR_HANDLER(7)
356         INTR_HANDLER(8)
357         INTR_HANDLER(9)
358         INTR_HANDLER(10)
359         INTR_HANDLER(11)
360         INTR_HANDLER(12)
361         INTR_HANDLER(13)
362         INTR_HANDLER(14)
363         INTR_HANDLER(15)
364         INTR_HANDLER(16)
365         INTR_HANDLER(17)
366         INTR_HANDLER(18)
367         INTR_HANDLER(19)
368         INTR_HANDLER(20)
369         INTR_HANDLER(21)
370         INTR_HANDLER(22)
371         INTR_HANDLER(23)
372         INTR_HANDLER(24)
373         INTR_HANDLER(25)
374         INTR_HANDLER(26)
375         INTR_HANDLER(27)
376         INTR_HANDLER(28)
377         INTR_HANDLER(29)
378         INTR_HANDLER(30)
379         INTR_HANDLER(31)
380         INTR_HANDLER(32)
381         INTR_HANDLER(33)
382         INTR_HANDLER(34)
383         INTR_HANDLER(35)
384         INTR_HANDLER(36)
385         INTR_HANDLER(37)
386         INTR_HANDLER(38)
387         INTR_HANDLER(39)
388         INTR_HANDLER(40)
389         INTR_HANDLER(41)
390         INTR_HANDLER(42)
391         INTR_HANDLER(43)
392         INTR_HANDLER(44)
393         INTR_HANDLER(45)
394         INTR_HANDLER(46)
395         INTR_HANDLER(47)
396         INTR_HANDLER(48)
397         INTR_HANDLER(49)
398         INTR_HANDLER(50)
399         INTR_HANDLER(51)
400         INTR_HANDLER(52)
401         INTR_HANDLER(53)
402         INTR_HANDLER(54)
403         INTR_HANDLER(55)
404         INTR_HANDLER(56)
405         INTR_HANDLER(57)
406         INTR_HANDLER(58)
407         INTR_HANDLER(59)
408         INTR_HANDLER(60)
409         INTR_HANDLER(61)
410         INTR_HANDLER(62)
411         INTR_HANDLER(63)
412         INTR_HANDLER(64)
413         INTR_HANDLER(65)
414         INTR_HANDLER(66)
415         INTR_HANDLER(67)
416         INTR_HANDLER(68)
417         INTR_HANDLER(69)
418         INTR_HANDLER(70)
419         INTR_HANDLER(71)
420         INTR_HANDLER(72)
421         INTR_HANDLER(73)
422         INTR_HANDLER(74)
423         INTR_HANDLER(75)
424         INTR_HANDLER(76)
425         INTR_HANDLER(77)
426         INTR_HANDLER(78)
427         INTR_HANDLER(79)
428         INTR_HANDLER(80)
429         INTR_HANDLER(81)
430         INTR_HANDLER(82)
431         INTR_HANDLER(83)
432         INTR_HANDLER(84)
433         INTR_HANDLER(85)
434         INTR_HANDLER(86)
435         INTR_HANDLER(87)
436         INTR_HANDLER(88)
437         INTR_HANDLER(89)
438         INTR_HANDLER(90)
439         INTR_HANDLER(91)
440         INTR_HANDLER(92)
441         INTR_HANDLER(93)
442         INTR_HANDLER(94)
443         INTR_HANDLER(95)
444         INTR_HANDLER(96)
445         INTR_HANDLER(97)
446         INTR_HANDLER(98)
447         INTR_HANDLER(99)
448         INTR_HANDLER(100)
449         INTR_HANDLER(101)
450         INTR_HANDLER(102)
451         INTR_HANDLER(103)
452         INTR_HANDLER(104)
453         INTR_HANDLER(105)
454         INTR_HANDLER(106)
455         INTR_HANDLER(107)
456         INTR_HANDLER(108)
457         INTR_HANDLER(109)
458         INTR_HANDLER(110)
459         INTR_HANDLER(111)
460         INTR_HANDLER(112)
461         INTR_HANDLER(113)
462         INTR_HANDLER(114)
463         INTR_HANDLER(115)
464         INTR_HANDLER(116)
465         INTR_HANDLER(117)
466         INTR_HANDLER(118)
467         INTR_HANDLER(119)
468         INTR_HANDLER(120)
469         INTR_HANDLER(121)
470         INTR_HANDLER(122)
471         INTR_HANDLER(123)
472         INTR_HANDLER(124)
473         INTR_HANDLER(125)
474         INTR_HANDLER(126)
475         INTR_HANDLER(127)
476         INTR_HANDLER(128)
477         INTR_HANDLER(129)
478         INTR_HANDLER(130)
479         INTR_HANDLER(131)
480         INTR_HANDLER(132)
481         INTR_HANDLER(133)
482         INTR_HANDLER(134)
483         INTR_HANDLER(135)
484         INTR_HANDLER(136)
485         INTR_HANDLER(137)
486         INTR_HANDLER(138)
487         INTR_HANDLER(139)
488         INTR_HANDLER(140)
489         INTR_HANDLER(141)
490         INTR_HANDLER(142)
491         INTR_HANDLER(143)
492         INTR_HANDLER(144)
493         INTR_HANDLER(145)
494         INTR_HANDLER(146)
495         INTR_HANDLER(147)
496         INTR_HANDLER(148)
497         INTR_HANDLER(149)
498         INTR_HANDLER(150)
499         INTR_HANDLER(151)
500         INTR_HANDLER(152)
501         INTR_HANDLER(153)
502         INTR_HANDLER(154)
503         INTR_HANDLER(155)
504         INTR_HANDLER(156)
505         INTR_HANDLER(157)
506         INTR_HANDLER(158)
507         INTR_HANDLER(159)
508         INTR_HANDLER(160)
509         INTR_HANDLER(161)
510         INTR_HANDLER(162)
511         INTR_HANDLER(163)
512         INTR_HANDLER(164)
513         INTR_HANDLER(165)
514         INTR_HANDLER(166)
515         INTR_HANDLER(167)
516         INTR_HANDLER(168)
517         INTR_HANDLER(169)
518         INTR_HANDLER(170)
519         INTR_HANDLER(171)
520         INTR_HANDLER(172)
521         INTR_HANDLER(173)
522         INTR_HANDLER(174)
523         INTR_HANDLER(175)
524         INTR_HANDLER(176)
525         INTR_HANDLER(177)
526         INTR_HANDLER(178)
527         INTR_HANDLER(179)
528         INTR_HANDLER(180)
529         INTR_HANDLER(181)
530         INTR_HANDLER(182)
531         INTR_HANDLER(183)
532         INTR_HANDLER(184)
533         INTR_HANDLER(185)
534         INTR_HANDLER(186)
535         INTR_HANDLER(187)
536         INTR_HANDLER(188)
537         INTR_HANDLER(189)
538         INTR_HANDLER(190)
539         INTR_HANDLER(191)
540 MCOUNT_LABEL(eintr)
541
542         .data
543
544 /* variables used by stop_cpus()/restart_cpus()/Xcpustop */
545         .globl stopped_cpus, started_cpus
546 stopped_cpus:
547         .quad   0
548 started_cpus:
549         .quad   0
550
551         .globl CNAME(cpustop_restartfunc)
552 CNAME(cpustop_restartfunc):
553         .quad 0
554                 
555         .text
556