c12a03ad606e7ef1cc0e8efdf76aebf5855f71d6
[dragonfly.git] / sys / platform / pc64 / apic / apic_vector.s
1 /*
2  *      from: vector.s, 386BSD 0.1 unknown origin
3  * $FreeBSD: src/sys/i386/isa/apic_vector.s,v 1.47.2.5 2001/09/01 22:33:38 tegge Exp $
4  */
5
6 #if 0
7 #include "opt_auto_eoi.h"
8 #endif
9
10 #include <machine/asmacros.h>
11 #include <machine/lock.h>
12 #include <machine/psl.h>
13 #include <machine/trap.h>
14 #include <machine/segments.h>
15
16 #include <machine_base/icu/icu.h>
17 #include <bus/isa/isa.h>
18
19 #include "assym.s"
20
21 #include "apicreg.h"
22 #include <machine_base/apic/ioapic_ipl.h>
23 #include <machine/intr_machdep.h>
24
25 #ifdef foo
26 /* convert an absolute IRQ# into bitmask */
27 #define IRQ_LBIT(irq_num)       (1UL << (irq_num & 0x3f))
28 #endif
29
30 #define IRQ_SBITS(irq_num)      ((irq_num) & 0x3f)
31
32 /* convert an absolute IRQ# into gd_ipending index */
33 #define IRQ_LIDX(irq_num)       ((irq_num) >> 6)
34
35 #ifdef SMP
36 #define MPLOCKED     lock ;
37 #else
38 #define MPLOCKED
39 #endif
40
41 #define APIC_PUSH_FRAME                                                 \
42         PUSH_FRAME ;            /* 15 regs + space for 5 extras */      \
43         movq $0,TF_XFLAGS(%rsp) ;                                       \
44         movq $0,TF_TRAPNO(%rsp) ;                                       \
45         movq $0,TF_ADDR(%rsp) ;                                         \
46         movq $0,TF_FLAGS(%rsp) ;                                        \
47         movq $0,TF_ERR(%rsp) ;                                          \
48         cld ;                                                           \
49
50 /*
51  * JG stale? Warning: POP_FRAME can only be used if there is no chance of a
52  * segment register being changed (e.g. by procfs), which is why syscalls
53  * have to use doreti.
54  */
55 #define APIC_POP_FRAME POP_FRAME
56
57 #define IOAPICADDR(irq_num) \
58         CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_ADDR
59 #define REDIRIDX(irq_num) \
60         CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_ENTIDX
61 #define IOAPICFLAGS(irq_num) \
62         CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_FLAGS
63  
64 #define MASK_IRQ(irq_num)                                               \
65         IOAPIC_IMASK_LOCK ;                     /* into critical reg */ \
66         testl   $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ;          \
67         jne     7f ;                    /* masked, don't mask */        \
68         orl     $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ;          \
69                                                 /* set the mask bit */  \
70         movq    IOAPICADDR(irq_num), %rcx ;     /* ioapic addr */       \
71         movl    REDIRIDX(irq_num), %eax ;       /* get the index */     \
72         movl    %eax, (%rcx) ;                  /* write the index */   \
73         orl     $IOART_INTMASK,IOAPIC_WINDOW(%rcx) ;/* set the mask */  \
74 7: ;                                            /* already masked */    \
75         IOAPIC_IMASK_UNLOCK ;                                           \
76
77 /*
78  * Test to see whether we are handling an edge or level triggered INT.
79  *  Level-triggered INTs must still be masked as we don't clear the source,
80  *  and the EOI cycle would cause redundant INTs to occur.
81  */
82 #define MASK_LEVEL_IRQ(irq_num)                                         \
83         testl   $IOAPIC_IM_FLAG_LEVEL, IOAPICFLAGS(irq_num) ;           \
84         jz      9f ;                            /* edge, don't mask */  \
85         MASK_IRQ(irq_num) ;                                             \
86 9: ;                                                                    \
87
88 /*
89  * Test to see if the source is currntly masked, clear if so.
90  */
91 #define UNMASK_IRQ(irq_num)                                     \
92         cmpl    $0,%eax ;                                               \
93         jnz     8f ;                                                    \
94         IOAPIC_IMASK_LOCK ;                     /* into critical reg */ \
95         testl   $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ;          \
96         je      7f ;                    /* bit clear, not masked */     \
97         andl    $~IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ;         \
98                                                 /* clear mask bit */    \
99         movq    IOAPICADDR(irq_num),%rcx ;      /* ioapic addr */       \
100         movl    REDIRIDX(irq_num), %eax ;       /* get the index */     \
101         movl    %eax,(%rcx) ;                   /* write the index */   \
102         andl    $~IOART_INTMASK,IOAPIC_WINDOW(%rcx) ;/* clear the mask */ \
103 7: ;                                                                    \
104         IOAPIC_IMASK_UNLOCK ;                                           \
105 8: ;                                                                    \
106
107 /*
108  * Interrupt call handlers run in the following sequence:
109  *
110  *      - Push the trap frame required by doreti
111  *      - Mask the interrupt and reenable its source
112  *      - If we cannot take the interrupt set its ipending bit and
113  *        doreti.
114  *      - If we can take the interrupt clear its ipending bit,
115  *        call the handler, then unmask and doreti.
116  *
117  * YYY can cache gd base opitner instead of using hidden %fs prefixes.
118  */
119
120 #define INTR_HANDLER(irq_num)                                           \
121         .text ;                                                         \
122         SUPERALIGN_TEXT ;                                               \
123 IDTVEC(ioapic_intr##irq_num) ;                                          \
124         APIC_PUSH_FRAME ;                                               \
125         FAKE_MCOUNT(TF_RIP(%rsp)) ;                                     \
126         MASK_LEVEL_IRQ(irq_num) ;                                       \
127         movq    lapic, %rax ;                                           \
128         movl    $0, LA_EOI(%rax) ;                                      \
129         movq    PCPU(curthread),%rbx ;                                  \
130         testl   $-1,TD_NEST_COUNT(%rbx) ;                               \
131         jne     1f ;                                                    \
132         testl   $-1,TD_CRITCOUNT(%rbx) ;                                \
133         je      2f ;                                                    \
134 1: ;                                                                    \
135         /* in critical section, make interrupt pending */               \
136         /* set the pending bit and return, leave interrupt masked */    \
137         movq    $1,%rcx ;                                               \
138         shlq    $IRQ_SBITS(irq_num),%rcx ;                              \
139         movq    $IRQ_LIDX(irq_num),%rdx ;                               \
140         orq     %rcx,PCPU_E8(ipending,%rdx) ;                           \
141         orl     $RQF_INTPEND,PCPU(reqflags) ;                           \
142         jmp     5f ;                                                    \
143 2: ;                                                                    \
144         /* clear pending bit, run handler */                            \
145         movq    $1,%rcx ;                                               \
146         shlq    $IRQ_SBITS(irq_num),%rcx ;                              \
147         notq    %rcx ;                                                  \
148         movq    $IRQ_LIDX(irq_num),%rdx ;                               \
149         andq    %rcx,PCPU_E8(ipending,%rdx) ;                           \
150         pushq   $irq_num ;              /* trapframe -> intrframe */    \
151         movq    %rsp, %rdi ;            /* pass frame by reference */   \
152         incl    TD_CRITCOUNT(%rbx) ;                                    \
153         sti ;                                                           \
154         call    ithread_fast_handler ;  /* returns 0 to unmask */       \
155         decl    TD_CRITCOUNT(%rbx) ;                                    \
156         addq    $8, %rsp ;              /* intrframe -> trapframe */    \
157         UNMASK_IRQ(irq_num) ;                                           \
158 5: ;                                                                    \
159         MEXITCOUNT ;                                                    \
160         jmp     doreti ;                                                \
161
162 /*
163  * Handle "spurious INTerrupts".
164  * Notes:
165  *  This is different than the "spurious INTerrupt" generated by an
166  *   8259 PIC for missing INTs.  See the APIC documentation for details.
167  *  This routine should NOT do an 'EOI' cycle.
168  */
169         .text
170         SUPERALIGN_TEXT
171         .globl Xspuriousint
172 Xspuriousint:
173
174         /* No EOI cycle used here */
175
176         jmp     doreti_iret
177
178
179 /*
180  * Handle TLB shootdowns.
181  *
182  * NOTE: interrupts are left disabled.
183  */
184         .text
185         SUPERALIGN_TEXT
186         .globl  Xinvltlb
187 Xinvltlb:
188         APIC_PUSH_FRAME
189         movq    lapic, %rax
190         movl    $0, LA_EOI(%rax)        /* End Of Interrupt to APIC */
191         FAKE_MCOUNT(TF_RIP(%rsp))
192         subq    $8,%rsp                 /* make same as interrupt frame */
193         movq    %rsp,%rdi               /* pass frame by reference */
194         call    smp_invltlb_intr
195         addq    $8,%rsp                 /* turn into trapframe */
196         MEXITCOUNT
197         APIC_POP_FRAME
198         jmp     doreti_iret
199
200 /*
201  * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
202  *
203  *  - We cannot call doreti
204  *  - Signals its receipt.
205  *  - Waits for permission to restart.
206  *  - Processing pending IPIQ events while waiting.
207  *  - Signals its restart.
208  */
209
210         .text
211         SUPERALIGN_TEXT
212         .globl Xcpustop
213 Xcpustop:
214         APIC_PUSH_FRAME
215         movq    lapic, %rax
216         movl    $0, LA_EOI(%rax)        /* End Of Interrupt to APIC */
217
218         movl    PCPU(cpuid), %eax
219         imull   $PCB_SIZE, %eax
220         leaq    CNAME(stoppcbs), %rdi
221         addq    %rax, %rdi
222         call    CNAME(savectx)          /* Save process context */
223
224         movslq  PCPU(cpuid), %rax
225
226         /*
227          * Indicate that we have stopped and loop waiting for permission
228          * to start again.  We must still process IPI events while in a
229          * stopped state.
230          *
231          * Interrupts must remain enabled for non-IPI'd per-cpu interrupts
232          * (e.g. Xtimer, Xinvltlb).
233          */
234         MPLOCKED
235         btsq    %rax, stopped_cpus      /* stopped_cpus |= (1<<id) */
236         sti
237 1:
238         andl    $~RQF_IPIQ,PCPU(reqflags)
239         pushq   %rax
240         call    lwkt_smp_stopped
241         popq    %rax
242         pause
243         btq     %rax, started_cpus      /* while (!(started_cpus & (1<<id))) */
244         jnc     1b
245
246         MPLOCKED
247         btrq    %rax, started_cpus      /* started_cpus &= ~(1<<id) */
248         MPLOCKED
249         btrq    %rax, stopped_cpus      /* stopped_cpus &= ~(1<<id) */
250
251         test    %eax, %eax
252         jnz     2f
253
254         movq    CNAME(cpustop_restartfunc), %rax
255         test    %rax, %rax
256         jz      2f
257         movq    $0, CNAME(cpustop_restartfunc)  /* One-shot */
258
259         call    *%rax
260 2:
261         MEXITCOUNT
262         APIC_POP_FRAME
263         jmp     doreti_iret
264
265         /*
266          * For now just have one ipiq IPI, but what we really want is
267          * to have one for each source cpu to the APICs don't get stalled
268          * backlogging the requests.
269          */
270         .text
271         SUPERALIGN_TEXT
272         .globl Xipiq
273 Xipiq:
274         APIC_PUSH_FRAME
275         movq    lapic, %rax
276         movl    $0, LA_EOI(%rax)        /* End Of Interrupt to APIC */
277         FAKE_MCOUNT(TF_RIP(%rsp))
278
279         incl    PCPU(cnt) + V_IPI
280         movq    PCPU(curthread),%rbx
281         testl   $-1,TD_CRITCOUNT(%rbx)
282         jne     1f
283         subq    $8,%rsp                 /* make same as interrupt frame */
284         movq    %rsp,%rdi               /* pass frame by reference */
285         incl    PCPU(intr_nesting_level)
286         incl    TD_CRITCOUNT(%rbx)
287         sti
288         call    lwkt_process_ipiq_frame
289         decl    TD_CRITCOUNT(%rbx)
290         decl    PCPU(intr_nesting_level)
291         addq    $8,%rsp                 /* turn into trapframe */
292         MEXITCOUNT
293         jmp     doreti
294 1:
295         orl     $RQF_IPIQ,PCPU(reqflags)
296         MEXITCOUNT
297         APIC_POP_FRAME
298         jmp     doreti_iret
299
300         .text
301         SUPERALIGN_TEXT
302         .globl Xtimer
303 Xtimer:
304         APIC_PUSH_FRAME
305         movq    lapic, %rax
306         movl    $0, LA_EOI(%rax)        /* End Of Interrupt to APIC */
307         FAKE_MCOUNT(TF_RIP(%rsp))
308
309         subq    $8,%rsp                 /* make same as interrupt frame */
310         movq    %rsp,%rdi               /* pass frame by reference */
311         call    lapic_timer_always
312         addq    $8,%rsp                 /* turn into trapframe */
313
314         incl    PCPU(cnt) + V_TIMER
315         movq    PCPU(curthread),%rbx
316         testl   $-1,TD_CRITCOUNT(%rbx)
317         jne     1f
318         testl   $-1,TD_NEST_COUNT(%rbx)
319         jne     1f
320         subq    $8,%rsp                 /* make same as interrupt frame */
321         movq    %rsp,%rdi               /* pass frame by reference */
322         incl    PCPU(intr_nesting_level)
323         incl    TD_CRITCOUNT(%rbx)
324         sti
325         call    lapic_timer_process_frame
326         decl    TD_CRITCOUNT(%rbx)
327         decl    PCPU(intr_nesting_level)
328         addq    $8,%rsp                 /* turn into trapframe */
329         MEXITCOUNT
330         jmp     doreti
331 1:
332         orl     $RQF_TIMER,PCPU(reqflags)
333         MEXITCOUNT
334         APIC_POP_FRAME
335         jmp     doreti_iret
336
337 MCOUNT_LABEL(bintr)
338         INTR_HANDLER(0)
339         INTR_HANDLER(1)
340         INTR_HANDLER(2)
341         INTR_HANDLER(3)
342         INTR_HANDLER(4)
343         INTR_HANDLER(5)
344         INTR_HANDLER(6)
345         INTR_HANDLER(7)
346         INTR_HANDLER(8)
347         INTR_HANDLER(9)
348         INTR_HANDLER(10)
349         INTR_HANDLER(11)
350         INTR_HANDLER(12)
351         INTR_HANDLER(13)
352         INTR_HANDLER(14)
353         INTR_HANDLER(15)
354         INTR_HANDLER(16)
355         INTR_HANDLER(17)
356         INTR_HANDLER(18)
357         INTR_HANDLER(19)
358         INTR_HANDLER(20)
359         INTR_HANDLER(21)
360         INTR_HANDLER(22)
361         INTR_HANDLER(23)
362         INTR_HANDLER(24)
363         INTR_HANDLER(25)
364         INTR_HANDLER(26)
365         INTR_HANDLER(27)
366         INTR_HANDLER(28)
367         INTR_HANDLER(29)
368         INTR_HANDLER(30)
369         INTR_HANDLER(31)
370         INTR_HANDLER(32)
371         INTR_HANDLER(33)
372         INTR_HANDLER(34)
373         INTR_HANDLER(35)
374         INTR_HANDLER(36)
375         INTR_HANDLER(37)
376         INTR_HANDLER(38)
377         INTR_HANDLER(39)
378         INTR_HANDLER(40)
379         INTR_HANDLER(41)
380         INTR_HANDLER(42)
381         INTR_HANDLER(43)
382         INTR_HANDLER(44)
383         INTR_HANDLER(45)
384         INTR_HANDLER(46)
385         INTR_HANDLER(47)
386         INTR_HANDLER(48)
387         INTR_HANDLER(49)
388         INTR_HANDLER(50)
389         INTR_HANDLER(51)
390         INTR_HANDLER(52)
391         INTR_HANDLER(53)
392         INTR_HANDLER(54)
393         INTR_HANDLER(55)
394         INTR_HANDLER(56)
395         INTR_HANDLER(57)
396         INTR_HANDLER(58)
397         INTR_HANDLER(59)
398         INTR_HANDLER(60)
399         INTR_HANDLER(61)
400         INTR_HANDLER(62)
401         INTR_HANDLER(63)
402         INTR_HANDLER(64)
403         INTR_HANDLER(65)
404         INTR_HANDLER(66)
405         INTR_HANDLER(67)
406         INTR_HANDLER(68)
407         INTR_HANDLER(69)
408         INTR_HANDLER(70)
409         INTR_HANDLER(71)
410         INTR_HANDLER(72)
411         INTR_HANDLER(73)
412         INTR_HANDLER(74)
413         INTR_HANDLER(75)
414         INTR_HANDLER(76)
415         INTR_HANDLER(77)
416         INTR_HANDLER(78)
417         INTR_HANDLER(79)
418         INTR_HANDLER(80)
419         INTR_HANDLER(81)
420         INTR_HANDLER(82)
421         INTR_HANDLER(83)
422         INTR_HANDLER(84)
423         INTR_HANDLER(85)
424         INTR_HANDLER(86)
425         INTR_HANDLER(87)
426         INTR_HANDLER(88)
427         INTR_HANDLER(89)
428         INTR_HANDLER(90)
429         INTR_HANDLER(91)
430         INTR_HANDLER(92)
431         INTR_HANDLER(93)
432         INTR_HANDLER(94)
433         INTR_HANDLER(95)
434         INTR_HANDLER(96)
435         INTR_HANDLER(97)
436         INTR_HANDLER(98)
437         INTR_HANDLER(99)
438         INTR_HANDLER(100)
439         INTR_HANDLER(101)
440         INTR_HANDLER(102)
441         INTR_HANDLER(103)
442         INTR_HANDLER(104)
443         INTR_HANDLER(105)
444         INTR_HANDLER(106)
445         INTR_HANDLER(107)
446         INTR_HANDLER(108)
447         INTR_HANDLER(109)
448         INTR_HANDLER(110)
449         INTR_HANDLER(111)
450         INTR_HANDLER(112)
451         INTR_HANDLER(113)
452         INTR_HANDLER(114)
453         INTR_HANDLER(115)
454         INTR_HANDLER(116)
455         INTR_HANDLER(117)
456         INTR_HANDLER(118)
457         INTR_HANDLER(119)
458         INTR_HANDLER(120)
459         INTR_HANDLER(121)
460         INTR_HANDLER(122)
461         INTR_HANDLER(123)
462         INTR_HANDLER(124)
463         INTR_HANDLER(125)
464         INTR_HANDLER(126)
465         INTR_HANDLER(127)
466         INTR_HANDLER(128)
467         INTR_HANDLER(129)
468         INTR_HANDLER(130)
469         INTR_HANDLER(131)
470         INTR_HANDLER(132)
471         INTR_HANDLER(133)
472         INTR_HANDLER(134)
473         INTR_HANDLER(135)
474         INTR_HANDLER(136)
475         INTR_HANDLER(137)
476         INTR_HANDLER(138)
477         INTR_HANDLER(139)
478         INTR_HANDLER(140)
479         INTR_HANDLER(141)
480         INTR_HANDLER(142)
481         INTR_HANDLER(143)
482         INTR_HANDLER(144)
483         INTR_HANDLER(145)
484         INTR_HANDLER(146)
485         INTR_HANDLER(147)
486         INTR_HANDLER(148)
487         INTR_HANDLER(149)
488         INTR_HANDLER(150)
489         INTR_HANDLER(151)
490         INTR_HANDLER(152)
491         INTR_HANDLER(153)
492         INTR_HANDLER(154)
493         INTR_HANDLER(155)
494         INTR_HANDLER(156)
495         INTR_HANDLER(157)
496         INTR_HANDLER(158)
497         INTR_HANDLER(159)
498         INTR_HANDLER(160)
499         INTR_HANDLER(161)
500         INTR_HANDLER(162)
501         INTR_HANDLER(163)
502         INTR_HANDLER(164)
503         INTR_HANDLER(165)
504         INTR_HANDLER(166)
505         INTR_HANDLER(167)
506         INTR_HANDLER(168)
507         INTR_HANDLER(169)
508         INTR_HANDLER(170)
509         INTR_HANDLER(171)
510         INTR_HANDLER(172)
511         INTR_HANDLER(173)
512         INTR_HANDLER(174)
513         INTR_HANDLER(175)
514         INTR_HANDLER(176)
515         INTR_HANDLER(177)
516         INTR_HANDLER(178)
517         INTR_HANDLER(179)
518         INTR_HANDLER(180)
519         INTR_HANDLER(181)
520         INTR_HANDLER(182)
521         INTR_HANDLER(183)
522         INTR_HANDLER(184)
523         INTR_HANDLER(185)
524         INTR_HANDLER(186)
525         INTR_HANDLER(187)
526         INTR_HANDLER(188)
527         INTR_HANDLER(189)
528         INTR_HANDLER(190)
529         INTR_HANDLER(191)
530 MCOUNT_LABEL(eintr)
531
532         .data
533
534 /* variables used by stop_cpus()/restart_cpus()/Xcpustop */
535         .globl stopped_cpus, started_cpus
536 stopped_cpus:
537         .quad   0
538 started_cpus:
539         .quad   0
540
541         .globl CNAME(cpustop_restartfunc)
542 CNAME(cpustop_restartfunc):
543         .quad 0
544                 
545         .text
546