f9de7807391261cbd46130a837b48643def7a334
[dragonfly.git] / sys / platform / pc64 / apic / apic_vector.s
1 /*
2  *      from: vector.s, 386BSD 0.1 unknown origin
3  * $FreeBSD: src/sys/i386/isa/apic_vector.s,v 1.47.2.5 2001/09/01 22:33:38 tegge Exp $
4  */
5
6 #if 0
7 #include "opt_auto_eoi.h"
8 #endif
9
10 #include <machine/asmacros.h>
11 #include <machine/lock.h>
12 #include <machine/psl.h>
13 #include <machine/trap.h>
14 #include <machine/segments.h>
15
16 #include <machine_base/icu/icu.h>
17 #include <bus/isa/isa.h>
18
19 #include "assym.s"
20
21 #include "apicreg.h"
22 #include <machine_base/apic/ioapic_ipl.h>
23 #include <machine/intr_machdep.h>
24
25 #ifdef foo
26 /* convert an absolute IRQ# into bitmask */
27 #define IRQ_LBIT(irq_num)       (1UL << (irq_num & 0x3f))
28 #endif
29
30 #define IRQ_SBITS(irq_num)      ((irq_num) & 0x3f)
31
32 /* convert an absolute IRQ# into gd_ipending index */
33 #define IRQ_LIDX(irq_num)       ((irq_num) >> 6)
34
35 #ifdef SMP
36 #define MPLOCKED     lock ;
37 #else
38 #define MPLOCKED
39 #endif
40
41 #define APIC_PUSH_FRAME                                                 \
42         PUSH_FRAME ;            /* 15 regs + space for 5 extras */      \
43         movq $0,TF_XFLAGS(%rsp) ;                                       \
44         movq $0,TF_TRAPNO(%rsp) ;                                       \
45         movq $0,TF_ADDR(%rsp) ;                                         \
46         movq $0,TF_FLAGS(%rsp) ;                                        \
47         movq $0,TF_ERR(%rsp) ;                                          \
48         cld ;                                                           \
49
50 /*
51  * JG stale? Warning: POP_FRAME can only be used if there is no chance of a
52  * segment register being changed (e.g. by procfs), which is why syscalls
53  * have to use doreti.
54  */
55 #define APIC_POP_FRAME POP_FRAME
56
57 #define IOAPICADDR(irq_num) \
58         CNAME(ioapic_irqs) + IOAPIC_IRQI_SIZE * (irq_num) + IOAPIC_IRQI_ADDR
59 #define REDIRIDX(irq_num) \
60         CNAME(ioapic_irqs) + IOAPIC_IRQI_SIZE * (irq_num) + IOAPIC_IRQI_IDX
61 #define IOAPICFLAGS(irq_num) \
62         CNAME(ioapic_irqs) + IOAPIC_IRQI_SIZE * (irq_num) + IOAPIC_IRQI_FLAGS
63  
64 #define MASK_IRQ(irq_num)                                               \
65         IOAPIC_IMASK_LOCK ;                     /* into critical reg */ \
66         testl   $IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ;        \
67         jne     7f ;                    /* masked, don't mask */        \
68         orl     $IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ;        \
69                                                 /* set the mask bit */  \
70         movq    IOAPICADDR(irq_num), %rcx ;     /* ioapic addr */       \
71         movl    REDIRIDX(irq_num), %eax ;       /* get the index */     \
72         movl    %eax, (%rcx) ;                  /* write the index */   \
73         orl     $IOART_INTMASK,IOAPIC_WINDOW(%rcx) ;/* set the mask */  \
74 7: ;                                            /* already masked */    \
75         IOAPIC_IMASK_UNLOCK ;                                           \
76
77 /*
78  * Test to see whether we are handling an edge or level triggered INT.
79  *  Level-triggered INTs must still be masked as we don't clear the source,
80  *  and the EOI cycle would cause redundant INTs to occur.
81  */
82 #define MASK_LEVEL_IRQ(irq_num)                                         \
83         testl   $IOAPIC_IRQI_FLAG_LEVEL, IOAPICFLAGS(irq_num) ;         \
84         jz      9f ;                            /* edge, don't mask */  \
85         MASK_IRQ(irq_num) ;                                             \
86 9: ;                                                                    \
87
88 /*
89  * Test to see if the source is currntly masked, clear if so.
90  */
91 #define UNMASK_IRQ(irq_num)                                     \
92         cmpl    $0,%eax ;                                               \
93         jnz     8f ;                                                    \
94         IOAPIC_IMASK_LOCK ;                     /* into critical reg */ \
95         testl   $IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ;        \
96         je      7f ;                    /* bit clear, not masked */     \
97         andl    $~IOAPIC_IRQI_FLAG_MASKED, IOAPICFLAGS(irq_num) ;       \
98                                                 /* clear mask bit */    \
99         movq    IOAPICADDR(irq_num),%rcx ;      /* ioapic addr */       \
100         movl    REDIRIDX(irq_num), %eax ;       /* get the index */     \
101         movl    %eax,(%rcx) ;                   /* write the index */   \
102         andl    $~IOART_INTMASK,IOAPIC_WINDOW(%rcx) ;/* clear the mask */ \
103 7: ;                                                                    \
104         IOAPIC_IMASK_UNLOCK ;                                           \
105 8: ;                                                                    \
106
107 /*
108  * Interrupt call handlers run in the following sequence:
109  *
110  *      - Push the trap frame required by doreti
111  *      - Mask the interrupt and reenable its source
112  *      - If we cannot take the interrupt set its ipending bit and
113  *        doreti.
114  *      - If we can take the interrupt clear its ipending bit,
115  *        call the handler, then unmask and doreti.
116  *
117  * YYY can cache gd base opitner instead of using hidden %fs prefixes.
118  */
119
120 #define INTR_HANDLER(irq_num)                                           \
121         .text ;                                                         \
122         SUPERALIGN_TEXT ;                                               \
123 IDTVEC(ioapic_intr##irq_num) ;                                          \
124         APIC_PUSH_FRAME ;                                               \
125         FAKE_MCOUNT(TF_RIP(%rsp)) ;                                     \
126         MASK_LEVEL_IRQ(irq_num) ;                                       \
127         movq    lapic, %rax ;                                           \
128         movl    $0, LA_EOI(%rax) ;                                      \
129         movq    PCPU(curthread),%rbx ;                                  \
130         testl   $-1,TD_NEST_COUNT(%rbx) ;                               \
131         jne     1f ;                                                    \
132         testl   $-1,TD_CRITCOUNT(%rbx) ;                                \
133         je      2f ;                                                    \
134 1: ;                                                                    \
135         /* in critical section, make interrupt pending */               \
136         /* set the pending bit and return, leave interrupt masked */    \
137         movq    $1,%rcx ;                                               \
138         shlq    $IRQ_SBITS(irq_num),%rcx ;                              \
139         movq    $IRQ_LIDX(irq_num),%rdx ;                               \
140         orq     %rcx,PCPU_E8(ipending,%rdx) ;                           \
141         orl     $RQF_INTPEND,PCPU(reqflags) ;                           \
142         jmp     5f ;                                                    \
143 2: ;                                                                    \
144         /* clear pending bit, run handler */                            \
145         movq    $1,%rcx ;                                               \
146         shlq    $IRQ_SBITS(irq_num),%rcx ;                              \
147         notq    %rcx ;                                                  \
148         movq    $IRQ_LIDX(irq_num),%rdx ;                               \
149         andq    %rcx,PCPU_E8(ipending,%rdx) ;                           \
150         pushq   $irq_num ;              /* trapframe -> intrframe */    \
151         movq    %rsp, %rdi ;            /* pass frame by reference */   \
152         incl    TD_CRITCOUNT(%rbx) ;                                    \
153         sti ;                                                           \
154         call    ithread_fast_handler ;  /* returns 0 to unmask */       \
155         decl    TD_CRITCOUNT(%rbx) ;                                    \
156         addq    $8, %rsp ;              /* intrframe -> trapframe */    \
157         UNMASK_IRQ(irq_num) ;                                           \
158 5: ;                                                                    \
159         MEXITCOUNT ;                                                    \
160         jmp     doreti ;                                                \
161
162 /*
163  * Handle "spurious INTerrupts".
164  * Notes:
165  *  This is different than the "spurious INTerrupt" generated by an
166  *   8259 PIC for missing INTs.  See the APIC documentation for details.
167  *  This routine should NOT do an 'EOI' cycle.
168  */
169         .text
170         SUPERALIGN_TEXT
171         .globl Xspuriousint
172 Xspuriousint:
173
174         /* No EOI cycle used here */
175
176         jmp     doreti_iret
177
178 #ifdef SMP
179
180 /*
181  * Handle TLB shootdowns.
182  *
183  * NOTE: interrupts are left disabled.
184  */
185         .text
186         SUPERALIGN_TEXT
187         .globl  Xinvltlb
188 Xinvltlb:
189         APIC_PUSH_FRAME
190         movq    lapic, %rax
191         movl    $0, LA_EOI(%rax)        /* End Of Interrupt to APIC */
192         FAKE_MCOUNT(TF_RIP(%rsp))
193         subq    $8,%rsp                 /* make same as interrupt frame */
194         movq    %rsp,%rdi               /* pass frame by reference */
195         call    smp_invltlb_intr
196         addq    $8,%rsp                 /* turn into trapframe */
197         MEXITCOUNT
198         APIC_POP_FRAME
199         jmp     doreti_iret
200
201 /*
202  * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
203  *
204  *  - We cannot call doreti
205  *  - Signals its receipt.
206  *  - Waits for permission to restart.
207  *  - Processing pending IPIQ events while waiting.
208  *  - Signals its restart.
209  */
210
211         .text
212         SUPERALIGN_TEXT
213         .globl Xcpustop
214 Xcpustop:
215         APIC_PUSH_FRAME
216         movq    lapic, %rax
217         movl    $0, LA_EOI(%rax)        /* End Of Interrupt to APIC */
218
219         movl    PCPU(cpuid), %eax
220         imull   $PCB_SIZE, %eax
221         leaq    CNAME(stoppcbs), %rdi
222         addq    %rax, %rdi
223         call    CNAME(savectx)          /* Save process context */
224
225         movslq  PCPU(cpuid), %rax
226
227         /*
228          * Indicate that we have stopped and loop waiting for permission
229          * to start again.  We must still process IPI events while in a
230          * stopped state.
231          *
232          * Interrupts must remain enabled for non-IPI'd per-cpu interrupts
233          * (e.g. Xtimer, Xinvltlb).
234          */
235         MPLOCKED
236         btsq    %rax, stopped_cpus      /* stopped_cpus |= (1<<id) */
237         sti
238 1:
239         andl    $~RQF_IPIQ,PCPU(reqflags)
240         pushq   %rax
241         call    lwkt_smp_stopped
242         popq    %rax
243         pause
244         btq     %rax, started_cpus      /* while (!(started_cpus & (1<<id))) */
245         jnc     1b
246
247         MPLOCKED
248         btrq    %rax, started_cpus      /* started_cpus &= ~(1<<id) */
249         MPLOCKED
250         btrq    %rax, stopped_cpus      /* stopped_cpus &= ~(1<<id) */
251
252         test    %eax, %eax
253         jnz     2f
254
255         movq    CNAME(cpustop_restartfunc), %rax
256         test    %rax, %rax
257         jz      2f
258         movq    $0, CNAME(cpustop_restartfunc)  /* One-shot */
259
260         call    *%rax
261 2:
262         MEXITCOUNT
263         APIC_POP_FRAME
264         jmp     doreti_iret
265
266         /*
267          * For now just have one ipiq IPI, but what we really want is
268          * to have one for each source cpu to the APICs don't get stalled
269          * backlogging the requests.
270          */
271         .text
272         SUPERALIGN_TEXT
273         .globl Xipiq
274 Xipiq:
275         APIC_PUSH_FRAME
276         movq    lapic, %rax
277         movl    $0, LA_EOI(%rax)        /* End Of Interrupt to APIC */
278         FAKE_MCOUNT(TF_RIP(%rsp))
279
280         incl    PCPU(cnt) + V_IPI
281         movq    PCPU(curthread),%rbx
282         testl   $-1,TD_CRITCOUNT(%rbx)
283         jne     1f
284         subq    $8,%rsp                 /* make same as interrupt frame */
285         movq    %rsp,%rdi               /* pass frame by reference */
286         incl    PCPU(intr_nesting_level)
287         incl    TD_CRITCOUNT(%rbx)
288         sti
289         call    lwkt_process_ipiq_frame
290         decl    TD_CRITCOUNT(%rbx)
291         decl    PCPU(intr_nesting_level)
292         addq    $8,%rsp                 /* turn into trapframe */
293         MEXITCOUNT
294         jmp     doreti
295 1:
296         orl     $RQF_IPIQ,PCPU(reqflags)
297         MEXITCOUNT
298         APIC_POP_FRAME
299         jmp     doreti_iret
300
301 #endif  /* SMP */
302
303         .text
304         SUPERALIGN_TEXT
305         .globl Xtimer
306 Xtimer:
307         APIC_PUSH_FRAME
308         movq    lapic, %rax
309         movl    $0, LA_EOI(%rax)        /* End Of Interrupt to APIC */
310         FAKE_MCOUNT(TF_RIP(%rsp))
311
312         subq    $8,%rsp                 /* make same as interrupt frame */
313         movq    %rsp,%rdi               /* pass frame by reference */
314         call    lapic_timer_always
315         addq    $8,%rsp                 /* turn into trapframe */
316
317         incl    PCPU(cnt) + V_TIMER
318         movq    PCPU(curthread),%rbx
319         testl   $-1,TD_CRITCOUNT(%rbx)
320         jne     1f
321         testl   $-1,TD_NEST_COUNT(%rbx)
322         jne     1f
323         subq    $8,%rsp                 /* make same as interrupt frame */
324         movq    %rsp,%rdi               /* pass frame by reference */
325         incl    PCPU(intr_nesting_level)
326         incl    TD_CRITCOUNT(%rbx)
327         sti
328         call    lapic_timer_process_frame
329         decl    TD_CRITCOUNT(%rbx)
330         decl    PCPU(intr_nesting_level)
331         addq    $8,%rsp                 /* turn into trapframe */
332         MEXITCOUNT
333         jmp     doreti
334 1:
335         orl     $RQF_TIMER,PCPU(reqflags)
336         MEXITCOUNT
337         APIC_POP_FRAME
338         jmp     doreti_iret
339
340 MCOUNT_LABEL(bintr)
341         INTR_HANDLER(0)
342         INTR_HANDLER(1)
343         INTR_HANDLER(2)
344         INTR_HANDLER(3)
345         INTR_HANDLER(4)
346         INTR_HANDLER(5)
347         INTR_HANDLER(6)
348         INTR_HANDLER(7)
349         INTR_HANDLER(8)
350         INTR_HANDLER(9)
351         INTR_HANDLER(10)
352         INTR_HANDLER(11)
353         INTR_HANDLER(12)
354         INTR_HANDLER(13)
355         INTR_HANDLER(14)
356         INTR_HANDLER(15)
357         INTR_HANDLER(16)
358         INTR_HANDLER(17)
359         INTR_HANDLER(18)
360         INTR_HANDLER(19)
361         INTR_HANDLER(20)
362         INTR_HANDLER(21)
363         INTR_HANDLER(22)
364         INTR_HANDLER(23)
365         INTR_HANDLER(24)
366         INTR_HANDLER(25)
367         INTR_HANDLER(26)
368         INTR_HANDLER(27)
369         INTR_HANDLER(28)
370         INTR_HANDLER(29)
371         INTR_HANDLER(30)
372         INTR_HANDLER(31)
373         INTR_HANDLER(32)
374         INTR_HANDLER(33)
375         INTR_HANDLER(34)
376         INTR_HANDLER(35)
377         INTR_HANDLER(36)
378         INTR_HANDLER(37)
379         INTR_HANDLER(38)
380         INTR_HANDLER(39)
381         INTR_HANDLER(40)
382         INTR_HANDLER(41)
383         INTR_HANDLER(42)
384         INTR_HANDLER(43)
385         INTR_HANDLER(44)
386         INTR_HANDLER(45)
387         INTR_HANDLER(46)
388         INTR_HANDLER(47)
389         INTR_HANDLER(48)
390         INTR_HANDLER(49)
391         INTR_HANDLER(50)
392         INTR_HANDLER(51)
393         INTR_HANDLER(52)
394         INTR_HANDLER(53)
395         INTR_HANDLER(54)
396         INTR_HANDLER(55)
397         INTR_HANDLER(56)
398         INTR_HANDLER(57)
399         INTR_HANDLER(58)
400         INTR_HANDLER(59)
401         INTR_HANDLER(60)
402         INTR_HANDLER(61)
403         INTR_HANDLER(62)
404         INTR_HANDLER(63)
405         INTR_HANDLER(64)
406         INTR_HANDLER(65)
407         INTR_HANDLER(66)
408         INTR_HANDLER(67)
409         INTR_HANDLER(68)
410         INTR_HANDLER(69)
411         INTR_HANDLER(70)
412         INTR_HANDLER(71)
413         INTR_HANDLER(72)
414         INTR_HANDLER(73)
415         INTR_HANDLER(74)
416         INTR_HANDLER(75)
417         INTR_HANDLER(76)
418         INTR_HANDLER(77)
419         INTR_HANDLER(78)
420         INTR_HANDLER(79)
421         INTR_HANDLER(80)
422         INTR_HANDLER(81)
423         INTR_HANDLER(82)
424         INTR_HANDLER(83)
425         INTR_HANDLER(84)
426         INTR_HANDLER(85)
427         INTR_HANDLER(86)
428         INTR_HANDLER(87)
429         INTR_HANDLER(88)
430         INTR_HANDLER(89)
431         INTR_HANDLER(90)
432         INTR_HANDLER(91)
433         INTR_HANDLER(92)
434         INTR_HANDLER(93)
435         INTR_HANDLER(94)
436         INTR_HANDLER(95)
437         INTR_HANDLER(96)
438         INTR_HANDLER(97)
439         INTR_HANDLER(98)
440         INTR_HANDLER(99)
441         INTR_HANDLER(100)
442         INTR_HANDLER(101)
443         INTR_HANDLER(102)
444         INTR_HANDLER(103)
445         INTR_HANDLER(104)
446         INTR_HANDLER(105)
447         INTR_HANDLER(106)
448         INTR_HANDLER(107)
449         INTR_HANDLER(108)
450         INTR_HANDLER(109)
451         INTR_HANDLER(110)
452         INTR_HANDLER(111)
453         INTR_HANDLER(112)
454         INTR_HANDLER(113)
455         INTR_HANDLER(114)
456         INTR_HANDLER(115)
457         INTR_HANDLER(116)
458         INTR_HANDLER(117)
459         INTR_HANDLER(118)
460         INTR_HANDLER(119)
461         INTR_HANDLER(120)
462         INTR_HANDLER(121)
463         INTR_HANDLER(122)
464         INTR_HANDLER(123)
465         INTR_HANDLER(124)
466         INTR_HANDLER(125)
467         INTR_HANDLER(126)
468         INTR_HANDLER(127)
469         INTR_HANDLER(128)
470         INTR_HANDLER(129)
471         INTR_HANDLER(130)
472         INTR_HANDLER(131)
473         INTR_HANDLER(132)
474         INTR_HANDLER(133)
475         INTR_HANDLER(134)
476         INTR_HANDLER(135)
477         INTR_HANDLER(136)
478         INTR_HANDLER(137)
479         INTR_HANDLER(138)
480         INTR_HANDLER(139)
481         INTR_HANDLER(140)
482         INTR_HANDLER(141)
483         INTR_HANDLER(142)
484         INTR_HANDLER(143)
485         INTR_HANDLER(144)
486         INTR_HANDLER(145)
487         INTR_HANDLER(146)
488         INTR_HANDLER(147)
489         INTR_HANDLER(148)
490         INTR_HANDLER(149)
491         INTR_HANDLER(150)
492         INTR_HANDLER(151)
493         INTR_HANDLER(152)
494         INTR_HANDLER(153)
495         INTR_HANDLER(154)
496         INTR_HANDLER(155)
497         INTR_HANDLER(156)
498         INTR_HANDLER(157)
499         INTR_HANDLER(158)
500         INTR_HANDLER(159)
501         INTR_HANDLER(160)
502         INTR_HANDLER(161)
503         INTR_HANDLER(162)
504         INTR_HANDLER(163)
505         INTR_HANDLER(164)
506         INTR_HANDLER(165)
507         INTR_HANDLER(166)
508         INTR_HANDLER(167)
509         INTR_HANDLER(168)
510         INTR_HANDLER(169)
511         INTR_HANDLER(170)
512         INTR_HANDLER(171)
513         INTR_HANDLER(172)
514         INTR_HANDLER(173)
515         INTR_HANDLER(174)
516         INTR_HANDLER(175)
517         INTR_HANDLER(176)
518         INTR_HANDLER(177)
519         INTR_HANDLER(178)
520         INTR_HANDLER(179)
521         INTR_HANDLER(180)
522         INTR_HANDLER(181)
523         INTR_HANDLER(182)
524         INTR_HANDLER(183)
525         INTR_HANDLER(184)
526         INTR_HANDLER(185)
527         INTR_HANDLER(186)
528         INTR_HANDLER(187)
529         INTR_HANDLER(188)
530         INTR_HANDLER(189)
531         INTR_HANDLER(190)
532         INTR_HANDLER(191)
533 MCOUNT_LABEL(eintr)
534
535         .data
536
537 /* variables used by stop_cpus()/restart_cpus()/Xcpustop */
538         .globl stopped_cpus, started_cpus
539 stopped_cpus:
540         .quad   0
541 started_cpus:
542         .quad   0
543
544         .globl CNAME(cpustop_restartfunc)
545 CNAME(cpustop_restartfunc):
546         .quad 0
547                 
548         .text
549