Merge branch 'apic_io'
[dragonfly.git] / sys / platform / pc64 / apic / apic_vector.s
1 /*
2  *      from: vector.s, 386BSD 0.1 unknown origin
3  * $FreeBSD: src/sys/i386/isa/apic_vector.s,v 1.47.2.5 2001/09/01 22:33:38 tegge Exp $
4  */
5
6 #if 0
7 #include "opt_auto_eoi.h"
8 #endif
9
10 #include <machine/asmacros.h>
11 #include <machine/lock.h>
12 #include <machine/psl.h>
13 #include <machine/trap.h>
14 #include <machine/segments.h>
15
16 #include <machine_base/icu/icu.h>
17 #include <bus/isa/isa.h>
18
19 #include "assym.s"
20
21 #include "apicreg.h"
22 #include "apic_ipl.h"
23 #include <machine/smp.h>
24 #include <machine_base/isa/intr_machdep.h>
25
26 /* convert an absolute IRQ# into a bitmask */
27 #define IRQ_LBIT(irq_num)       (1 << (irq_num))
28
29 /* make an index into the IO APIC from the IRQ# */
30 #define REDTBL_IDX(irq_num)     (0x10 + ((irq_num) * 2))
31
32 #ifdef SMP
33 #define MPLOCKED     lock ;
34 #else
35 #define MPLOCKED
36 #endif
37
38 #define APIC_PUSH_FRAME                                                 \
39         PUSH_FRAME ;            /* 15 regs + space for 5 extras */      \
40         movq $0,TF_XFLAGS(%rsp) ;                                       \
41         movq $0,TF_TRAPNO(%rsp) ;                                       \
42         movq $0,TF_ADDR(%rsp) ;                                         \
43         movq $0,TF_FLAGS(%rsp) ;                                        \
44         movq $0,TF_ERR(%rsp) ;                                          \
45         cld ;                                                           \
46
47 /*
48  * JG stale? Warning: POP_FRAME can only be used if there is no chance of a
49  * segment register being changed (e.g. by procfs), which is why syscalls
50  * have to use doreti.
51  */
52 #define APIC_POP_FRAME POP_FRAME
53
54 #define IOAPICADDR(irq_num) \
55         CNAME(int_to_apicintpin) + AIMI_SIZE * (irq_num) + AIMI_APIC_ADDRESS
56 #define REDIRIDX(irq_num) \
57         CNAME(int_to_apicintpin) + AIMI_SIZE * (irq_num) + AIMI_REDIRINDEX
58 #define IOAPICFLAGS(irq_num) \
59         CNAME(int_to_apicintpin) + AIMI_SIZE * (irq_num) + AIMI_FLAGS
60  
61 #define MASK_IRQ(irq_num)                                               \
62         APIC_IMASK_LOCK ;                       /* into critical reg */ \
63         testl   $AIMI_FLAG_MASKED, IOAPICFLAGS(irq_num) ;               \
64         jne     7f ;                    /* masked, don't mask */        \
65         orl     $AIMI_FLAG_MASKED, IOAPICFLAGS(irq_num) ;               \
66                                                 /* set the mask bit */  \
67         movq    IOAPICADDR(irq_num), %rcx ;     /* ioapic addr */       \
68         movl    REDIRIDX(irq_num), %eax ;       /* get the index */     \
69         movl    %eax, (%rcx) ;                  /* write the index */   \
70         orl     $IOART_INTMASK,IOAPIC_WINDOW(%rcx) ;/* set the mask */  \
71 7: ;                                            /* already masked */    \
72         APIC_IMASK_UNLOCK ;                                             \
73
74 /*
75  * Test to see whether we are handling an edge or level triggered INT.
76  *  Level-triggered INTs must still be masked as we don't clear the source,
77  *  and the EOI cycle would cause redundant INTs to occur.
78  */
79 #define MASK_LEVEL_IRQ(irq_num)                                         \
80         testl   $AIMI_FLAG_LEVEL, IOAPICFLAGS(irq_num) ;                \
81         jz      9f ;                            /* edge, don't mask */  \
82         MASK_IRQ(irq_num) ;                                             \
83 9: ;                                                                    \
84
85 /*
86  * Test to see if the source is currntly masked, clear if so.
87  */
88 #define UNMASK_IRQ(irq_num)                                     \
89         cmpl    $0,%eax ;                                               \
90         jnz     8f ;                                                    \
91         APIC_IMASK_LOCK ;                       /* into critical reg */ \
92         testl   $AIMI_FLAG_MASKED, IOAPICFLAGS(irq_num) ;               \
93         je      7f ;                    /* bit clear, not masked */     \
94         andl    $~AIMI_FLAG_MASKED, IOAPICFLAGS(irq_num) ;              \
95                                                 /* clear mask bit */    \
96         movq    IOAPICADDR(irq_num),%rcx ;      /* ioapic addr */       \
97         movl    REDIRIDX(irq_num), %eax ;       /* get the index */     \
98         movl    %eax,(%rcx) ;                   /* write the index */   \
99         andl    $~IOART_INTMASK,IOAPIC_WINDOW(%rcx) ;/* clear the mask */ \
100 7: ;                                                                    \
101         APIC_IMASK_UNLOCK ;                                             \
102 8: ;                                                                    \
103
104 #ifdef SMP /* APIC-IO */
105
106 /*
107  * Fast interrupt call handlers run in the following sequence:
108  *
109  *      - Push the trap frame required by doreti
110  *      - Mask the interrupt and reenable its source
111  *      - If we cannot take the interrupt set its fpending bit and
112  *        doreti.  Note that we cannot mess with mp_lock at all
113  *        if we entered from a critical section!
114  *      - If we can take the interrupt clear its fpending bit,
115  *        call the handler, then unmask and doreti.
116  *
117  * YYY can cache gd base opitner instead of using hidden %fs prefixes.
118  */
119
120 #define FAST_INTR(irq_num, vec_name)                                    \
121         .text ;                                                         \
122         SUPERALIGN_TEXT ;                                               \
123 IDTVEC(vec_name) ;                                                      \
124         APIC_PUSH_FRAME ;                                               \
125         FAKE_MCOUNT(TF_RIP(%rsp)) ;                                     \
126         MASK_LEVEL_IRQ(irq_num) ;                                       \
127         movq    lapic, %rax ;                                           \
128         movl    $0, LA_EOI(%rax) ;                                      \
129         movq    PCPU(curthread),%rbx ;                                  \
130         testl   $-1,TD_NEST_COUNT(%rbx) ;                               \
131         jne     1f ;                                                    \
132         testl   $-1,TD_CRITCOUNT(%rbx) ;                                \
133         je      2f ;                                                    \
134 1: ;                                                                    \
135         /* in critical section, make interrupt pending */               \
136         /* set the pending bit and return, leave interrupt masked */    \
137         orl     $IRQ_LBIT(irq_num),PCPU(fpending) ;                     \
138         orl     $RQF_INTPEND,PCPU(reqflags) ;                           \
139         jmp     5f ;                                                    \
140 2: ;                                                                    \
141         /* clear pending bit, run handler */                            \
142         andl    $~IRQ_LBIT(irq_num),PCPU(fpending) ;                    \
143         pushq   $irq_num ;              /* trapframe -> intrframe */    \
144         movq    %rsp, %rdi ;            /* pass frame by reference */   \
145         incl    TD_CRITCOUNT(%rbx) ;                                    \
146         sti ;                                                           \
147         call    ithread_fast_handler ;  /* returns 0 to unmask */       \
148         decl    TD_CRITCOUNT(%rbx) ;                                    \
149         addq    $8, %rsp ;              /* intrframe -> trapframe */    \
150         UNMASK_IRQ(irq_num) ;                                           \
151 5: ;                                                                    \
152         MEXITCOUNT ;                                                    \
153         jmp     doreti ;                                                \
154
155 #endif
156
157 /*
158  * Handle "spurious INTerrupts".
159  * Notes:
160  *  This is different than the "spurious INTerrupt" generated by an
161  *   8259 PIC for missing INTs.  See the APIC documentation for details.
162  *  This routine should NOT do an 'EOI' cycle.
163  */
164         .text
165         SUPERALIGN_TEXT
166         .globl Xspuriousint
167 Xspuriousint:
168
169         /* No EOI cycle used here */
170
171         jmp     doreti_iret
172
173
174 /*
175  * Handle TLB shootdowns.
176  *
177  * NOTE: interrupts are left disabled.
178  */
179         .text
180         SUPERALIGN_TEXT
181         .globl  Xinvltlb
182 Xinvltlb:
183         APIC_PUSH_FRAME
184         movq    lapic, %rax
185         movl    $0, LA_EOI(%rax)        /* End Of Interrupt to APIC */
186         FAKE_MCOUNT(TF_RIP(%rsp))
187         subq    $8,%rsp                 /* make same as interrupt frame */
188         movq    %rsp,%rdi               /* pass frame by reference */
189         call    smp_invltlb_intr
190         addq    $8,%rsp                 /* turn into trapframe */
191         MEXITCOUNT
192         APIC_POP_FRAME
193         jmp     doreti_iret
194
195 /*
196  * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
197  *
198  *  - We cannot call doreti
199  *  - Signals its receipt.
200  *  - Waits for permission to restart.
201  *  - Processing pending IPIQ events while waiting.
202  *  - Signals its restart.
203  */
204
205         .text
206         SUPERALIGN_TEXT
207         .globl Xcpustop
208 Xcpustop:
209         APIC_PUSH_FRAME
210         movq    lapic, %rax
211         movl    $0, LA_EOI(%rax)        /* End Of Interrupt to APIC */
212
213         movl    PCPU(cpuid), %eax
214         imull   $PCB_SIZE, %eax
215         leaq    CNAME(stoppcbs), %rdi
216         addq    %rax, %rdi
217         call    CNAME(savectx)          /* Save process context */
218
219         movl    PCPU(cpuid), %eax
220
221         /*
222          * Indicate that we have stopped and loop waiting for permission
223          * to start again.  We must still process IPI events while in a
224          * stopped state.
225          *
226          * Interrupts must remain enabled for non-IPI'd per-cpu interrupts
227          * (e.g. Xtimer, Xinvltlb).
228          */
229         MPLOCKED
230         btsl    %eax, stopped_cpus      /* stopped_cpus |= (1<<id) */
231         sti
232 1:
233         andl    $~RQF_IPIQ,PCPU(reqflags)
234         pushq   %rax
235         call    lwkt_smp_stopped
236         popq    %rax
237         pause
238         btl     %eax, started_cpus      /* while (!(started_cpus & (1<<id))) */
239         jnc     1b
240
241         MPLOCKED
242         btrl    %eax, started_cpus      /* started_cpus &= ~(1<<id) */
243         MPLOCKED
244         btrl    %eax, stopped_cpus      /* stopped_cpus &= ~(1<<id) */
245
246         test    %eax, %eax
247         jnz     2f
248
249         movq    CNAME(cpustop_restartfunc), %rax
250         test    %rax, %rax
251         jz      2f
252         movq    $0, CNAME(cpustop_restartfunc)  /* One-shot */
253
254         call    *%rax
255 2:
256         MEXITCOUNT
257         APIC_POP_FRAME
258         jmp     doreti_iret
259
260         /*
261          * For now just have one ipiq IPI, but what we really want is
262          * to have one for each source cpu to the APICs don't get stalled
263          * backlogging the requests.
264          */
265         .text
266         SUPERALIGN_TEXT
267         .globl Xipiq
268 Xipiq:
269         APIC_PUSH_FRAME
270         movq    lapic, %rax
271         movl    $0, LA_EOI(%rax)        /* End Of Interrupt to APIC */
272         FAKE_MCOUNT(TF_RIP(%rsp))
273
274         incl    PCPU(cnt) + V_IPI
275         movq    PCPU(curthread),%rbx
276         testl   $-1,TD_CRITCOUNT(%rbx)
277         jne     1f
278         subq    $8,%rsp                 /* make same as interrupt frame */
279         movq    %rsp,%rdi               /* pass frame by reference */
280         incl    PCPU(intr_nesting_level)
281         incl    TD_CRITCOUNT(%rbx)
282         sti
283         call    lwkt_process_ipiq_frame
284         decl    TD_CRITCOUNT(%rbx)
285         decl    PCPU(intr_nesting_level)
286         addq    $8,%rsp                 /* turn into trapframe */
287         MEXITCOUNT
288         jmp     doreti
289 1:
290         orl     $RQF_IPIQ,PCPU(reqflags)
291         MEXITCOUNT
292         APIC_POP_FRAME
293         jmp     doreti_iret
294
295         .text
296         SUPERALIGN_TEXT
297         .globl Xtimer
298 Xtimer:
299         APIC_PUSH_FRAME
300         movq    lapic, %rax
301         movl    $0, LA_EOI(%rax)        /* End Of Interrupt to APIC */
302         FAKE_MCOUNT(TF_RIP(%rsp))
303
304         incl    PCPU(cnt) + V_TIMER
305         movq    PCPU(curthread),%rbx
306         testl   $-1,TD_CRITCOUNT(%rbx)
307         jne     1f
308         testl   $-1,TD_NEST_COUNT(%rbx)
309         jne     1f
310         subq    $8,%rsp                 /* make same as interrupt frame */
311         movq    %rsp,%rdi               /* pass frame by reference */
312         incl    PCPU(intr_nesting_level)
313         incl    TD_CRITCOUNT(%rbx)
314         sti
315         call    lapic_timer_process_frame
316         decl    TD_CRITCOUNT(%rbx)
317         decl    PCPU(intr_nesting_level)
318         addq    $8,%rsp                 /* turn into trapframe */
319         MEXITCOUNT
320         jmp     doreti
321 1:
322         orl     $RQF_TIMER,PCPU(reqflags)
323         MEXITCOUNT
324         APIC_POP_FRAME
325         jmp     doreti_iret
326
327 #ifdef SMP /* APIC-IO */
328
329 MCOUNT_LABEL(bintr)
330         FAST_INTR(0,apic_fastintr0)
331         FAST_INTR(1,apic_fastintr1)
332         FAST_INTR(2,apic_fastintr2)
333         FAST_INTR(3,apic_fastintr3)
334         FAST_INTR(4,apic_fastintr4)
335         FAST_INTR(5,apic_fastintr5)
336         FAST_INTR(6,apic_fastintr6)
337         FAST_INTR(7,apic_fastintr7)
338         FAST_INTR(8,apic_fastintr8)
339         FAST_INTR(9,apic_fastintr9)
340         FAST_INTR(10,apic_fastintr10)
341         FAST_INTR(11,apic_fastintr11)
342         FAST_INTR(12,apic_fastintr12)
343         FAST_INTR(13,apic_fastintr13)
344         FAST_INTR(14,apic_fastintr14)
345         FAST_INTR(15,apic_fastintr15)
346         FAST_INTR(16,apic_fastintr16)
347         FAST_INTR(17,apic_fastintr17)
348         FAST_INTR(18,apic_fastintr18)
349         FAST_INTR(19,apic_fastintr19)
350         FAST_INTR(20,apic_fastintr20)
351         FAST_INTR(21,apic_fastintr21)
352         FAST_INTR(22,apic_fastintr22)
353         FAST_INTR(23,apic_fastintr23)
354 MCOUNT_LABEL(eintr)
355
356 #endif
357
358         .data
359
360 /* variables used by stop_cpus()/restart_cpus()/Xcpustop */
361         .globl stopped_cpus, started_cpus
362 stopped_cpus:
363         .long   0
364 started_cpus:
365         .long   0
366
367         .globl CNAME(cpustop_restartfunc)
368 CNAME(cpustop_restartfunc):
369         .quad 0
370                 
371         .text
372