i386: Allow UP kernel to use LAPIC timer and I/O APIC
[dragonfly.git] / sys / platform / pc32 / apic / apic_vector.s
1 /*
2  *      from: vector.s, 386BSD 0.1 unknown origin
3  * $FreeBSD: src/sys/i386/isa/apic_vector.s,v 1.47.2.5 2001/09/01 22:33:38 tegge Exp $
4  */
5
6 #include "opt_auto_eoi.h"
7
8 #include <machine/asmacros.h>
9 #include <machine/lock.h>
10 #include <machine/psl.h>
11 #include <machine/trap.h>
12
13 #include <machine_base/icu/icu.h>
14 #include <bus/isa/isa.h>
15
16 #include "assym.s"
17
18 #include "apicreg.h"
19 #include <machine_base/apic/ioapic_ipl.h>
20 #include <machine/intr_machdep.h>
21
22 /* convert an absolute IRQ# into bitmask */
23 #define IRQ_LBIT(irq_num)       (1 << ((irq_num) & 0x1f))
24
25 /* convert an absolute IRQ# into ipending index */
26 #define IRQ_LIDX(irq_num)       ((irq_num) >> 5)
27
28 #ifdef SMP
29 #define MPLOCKED     lock ;
30 #else
31 #define MPLOCKED
32 #endif
33
34 /*
35  * Push an interrupt frame in a format acceptable to doreti, reload
36  * the segment registers for the kernel.
37  */
38 #define PUSH_FRAME                                                      \
39         pushl   $0 ;            /* dummy error code */                  \
40         pushl   $0 ;            /* dummy trap type */                   \
41         pushl   $0 ;            /* dummy xflags type */                 \
42         pushal ;                                                        \
43         pushl   %ds ;           /* save data and extra segments ... */  \
44         pushl   %es ;                                                   \
45         pushl   %fs ;                                                   \
46         pushl   %gs ;                                                   \
47         cld ;                                                           \
48         mov     $KDSEL,%ax ;                                            \
49         mov     %ax,%ds ;                                               \
50         mov     %ax,%es ;                                               \
51         mov     %ax,%gs ;                                               \
52         mov     $KPSEL,%ax ;                                            \
53         mov     %ax,%fs ;                                               \
54
55 /*
56  * Warning: POP_FRAME can only be used if there is no chance of a
57  * segment register being changed (e.g. by procfs), which is why syscalls
58  * have to use doreti.
59  */
60 #define POP_FRAME                                                       \
61         popl    %gs ;                                                   \
62         popl    %fs ;                                                   \
63         popl    %es ;                                                   \
64         popl    %ds ;                                                   \
65         popal ;                                                         \
66         addl    $3*4,%esp ;     /* dummy xflags, trap & error codes */  \
67
68 #define IOAPICADDR(irq_num) \
69         CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_ADDR
70 #define REDIRIDX(irq_num) \
71         CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_ENTIDX
72 #define IOAPICFLAGS(irq_num) \
73         CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_FLAGS
74
75 #define MASK_IRQ(irq_num)                                               \
76         IOAPIC_IMASK_LOCK ;                     /* into critical reg */ \
77         testl   $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ;          \
78         jne     7f ;                    /* masked, don't mask */        \
79         orl     $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ;          \
80                                                 /* set the mask bit */  \
81         movl    IOAPICADDR(irq_num), %ecx ;     /* ioapic addr */       \
82         movl    REDIRIDX(irq_num), %eax ;       /* get the index */     \
83         movl    %eax, (%ecx) ;                  /* write the index */   \
84         orl     $IOART_INTMASK,IOAPIC_WINDOW(%ecx) ;/* set the mask */  \
85 7: ;                                            /* already masked */    \
86         IOAPIC_IMASK_UNLOCK ;                                           \
87
88 /*
89  * Test to see whether we are handling an edge or level triggered INT.
90  *  Level-triggered INTs must still be masked as we don't clear the source,
91  *  and the EOI cycle would cause redundant INTs to occur.
92  */
93 #define MASK_LEVEL_IRQ(irq_num)                                         \
94         testl   $IOAPIC_IM_FLAG_LEVEL, IOAPICFLAGS(irq_num) ;           \
95         jz      9f ;                            /* edge, don't mask */  \
96         MASK_IRQ(irq_num) ;                                             \
97 9: ;                                                                    \
98
99 /*
100  * Test to see if the source is currntly masked, clear if so.
101  */
102 #define UNMASK_IRQ(irq_num)                                             \
103         cmpl    $0,%eax ;                                               \
104         jnz     8f ;                                                    \
105         IOAPIC_IMASK_LOCK ;                     /* into critical reg */ \
106         testl   $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ;          \
107         je      7f ;                    /* bit clear, not masked */     \
108         andl    $~IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ;         \
109                                                 /* clear mask bit */    \
110         movl    IOAPICADDR(irq_num),%ecx ;      /* ioapic addr */       \
111         movl    REDIRIDX(irq_num), %eax ;       /* get the index */     \
112         movl    %eax,(%ecx) ;                   /* write the index */   \
113         andl    $~IOART_INTMASK,IOAPIC_WINDOW(%ecx) ;/* clear the mask */ \
114 7: ;                                                                    \
115         IOAPIC_IMASK_UNLOCK ;                                           \
116 8: ;                                                                    \
117
118 /*
119  * Interrupt call handlers run in the following sequence:
120  *
121  *      - Push the trap frame required by doreti
122  *      - Mask the interrupt and reenable its source
123  *      - If we cannot take the interrupt set its ipending bit and
124  *        doreti.
125  *      - If we can take the interrupt clear its ipending bit,
126  *        call the handler, then unmask and doreti.
127  *
128  * YYY can cache gd base opitner instead of using hidden %fs prefixes.
129  */
130
131 #define INTR_HANDLER(irq_num)                                           \
132         .text ;                                                         \
133         SUPERALIGN_TEXT ;                                               \
134 IDTVEC(ioapic_intr##irq_num) ;                                          \
135         PUSH_FRAME ;                                                    \
136         FAKE_MCOUNT(15*4(%esp)) ;                                       \
137         MASK_LEVEL_IRQ(irq_num) ;                                       \
138         movl    lapic,%eax ;                                            \
139         movl    $0,LA_EOI(%eax) ;                                       \
140         movl    PCPU(curthread),%ebx ;                                  \
141         movl    $0,%eax ;       /* CURRENT CPL IN FRAME (REMOVED) */    \
142         pushl   %eax ;                                                  \
143         testl   $-1,TD_NEST_COUNT(%ebx) ;                               \
144         jne     1f ;                                                    \
145         testl   $-1,TD_CRITCOUNT(%ebx) ;                                \
146         je      2f ;                                                    \
147 1: ;                                                                    \
148         /* in critical section, make interrupt pending */               \
149         /* set the pending bit and return, leave interrupt masked */    \
150         movl    $IRQ_LIDX(irq_num),%edx ;                               \
151         orl     $IRQ_LBIT(irq_num),PCPU_E4(ipending,%edx) ;             \
152         orl     $RQF_INTPEND,PCPU(reqflags) ;                           \
153         jmp     5f ;                                                    \
154 2: ;                                                                    \
155         /* clear pending bit, run handler */                            \
156         movl    $IRQ_LIDX(irq_num),%edx ;                               \
157         andl    $~IRQ_LBIT(irq_num),PCPU_E4(ipending,%edx) ;            \
158         pushl   $irq_num ;                                              \
159         pushl   %esp ;                   /* pass frame by reference */  \
160         incl    TD_CRITCOUNT(%ebx) ;                                    \
161         sti ;                                                           \
162         call    ithread_fast_handler ;   /* returns 0 to unmask */      \
163         decl    TD_CRITCOUNT(%ebx) ;                                    \
164         addl    $8, %esp ;                                              \
165         UNMASK_IRQ(irq_num) ;                                           \
166 5: ;                                                                    \
167         MEXITCOUNT ;                                                    \
168         jmp     doreti ;                                                \
169
170 /*
171  * Handle "spurious INTerrupts".
172  * Notes:
173  *  This is different than the "spurious INTerrupt" generated by an
174  *   8259 PIC for missing INTs.  See the APIC documentation for details.
175  *  This routine should NOT do an 'EOI' cycle.
176  */
177         .text
178         SUPERALIGN_TEXT
179         .globl Xspuriousint
180 Xspuriousint:
181
182         /* No EOI cycle used here */
183
184         iret
185
186 #ifdef SMP
187
188 /*
189  * Handle TLB shootdowns.
190  *
191  * NOTE: Interrupts remain disabled.
192  */
193         .text
194         SUPERALIGN_TEXT
195         .globl  Xinvltlb
196 Xinvltlb:
197         PUSH_FRAME
198         movl    lapic,%eax
199         movl    $0,LA_EOI(%eax)         /* End Of Interrupt to APIC */
200         FAKE_MCOUNT(15*4(%esp))
201
202         subl    $8,%esp                 /* make same as interrupt frame */
203         pushl   %esp                    /* pass frame by reference */
204         call    smp_invltlb_intr
205         addl    $12,%esp
206
207         MEXITCOUNT
208         jmp     doreti_syscall_ret
209
210 /*
211  * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
212  *
213  *  - Signals its receipt.
214  *  - Waits for permission to restart.
215  *  - Processing pending IPIQ events while waiting.
216  *  - Signals its restart.
217  */
218
219         .text
220         SUPERALIGN_TEXT
221         .globl Xcpustop
222 Xcpustop:
223         pushl   %ebp
224         movl    %esp, %ebp
225         pushl   %eax
226         pushl   %ecx
227         pushl   %edx
228         pushl   %ds                     /* save current data segment */
229         pushl   %fs
230
231         movl    $KDSEL, %eax
232         mov     %ax, %ds                /* use KERNEL data segment */
233         movl    $KPSEL, %eax
234         mov     %ax, %fs
235
236         movl    lapic, %eax
237         movl    $0, LA_EOI(%eax)        /* End Of Interrupt to APIC */
238
239         movl    PCPU(cpuid), %eax
240         imull   $PCB_SIZE, %eax
241         leal    CNAME(stoppcbs)(%eax), %eax
242         pushl   %eax
243         call    CNAME(savectx)          /* Save process context */
244         addl    $4, %esp
245         
246                 
247         movl    PCPU(cpuid), %eax
248
249         /*
250          * Indicate that we have stopped and loop waiting for permission
251          * to start again.  We must still process IPI events while in a
252          * stopped state.
253          *
254          * Interrupts must remain enabled for non-IPI'd per-cpu interrupts
255          * (e.g. Xtimer, Xinvltlb).
256          */
257         MPLOCKED
258         btsl    %eax, stopped_cpus      /* stopped_cpus |= (1<<id) */
259         sti
260 1:
261         andl    $~RQF_IPIQ,PCPU(reqflags)
262         pushl   %eax
263         call    lwkt_smp_stopped
264         popl    %eax
265         btl     %eax, started_cpus      /* while (!(started_cpus & (1<<id))) */
266         jnc     1b
267
268         MPLOCKED
269         btrl    %eax, started_cpus      /* started_cpus &= ~(1<<id) */
270         MPLOCKED
271         btrl    %eax, stopped_cpus      /* stopped_cpus &= ~(1<<id) */
272
273         test    %eax, %eax
274         jnz     2f
275
276         movl    CNAME(cpustop_restartfunc), %eax
277         test    %eax, %eax
278         jz      2f
279         movl    $0, CNAME(cpustop_restartfunc)  /* One-shot */
280
281         call    *%eax
282 2:
283         popl    %fs
284         popl    %ds                     /* restore previous data segment */
285         popl    %edx
286         popl    %ecx
287         popl    %eax
288         movl    %ebp, %esp
289         popl    %ebp
290         iret
291
292         /*
293          * For now just have one ipiq IPI, but what we really want is
294          * to have one for each source cpu to the APICs don't get stalled
295          * backlogging the requests.
296          */
297         .text
298         SUPERALIGN_TEXT
299         .globl Xipiq
300 Xipiq:
301         PUSH_FRAME
302         movl    lapic,%eax
303         movl    $0,LA_EOI(%eax)         /* End Of Interrupt to APIC */
304         FAKE_MCOUNT(15*4(%esp))
305
306         incl    PCPU(cnt) + V_IPI
307         movl    PCPU(curthread),%ebx
308         testl   $-1,TD_CRITCOUNT(%ebx)
309         jne     1f
310         subl    $8,%esp                 /* make same as interrupt frame */
311         pushl   %esp                    /* pass frame by reference */
312         incl    PCPU(intr_nesting_level)
313         incl    TD_CRITCOUNT(%ebx)
314         sti
315         call    lwkt_process_ipiq_frame
316         decl    TD_CRITCOUNT(%ebx)
317         decl    PCPU(intr_nesting_level)
318         addl    $12,%esp
319         pushl   $0                      /* CPL for frame (REMOVED) */
320         MEXITCOUNT
321         jmp     doreti
322 1:
323         orl     $RQF_IPIQ,PCPU(reqflags)
324         MEXITCOUNT
325         jmp     doreti_syscall_ret
326
327 #endif  /* SMP */
328
329         .text
330         SUPERALIGN_TEXT
331         .globl Xtimer
332 Xtimer:
333         PUSH_FRAME
334         movl    lapic,%eax
335         movl    $0,LA_EOI(%eax)         /* End Of Interrupt to APIC */
336         FAKE_MCOUNT(15*4(%esp))
337
338         incl    PCPU(cnt) + V_TIMER
339         movl    PCPU(curthread),%ebx
340         testl   $-1,TD_CRITCOUNT(%ebx)
341         jne     1f
342         testl   $-1,TD_NEST_COUNT(%ebx)
343         jne     1f
344         subl    $8,%esp                 /* make same as interrupt frame */
345         pushl   %esp                    /* pass frame by reference */
346         incl    PCPU(intr_nesting_level)
347         incl    TD_CRITCOUNT(%ebx)
348         sti
349         call    lapic_timer_process_frame
350         decl    TD_CRITCOUNT(%ebx)
351         decl    PCPU(intr_nesting_level)
352         addl    $12,%esp
353         pushl   $0                      /* CPL for frame (REMOVED) */
354         MEXITCOUNT
355         jmp     doreti
356 1:
357         orl     $RQF_TIMER,PCPU(reqflags)
358         MEXITCOUNT
359         jmp     doreti_syscall_ret
360
361 MCOUNT_LABEL(bintr)
362         INTR_HANDLER(0)
363         INTR_HANDLER(1)
364         INTR_HANDLER(2)
365         INTR_HANDLER(3)
366         INTR_HANDLER(4)
367         INTR_HANDLER(5)
368         INTR_HANDLER(6)
369         INTR_HANDLER(7)
370         INTR_HANDLER(8)
371         INTR_HANDLER(9)
372         INTR_HANDLER(10)
373         INTR_HANDLER(11)
374         INTR_HANDLER(12)
375         INTR_HANDLER(13)
376         INTR_HANDLER(14)
377         INTR_HANDLER(15)
378         INTR_HANDLER(16)
379         INTR_HANDLER(17)
380         INTR_HANDLER(18)
381         INTR_HANDLER(19)
382         INTR_HANDLER(20)
383         INTR_HANDLER(21)
384         INTR_HANDLER(22)
385         INTR_HANDLER(23)
386         INTR_HANDLER(24)
387         INTR_HANDLER(25)
388         INTR_HANDLER(26)
389         INTR_HANDLER(27)
390         INTR_HANDLER(28)
391         INTR_HANDLER(29)
392         INTR_HANDLER(30)
393         INTR_HANDLER(31)
394         INTR_HANDLER(32)
395         INTR_HANDLER(33)
396         INTR_HANDLER(34)
397         INTR_HANDLER(35)
398         INTR_HANDLER(36)
399         INTR_HANDLER(37)
400         INTR_HANDLER(38)
401         INTR_HANDLER(39)
402         INTR_HANDLER(40)
403         INTR_HANDLER(41)
404         INTR_HANDLER(42)
405         INTR_HANDLER(43)
406         INTR_HANDLER(44)
407         INTR_HANDLER(45)
408         INTR_HANDLER(46)
409         INTR_HANDLER(47)
410         INTR_HANDLER(48)
411         INTR_HANDLER(49)
412         INTR_HANDLER(50)
413         INTR_HANDLER(51)
414         INTR_HANDLER(52)
415         INTR_HANDLER(53)
416         INTR_HANDLER(54)
417         INTR_HANDLER(55)
418         INTR_HANDLER(56)
419         INTR_HANDLER(57)
420         INTR_HANDLER(58)
421         INTR_HANDLER(59)
422         INTR_HANDLER(60)
423         INTR_HANDLER(61)
424         INTR_HANDLER(62)
425         INTR_HANDLER(63)
426         INTR_HANDLER(64)
427         INTR_HANDLER(65)
428         INTR_HANDLER(66)
429         INTR_HANDLER(67)
430         INTR_HANDLER(68)
431         INTR_HANDLER(69)
432         INTR_HANDLER(70)
433         INTR_HANDLER(71)
434         INTR_HANDLER(72)
435         INTR_HANDLER(73)
436         INTR_HANDLER(74)
437         INTR_HANDLER(75)
438         INTR_HANDLER(76)
439         INTR_HANDLER(77)
440         INTR_HANDLER(78)
441         INTR_HANDLER(79)
442         INTR_HANDLER(80)
443         INTR_HANDLER(81)
444         INTR_HANDLER(82)
445         INTR_HANDLER(83)
446         INTR_HANDLER(84)
447         INTR_HANDLER(85)
448         INTR_HANDLER(86)
449         INTR_HANDLER(87)
450         INTR_HANDLER(88)
451         INTR_HANDLER(89)
452         INTR_HANDLER(90)
453         INTR_HANDLER(91)
454         INTR_HANDLER(92)
455         INTR_HANDLER(93)
456         INTR_HANDLER(94)
457         INTR_HANDLER(95)
458         INTR_HANDLER(96)
459         INTR_HANDLER(97)
460         INTR_HANDLER(98)
461         INTR_HANDLER(99)
462         INTR_HANDLER(100)
463         INTR_HANDLER(101)
464         INTR_HANDLER(102)
465         INTR_HANDLER(103)
466         INTR_HANDLER(104)
467         INTR_HANDLER(105)
468         INTR_HANDLER(106)
469         INTR_HANDLER(107)
470         INTR_HANDLER(108)
471         INTR_HANDLER(109)
472         INTR_HANDLER(110)
473         INTR_HANDLER(111)
474         INTR_HANDLER(112)
475         INTR_HANDLER(113)
476         INTR_HANDLER(114)
477         INTR_HANDLER(115)
478         INTR_HANDLER(116)
479         INTR_HANDLER(117)
480         INTR_HANDLER(118)
481         INTR_HANDLER(119)
482         INTR_HANDLER(120)
483         INTR_HANDLER(121)
484         INTR_HANDLER(122)
485         INTR_HANDLER(123)
486         INTR_HANDLER(124)
487         INTR_HANDLER(125)
488         INTR_HANDLER(126)
489         INTR_HANDLER(127)
490         INTR_HANDLER(128)
491         INTR_HANDLER(129)
492         INTR_HANDLER(130)
493         INTR_HANDLER(131)
494         INTR_HANDLER(132)
495         INTR_HANDLER(133)
496         INTR_HANDLER(134)
497         INTR_HANDLER(135)
498         INTR_HANDLER(136)
499         INTR_HANDLER(137)
500         INTR_HANDLER(138)
501         INTR_HANDLER(139)
502         INTR_HANDLER(140)
503         INTR_HANDLER(141)
504         INTR_HANDLER(142)
505         INTR_HANDLER(143)
506         INTR_HANDLER(144)
507         INTR_HANDLER(145)
508         INTR_HANDLER(146)
509         INTR_HANDLER(147)
510         INTR_HANDLER(148)
511         INTR_HANDLER(149)
512         INTR_HANDLER(150)
513         INTR_HANDLER(151)
514         INTR_HANDLER(152)
515         INTR_HANDLER(153)
516         INTR_HANDLER(154)
517         INTR_HANDLER(155)
518         INTR_HANDLER(156)
519         INTR_HANDLER(157)
520         INTR_HANDLER(158)
521         INTR_HANDLER(159)
522         INTR_HANDLER(160)
523         INTR_HANDLER(161)
524         INTR_HANDLER(162)
525         INTR_HANDLER(163)
526         INTR_HANDLER(164)
527         INTR_HANDLER(165)
528         INTR_HANDLER(166)
529         INTR_HANDLER(167)
530         INTR_HANDLER(168)
531         INTR_HANDLER(169)
532         INTR_HANDLER(170)
533         INTR_HANDLER(171)
534         INTR_HANDLER(172)
535         INTR_HANDLER(173)
536         INTR_HANDLER(174)
537         INTR_HANDLER(175)
538         INTR_HANDLER(176)
539         INTR_HANDLER(177)
540         INTR_HANDLER(178)
541         INTR_HANDLER(179)
542         INTR_HANDLER(180)
543         INTR_HANDLER(181)
544         INTR_HANDLER(182)
545         INTR_HANDLER(183)
546         INTR_HANDLER(184)
547         INTR_HANDLER(185)
548         INTR_HANDLER(186)
549         INTR_HANDLER(187)
550         INTR_HANDLER(188)
551         INTR_HANDLER(189)
552         INTR_HANDLER(190)
553         INTR_HANDLER(191)
554 MCOUNT_LABEL(eintr)
555
556         .data
557
558 /* variables used by stop_cpus()/restart_cpus()/Xcpustop */
559         .globl stopped_cpus, started_cpus
560 stopped_cpus:
561         .long   0
562 started_cpus:
563         .long   0
564
565         .globl CNAME(cpustop_restartfunc)
566 CNAME(cpustop_restartfunc):
567         .long 0
568                 
569         .text
570