Merge branch 'vendor/OPENSSH' (early part)
[dragonfly.git] / sys / platform / pc64 / apic / apic_vector.s
1 /*
2  * Copyright (c) 2008 The DragonFly Project.  All rights reserved.
3  * 
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in
12  *    the documentation and/or other materials provided with the
13  *    distribution.
14  * 3. Neither the name of The DragonFly Project nor the names of its
15  *    contributors may be used to endorse or promote products derived
16  *    from this software without specific, prior written permission.
17  * 
18  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
21  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
22  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
23  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
24  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
25  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
27  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
28  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29  * SUCH DAMAGE.
30  * 
31  * from: vector.s, 386BSD 0.1 unknown origin
32  * $FreeBSD: src/sys/i386/isa/apic_vector.s,v 1.47.2.5 2001/09/01 22:33:38 tegge Exp $
33  * $DragonFly: src/sys/platform/pc64/apic/apic_vector.s,v 1.1 2008/08/29 17:07:12 dillon Exp $
34  */
35
36 #include "use_npx.h"
37 #include "opt_auto_eoi.h"
38
39 #include <machine/asmacros.h>
40 #include <machine/lock.h>
41 #include <machine/psl.h>
42 #include <machine/trap.h>
43
44 #include <machine_base/icu/icu.h>
45 #include <bus/isa/isa.h>
46
47 #include "assym.s"
48
49 #include "apicreg.h"
50 #include "apic_ipl.h"
51 #include <machine/smp.h>
52 #include <machine_base/isa/intr_machdep.h>
53
54 /* convert an absolute IRQ# into a bitmask */
55 #define IRQ_LBIT(irq_num)       (1 << (irq_num))
56
57 /* make an index into the IO APIC from the IRQ# */
58 #define REDTBL_IDX(irq_num)     (0x10 + ((irq_num) * 2))
59
60 #ifdef SMP
61 #define MPLOCKED     lock ;
62 #else
63 #define MPLOCKED
64 #endif
65
66 /*
67  * Push an interrupt frame in a format acceptable to doreti, reload
68  * the segment registers for the kernel.
69  */
70 #define PUSH_FRAME                                                      \
71         pushl   $0 ;            /* dummy error code */                  \
72         pushl   $0 ;            /* dummy trap type */                   \
73         pushl   $0 ;            /* dummy xflags type */                 \
74         pushal ;                                                        \
75         pushl   %ds ;           /* save data and extra segments ... */  \
76         pushl   %es ;                                                   \
77         pushl   %fs ;                                                   \
78         pushl   %gs ;                                                   \
79         cld ;                                                           \
80         mov     $KDSEL,%ax ;                                            \
81         mov     %ax,%ds ;                                               \
82         mov     %ax,%es ;                                               \
83         mov     %ax,%gs ;                                               \
84         mov     $KPSEL,%ax ;                                            \
85         mov     %ax,%fs ;                                               \
86
87 #define PUSH_DUMMY                                                      \
88         pushfl ;                /* phys int frame / flags */            \
89         pushl %cs ;             /* phys int frame / cs */               \
90         pushl   12(%esp) ;      /* original caller eip */               \
91         pushl   $0 ;            /* dummy error code */                  \
92         pushl   $0 ;            /* dummy trap type */                   \
93         pushl   $0 ;            /* dummy xflags type */                 \
94         subl    $13*4,%esp ;    /* pushal + 4 seg regs (dummy) + CPL */ \
95
96 /*
97  * Warning: POP_FRAME can only be used if there is no chance of a
98  * segment register being changed (e.g. by procfs), which is why syscalls
99  * have to use doreti.
100  */
101 #define POP_FRAME                                                       \
102         popl    %gs ;                                                   \
103         popl    %fs ;                                                   \
104         popl    %es ;                                                   \
105         popl    %ds ;                                                   \
106         popal ;                                                         \
107         addl    $3*4,%esp ;     /* dummy xflags, trap & error codes */  \
108
109 #define POP_DUMMY                                                       \
110         addl    $19*4,%esp ;                                            \
111
112 #define IOAPICADDR(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 8
113 #define REDIRIDX(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 12
114
115 #define MASK_IRQ(irq_num)                                               \
116         APIC_IMASK_LOCK ;                       /* into critical reg */ \
117         testl   $IRQ_LBIT(irq_num), apic_imen ;                         \
118         jne     7f ;                    /* masked, don't mask */        \
119         orl     $IRQ_LBIT(irq_num), apic_imen ; /* set the mask bit */  \
120         movl    IOAPICADDR(irq_num), %ecx ;     /* ioapic addr */       \
121         movl    REDIRIDX(irq_num), %eax ;       /* get the index */     \
122         movl    %eax, (%ecx) ;                  /* write the index */   \
123         movl    IOAPIC_WINDOW(%ecx), %eax ;     /* current value */     \
124         orl     $IOART_INTMASK, %eax ;          /* set the mask */      \
125         movl    %eax, IOAPIC_WINDOW(%ecx) ;     /* new value */         \
126 7: ;                                            /* already masked */    \
127         APIC_IMASK_UNLOCK ;                                             \
128
129 /*
130  * Test to see whether we are handling an edge or level triggered INT.
131  *  Level-triggered INTs must still be masked as we don't clear the source,
132  *  and the EOI cycle would cause redundant INTs to occur.
133  */
134 #define MASK_LEVEL_IRQ(irq_num)                                         \
135         testl   $IRQ_LBIT(irq_num), apic_pin_trigger ;                  \
136         jz      9f ;                            /* edge, don't mask */  \
137         MASK_IRQ(irq_num) ;                                             \
138 9: ;                                                                    \
139
140 /*
141  * Test to see if the source is currntly masked, clear if so.
142  */
143 #define UNMASK_IRQ(irq_num)                                     \
144         cmpl    $0,%eax ;                                               \
145         jnz     8f ;                                                    \
146         APIC_IMASK_LOCK ;                       /* into critical reg */ \
147         testl   $IRQ_LBIT(irq_num), apic_imen ;                         \
148         je      7f ;                    /* bit clear, not masked */     \
149         andl    $~IRQ_LBIT(irq_num), apic_imen ;/* clear mask bit */    \
150         movl    IOAPICADDR(irq_num),%ecx ;      /* ioapic addr */       \
151         movl    REDIRIDX(irq_num), %eax ;       /* get the index */     \
152         movl    %eax,(%ecx) ;                   /* write the index */   \
153         movl    IOAPIC_WINDOW(%ecx),%eax ;      /* current value */     \
154         andl    $~IOART_INTMASK,%eax ;          /* clear the mask */    \
155         movl    %eax,IOAPIC_WINDOW(%ecx) ;      /* new value */         \
156 7: ;                                                                    \
157         APIC_IMASK_UNLOCK ;                                             \
158 8: ;                                                                    \
159
160 #ifdef APIC_IO
161
162 /*
163  * Fast interrupt call handlers run in the following sequence:
164  *
165  *      - Push the trap frame required by doreti
166  *      - Mask the interrupt and reenable its source
167  *      - If we cannot take the interrupt set its fpending bit and
168  *        doreti.  Note that we cannot mess with mp_lock at all
169  *        if we entered from a critical section!
170  *      - If we can take the interrupt clear its fpending bit,
171  *        call the handler, then unmask and doreti.
172  *
173  * YYY can cache gd base opitner instead of using hidden %fs prefixes.
174  */
175
176 #define FAST_INTR(irq_num, vec_name)                                    \
177         .text ;                                                         \
178         SUPERALIGN_TEXT ;                                               \
179 IDTVEC(vec_name) ;                                                      \
180         PUSH_FRAME ;                                                    \
181         FAKE_MCOUNT(15*4(%esp)) ;                                       \
182         MASK_LEVEL_IRQ(irq_num) ;                                       \
183         movl    $0, lapic_eoi ;                                         \
184         movl    PCPU(curthread),%ebx ;                                  \
185         movl    $0,%eax ;       /* CURRENT CPL IN FRAME (REMOVED) */    \
186         pushl   %eax ;                                                  \
187         testl   $-1,TD_NEST_COUNT(%ebx) ;                               \
188         jne     1f ;                                                    \
189         cmpl    $TDPRI_CRIT,TD_PRI(%ebx) ;                              \
190         jl      2f ;                                                    \
191 1: ;                                                                    \
192         /* in critical section, make interrupt pending */               \
193         /* set the pending bit and return, leave interrupt masked */    \
194         orl     $IRQ_LBIT(irq_num),PCPU(fpending) ;                     \
195         orl     $RQF_INTPEND,PCPU(reqflags) ;                           \
196         jmp     5f ;                                                    \
197 2: ;                                                                    \
198         /* clear pending bit, run handler */                            \
199         andl    $~IRQ_LBIT(irq_num),PCPU(fpending) ;                    \
200         pushl   $irq_num ;                                              \
201         pushl   %esp ;                   /* pass frame by reference */  \
202         call    ithread_fast_handler ;   /* returns 0 to unmask */      \
203         addl    $8, %esp ;                                              \
204         UNMASK_IRQ(irq_num) ;                                           \
205 5: ;                                                                    \
206         MEXITCOUNT ;                                                    \
207         jmp     doreti ;                                                \
208
209 /*
210  * Slow interrupt call handlers run in the following sequence:
211  *
212  *      - Push the trap frame required by doreti.
213  *      - Mask the interrupt and reenable its source.
214  *      - If we cannot take the interrupt set its ipending bit and
215  *        doreti.  In addition to checking for a critical section
216  *        and cpl mask we also check to see if the thread is still
217  *        running.  Note that we cannot mess with mp_lock at all
218  *        if we entered from a critical section!
219  *      - If we can take the interrupt clear its ipending bit
220  *        and schedule the thread.  Leave interrupts masked and doreti.
221  *
222  *      Note that calls to sched_ithd() are made with interrupts enabled
223  *      and outside a critical section.  YYY sched_ithd may preempt us
224  *      synchronously (fix interrupt stacking).
225  *
226  *      YYY can cache gd base pointer instead of using hidden %fs
227  *      prefixes.
228  */
229
230 #define SLOW_INTR(irq_num, vec_name, maybe_extra_ipending)              \
231         .text ;                                                         \
232         SUPERALIGN_TEXT ;                                               \
233 IDTVEC(vec_name) ;                                                      \
234         PUSH_FRAME ;                                                    \
235         maybe_extra_ipending ;                                          \
236 ;                                                                       \
237         MASK_LEVEL_IRQ(irq_num) ;                                       \
238         incl    PCPU(cnt) + V_INTR ;                                    \
239         movl    $0, lapic_eoi ;                                         \
240         movl    PCPU(curthread),%ebx ;                                  \
241         movl    $0,%eax ;       /* CURRENT CPL IN FRAME (REMOVED) */    \
242         pushl   %eax ;          /* cpl do restore */                    \
243         testl   $-1,TD_NEST_COUNT(%ebx) ;                               \
244         jne     1f ;                                                    \
245         cmpl    $TDPRI_CRIT,TD_PRI(%ebx) ;                              \
246         jl      2f ;                                                    \
247 1: ;                                                                    \
248         /* set the pending bit and return, leave the interrupt masked */ \
249         orl     $IRQ_LBIT(irq_num), PCPU(ipending) ;                    \
250         orl     $RQF_INTPEND,PCPU(reqflags) ;                           \
251         jmp     5f ;                                                    \
252 2: ;                                                                    \
253         /* set running bit, clear pending bit, run handler */           \
254         andl    $~IRQ_LBIT(irq_num), PCPU(ipending) ;                   \
255         incl    TD_NEST_COUNT(%ebx) ;                                   \
256         sti ;                                                           \
257         pushl   $irq_num ;                                              \
258         call    sched_ithd ;                                            \
259         addl    $4,%esp ;                                               \
260         cli ;                                                           \
261         decl    TD_NEST_COUNT(%ebx) ;                                   \
262 5: ;                                                                    \
263         MEXITCOUNT ;                                                    \
264         jmp     doreti ;                                                \
265
266 /*
267  * Wrong interrupt call handlers.  We program these into APIC vectors
268  * that should otherwise never occur.  For example, we program the SLOW
269  * vector for irq N with this when we program the FAST vector with the
270  * real interrupt.
271  *
272  * XXX for now all we can do is EOI it.  We can't call do_wrongintr
273  * (yet) because we could be in a critical section.
274  */
275 #define WRONGINTR(irq_num,vec_name)                                     \
276         .text ;                                                         \
277         SUPERALIGN_TEXT  ;                                              \
278 IDTVEC(vec_name) ;                                                      \
279         PUSH_FRAME ;                                                    \
280         movl    $0, lapic_eoi ; /* End Of Interrupt to APIC */          \
281         /*pushl $irq_num ;*/                                            \
282         /*call  do_wrongintr ;*/                                        \
283         /*addl  $4,%esp ;*/                                             \
284         POP_FRAME ;                                                     \
285         iret  ;                                                         \
286
287 #endif
288
289 /*
290  * Handle "spurious INTerrupts".
291  * Notes:
292  *  This is different than the "spurious INTerrupt" generated by an
293  *   8259 PIC for missing INTs.  See the APIC documentation for details.
294  *  This routine should NOT do an 'EOI' cycle.
295  */
296         .text
297         SUPERALIGN_TEXT
298         .globl Xspuriousint
299 Xspuriousint:
300
301         /* No EOI cycle used here */
302
303         iret
304
305
306 /*
307  * Handle TLB shootdowns.
308  */
309         .text
310         SUPERALIGN_TEXT
311         .globl  Xinvltlb
312 Xinvltlb:
313         pushl   %eax
314
315         movl    %cr3, %eax              /* invalidate the TLB */
316         movl    %eax, %cr3
317
318         ss                              /* stack segment, avoid %ds load */
319         movl    $0, lapic_eoi           /* End Of Interrupt to APIC */
320
321         popl    %eax
322         iret
323
324
325 /*
326  * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
327  *
328  *  - Signals its receipt.
329  *  - Waits for permission to restart.
330  *  - Processing pending IPIQ events while waiting.
331  *  - Signals its restart.
332  */
333
334         .text
335         SUPERALIGN_TEXT
336         .globl Xcpustop
337 Xcpustop:
338         pushl   %ebp
339         movl    %esp, %ebp
340         pushl   %eax
341         pushl   %ecx
342         pushl   %edx
343         pushl   %ds                     /* save current data segment */
344         pushl   %fs
345
346         movl    $KDSEL, %eax
347         mov     %ax, %ds                /* use KERNEL data segment */
348         movl    $KPSEL, %eax
349         mov     %ax, %fs
350
351         movl    $0, lapic_eoi           /* End Of Interrupt to APIC */
352
353         movl    PCPU(cpuid), %eax
354         imull   $PCB_SIZE, %eax
355         leal    CNAME(stoppcbs)(%eax), %eax
356         pushl   %eax
357         call    CNAME(savectx)          /* Save process context */
358         addl    $4, %esp
359         
360                 
361         movl    PCPU(cpuid), %eax
362
363         /*
364          * Indicate that we have stopped and loop waiting for permission
365          * to start again.  We must still process IPI events while in a
366          * stopped state.
367          */
368         MPLOCKED
369         btsl    %eax, stopped_cpus      /* stopped_cpus |= (1<<id) */
370 1:
371         andl    $~RQF_IPIQ,PCPU(reqflags)
372         pushl   %eax
373         call    lwkt_smp_stopped
374         popl    %eax
375         btl     %eax, started_cpus      /* while (!(started_cpus & (1<<id))) */
376         jnc     1b
377
378         MPLOCKED
379         btrl    %eax, started_cpus      /* started_cpus &= ~(1<<id) */
380         MPLOCKED
381         btrl    %eax, stopped_cpus      /* stopped_cpus &= ~(1<<id) */
382
383         test    %eax, %eax
384         jnz     2f
385
386         movl    CNAME(cpustop_restartfunc), %eax
387         test    %eax, %eax
388         jz      2f
389         movl    $0, CNAME(cpustop_restartfunc)  /* One-shot */
390
391         call    *%eax
392 2:
393         popl    %fs
394         popl    %ds                     /* restore previous data segment */
395         popl    %edx
396         popl    %ecx
397         popl    %eax
398         movl    %ebp, %esp
399         popl    %ebp
400         iret
401
402         /*
403          * For now just have one ipiq IPI, but what we really want is
404          * to have one for each source cpu to the APICs don't get stalled
405          * backlogging the requests.
406          */
407         .text
408         SUPERALIGN_TEXT
409         .globl Xipiq
410 Xipiq:
411         PUSH_FRAME
412         movl    $0, lapic_eoi           /* End Of Interrupt to APIC */
413         FAKE_MCOUNT(15*4(%esp))
414
415         movl    PCPU(curthread),%ebx
416         cmpl    $TDPRI_CRIT,TD_PRI(%ebx)
417         jge     1f
418         subl    $8,%esp                 /* make same as interrupt frame */
419         pushl   %esp                    /* pass frame by reference */
420         incl    PCPU(intr_nesting_level)
421         addl    $TDPRI_CRIT,TD_PRI(%ebx)
422         call    lwkt_process_ipiq_frame
423         subl    $TDPRI_CRIT,TD_PRI(%ebx)
424         decl    PCPU(intr_nesting_level)
425         addl    $12,%esp
426         pushl   $0                      /* CPL for frame (REMOVED) */
427         MEXITCOUNT
428         jmp     doreti
429 1:
430         orl     $RQF_IPIQ,PCPU(reqflags)
431         MEXITCOUNT
432         POP_FRAME
433         iret
434
435 #ifdef APIC_IO
436
437 MCOUNT_LABEL(bintr)
438         FAST_INTR(0,apic_fastintr0)
439         FAST_INTR(1,apic_fastintr1)
440         FAST_INTR(2,apic_fastintr2)
441         FAST_INTR(3,apic_fastintr3)
442         FAST_INTR(4,apic_fastintr4)
443         FAST_INTR(5,apic_fastintr5)
444         FAST_INTR(6,apic_fastintr6)
445         FAST_INTR(7,apic_fastintr7)
446         FAST_INTR(8,apic_fastintr8)
447         FAST_INTR(9,apic_fastintr9)
448         FAST_INTR(10,apic_fastintr10)
449         FAST_INTR(11,apic_fastintr11)
450         FAST_INTR(12,apic_fastintr12)
451         FAST_INTR(13,apic_fastintr13)
452         FAST_INTR(14,apic_fastintr14)
453         FAST_INTR(15,apic_fastintr15)
454         FAST_INTR(16,apic_fastintr16)
455         FAST_INTR(17,apic_fastintr17)
456         FAST_INTR(18,apic_fastintr18)
457         FAST_INTR(19,apic_fastintr19)
458         FAST_INTR(20,apic_fastintr20)
459         FAST_INTR(21,apic_fastintr21)
460         FAST_INTR(22,apic_fastintr22)
461         FAST_INTR(23,apic_fastintr23)
462         
463         /* YYY what is this garbage? */
464
465         SLOW_INTR(0,apic_slowintr0,)
466         SLOW_INTR(1,apic_slowintr1,)
467         SLOW_INTR(2,apic_slowintr2,)
468         SLOW_INTR(3,apic_slowintr3,)
469         SLOW_INTR(4,apic_slowintr4,)
470         SLOW_INTR(5,apic_slowintr5,)
471         SLOW_INTR(6,apic_slowintr6,)
472         SLOW_INTR(7,apic_slowintr7,)
473         SLOW_INTR(8,apic_slowintr8,)
474         SLOW_INTR(9,apic_slowintr9,)
475         SLOW_INTR(10,apic_slowintr10,)
476         SLOW_INTR(11,apic_slowintr11,)
477         SLOW_INTR(12,apic_slowintr12,)
478         SLOW_INTR(13,apic_slowintr13,)
479         SLOW_INTR(14,apic_slowintr14,)
480         SLOW_INTR(15,apic_slowintr15,)
481         SLOW_INTR(16,apic_slowintr16,)
482         SLOW_INTR(17,apic_slowintr17,)
483         SLOW_INTR(18,apic_slowintr18,)
484         SLOW_INTR(19,apic_slowintr19,)
485         SLOW_INTR(20,apic_slowintr20,)
486         SLOW_INTR(21,apic_slowintr21,)
487         SLOW_INTR(22,apic_slowintr22,)
488         SLOW_INTR(23,apic_slowintr23,)
489
490         WRONGINTR(0,apic_wrongintr0)
491         WRONGINTR(1,apic_wrongintr1)
492         WRONGINTR(2,apic_wrongintr2)
493         WRONGINTR(3,apic_wrongintr3)
494         WRONGINTR(4,apic_wrongintr4)
495         WRONGINTR(5,apic_wrongintr5)
496         WRONGINTR(6,apic_wrongintr6)
497         WRONGINTR(7,apic_wrongintr7)
498         WRONGINTR(8,apic_wrongintr8)
499         WRONGINTR(9,apic_wrongintr9)
500         WRONGINTR(10,apic_wrongintr10)
501         WRONGINTR(11,apic_wrongintr11)
502         WRONGINTR(12,apic_wrongintr12)
503         WRONGINTR(13,apic_wrongintr13)
504         WRONGINTR(14,apic_wrongintr14)
505         WRONGINTR(15,apic_wrongintr15)
506         WRONGINTR(16,apic_wrongintr16)
507         WRONGINTR(17,apic_wrongintr17)
508         WRONGINTR(18,apic_wrongintr18)
509         WRONGINTR(19,apic_wrongintr19)
510         WRONGINTR(20,apic_wrongintr20)
511         WRONGINTR(21,apic_wrongintr21)
512         WRONGINTR(22,apic_wrongintr22)
513         WRONGINTR(23,apic_wrongintr23)
514 MCOUNT_LABEL(eintr)
515
516 #endif
517
518         .data
519
520 /* variables used by stop_cpus()/restart_cpus()/Xcpustop */
521         .globl stopped_cpus, started_cpus
522 stopped_cpus:
523         .long   0
524 started_cpus:
525         .long   0
526
527         .globl CNAME(cpustop_restartfunc)
528 CNAME(cpustop_restartfunc):
529         .long 0
530                 
531         .globl  apic_pin_trigger
532 apic_pin_trigger:
533         .long   0
534
535         .text
536