sys/platform/pc32/apic/apic_vector.s

   1 /*
   2  *      from: vector.s, 386BSD 0.1 unknown origin
   3  * $FreeBSD: src/sys/i386/isa/apic_vector.s,v 1.47.2.5 2001/09/01 22:33:38 tegge Exp $
   4  */
   5
   6 #include "opt_auto_eoi.h"
   7
   8 #include <machine/asmacros.h>
   9 #include <machine/lock.h>
  10 #include <machine/psl.h>
  11 #include <machine/trap.h>
  12
  13 #include <machine_base/icu/icu.h>
  14 #include <bus/isa/isa.h>
  15
  16 #include "assym.s"
  17
  18 #include "apicreg.h"
  19 #include "apic_ipl.h"
  20 #include <machine/smp.h>
  21 #include <machine_base/isa/intr_machdep.h>
  22
  23 /* convert an absolute IRQ# into a bitmask */
  24 #define IRQ_LBIT(irq_num)       (1 << (irq_num))
  25
  26 /* make an index into the IO APIC from the IRQ# */
  27 #define REDTBL_IDX(irq_num)     (0x10 + ((irq_num) * 2))
  28
  29 #ifdef SMP
  30 #define MPLOCKED     lock ;
  31 #else
  32 #define MPLOCKED
  33 #endif
  34
  35 /*
  36  * Push an interrupt frame in a format acceptable to doreti, reload
  37  * the segment registers for the kernel.
  38  */
  39 #define PUSH_FRAME                                                      \
  40         pushl   $0 ;            /* dummy error code */                  \
  41         pushl   $0 ;            /* dummy trap type */                   \
  42         pushl   $0 ;            /* dummy xflags type */                 \
  43         pushal ;                                                        \
  44         pushl   %ds ;           /* save data and extra segments ... */  \
  45         pushl   %es ;                                                   \
  46         pushl   %fs ;                                                   \
  47         pushl   %gs ;                                                   \
  48         cld ;                                                           \
  49         mov     $KDSEL,%ax ;                                            \
  50         mov     %ax,%ds ;                                               \
  51         mov     %ax,%es ;                                               \
  52         mov     %ax,%gs ;                                               \
  53         mov     $KPSEL,%ax ;                                            \
  54         mov     %ax,%fs ;                                               \
  55
  56 /*
  57  * Warning: POP_FRAME can only be used if there is no chance of a
  58  * segment register being changed (e.g. by procfs), which is why syscalls
  59  * have to use doreti.
  60  */
  61 #define POP_FRAME                                                       \
  62         popl    %gs ;                                                   \
  63         popl    %fs ;                                                   \
  64         popl    %es ;                                                   \
  65         popl    %ds ;                                                   \
  66         popal ;                                                         \
  67         addl    $3*4,%esp ;     /* dummy xflags, trap & error codes */  \
  68
  69 #define IOAPICADDR(irq_num) \
  70         CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_ADDR
  71 #define REDIRIDX(irq_num) \
  72         CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_ENTIDX
  73 #define IOAPICFLAGS(irq_num) \
  74         CNAME(int_to_apicintpin) + IOAPIC_IM_SIZE * (irq_num) + IOAPIC_IM_FLAGS
  75
  76 #define MASK_IRQ(irq_num)                                               \
  77         APIC_IMASK_LOCK ;                       /* into critical reg */ \
  78         testl   $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ;          \
  79         jne     7f ;                    /* masked, don't mask */        \
  80         orl     $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ;          \
  81                                                 /* set the mask bit */  \
  82         movl    IOAPICADDR(irq_num), %ecx ;     /* ioapic addr */       \
  83         movl    REDIRIDX(irq_num), %eax ;       /* get the index */     \
  84         movl    %eax, (%ecx) ;                  /* write the index */   \
  85         orl     $IOART_INTMASK,IOAPIC_WINDOW(%ecx) ;/* set the mask */  \
  86 7: ;                                            /* already masked */    \
  87         APIC_IMASK_UNLOCK ;                                             \
  88
  89 /*
  90  * Test to see whether we are handling an edge or level triggered INT.
  91  *  Level-triggered INTs must still be masked as we don't clear the source,
  92  *  and the EOI cycle would cause redundant INTs to occur.
  93  */
  94 #define MASK_LEVEL_IRQ(irq_num)                                         \
  95         testl   $IOAPIC_IM_FLAG_LEVEL, IOAPICFLAGS(irq_num) ;           \
  96         jz      9f ;                            /* edge, don't mask */  \
  97         MASK_IRQ(irq_num) ;                                             \
  98 9: ;                                                                    \
  99
 100 /*
 101  * Test to see if the source is currntly masked, clear if so.
 102  */
 103 #define UNMASK_IRQ(irq_num)                                             \
 104         cmpl    $0,%eax ;                                               \
 105         jnz     8f ;                                                    \
 106         APIC_IMASK_LOCK ;                       /* into critical reg */ \
 107         testl   $IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ;          \
 108         je      7f ;                    /* bit clear, not masked */     \
 109         andl    $~IOAPIC_IM_FLAG_MASKED, IOAPICFLAGS(irq_num) ;         \
 110                                                 /* clear mask bit */    \
 111         movl    IOAPICADDR(irq_num),%ecx ;      /* ioapic addr */       \
 112         movl    REDIRIDX(irq_num), %eax ;       /* get the index */     \
 113         movl    %eax,(%ecx) ;                   /* write the index */   \
 114         andl    $~IOART_INTMASK,IOAPIC_WINDOW(%ecx) ;/* clear the mask */ \
 115 7: ;                                                                    \
 116         APIC_IMASK_UNLOCK ;                                             \
 117 8: ;                                                                    \
 118
 119 #ifdef SMP /* APIC-IO */
 120
 121 /*
 122  * Fast interrupt call handlers run in the following sequence:
 123  *
 124  *      - Push the trap frame required by doreti
 125  *      - Mask the interrupt and reenable its source
 126  *      - If we cannot take the interrupt set its fpending bit and
 127  *        doreti.  Note that we cannot mess with mp_lock at all
 128  *        if we entered from a critical section!
 129  *      - If we can take the interrupt clear its fpending bit,
 130  *        call the handler, then unmask and doreti.
 131  *
 132  * YYY can cache gd base opitner instead of using hidden %fs prefixes.
 133  */
 134
 135 #define FAST_INTR(irq_num, vec_name)                                    \
 136         .text ;                                                         \
 137         SUPERALIGN_TEXT ;                                               \
 138 IDTVEC(vec_name) ;                                                      \
 139         PUSH_FRAME ;                                                    \
 140         FAKE_MCOUNT(15*4(%esp)) ;                                       \
 141         MASK_LEVEL_IRQ(irq_num) ;                                       \
 142         movl    $0, lapic_eoi ;                                         \
 143         movl    PCPU(curthread),%ebx ;                                  \
 144         movl    $0,%eax ;       /* CURRENT CPL IN FRAME (REMOVED) */    \
 145         pushl   %eax ;                                                  \
 146         testl   $-1,TD_NEST_COUNT(%ebx) ;                               \
 147         jne     1f ;                                                    \
 148         testl   $-1,TD_CRITCOUNT(%ebx) ;                                \
 149         je      2f ;                                                    \
 150 1: ;                                                                    \
 151         /* in critical section, make interrupt pending */               \
 152         /* set the pending bit and return, leave interrupt masked */    \
 153         orl     $IRQ_LBIT(irq_num),PCPU(fpending) ;                     \
 154         orl     $RQF_INTPEND,PCPU(reqflags) ;                           \
 155         jmp     5f ;                                                    \
 156 2: ;                                                                    \
 157         /* clear pending bit, run handler */                            \
 158         andl    $~IRQ_LBIT(irq_num),PCPU(fpending) ;                    \
 159         pushl   $irq_num ;                                              \
 160         pushl   %esp ;                   /* pass frame by reference */  \
 161         incl    TD_CRITCOUNT(%ebx) ;                                    \
 162         sti ;                                                           \
 163         call    ithread_fast_handler ;   /* returns 0 to unmask */      \
 164         decl    TD_CRITCOUNT(%ebx) ;                                    \
 165         addl    $8, %esp ;                                              \
 166         UNMASK_IRQ(irq_num) ;                                           \
 167 5: ;                                                                    \
 168         MEXITCOUNT ;                                                    \
 169         jmp     doreti ;                                                \
 170
 171 #endif
 172
 173 /*
 174  * Handle "spurious INTerrupts".
 175  * Notes:
 176  *  This is different than the "spurious INTerrupt" generated by an
 177  *   8259 PIC for missing INTs.  See the APIC documentation for details.
 178  *  This routine should NOT do an 'EOI' cycle.
 179  */
 180         .text
 181         SUPERALIGN_TEXT
 182         .globl Xspuriousint
 183 Xspuriousint:
 184
 185         /* No EOI cycle used here */
 186
 187         iret
 188
 189
 190 /*
 191  * Handle TLB shootdowns.
 192  *
 193  * NOTE: Interrupts remain disabled.
 194  */
 195         .text
 196         SUPERALIGN_TEXT
 197         .globl  Xinvltlb
 198 Xinvltlb:
 199         PUSH_FRAME
 200         movl    $0, lapic_eoi           /* End Of Interrupt to APIC */
 201         FAKE_MCOUNT(15*4(%esp))
 202
 203         subl    $8,%esp                 /* make same as interrupt frame */
 204         pushl   %esp                    /* pass frame by reference */
 205         call    smp_invltlb_intr
 206         addl    $12,%esp
 207
 208         MEXITCOUNT
 209         jmp     doreti_syscall_ret
 210
 211 /*
 212  * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
 213  *
 214  *  - Signals its receipt.
 215  *  - Waits for permission to restart.
 216  *  - Processing pending IPIQ events while waiting.
 217  *  - Signals its restart.
 218  */
 219
 220         .text
 221         SUPERALIGN_TEXT
 222         .globl Xcpustop
 223 Xcpustop:
 224         pushl   %ebp
 225         movl    %esp, %ebp
 226         pushl   %eax
 227         pushl   %ecx
 228         pushl   %edx
 229         pushl   %ds                     /* save current data segment */
 230         pushl   %fs
 231
 232         movl    $KDSEL, %eax
 233         mov     %ax, %ds                /* use KERNEL data segment */
 234         movl    $KPSEL, %eax
 235         mov     %ax, %fs
 236
 237         movl    $0, lapic_eoi           /* End Of Interrupt to APIC */
 238
 239         movl    PCPU(cpuid), %eax
 240         imull   $PCB_SIZE, %eax
 241         leal    CNAME(stoppcbs)(%eax), %eax
 242         pushl   %eax
 243         call    CNAME(savectx)          /* Save process context */
 244         addl    $4, %esp
 245
 246
 247         movl    PCPU(cpuid), %eax
 248
 249         /*
 250          * Indicate that we have stopped and loop waiting for permission
 251          * to start again.  We must still process IPI events while in a
 252          * stopped state.
 253          *
 254          * Interrupts must remain enabled for non-IPI'd per-cpu interrupts
 255          * (e.g. Xtimer, Xinvltlb).
 256          */
 257         MPLOCKED
 258         btsl    %eax, stopped_cpus      /* stopped_cpus |= (1<<id) */
 259         sti
 260 1:
 261         andl    $~RQF_IPIQ,PCPU(reqflags)
 262         pushl   %eax
 263         call    lwkt_smp_stopped
 264         popl    %eax
 265         btl     %eax, started_cpus      /* while (!(started_cpus & (1<<id))) */
 266         jnc     1b
 267
 268         MPLOCKED
 269         btrl    %eax, started_cpus      /* started_cpus &= ~(1<<id) */
 270         MPLOCKED
 271         btrl    %eax, stopped_cpus      /* stopped_cpus &= ~(1<<id) */
 272
 273         test    %eax, %eax
 274         jnz     2f
 275
 276         movl    CNAME(cpustop_restartfunc), %eax
 277         test    %eax, %eax
 278         jz      2f
 279         movl    $0, CNAME(cpustop_restartfunc)  /* One-shot */
 280
 281         call    *%eax
 282 2:
 283         popl    %fs
 284         popl    %ds                     /* restore previous data segment */
 285         popl    %edx
 286         popl    %ecx
 287         popl    %eax
 288         movl    %ebp, %esp
 289         popl    %ebp
 290         iret
 291
 292         /*
 293          * For now just have one ipiq IPI, but what we really want is
 294          * to have one for each source cpu to the APICs don't get stalled
 295          * backlogging the requests.
 296          */
 297         .text
 298         SUPERALIGN_TEXT
 299         .globl Xipiq
 300 Xipiq:
 301         PUSH_FRAME
 302         movl    $0, lapic_eoi           /* End Of Interrupt to APIC */
 303         FAKE_MCOUNT(15*4(%esp))
 304
 305         incl    PCPU(cnt) + V_IPI
 306         movl    PCPU(curthread),%ebx
 307         testl   $-1,TD_CRITCOUNT(%ebx)
 308         jne     1f
 309         subl    $8,%esp                 /* make same as interrupt frame */
 310         pushl   %esp                    /* pass frame by reference */
 311         incl    PCPU(intr_nesting_level)
 312         incl    TD_CRITCOUNT(%ebx)
 313         sti
 314         call    lwkt_process_ipiq_frame
 315         decl    TD_CRITCOUNT(%ebx)
 316         decl    PCPU(intr_nesting_level)
 317         addl    $12,%esp
 318         pushl   $0                      /* CPL for frame (REMOVED) */
 319         MEXITCOUNT
 320         jmp     doreti
 321 1:
 322         orl     $RQF_IPIQ,PCPU(reqflags)
 323         MEXITCOUNT
 324         jmp     doreti_syscall_ret
 325
 326         .text
 327         SUPERALIGN_TEXT
 328         .globl Xtimer
 329 Xtimer:
 330         PUSH_FRAME
 331         movl    $0, lapic_eoi           /* End Of Interrupt to APIC */
 332         FAKE_MCOUNT(15*4(%esp))
 333
 334         incl    PCPU(cnt) + V_TIMER
 335         movl    PCPU(curthread),%ebx
 336         testl   $-1,TD_CRITCOUNT(%ebx)
 337         jne     1f
 338         testl   $-1,TD_NEST_COUNT(%ebx)
 339         jne     1f
 340         subl    $8,%esp                 /* make same as interrupt frame */
 341         pushl   %esp                    /* pass frame by reference */
 342         incl    PCPU(intr_nesting_level)
 343         incl    TD_CRITCOUNT(%ebx)
 344         sti
 345         call    lapic_timer_process_frame
 346         decl    TD_CRITCOUNT(%ebx)
 347         decl    PCPU(intr_nesting_level)
 348         addl    $12,%esp
 349         pushl   $0                      /* CPL for frame (REMOVED) */
 350         MEXITCOUNT
 351         jmp     doreti
 352 1:
 353         orl     $RQF_TIMER,PCPU(reqflags)
 354         MEXITCOUNT
 355         jmp     doreti_syscall_ret
 356
 357 #ifdef SMP /* APIC-IO */
 358
 359 MCOUNT_LABEL(bintr)
 360         FAST_INTR(0,apic_fastintr0)
 361         FAST_INTR(1,apic_fastintr1)
 362         FAST_INTR(2,apic_fastintr2)
 363         FAST_INTR(3,apic_fastintr3)
 364         FAST_INTR(4,apic_fastintr4)
 365         FAST_INTR(5,apic_fastintr5)
 366         FAST_INTR(6,apic_fastintr6)
 367         FAST_INTR(7,apic_fastintr7)
 368         FAST_INTR(8,apic_fastintr8)
 369         FAST_INTR(9,apic_fastintr9)
 370         FAST_INTR(10,apic_fastintr10)
 371         FAST_INTR(11,apic_fastintr11)
 372         FAST_INTR(12,apic_fastintr12)
 373         FAST_INTR(13,apic_fastintr13)
 374         FAST_INTR(14,apic_fastintr14)
 375         FAST_INTR(15,apic_fastintr15)
 376         FAST_INTR(16,apic_fastintr16)
 377         FAST_INTR(17,apic_fastintr17)
 378         FAST_INTR(18,apic_fastintr18)
 379         FAST_INTR(19,apic_fastintr19)
 380         FAST_INTR(20,apic_fastintr20)
 381         FAST_INTR(21,apic_fastintr21)
 382         FAST_INTR(22,apic_fastintr22)
 383         FAST_INTR(23,apic_fastintr23)
 384         FAST_INTR(24,apic_fastintr24)
 385         FAST_INTR(25,apic_fastintr25)
 386         FAST_INTR(26,apic_fastintr26)
 387         FAST_INTR(27,apic_fastintr27)
 388         FAST_INTR(28,apic_fastintr28)
 389         FAST_INTR(29,apic_fastintr29)
 390         FAST_INTR(30,apic_fastintr30)
 391         FAST_INTR(31,apic_fastintr31)
 392 MCOUNT_LABEL(eintr)
 393
 394 #endif
 395
 396         .data
 397
 398 /* variables used by stop_cpus()/restart_cpus()/Xcpustop */
 399         .globl stopped_cpus, started_cpus
 400 stopped_cpus:
 401         .long   0
 402 started_cpus:
 403         .long   0
 404
 405         .globl CNAME(cpustop_restartfunc)
 406 CNAME(cpustop_restartfunc):
 407         .long 0
 408
 409         .text
 410